mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-23 06:37:15 +00:00
Getting Started: Initial configuration working on the laptop
This commit is contained in:
parent
a44bdfa7d4
commit
3dc356dc5e
7 changed files with 43 additions and 9 deletions
|
@ -66,7 +66,7 @@ if __name__ == "__main__":
|
|||
# FUNCTIONS #
|
||||
publisher.info("Script Categ started")
|
||||
|
||||
categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential']
|
||||
categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve']
|
||||
tmp_dict = {}
|
||||
for filename in categories:
|
||||
bname = os.path.basename(filename)
|
||||
|
|
|
@ -39,7 +39,7 @@ def main():
|
|||
if message is not None:
|
||||
PST = Paste.Paste(message)
|
||||
else:
|
||||
publisher.debug("Script DomClassifier is idling 10s")
|
||||
publisher.debug("Script DomClassifier is idling 1s")
|
||||
time.sleep(1)
|
||||
continue
|
||||
paste = PST.get_p_content()
|
||||
|
|
|
@ -33,11 +33,12 @@ if __name__ == "__main__":
|
|||
# DB OBJECT & HASHS ( DISK )
|
||||
# FIXME increase flexibility
|
||||
dico_redis = {}
|
||||
for year in xrange(2013, 2015):
|
||||
for year in xrange(2013, 2017):
|
||||
for month in xrange(0, 16):
|
||||
dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Level_DB", "host"), port=year,
|
||||
db=month)
|
||||
#print("dup: "+str(year)+str(month).zfill(2)+"\n")
|
||||
|
||||
# FUNCTIONS #
|
||||
publisher.info("Script duplicate started")
|
||||
|
@ -81,7 +82,7 @@ if __name__ == "__main__":
|
|||
bloop_path_set.add(filebloompath)
|
||||
|
||||
# UNIQUE INDEX HASHS TABLE
|
||||
r_serv0 = dico_redis["201300"]
|
||||
r_serv0 = dico_redis["201600"]
|
||||
r_serv0.incr("current_index")
|
||||
index = r_serv0.get("current_index")+str(PST.p_date)
|
||||
# HASHTABLES PER MONTH (because of r_serv1 changing db)
|
||||
|
|
|
@ -66,6 +66,7 @@ function launching_lvldb {
|
|||
lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
|
||||
db1_y='2013'
|
||||
db2_y='2014'
|
||||
db3_y='2016'
|
||||
nb_db=13
|
||||
|
||||
screen -dmS "LevelDB"
|
||||
|
@ -75,6 +76,8 @@ function launching_lvldb {
|
|||
screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
|
||||
sleep 0.1
|
||||
screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
|
||||
sleep 0.1
|
||||
screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
|
||||
}
|
||||
|
||||
function launching_logs {
|
||||
|
@ -104,11 +107,11 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x'
|
||||
#screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Line" bash -c './Line.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x'
|
||||
#screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x'
|
||||
sleep 0.1
|
||||
|
@ -127,6 +130,16 @@ function launching_scripts {
|
|||
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
|
||||
sleep 0.1
|
||||
#screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
|
||||
}
|
||||
|
||||
#If no params, display the help
|
||||
|
|
16
bin/Url.py
16
bin/Url.py
|
@ -57,7 +57,21 @@ if __name__ == "__main__":
|
|||
port, resource_path, query_string, f1, f2, f3, \
|
||||
f4 = x
|
||||
domains_list.append(domain)
|
||||
p.populate_set_out(x, 'Url')
|
||||
# p.populate_set_out(x, 'Url')
|
||||
temp_x = ()
|
||||
for i in range(0,13):
|
||||
if x[i] == '':
|
||||
temp_x += ('None', )
|
||||
else:
|
||||
temp_x += (x[i], )
|
||||
temp_scheme, temp_credential, temp_subdomain, temp_domain, temp_host, temp_tld, \
|
||||
temp_port, temp_resource_path, temp_query_string, temp_f1, temp_f2, temp_f3, \
|
||||
temp_f4 = temp_x
|
||||
|
||||
to_send = '{} {} {} {} {} {} {} {} {} {} {} {} {} {}'.format(temp_scheme, \
|
||||
temp_subdomain, temp_credential, temp_domain, temp_host, temp_tld, temp_port, temp_resource_path,\
|
||||
temp_query_string, temp_f1, temp_f2, temp_f3, temp_f4, PST._get_p_date())
|
||||
p.populate_set_out(to_send , 'Url')
|
||||
publisher.debug('{} Published'.format(x))
|
||||
|
||||
if f1 == "onion":
|
||||
|
|
|
@ -27,7 +27,7 @@ subscribe = Redis_Words
|
|||
|
||||
[Categ]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode
|
||||
publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve
|
||||
|
||||
[CreditCards]
|
||||
subscribe = Redis_CreditCards
|
||||
|
@ -47,12 +47,18 @@ subscribe = Redis_ValidOnion
|
|||
subscribe = Redis_Web
|
||||
publish = Redis_Url,ZMQ_Url
|
||||
|
||||
[WebStats]
|
||||
subscribe = Redis_Url
|
||||
|
||||
[Release]
|
||||
subscribe = Redis_Global
|
||||
|
||||
[Credential]
|
||||
subscribe = Redis_Credential
|
||||
|
||||
[Cve]
|
||||
subscribe = Redis_Cve
|
||||
|
||||
[Phone]
|
||||
subscribe = Redis_Global
|
||||
|
||||
|
|
|
@ -17,7 +17,7 @@ mv temp/startbootstrap-sb-admin-2-1.0.2 temp/sb-admin-2
|
|||
JQVERSION="1.11.1"
|
||||
wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js
|
||||
|
||||
wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js
|
||||
#wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js
|
||||
|
||||
#Here to fix an error about an hard dependency in a obscur script of bootstrap..
|
||||
wget http://code.jquery.com/jquery-1.4.2.js -O ./static/js/jquery-1.4.2.js
|
||||
|
|
Loading…
Reference in a new issue