From 3dc356dc5e04abfba56eb21c4183a02c6fa37501 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Thu, 30 Jun 2016 14:36:47 +0200 Subject: [PATCH] Getting Started: Initial configuration working on the laptop --- bin/Categ.py | 2 +- bin/DomClassifier.py | 2 +- bin/Duplicate.py | 5 +++-- bin/LAUNCH.sh | 17 +++++++++++++++-- bin/Url.py | 16 +++++++++++++++- bin/packages/modules.cfg | 8 +++++++- var/www/update_thirdparty.sh | 2 +- 7 files changed, 43 insertions(+), 9 deletions(-) diff --git a/bin/Categ.py b/bin/Categ.py index 74d568d7..9449e87c 100755 --- a/bin/Categ.py +++ b/bin/Categ.py @@ -66,7 +66,7 @@ if __name__ == "__main__": # FUNCTIONS # publisher.info("Script Categ started") - categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential'] + categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve'] tmp_dict = {} for filename in categories: bname = os.path.basename(filename) diff --git a/bin/DomClassifier.py b/bin/DomClassifier.py index 0712e959..14a417f2 100755 --- a/bin/DomClassifier.py +++ b/bin/DomClassifier.py @@ -39,7 +39,7 @@ def main(): if message is not None: PST = Paste.Paste(message) else: - publisher.debug("Script DomClassifier is idling 10s") + publisher.debug("Script DomClassifier is idling 1s") time.sleep(1) continue paste = PST.get_p_content() diff --git a/bin/Duplicate.py b/bin/Duplicate.py index ed62be0d..a7a41dc1 100755 --- a/bin/Duplicate.py +++ b/bin/Duplicate.py @@ -33,11 +33,12 @@ if __name__ == "__main__": # DB OBJECT & HASHS ( DISK ) # FIXME increase flexibility dico_redis = {} - for year in xrange(2013, 2015): + for year in xrange(2013, 2017): for month in xrange(0, 16): dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis( host=p.config.get("Redis_Level_DB", "host"), port=year, db=month) + #print("dup: "+str(year)+str(month).zfill(2)+"\n") # FUNCTIONS # publisher.info("Script duplicate started") @@ -81,7 +82,7 @@ if __name__ == "__main__": bloop_path_set.add(filebloompath) # UNIQUE INDEX HASHS TABLE - r_serv0 = dico_redis["201300"] + r_serv0 = dico_redis["201600"] r_serv0.incr("current_index") index = r_serv0.get("current_index")+str(PST.p_date) # HASHTABLES PER MONTH (because of r_serv1 changing db) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index d033273a..330a23ce 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -66,6 +66,7 @@ function launching_lvldb { lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/" db1_y='2013' db2_y='2014' + db3_y='2016' nb_db=13 screen -dmS "LevelDB" @@ -75,6 +76,8 @@ function launching_lvldb { screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x' sleep 0.1 screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x' + sleep 0.1 + screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x' } function launching_logs { @@ -104,11 +107,11 @@ function launching_scripts { sleep 0.1 screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate.py; read x' sleep 0.1 - screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x' + #screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Line" bash -c './Line.py; read x' sleep 0.1 - screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x' + #screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x' sleep 0.1 @@ -127,6 +130,16 @@ function launching_scripts { screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' sleep 0.1 screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x' + sleep 0.1 + screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x' + sleep 0.1 + #screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x' } #If no params, display the help diff --git a/bin/Url.py b/bin/Url.py index 472d73b3..d33e7e9f 100755 --- a/bin/Url.py +++ b/bin/Url.py @@ -57,7 +57,21 @@ if __name__ == "__main__": port, resource_path, query_string, f1, f2, f3, \ f4 = x domains_list.append(domain) - p.populate_set_out(x, 'Url') +# p.populate_set_out(x, 'Url') + temp_x = () + for i in range(0,13): + if x[i] == '': + temp_x += ('None', ) + else: + temp_x += (x[i], ) + temp_scheme, temp_credential, temp_subdomain, temp_domain, temp_host, temp_tld, \ + temp_port, temp_resource_path, temp_query_string, temp_f1, temp_f2, temp_f3, \ + temp_f4 = temp_x + + to_send = '{} {} {} {} {} {} {} {} {} {} {} {} {} {}'.format(temp_scheme, \ + temp_subdomain, temp_credential, temp_domain, temp_host, temp_tld, temp_port, temp_resource_path,\ + temp_query_string, temp_f1, temp_f2, temp_f3, temp_f4, PST._get_p_date()) + p.populate_set_out(to_send , 'Url') publisher.debug('{} Published'.format(x)) if f1 == "onion": diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 94a2fee4..9d8d6637 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -27,7 +27,7 @@ subscribe = Redis_Words [Categ] subscribe = Redis_Global -publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode +publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve [CreditCards] subscribe = Redis_CreditCards @@ -47,12 +47,18 @@ subscribe = Redis_ValidOnion subscribe = Redis_Web publish = Redis_Url,ZMQ_Url +[WebStats] +subscribe = Redis_Url + [Release] subscribe = Redis_Global [Credential] subscribe = Redis_Credential +[Cve] +subscribe = Redis_Cve + [Phone] subscribe = Redis_Global diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh index 2b5f7703..b666a5d5 100755 --- a/var/www/update_thirdparty.sh +++ b/var/www/update_thirdparty.sh @@ -17,7 +17,7 @@ mv temp/startbootstrap-sb-admin-2-1.0.2 temp/sb-admin-2 JQVERSION="1.11.1" wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js -wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js +#wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js #Here to fix an error about an hard dependency in a obscur script of bootstrap.. wget http://code.jquery.com/jquery-1.4.2.js -O ./static/js/jquery-1.4.2.js