Getting Started: Initial configuration working on the laptop

bin/Categ.py

@@ -66,7 +66,7 @@ if __name__ == "__main__":
     # FUNCTIONS #
     publisher.info("Script Categ started")
 
-    categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential']
+    categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve']
     tmp_dict = {}
     for filename in categories:
         bname = os.path.basename(filename)

bin/DomClassifier.py

@@ -39,7 +39,7 @@ def main():
             if message is not None:
                 PST = Paste.Paste(message)
             else:
-                publisher.debug("Script DomClassifier is idling 10s")
+                publisher.debug("Script DomClassifier is idling 1s")
                 time.sleep(1)
                 continue
             paste = PST.get_p_content()

bin/Duplicate.py

@@ -33,11 +33,12 @@ if __name__ == "__main__":
     # DB OBJECT & HASHS ( DISK )
     # FIXME increase flexibility
     dico_redis = {}
-    for year in xrange(2013, 2015):
+    for year in xrange(2013, 2017):
         for month in xrange(0, 16):
             dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis(
                 host=p.config.get("Redis_Level_DB", "host"), port=year,
                 db=month)
+	    #print("dup: "+str(year)+str(month).zfill(2)+"\n")
 
     # FUNCTIONS #
     publisher.info("Script duplicate started")
@@ -81,7 +82,7 @@ if __name__ == "__main__":
                 bloop_path_set.add(filebloompath)
 
             # UNIQUE INDEX HASHS TABLE
-            r_serv0 = dico_redis["201300"]
+            r_serv0 = dico_redis["201600"]
             r_serv0.incr("current_index")
             index = r_serv0.get("current_index")+str(PST.p_date)
             # HASHTABLES PER MONTH (because of r_serv1 changing db)

bin/LAUNCH.sh

@@ -66,6 +66,7 @@ function launching_lvldb {
     lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
     db1_y='2013'
     db2_y='2014'
+    db3_y='2016'
     nb_db=13
 
     screen -dmS "LevelDB"
@@ -75,6 +76,8 @@ function launching_lvldb {
     screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
     sleep 0.1
     screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
+    sleep 0.1
+    screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
 }
 
 function launching_logs {
@@ -104,11 +107,11 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate.py; read x'
     sleep 0.1
-    screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x'
+    #screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x'
     sleep 0.1
     screen -S "Script" -X screen -t "Line" bash -c './Line.py; read x'
     sleep 0.1
-    screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x'
+    #screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x'
     sleep 0.1
     screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x'
     sleep 0.1
@@ -127,6 +130,16 @@ function launching_scripts {
     screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
     sleep 0.1
     screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
+    sleep 0.1
+    screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
+    sleep 0.1
+    screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x'
+    sleep 0.1
+    screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x'
+    sleep 0.1
+    screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
+    sleep 0.1
+    #screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
 }
 
 #If no params, display the help

bin/Url.py

@@ -57,7 +57,21 @@ if __name__ == "__main__":
                         port, resource_path, query_string, f1, f2, f3, \
                         f4 = x
                     domains_list.append(domain)
-                    p.populate_set_out(x, 'Url')
+#                    p.populate_set_out(x, 'Url')
+                    temp_x = ()
+                    for i in range(0,13):
+                        if x[i] == '':
+                            temp_x += ('None', )
+                        else:
+                            temp_x += (x[i], )
+                    temp_scheme, temp_credential, temp_subdomain, temp_domain, temp_host, temp_tld, \
+                        temp_port, temp_resource_path, temp_query_string, temp_f1, temp_f2, temp_f3, \
+                        temp_f4 = temp_x
+
+                    to_send = '{} {} {} {} {} {} {} {} {} {} {} {} {} {}'.format(temp_scheme, \
+                        temp_subdomain, temp_credential, temp_domain, temp_host, temp_tld, temp_port, temp_resource_path,\
+                        temp_query_string, temp_f1, temp_f2, temp_f3, temp_f4, PST._get_p_date())
+                    p.populate_set_out(to_send , 'Url')
                     publisher.debug('{} Published'.format(x))
 
                     if f1 == "onion":

bin/packages/modules.cfg

@@ -27,7 +27,7 @@ subscribe = Redis_Words
 
 [Categ]
 subscribe = Redis_Global
-publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode
+publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve
 
 [CreditCards]
 subscribe = Redis_CreditCards
@@ -47,12 +47,18 @@ subscribe = Redis_ValidOnion
 subscribe = Redis_Web
 publish = Redis_Url,ZMQ_Url
 
+[WebStats]
+subscribe = Redis_Url
+
 [Release]
 subscribe = Redis_Global
 
 [Credential]
 subscribe = Redis_Credential
 
+[Cve]
+subscribe = Redis_Cve
+
 [Phone]
 subscribe = Redis_Global
 

var/www/update_thirdparty.sh

@@ -17,7 +17,7 @@ mv temp/startbootstrap-sb-admin-2-1.0.2 temp/sb-admin-2
 JQVERSION="1.11.1"
 wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js
 
-wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js
+#wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js
 
 #Here to fix an error about an hard dependency in a obscur script of bootstrap..
 wget http://code.jquery.com/jquery-1.4.2.js -O ./static/js/jquery-1.4.2.js