python 3 backend upgrade

bin/Attributes.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 """
@@ -33,7 +33,7 @@ if __name__ == "__main__":
                 PST = Paste.Paste(message)
             else:
                 publisher.debug("Script Attribute is idling 1s")
-                print 'sleeping'
+                print('sleeping')
                 time.sleep(1)
                 continue
 
@@ -45,6 +45,6 @@ if __name__ == "__main__":
             # FIXME Not used.
             PST.store.sadd("Pastes_Objects", PST.p_path)
         except IOError:
-            print "CRC Checksum Failed on :", PST.p_path
+            print("CRC Checksum Failed on :", PST.p_path)
             publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(
                 PST.p_source, PST.p_date, PST.p_name))

bin/Categ.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 """
 The ZMQ_PubSub_Categ Module
@@ -73,7 +73,7 @@ if __name__ == "__main__":
         bname = os.path.basename(filename)
         tmp_dict[bname] = []
         with open(os.path.join(args.d, filename), 'r') as f:
-            patterns = [r'%s' % re.escape(s.strip()) for s in f]
+            patterns = [r'%s' % ( re.escape(s.strip()) ) for s in f]
             tmp_dict[bname] = re.compile('|'.join(patterns), re.IGNORECASE)
 
     prec_filename = None
@@ -82,18 +82,25 @@ if __name__ == "__main__":
         filename = p.get_from_set()
         if filename is None:
             publisher.debug("Script Categ is Idling 10s")
-            print 'Sleeping'
+            print('Sleeping')
             time.sleep(10)
             continue
 
         paste = Paste.Paste(filename)
         content = paste.get_p_content()
 
+        print('-----------------------------------------------------')
+        print(filename)
+        print(content)
+        print('-----------------------------------------------------')
+
         for categ, pattern in tmp_dict.items():
             found = set(re.findall(pattern, content))
             if len(found) >= matchingThreshold:
                 msg = '{} {}'.format(paste.p_path, len(found))
-                print msg, categ
+                #msg = " ".join( [paste.p_path, bytes(len(found))] )
+
+                print(msg, categ)
                 p.populate_set_out(msg, categ)
 
                 publisher.info(

bin/Credential.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 """
@@ -61,26 +61,34 @@ if __name__ == "__main__":
     minTopPassList = p.config.getint("Credential", "minTopPassList")
 
     regex_web = "((?:https?:\/\/)[-_0-9a-zA-Z]+\.[0-9a-zA-Z]+)"
-    regex_cred = "[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}:[a-zA-Z0-9\_\-]+"
+    #regex_cred = "[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}:[a-zA-Z0-9\_\-]+"
+    regex_cred = "[a-zA-Z0-9\\._-]+@[a-zA-Z0-9\\.-]+\.[a-zA-Z]{2,6}[\\rn :\_\-]{1,10}[a-zA-Z0-9\_\-]+"
     regex_site_for_stats = "@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}:"
     while True:
         message = p.get_from_set()
         if message is None:
             publisher.debug("Script Credential is Idling 10s")
             print('sleeping 10s')
-            time.sleep(10)
+            time.sleep(1)
             continue
 
-        filepath, count = message.split()
+        filepath, count = message.split(' ')
 
-        if count < minTopPassList:
+        #if count < minTopPassList:
             # Less than 5 matches from the top password list, false positive.
-            print("false positive:", count)
+            #print("false positive:", count)
-            continue
+            #continue
 
         paste = Paste.Paste(filepath)
         content = paste.get_p_content()
         creds = set(re.findall(regex_cred, content))
+        print(len(creds))
+        print(creds)
+        print(content)
+        print('-----')
+
+        publisher.warning('to_print')
+
         if len(creds) == 0:
             continue
 
@@ -89,7 +97,7 @@ if __name__ == "__main__":
 
         message = 'Checked {} credentials found.'.format(len(creds))
         if sites_set:
-            message += ' Related websites: {}'.format(', '.join(sites_set))
+            message += ' Related websites: {}'.format( (', '.join(sites_set)) )
 
         to_print = 'Credential;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, message, paste.p_path)
 
@@ -97,12 +105,13 @@ if __name__ == "__main__":
 
         #num of creds above tresh, publish an alert
         if len(creds) > criticalNumberToAlert:
-            print("========> Found more than 10 credentials in this file : {}".format(filepath))
+            print("========> Found more than 10 credentials in this file : {}".format( filepath ))
             publisher.warning(to_print)
             #Send to duplicate
             p.populate_set_out(filepath, 'Duplicate')
             #Send to alertHandler
-            p.populate_set_out('credential;{}'.format(filepath), 'alertHandler')
+            msg = 'credential;{}'.format(filepath)
+            p.populate_set_out(msg, 'alertHandler')
 
             #Put in form, count occurences, then send to moduleStats
             creds_sites = {}
@@ -122,9 +131,10 @@ if __name__ == "__main__":
                 else:
                     creds_sites[domain] = 1
 
-            for site, num in creds_sites.iteritems(): # Send for each different site to moduleStats
+            for site, num in creds_sites.items(): # Send for each different site to moduleStats
-                print 'credential;{};{};{}'.format(num, site, paste.p_date)
+                mssg = 'credential;{};{};{}'.format(num, site, paste.p_date)
-                p.populate_set_out('credential;{};{};{}'.format(num, site, paste.p_date), 'ModuleStats')
+                print(mssg)
+                p.populate_set_out(mssg, 'ModuleStats')
 
             if sites_set:
                 print("=======> Probably on : {}".format(', '.join(sites_set)))
@@ -158,4 +168,3 @@ if __name__ == "__main__":
             for partCred in splitedCred:
                 if len(partCred) > minimumLengthThreshold:
                     server_cred.sadd(partCred, uniq_num_cred)
-

bin/CreditCards.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 """
@@ -18,6 +18,7 @@ from packages import Paste
 from packages import lib_refine
 from pubsublogger import publisher
 import re
+import sys
 
 from Helper import Process
 
@@ -58,13 +59,14 @@ if __name__ == "__main__":
             content = paste.get_p_content()
             all_cards = re.findall(regex, content)
             if len(all_cards) > 0:
-                print 'All matching', all_cards
+                print('All matching', all_cards)
                 creditcard_set = set([])
 
                 for card in all_cards:
                     clean_card = re.sub('[^0-9]', '', card)
+                    clean_card = clean_card
                     if lib_refine.is_luhn_valid(clean_card):
-                        print clean_card, 'is valid'
+                        print(clean_card, 'is valid')
                         creditcard_set.add(clean_card)
 
                 paste.__setattr__(channel, creditcard_set)
@@ -76,13 +78,15 @@ if __name__ == "__main__":
                 if (len(creditcard_set) > 0):
                     publisher.warning('{}Checked {} valid number(s);{}'.format(
                         to_print, len(creditcard_set), paste.p_path))
+                    print('{}Checked {} valid number(s);{}'.format(
+                        to_print, len(creditcard_set), paste.p_path))
                     #Send to duplicate
                     p.populate_set_out(filename, 'Duplicate')
                     #send to Browse_warning_paste
-                    p.populate_set_out('creditcard;{}'.format(filename), 'alertHandler')
+                    msg = 'creditcard;{}'.format(filename)
+                    p.populate_set_out(msg, 'alertHandler')
                 else:
                     publisher.info('{}CreditCard related;{}'.format(to_print, paste.p_path))
         else:
             publisher.debug("Script creditcard is idling 1m")
             time.sleep(10)
-

bin/Curve.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 """
 This module is consuming the Redis-list created by the ZMQ_Sub_Curve_Q Module.
@@ -53,7 +53,7 @@ def check_if_tracked_term(term, path):
         #add_paste to tracked_word_set
         set_name = "tracked_" + term
         server_term.sadd(set_name, path)
-        print term, 'addded', set_name, '->', path
+        print(term, 'addded', set_name, '->', path)
         p.populate_set_out("New Term added", 'CurveManageTopSets')
 
         # Send a notification only when the member is in the set
@@ -149,15 +149,16 @@ if __name__ == "__main__":
 
             if generate_new_graph:
                 generate_new_graph = False
-                print 'Building graph'
+                print('Building graph')
                 today = datetime.date.today()
                 year = today.year
                 month = today.month
+
                 lib_words.create_curve_with_word_file(r_serv1, csv_path,
                                                       wordfile_path, year,
                                                       month)
 
             publisher.debug("Script Curve is Idling")
-            print "sleeping"
+            print("sleeping")
             time.sleep(10)
         message = p.get_from_set()

bin/CurveManageTopSets.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 """
 
@@ -16,7 +16,7 @@ from packages import lib_words
 import datetime
 import calendar
 import os
-import ConfigParser
+import configparser
 
 # Config Variables
 Refresh_rate = 60*5 #sec
@@ -68,26 +68,26 @@ def manage_top_set():
 
     # convert dico into sorted array
     array_month = []
-    for w, v in dico.iteritems():
+    for w, v in dico.items():
         array_month.append((w, v))
     array_month.sort(key=lambda tup: -tup[1])
     array_month = array_month[0:20]
 
     array_week = []
-    for w, v in dico_week.iteritems():
+    for w, v in dico_week.items():
         array_week.append((w, v))
     array_week.sort(key=lambda tup: -tup[1])
     array_week = array_week[0:20]
 
     # convert dico_per_paste into sorted array
     array_month_per_paste = []
-    for w, v in dico_per_paste.iteritems():
+    for w, v in dico_per_paste.items():
         array_month_per_paste.append((w, v))
     array_month_per_paste.sort(key=lambda tup: -tup[1])
     array_month_per_paste = array_month_per_paste[0:20]
 
     array_week_per_paste = []
-    for w, v in dico_week_per_paste.iteritems():
+    for w, v in dico_week_per_paste.items():
         array_week_per_paste.append((w, v))
     array_week_per_paste.sort(key=lambda tup: -tup[1])
     array_week_per_paste = array_week_per_paste[0:20]
@@ -114,7 +114,7 @@ def manage_top_set():
     timestamp = int(time.mktime(datetime.datetime.now().timetuple()))
     value = str(timestamp) + ", " + "-"
     r_temp.set("MODULE_"+ "CurveManageTopSets" + "_" + str(os.getpid()), value)
-    print "refreshed module"
+    print("refreshed module")
 
 
 
@@ -131,7 +131,7 @@ if __name__ == '__main__':
                         Did you set environment variables? \
                         Or activate the virtualenv.')
 
-    cfg = ConfigParser.ConfigParser()
+    cfg = configparser.ConfigParser()
     cfg.read(configfile)
 
 
@@ -162,4 +162,3 @@ if __name__ == '__main__':
         # Get one message from the input queue (module only work if linked with a queue)
         time.sleep(Refresh_rate) # sleep a long time then manage the set
         manage_top_set()
-

bin/Cve.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 """
 The CVE Module
@@ -32,7 +32,8 @@ def search_cve(message):
         publisher.warning('{} contains CVEs'.format(paste.p_name))
 
         #send to Browse_warning_paste
-        p.populate_set_out('cve;{}'.format(filepath), 'alertHandler')
+        msg = 'cve;{}'.format(filepath)
+        p.populate_set_out(msg, 'alertHandler')
         #Send to duplicate
         p.populate_set_out(filepath, 'Duplicate')
 
@@ -63,4 +64,3 @@ if __name__ == '__main__':
 
         # Do something with the message from the queue
         search_cve(message)
-

bin/Dir.py

@@ -1,18 +1,18 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 import argparse
 import redis
 from pubsublogger import publisher
 from packages.lib_words import create_dirfile
-import ConfigParser
+import configparser
 
 
 def main():
     """Main Function"""
 
     # CONFIG #
-    cfg = ConfigParser.ConfigParser()
+    cfg = configparser.ConfigParser()
     cfg.read('./packages/config.cfg')
 
     parser = argparse.ArgumentParser(

bin/DomClassifier.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 """
@@ -44,6 +44,7 @@ def main():
                 continue
             paste = PST.get_p_content()
             mimetype = PST._get_p_encoding()
+    
             if mimetype == "text/plain":
                 c.text(rawtext=paste)
                 c.potentialdomain()
@@ -59,7 +60,7 @@ def main():
                     publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
                         PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path))
         except IOError:
-            print "CRC Checksum Failed on :", PST.p_path
+            print("CRC Checksum Failed on :", PST.p_path)
             publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(
                 PST.p_source, PST.p_date, PST.p_name))
 

bin/DumpValidOnion.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 from pubsublogger import publisher
@@ -23,7 +23,7 @@ if __name__ == "__main__":
         if message is not None:
             f = open(dump_file, 'a')
             while message is not None:
-                print message
+                print(message)
                 date = datetime.datetime.now()
                 if message is not None:
                     f.write(date.isoformat() + ' ' + message + '\n')

bin/Duplicates.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 """
@@ -49,8 +49,8 @@ if __name__ == "__main__":
     # REDIS #
     dico_redis = {}
     date_today = datetime.today()
-    for year in xrange(2013, date_today.year+1):
+    for year in range(2013, date_today.year+1):
-        for month in xrange(0, 13):
+        for month in range(0, 13):
             dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis(
                 host=p.config.get("Redis_Level_DB", "host"), port=year,
                 db=month)
@@ -90,7 +90,7 @@ if __name__ == "__main__":
             # Get the date of the range
             date_range = date_today - timedelta(days = maximum_month_range*30.4166666)
             num_of_month = (date_today.year - date_range.year)*12 + (date_today.month - date_range.month)
-            for diff_month in xrange(0, num_of_month+1):
+            for diff_month in range(0, num_of_month+1):
                 curr_date_range = date_today - timedelta(days = diff_month*30.4166666)
                 to_append = str(curr_date_range.year)+str(curr_date_range.month).zfill(2)
                 dico_range_list.append(to_append)
@@ -102,7 +102,7 @@ if __name__ == "__main__":
             yearly_index = str(date_today.year)+'00'
             r_serv0 = dico_redis[yearly_index]
             r_serv0.incr("current_index")
-            index = r_serv0.get("current_index")+str(PST.p_date)
+            index = (r_serv0.get("current_index")).decode('utf8') + str(PST.p_date)
 
             # Open selected dico range
             opened_dico = []
@@ -114,8 +114,10 @@ if __name__ == "__main__":
 
             # Go throught the Database of the dico (of the month)
             for curr_dico_name, curr_dico_redis in opened_dico:
-                for hash_type, paste_hash in paste_hashes.iteritems():
+                for hash_type, paste_hash in paste_hashes.items():
                     for dico_hash in curr_dico_redis.smembers('HASHS_'+hash_type):
+                        dico_hash = dico_hash.decode('utf8')
+
                         try:
                             if hash_type == 'ssdeep':
                                 percent = 100-ssdeep.compare(dico_hash, paste_hash)
@@ -130,15 +132,18 @@ if __name__ == "__main__":
 
                                 # index of paste
                                 index_current = r_serv_dico.get(dico_hash)
+                                index_current = index_current.decode('utf8')
                                 paste_path = r_serv_dico.get(index_current)
+                                paste_path = paste_path.decode('utf8')
                                 paste_date = r_serv_dico.get(index_current+'_date')
+                                paste_date = paste_date.decode('utf8')
                                 paste_date = paste_date if paste_date != None else "No date available"
                                 if paste_path != None:
                                     hash_dico[dico_hash] = (hash_type, paste_path, percent, paste_date)
 
-                                print '['+hash_type+'] '+'comparing: ' + str(PST.p_path[44:]) + '  and  ' + str(paste_path[44:]) + ' percentage: ' + str(percent)
+                                print('['+hash_type+'] '+'comparing: ' + str(PST.p_path[44:]) + '  and  ' + str(paste_path[44:]) + ' percentage: ' + str(percent))
-                        except Exception,e:
+                        except Exception:
-                            print str(e)
+                            print(str(e))
                             #print 'hash not comparable, bad hash: '+dico_hash+' , current_hash: '+paste_hash
 
             # Add paste in DB after checking to prevent its analysis twice
@@ -147,7 +152,7 @@ if __name__ == "__main__":
             r_serv1.set(index+'_date', PST._get_p_date())
             r_serv1.sadd("INDEX", index)
             # Adding hashes in Redis
-            for hash_type, paste_hash in paste_hashes.iteritems():
+            for hash_type, paste_hash in paste_hashes.items():
                 r_serv1.set(paste_hash, index)
                 r_serv1.sadd("HASHS_"+hash_type, paste_hash)
 
@@ -166,7 +171,7 @@ if __name__ == "__main__":
                     PST.__setattr__("p_duplicate", dupl)
                     PST.save_attribute_redis("p_duplicate", dupl)
                     publisher.info('{}Detected {};{}'.format(to_print, len(dupl), PST.p_path))
-                    print '{}Detected {}'.format(to_print, len(dupl))
+                    print('{}Detected {}'.format(to_print, len(dupl)))
 
                 y = time.time()
 
@@ -176,5 +181,5 @@ if __name__ == "__main__":
         except IOError:
             to_print = 'Duplicate;{};{};{};'.format(
                 PST.p_source, PST.p_date, PST.p_name)
-            print "CRC Checksum Failed on :", PST.p_path
+            print("CRC Checksum Failed on :", PST.p_path)
             publisher.error('{}CRC Checksum Failed'.format(to_print))

bin/Duplicates_old.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 """
@@ -33,8 +33,8 @@ if __name__ == "__main__":
     # DB OBJECT & HASHS ( DISK )
     # FIXME increase flexibility
     dico_redis = {}
-    for year in xrange(2013, 2017):
+    for year in range(2013, 2017):
-        for month in xrange(0, 16):
+        for month in range(0, 16):
             dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis(
                 host=p.config.get("Redis_Level_DB", "host"), port=year,
                 db=month)
@@ -147,7 +147,7 @@ if __name__ == "__main__":
                     if percentage >= 50:
                         dupl.append((paste, percentage))
                     else:
-                        print 'percentage: ' + str(percentage)
+                        print('percentage: ' + str(percentage))
 
                 # Creating the object attribute and save it.
                 to_print = 'Duplicate;{};{};{};'.format(
@@ -156,11 +156,11 @@ if __name__ == "__main__":
                     PST.__setattr__("p_duplicate", dupl)
                     PST.save_attribute_redis("p_duplicate", dupl)
                     publisher.info('{}Detected {}'.format(to_print, len(dupl)))
-                    print '{}Detected {}'.format(to_print, len(dupl))
+                    print('{}Detected {}'.format(to_print, len(dupl)))
 
                 y = time.time()
 
                 publisher.debug('{}Processed in {} sec'.format(to_print, y-x))
         except IOError:
-            print "CRC Checksum Failed on :", PST.p_path
+            print("CRC Checksum Failed on :", PST.p_path)
             publisher.error('{}CRC Checksum Failed'.format(to_print))

bin/Global.py

@@ -67,7 +67,7 @@ if __name__ == '__main__':
             continue
         # Creating the full filepath
         filename = os.path.join(os.environ['AIL_HOME'],
-                                p.config.get("Directories", "pastes"), paste.decode('utf8'))
+                                p.config.get("Directories", "pastes"), paste)
         #print(filename)
         dirname = os.path.dirname(filename)
         if not os.path.exists(dirname):
@@ -77,5 +77,5 @@ if __name__ == '__main__':
             f.write(base64.standard_b64decode(gzip64encoded))
 
         print(filename)
-        p.populate_set_out(filename.encode('utf8'))
+        p.populate_set_out(filename)
         processed_paste+=1

bin/Helper.py

@@ -58,7 +58,7 @@ class PubSub(object):
                 new_sub = context.socket(zmq.SUB)
                 new_sub.connect(address)
                 # bytes64 encode bytes to ascii only bytes
-                new_sub.setsockopt(zmq.SUBSCRIBE, channel.encode('ascii'))
+                new_sub.setsockopt_string(zmq.SUBSCRIBE, channel)
                 self.subscribers.append(new_sub)
 
     def setup_publish(self, conn_name):
@@ -78,15 +78,15 @@ class PubSub(object):
             self.publishers['ZMQ'].append((p, channel))
 
     def publish(self, message):
-        m = json.loads(message.decode('ascii'))
+        m = json.loads(message)
         channel_message = m.get('channel')
         for p, channel in self.publishers['Redis']:
             if channel_message is None or channel_message == channel:
-                p.publish(channel, ( m['message']).encode('ascii') )
+                p.publish(channel, ( m['message']) )
         for p, channel in self.publishers['ZMQ']:
             if channel_message is None or channel_message == channel:
-                mess = ( m['message'] ).encode('ascii')
+                p.send('{} {}'.format(channel, m['message']))
-                p.send(b' '.join( [channel,  mess] ) )
+                #p.send(b' '.join( [channel,  mess] ) )
 
 
     def subscribe(self):
@@ -99,7 +99,8 @@ class PubSub(object):
                 for sub in self.subscribers:
                     try:
                         msg = sub.recv(zmq.NOBLOCK)
-                        yield msg.split(b" ", 1)[1]
+                        msg = msg.decode('utf8')
+                        yield msg.split(" ", 1)[1]
                     except zmq.error.Again as e:
                         time.sleep(0.2)
                         pass
@@ -150,6 +151,12 @@ class Process(object):
         self.r_temp.hset('queues', self.subscriber_name,
                          int(self.r_temp.scard(in_set)))
         message = self.r_temp.spop(in_set)
+
+        try:
+            message = message.decode('utf8')
+        except AttributeError:
+            pass
+
         timestamp = int(time.mktime(datetime.datetime.now().timetuple()))
         dir_name = os.environ['AIL_HOME']+self.config.get('Directories', 'pastes')
 
@@ -158,12 +165,12 @@ class Process(object):
 
         else:
             #try:
-            if b'.gz' in message:
+            if '.gz' in message:
-                path = message.split(b".")[-2].split(b"/")[-1]
+                path = message.split(".")[-2].split("/")[-1]
                 #find start of path with AIL_HOME
-                index_s = (message.decode('ascii')).find(os.environ['AIL_HOME'])
+                index_s = message.find(os.environ['AIL_HOME'])
                 #Stop when .gz
-                index_e = message.find(b".gz")+3
+                index_e = message.find(".gz")+3
                 if(index_s == -1):
                     complete_path = message[0:index_e]
                 else:
@@ -173,7 +180,7 @@ class Process(object):
                 path = "-"
                 complete_path = "?"
 
-            value = str(timestamp) + ", " + path.decode('ascii')
+            value = str(timestamp) + ", " + path
             self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum), value)
             self.r_temp.set("MODULE_"+self.subscriber_name + "_" + str(self.moduleNum) + "_PATH", complete_path)
             self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
@@ -190,13 +197,12 @@ class Process(object):
 
     def populate_set_out(self, msg, channel=None):
         # multiproc
-        msg = msg.decode('ascii')
         msg = {'message': msg}
         if channel is not None:
             msg.update({'channel': channel})
 
         # bytes64 encode bytes to ascii only bytes
-        j = (json.dumps(msg)).encode('ascii')
+        j = json.dumps(msg)
         self.r_temp.sadd(self.subscriber_name + 'out', j)
 
     def publish(self):
@@ -209,6 +215,12 @@ class Process(object):
             self.pubsub.setup_publish(name)
         while True:
             message = self.r_temp.spop(self.subscriber_name + 'out')
+
+            try:
+                message = message.decode('utf8')
+            except AttributeError:
+                pass
+
             if message is None:
                 time.sleep(1)
                 continue

bin/Indexer.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 """
@@ -107,10 +107,11 @@ if __name__ == "__main__":
                 continue
             docpath = message.split(" ", -1)[-1]
             paste = PST.get_p_content()
-            print "Indexing - "+indexname+" :", docpath
+            print("Indexing - " + indexname + " :", docpath)
 
 
-            if time.time() - last_refresh > TIME_WAIT: #avoid calculating the index's size at each message
+            #avoid calculating the index's size at each message
+            if( time.time() - last_refresh > TIME_WAIT):
                 last_refresh = time.time()
                 if check_index_size(baseindexpath, indexname) >= INDEX_SIZE_THRESHOLD*(1000*1000):
                     timestamp = int(time.time())
@@ -128,11 +129,11 @@ if __name__ == "__main__":
             if indexertype == "whoosh":
                 indexwriter = ix.writer()
                 indexwriter.update_document(
-                    title=unicode(docpath, errors='ignore'),
+                    title=docpath,
-                    path=unicode(docpath, errors='ignore'),
+                    path=docpath,
-                    content=unicode(paste, errors='ignore'))
+                    content=paste)
                 indexwriter.commit()
         except IOError:
-            print "CRC Checksum Failed on :", PST.p_path
+            print("CRC Checksum Failed on :", PST.p_path)
             publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(
                 PST.p_source, PST.p_date, PST.p_name))

bin/Keys.py

@@ -15,46 +15,49 @@ RSA private key, certificate messages
 import time
 from pubsublogger import publisher
 
-from bin.packages import Paste
+#from bin.packages import Paste
-from bin.Helper import Process
+#from bin.Helper import Process
+
+from packages import Paste
+from Helper import Process
 
 
 def search_key(paste):
     content = paste.get_p_content()
     find = False
-    if b'-----BEGIN PGP MESSAGE-----' in content:
+    if '-----BEGIN PGP MESSAGE-----' in content:
         publisher.warning('{} has a PGP enc message'.format(paste.p_name))
         find = True
 
-    if b'-----BEGIN CERTIFICATE-----' in content:
+    if '-----BEGIN CERTIFICATE-----' in content:
         publisher.warning('{} has a certificate message'.format(paste.p_name))
         find = True
 
-    if b'-----BEGIN RSA PRIVATE KEY-----' in content:
+    if '-----BEGIN RSA PRIVATE KEY-----' in content:
         publisher.warning('{} has a RSA private key message'.format(paste.p_name))
         find = True
 
-    if b'-----BEGIN PRIVATE KEY-----' in content:
+    if '-----BEGIN PRIVATE KEY-----' in content:
         publisher.warning('{} has a private key message'.format(paste.p_name))
         find = True
 
-    if b'-----BEGIN ENCRYPTED PRIVATE KEY-----' in content:
+    if '-----BEGIN ENCRYPTED PRIVATE KEY-----' in content:
         publisher.warning('{} has an encrypted private key message'.format(paste.p_name))
         find = True
 
-    if b'-----BEGIN OPENSSH PRIVATE KEY-----' in content:
+    if '-----BEGIN OPENSSH PRIVATE KEY-----' in content:
         publisher.warning('{} has an openssh private key message'.format(paste.p_name))
         find = True
 
-    if b'-----BEGIN DSA PRIVATE KEY-----' in content:
+    if '-----BEGIN DSA PRIVATE KEY-----' in content:
         publisher.warning('{} has a dsa private key message'.format(paste.p_name))
         find = True
 
-    if b'-----BEGIN EC PRIVATE KEY-----' in content:
+    if '-----BEGIN EC PRIVATE KEY-----' in content:
         publisher.warning('{} has an ec private key message'.format(paste.p_name))
         find = True
 
-    if b'-----BEGIN PGP PRIVATE KEY BLOCK-----' in content:
+    if '-----BEGIN PGP PRIVATE KEY BLOCK-----' in content:
         publisher.warning('{} has a pgp private key block message'.format(paste.p_name))
         find = True
 
@@ -63,7 +66,9 @@ def search_key(paste):
         #Send to duplicate
         p.populate_set_out(message, 'Duplicate')
         #send to Browse_warning_paste
-        p.populate_set_out('keys;{}'.format(message), 'alertHandler')
+        msg = ('keys;{}'.format(message))
+        print(message)
+        p.populate_set_out( msg, 'alertHandler')
 
 
 if __name__ == '__main__':

bin/LAUNCH.sh

@@ -130,8 +130,8 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Lines" bash -c 'python3 Lines.py; read x'
     sleep 0.1
-    #screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
+    screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
-    #sleep 0.1
+    sleep 0.1
     screen -S "Script_AIL" -X screen -t "Categ" bash -c 'python3 Categ.py; read x'
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'python3 Tokenize.py; read x'
@@ -142,8 +142,8 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Mail" bash -c './Mail.py; read x'
     sleep 0.1
-    #screen -S "Script_AIL" -X screen -t "Web" bash -c './Web.py; read x'
+    screen -S "Script_AIL" -X screen -t "Web" bash -c './Web.py; read x'
-    #sleep 0.1
+    sleep 0.1
     screen -S "Script_AIL" -X screen -t "Credential" bash -c './Credential.py; read x'
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Curve" bash -c './Curve.py; read x'
@@ -168,8 +168,8 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x'
     sleep 0.1
-    #screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
+    screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
-    #sleep 0.1
+    sleep 0.1
     screen -S "Script_AIL" -X screen -t "alertHandler" bash -c './alertHandler.py; read x'
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'

bin/Lines.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 """
@@ -61,7 +61,7 @@ if __name__ == '__main__':
     while True:
         try:
             message = p.get_from_set()
-            print message
+            print(message)
             if message is not None:
                 PST = Paste.Paste(message)
             else:
@@ -77,8 +77,8 @@ if __name__ == '__main__':
             # FIXME Not used.
             PST.store.sadd("Pastes_Objects", PST.p_path)
             if lines_infos[1] < args.max:
-                p.populate_set_out(PST.p_path, 'LinesShort')
+                p.populate_set_out( PST.p_path , 'LinesShort')
             else:
-                p.populate_set_out(PST.p_path, 'LinesLong')
+                p.populate_set_out( PST.p_path , 'LinesLong')
         except IOError:
-            print "CRC Checksum Error on : ", PST.p_path
+            print("CRC Checksum Error on : ", PST.p_path)

bin/Mail.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 """
@@ -78,14 +78,14 @@ if __name__ == "__main__":
                         publisher.info(to_print)
                 #Send to ModuleStats
                 for mail in MX_values[1]:
-                    print 'mail;{};{};{}'.format(1, mail, PST.p_date)
+                    print('mail;{};{};{}'.format(1, mail, PST.p_date))
                     p.populate_set_out('mail;{};{};{}'.format(1, mail, PST.p_date), 'ModuleStats')
 
             prec_filename = filename
 
         else:
             publisher.debug("Script Mails is Idling 10s")
-            print 'Sleeping'
+            print('Sleeping')
             time.sleep(10)
 
         message = p.get_from_set()

bin/Mixer.py

@@ -91,7 +91,7 @@ if __name__ == '__main__':
                 complete_paste, gzip64encoded = splitted
 
                 try:
-                    feeder_name = ( complete_paste.replace(b"archive/",b"") ).split(b"/")[0]
+                    feeder_name = ( complete_paste.replace("archive/","") ).split("/")[0]
 
                     # TODO take real name ?
                     paste_name = complete_paste
@@ -109,9 +109,10 @@ if __name__ == '__main__':
                     processed_paste_per_feeder[feeder_name] = 1
                     duplicated_paste_per_feeder[feeder_name] = 0
 
-                relay_message = b" ".join( [paste_name, gzip64encoded] )
+                relay_message = "{0} {1}".format(paste_name, gzip64encoded)
+                #relay_message = b" ".join( [paste_name, gzip64encoded] )
 
-                digest = hashlib.sha1(gzip64encoded).hexdigest()
+                digest = hashlib.sha1(gzip64encoded.encode('utf8')).hexdigest()
 
                 # Avoid any duplicate coming from any sources
                 if operation_mode == 1:

bin/ModuleInformation.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 '''
@@ -20,7 +20,7 @@ import os
 import signal
 import argparse
 from subprocess import PIPE, Popen
-import ConfigParser
+import configparser
 import json
 from terminaltables import AsciiTable
 import textwrap
@@ -51,7 +51,7 @@ last_refresh = 0
 def getPid(module):
     p = Popen([command_search_pid.format(module+".py")], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
     for line in p.stdout:
-        print line
+        print(line)
         splittedLine = line.split()
         if 'python2' in splittedLine:
             return int(splittedLine[0])
@@ -76,7 +76,7 @@ def cleanRedis():
                     flag_pid_valid = True
 
             if not flag_pid_valid:
-                print flag_pid_valid, 'cleaning', pid, 'in', k
+                print(flag_pid_valid, 'cleaning', pid, 'in', k)
                 server.srem(k, pid)
                 inst_time = datetime.datetime.fromtimestamp(int(time.time()))
                 printarrayGlob.insert(1, [inst_time, moduleName, pid, "Cleared invalid pid in " + k])
@@ -85,11 +85,11 @@ def cleanRedis():
 
 
 def kill_module(module, pid):
-    print ''
+    print('')
-    print '-> trying to kill module:', module
+    print('-> trying to kill module:', module)
 
     if pid is None:
-        print 'pid was None'
+        print('pid was None')
         printarrayGlob.insert(1, [0, module, pid, "PID was None"])
         printarrayGlob.pop()
         pid = getPid(module)
@@ -102,15 +102,15 @@ def kill_module(module, pid):
         try:
             os.kill(pid, signal.SIGUSR1)
         except OSError:
-            print pid, 'already killed'
+            print(pid, 'already killed')
             inst_time = datetime.datetime.fromtimestamp(int(time.time()))
             printarrayGlob.insert(1, [inst_time, module, pid, "Already killed"])
             printarrayGlob.pop()
             return
         time.sleep(1)
         if getPid(module) is None:
-            print module, 'has been killed'
+            print(module, 'has been killed')
-            print 'restarting', module, '...'
+            print('restarting', module, '...')
             p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
             inst_time = datetime.datetime.fromtimestamp(int(time.time()))
             printarrayGlob.insert(1, [inst_time, module, pid, "Killed"])
@@ -119,7 +119,7 @@ def kill_module(module, pid):
             printarrayGlob.pop()
 
         else:
-            print 'killing failed, retrying...'
+            print('killing failed, retrying...')
             inst_time = datetime.datetime.fromtimestamp(int(time.time()))
             printarrayGlob.insert(1, [inst_time, module, pid, "Killing #1 failed."])
             printarrayGlob.pop()
@@ -128,8 +128,8 @@ def kill_module(module, pid):
             os.kill(pid, signal.SIGUSR1)
             time.sleep(1)
             if getPid(module) is None:
-                print module, 'has been killed'
+                print(module, 'has been killed')
-                print 'restarting', module, '...'
+                print('restarting', module, '...')
                 p2 = Popen([command_restart_module.format(module, module)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
                 inst_time = datetime.datetime.fromtimestamp(int(time.time()))
                 printarrayGlob.insert(1, [inst_time, module, pid, "Killed"])
@@ -137,12 +137,12 @@ def kill_module(module, pid):
                 printarrayGlob.pop()
                 printarrayGlob.pop()
             else:
-                print 'killing failed!'
+                print('killing failed!')
                 inst_time = datetime.datetime.fromtimestamp(int(time.time()))
                 printarrayGlob.insert(1, [inst_time, module, pid, "Killing failed!"])
                 printarrayGlob.pop()
     else:
-        print 'Module does not exist'
+        print('Module does not exist')
         inst_time = datetime.datetime.fromtimestamp(int(time.time()))
         printarrayGlob.insert(1, [inst_time, module, pid, "Killing failed, module not found"])
         printarrayGlob.pop()
@@ -192,7 +192,7 @@ if __name__ == "__main__":
                         Did you set environment variables? \
                         Or activate the virtualenv.')
 
-    cfg = ConfigParser.ConfigParser()
+    cfg = configparser.ConfigParser()
     cfg.read(configfile)
 
     # REDIS #
@@ -227,7 +227,7 @@ if __name__ == "__main__":
                 printarray1 = []
                 printarray2 = []
                 printarray3 = []
-                for queue, card in server.hgetall("queues").iteritems():
+                for queue, card in server.hgetall("queues").items():
                     all_queue.add(queue)
                     key = "MODULE_" + queue + "_"
                     keySet = "MODULE_TYPE_" + queue
@@ -337,15 +337,15 @@ if __name__ == "__main__":
                 legend = AsciiTable(legend_array, title="Legend")
                 legend.column_max_width(1)
 
-                print legend.table
+                print(legend.table)
-                print '\n'
+                print('\n')
-                print t1.table
+                print(t1.table)
-                print '\n'
+                print('\n')
-                print t2.table
+                print(t2.table)
-                print '\n'
+                print('\n')
-                print t3.table
+                print(t3.table)
-                print '\n'
+                print('\n')
-                print t4.table
+                print(t4.table9)
 
                 if (datetime.datetime.now() - lastTime).total_seconds() > args.refresh*5:
                     lastTime = datetime.datetime.now()

bin/ModuleStats.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 """
     This module makes statistics for some modules and providers
@@ -52,10 +52,10 @@ def compute_most_posted(server, message):
         # Member set is a list of (value, score) pairs
         if int(member_set[0][1]) < keyword_total_sum:
             #remove min from set and add the new one
-            print module + ': adding ' +keyword+ '(' +str(keyword_total_sum)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
+            print(module + ': adding ' +keyword+ '(' +str(keyword_total_sum)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')')
             server.zrem(redis_progression_name_set, member_set[0][0])
             server.zadd(redis_progression_name_set, float(keyword_total_sum), keyword)
-            print redis_progression_name_set
+            print(redis_progression_name_set)
 
 
 def compute_provider_info(server_trend, server_pasteName, path):
@@ -94,7 +94,7 @@ def compute_provider_info(server_trend, server_pasteName, path):
         # Member set is a list of (value, score) pairs
         if float(member_set[0][1]) < new_avg:
             #remove min from set and add the new one
-            print 'Size - adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
+            print('Size - adding ' +paste_provider+ '(' +str(new_avg)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')')
             server_trend.zrem(redis_sum_size_set, member_set[0][0])
             server_trend.zadd(redis_sum_size_set, float(sum_size), paste_provider)
             server_trend.zrem(redis_avg_size_name_set, member_set[0][0])
@@ -110,7 +110,7 @@ def compute_provider_info(server_trend, server_pasteName, path):
         # Member set is a list of (value, score) pairs
         if int(member_set[0][1]) < num_paste:
             #remove min from set and add the new one
-            print 'Num - adding ' +paste_provider+ '(' +str(num_paste)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')'
+            print('Num - adding ' +paste_provider+ '(' +str(num_paste)+') in set and removing '+member_set[0][0]+'('+str(member_set[0][1])+')')
             server_trend.zrem(member_set[0][0])
             server_trend.zadd(redis_providers_name_set, float(num_paste), paste_provider)
 
@@ -149,7 +149,7 @@ if __name__ == '__main__':
 
         if message is None:
             publisher.debug("{} queue is empty, waiting".format(config_section))
-            print 'sleeping'
+            print('sleeping')
             time.sleep(20)
             continue
 

bin/ModulesInformationV2.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 from asciimatics.widgets import Frame, ListBox, Layout, Divider, Text, \
@@ -10,7 +10,7 @@ from asciimatics.event import Event
 from asciimatics.event import KeyboardEvent, MouseEvent
 import sys, os
 import time, datetime
-import argparse, ConfigParser
+import argparse, configparser
 import json
 import redis
 import psutil
@@ -497,7 +497,7 @@ MANAGE MODULES AND GET INFOS
 def getPid(module):
     p = Popen([command_search_pid.format(module+".py")], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
     for line in p.stdout:
-        print line
+        print(line)
         splittedLine = line.split()
         if 'python2' in splittedLine:
             return int(splittedLine[0])
@@ -511,21 +511,21 @@ def clearRedisModuleInfo():
 
 def cleanRedis():
     for k in server.keys("MODULE_TYPE_*"):
-        moduleName = k[12:].split('_')[0]
+        moduleName = (k[12:].decode('utf8')).split('_')[0]
         for pid in server.smembers(k):
             flag_pid_valid = False
             proc = Popen([command_search_name.format(pid)], stdin=PIPE, stdout=PIPE, bufsize=1, shell=True)
             try:
                 for line in proc.stdout:
                     splittedLine = line.split()
-                    if ('python2' in splittedLine or 'python' in splittedLine) and "./"+moduleName+".py" in splittedLine:
+                    if ('python3' in splittedLine or 'python' in splittedLine) and "./"+moduleName + ".py" in splittedLine:
                         flag_pid_valid = True
 
                 if not flag_pid_valid:
                     #print flag_pid_valid, 'cleaning', pid, 'in', k
                     server.srem(k, pid)
                     inst_time = datetime.datetime.fromtimestamp(int(time.time()))
-                    log(([str(inst_time).split(' ')[1], moduleName, pid, "Cleared invalid pid in " + k], 0))
+                    log(([str(inst_time).split(' ')[1], moduleName, pid, "Cleared invalid pid in " + (k).decode('utf8')], 0))
 
             #Error due to resize, interrupted sys call
             except IOError as e:
@@ -601,7 +601,9 @@ def fetchQueueData():
     printarray_running = []
     printarray_idle = []
     printarray_notrunning = []
-    for queue, card in server.hgetall("queues").iteritems():
+    for queue, card in iter(server.hgetall("queues").items()):
+        queue = queue.decode('utf8')
+        card = card.decode('utf8')
         all_queue.add(queue)
         key = "MODULE_" + queue + "_"
         keySet = "MODULE_TYPE_" + queue
@@ -715,6 +717,11 @@ def format_string(tab, padding_row):
 
         text=""
         for ite, elem in enumerate(the_array):
+            try:
+                elem = elem.decode('utf8')
+            except AttributeError:
+                pass
+
             if len(elem) > padding_row[ite]:
                 text += "*" + elem[-padding_row[ite]+6:]
                 padd_off = " "*5
@@ -761,7 +768,7 @@ def demo(screen):
 
         if time.time() - time_cooldown > args.refresh:
             cleanRedis()
-            for key, val in fetchQueueData().iteritems(): #fetch data and put it into the tables
+            for key, val in iter(fetchQueueData().items()): #fetch data and put it into the tables
                 TABLES[key] = val
             TABLES["logs"] = format_string(printarrayLog, TABLES_PADDING["logs"])
 
@@ -790,7 +797,7 @@ if __name__ == "__main__":
                         Did you set environment variables? \
                         Or activate the virtualenv.')
 
-    cfg = ConfigParser.ConfigParser()
+    cfg = configparser.ConfigParser()
     cfg.read(configfile)
 
     # REDIS #

bin/NotificationHelper.py

@@ -1,10 +1,10 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
-import ConfigParser
+import configparser
 import os
 import smtplib
-from email.MIMEMultipart import MIMEMultipart
+from email.mime.multipart import MIMEMultipart
 from email.mime.text import MIMEText
 
 """
@@ -28,7 +28,7 @@ def sendEmailNotification(recipient, term):
                     Did you set environment variables? \
                     Or activate the virtualenv?')
 
-    cfg = ConfigParser.ConfigParser()
+    cfg = configparser.ConfigParser()
     cfg.read(configfile)
 
     sender = cfg.get("Notifications", "sender"),
@@ -76,8 +76,5 @@ def sendEmailNotification(recipient, term):
         smtp_server.quit()
 
     except Exception as e:
-        print str(e)
+        print(str(e))
         # raise e
-
-
-

bin/Onion.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 """
 The ZMQ_Sub_Onion Module
@@ -37,7 +37,7 @@ from Helper import Process
 def fetch(p, r_cache, urls, domains, path):
     failed = []
     downloaded = []
-    print len(urls), 'Urls to fetch.'
+    print(len(urls), 'Urls to fetch.')
     for url, domain in zip(urls, domains):
         if r_cache.exists(url) or url in failed:
             continue
@@ -73,9 +73,9 @@ def fetch(p, r_cache, urls, domains, path):
             r_cache.setbit(url, 0, 0)
             r_cache.expire(url, 3600)
             failed.append(url)
-            print 'Failed at downloading', url
+            print('Failed at downloading', url)
-            print process.stdout.read()
+            print(process.stdout.read())
-    print 'Failed:', len(failed), 'Downloaded:', len(downloaded)
+    print('Failed:', len(failed), 'Downloaded:', len(downloaded))
 
 
 if __name__ == "__main__":
@@ -109,7 +109,7 @@ if __name__ == "__main__":
 
     while True:
         if message is not None:
-            print message
+            print(message)
             filename, score = message.split()
 
             # "For each new paste"
@@ -152,6 +152,6 @@ if __name__ == "__main__":
             prec_filename = filename
         else:
             publisher.debug("Script url is Idling 10s")
-            print 'Sleeping'
+            print('Sleeping')
             time.sleep(10)
         message = p.get_from_set()

bin/Phone.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 """
@@ -30,10 +30,11 @@ def search_phone(message):
 
     # if the list is greater than 4, we consider the Paste may contain a list of phone numbers
     if len(results) > 4:
-        print results
+        print(results)
         publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
         #send to Browse_warning_paste
-        p.populate_set_out('phone;{}'.format(message), 'alertHandler')
+        msg = 'phone;{}'.format(message)
+        p.populate_set_out(msg, 'alertHandler')
         #Send to duplicate
         p.populate_set_out(message, 'Duplicate')
         stats = {}

bin/QueueIn.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 

bin/QueueOut.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 from pubsublogger import publisher

bin/Queues_Monitoring.py

@@ -1,9 +1,9 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 import redis
 import argparse
-import ConfigParser
+import configparser
 import time
 import os
 from pubsublogger import publisher
@@ -14,7 +14,7 @@ def main():
     """Main Function"""
 
     # CONFIG #
-    cfg = ConfigParser.ConfigParser()
+    cfg = configparser.ConfigParser()
     cfg.read('./packages/config.cfg')
 
     # SCRIPT PARSER #
@@ -49,7 +49,7 @@ def main():
         row.sort()
         table.add_rows(row, header=False)
         os.system('clear')
-        print table.draw()
+        print(table.draw())
 
 
 if __name__ == "__main__":

bin/RegexForTermsFrequency.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 """
 This Module is used for term frequency.
@@ -76,6 +76,9 @@ if __name__ == "__main__":
                 dico_regex, dico_regexname_to_redis = refresh_dicos()
                 print('dico got refreshed')
 
+            print(dico_regex)
+            print(dico_regexname_to_redis)
+
             filename = message
             temp = filename.split('/')
             timestamp = calendar.timegm((int(temp[-4]), int(temp[-3]), int(temp[-2]), 0, 0, 0))
@@ -115,6 +118,6 @@ if __name__ == "__main__":
 
         else:
             publisher.debug("Script RegexForTermsFrequency is Idling")
-            print "sleeping"
+            print("sleeping")
             time.sleep(5)
         message = p.get_from_set()

bin/Release.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 import time
 from packages import Paste
@@ -29,7 +29,7 @@ if __name__ == "__main__":
         filepath = p.get_from_set()
         if filepath is None:
             publisher.debug("Script Release is Idling 10s")
-            print 'Sleeping'
+            print('Sleeping')
             time.sleep(10)
             continue
 
@@ -40,6 +40,7 @@ if __name__ == "__main__":
             continue
 
         to_print = 'Release;{};{};{};{} releases;{}'.format(paste.p_source, paste.p_date, paste.p_name, len(releases), paste.p_path)
+        print(to_print)
         if len(releases) > 30:
             publisher.warning(to_print)
         else:

bin/Repartition_graph.py

@@ -1,9 +1,9 @@
-#!/usr/bin/python2.7
+#!/usr/bin/python3.5
 # -*-coding:UTF-8 -*
 
 import redis
 import argparse
-import ConfigParser
+import configparser
 from datetime import datetime
 from pubsublogger import publisher
 
@@ -14,7 +14,7 @@ def main():
     """Main Function"""
 
     # CONFIG #
-    cfg = ConfigParser.ConfigParser()
+    cfg = configparser.ConfigParser()
     cfg.read('./packages/config.cfg')
 
     # SCRIPT PARSER #

bin/SQLInjectionDetection.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 """
@@ -13,7 +13,7 @@ It test different possibility to makes some sqlInjection.
 
 import time
 import string
-import urllib2
+import urllib.request
 import re
 from pubsublogger import publisher
 from Helper import Process
@@ -74,8 +74,8 @@ def analyse(url, path):
     if (result_path > 0) or (result_query > 0):
         paste = Paste.Paste(path)
         if (result_path > 1) or (result_query > 1):
-            print "Detected SQL in URL: "
+            print("Detected SQL in URL: ")
-            print urllib2.unquote(url)
+            print(urllib.request.unquote(url))
             to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_path)
             publisher.warning(to_print)
             #Send to duplicate
@@ -83,8 +83,8 @@ def analyse(url, path):
             #send to Browse_warning_paste
             p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler')
         else:
-            print "Potential SQL injection:"
+            print("Potential SQL injection:")
-            print urllib2.unquote(url)
+            print(urllib.request.unquote(url))
             to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Potential SQL injection", paste.p_path)
             publisher.info(to_print)
 
@@ -92,7 +92,7 @@ def analyse(url, path):
 # Try to detect if the url passed might be an sql injection by appliying the regex
 # defined above on it.
 def is_sql_injection(url_parsed):
-    line = urllib2.unquote(url_parsed)
+    line = urllib.request.unquote(url_parsed)
     line = string.upper(line)
     result = []
     result_suspect = []
@@ -114,10 +114,10 @@ def is_sql_injection(url_parsed):
             result_suspect.append(line[temp_res:temp_res+len(word)])
 
     if len(result)>0:
-        print result
+        print(result)
         return 2
     elif len(result_suspect)>0:
-        print result_suspect
+        print(result_suspect)
         return 1
     else:
         return 0

bin/SentimentAnalysis.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 """
     Sentiment analyser module.
@@ -33,7 +33,7 @@ size_threshold = 250
 line_max_length_threshold = 1000
 
 import os
-import ConfigParser
+import configparser
 
 configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
 if not os.path.exists(configfile):
@@ -41,7 +41,7 @@ if not os.path.exists(configfile):
         Did you set environment variables? \
         Or activate the virtualenv.')
 
-cfg = ConfigParser.ConfigParser()
+cfg = configparser.ConfigParser()
 cfg.read(configfile)
 
 sentiment_lexicon_file = cfg.get("Directories", "sentiment_lexicon_file")
@@ -69,7 +69,7 @@ def Analyse(message, server):
         combined_datetime = datetime.datetime.combine(the_date, the_time)
         timestamp = calendar.timegm(combined_datetime.timetuple())
 
-        sentences = tokenize.sent_tokenize(p_content.decode('utf-8', 'ignore'))
+        sentences = tokenize.sent_tokenize(p_content)
 
         if len(sentences) > 0:
             avg_score = {'neg': 0.0, 'neu': 0.0, 'pos': 0.0, 'compoundPos': 0.0, 'compoundNeg': 0.0}
@@ -109,11 +109,11 @@ def Analyse(message, server):
             provider_timestamp = provider + '_' + str(timestamp)
             server.incr('UniqID')
             UniqID = server.get('UniqID')
-            print provider_timestamp, '->', UniqID, 'dropped', num_line_removed, 'lines'
+            print(provider_timestamp, '->', UniqID.decode('utf8'), 'dropped', num_line_removed, 'lines')
             server.sadd(provider_timestamp, UniqID)
             server.set(UniqID, avg_score)
     else:
-        print 'Dropped:', p_MimeType
+        print('Dropped:', p_MimeType)
 
 
 def isJSON(content):
@@ -121,7 +121,7 @@ def isJSON(content):
         json.loads(content)
         return True
 
-    except Exception,e:
+    except Exception:
         return False
 
 import signal
@@ -170,4 +170,3 @@ if __name__ == '__main__':
             continue
         else:
             signal.alarm(0)
-

bin/SetForTermsFrequency.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 """
 This Module is used for term frequency.
@@ -126,6 +126,6 @@ if __name__ == "__main__":
 
         else:
             publisher.debug("Script RegexForTermsFrequency is Idling")
-            print "sleeping"
+            print("sleeping")
             time.sleep(5)
         message = p.get_from_set()

bin/Shutdown.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 """
 The ZMQ_Feed_Q Module
@@ -21,7 +21,7 @@ Requirements
 
 """
 import redis
-import ConfigParser
+import configparser
 import os
 
 configfile = os.path.join(os.environ['AIL_BIN'], './packages/config.cfg')
@@ -31,7 +31,7 @@ def main():
     """Main Function"""
 
     # CONFIG #
-    cfg = ConfigParser.ConfigParser()
+    cfg = configparser.ConfigParser()
     cfg.read(configfile)
 
     # REDIS

bin/SourceCode.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 import time
 from packages import Paste
@@ -29,7 +29,7 @@ if __name__ == "__main__":
 
     languages = [shell, c, php, bash, python, javascript, bash, ruby, adr]
     regex = '|'.join(languages)
-    print regex
+    print(regex)
 
     while True:
         message = p.get_from_set()

bin/Tokenize.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 """
 The Tokenize Module
@@ -50,7 +50,7 @@ if __name__ == "__main__":
 
     while True:
         message = p.get_from_set()
-        print message
+        print(message)
         if message is not None:
             paste = Paste.Paste(message)
             signal.alarm(5)
@@ -67,4 +67,4 @@ if __name__ == "__main__":
         else:
             publisher.debug("Tokeniser is idling 10s")
             time.sleep(10)
-            print "sleepin"
+            print("Sleeping")

bin/Update-conf.py

@@ -1,8 +1,8 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
-import ConfigParser
+import configparser
-from ConfigParser import ConfigParser as cfgP
+from configparser import ConfigParser as cfgP
 import os
 from collections import OrderedDict
 import sys
@@ -20,9 +20,9 @@ def main():
                         Or activate the virtualenv.')
     configfileSample  = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg.sample')
 
-    cfg = ConfigParser.ConfigParser()
+    cfg = configparser.ConfigParser()
     cfg.read(configfile)
-    cfgSample = ConfigParser.ConfigParser()
+    cfgSample = configparser.ConfigParser()
     cfgSample.read(configfileSample)
 
     sections = cfgP.sections(cfg)
@@ -109,4 +109,3 @@ if __name__ == "__main__":
         sys.exit()
     else:
         sys.exit(1)
-

bin/Web.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 """
@@ -100,7 +100,7 @@ if __name__ == "__main__":
                     publisher.debug('{} Published'.format(url))
 
                     if f1 == "onion":
-                        print domain
+                        print(domain)
 
                     hostl = unicode(avoidNone(subdomain)+avoidNone(domain))
                     try:
@@ -121,16 +121,16 @@ if __name__ == "__main__":
                     # EU is not an official ISO 3166 code (but used by RIPE
                     # IP allocation)
                     if cc is not None and cc != "EU":
-                        print hostl, asn, cc, \
+                        print(hostl, asn, cc, \
-                            pycountry.countries.get(alpha_2=cc).name
+                            pycountry.countries.get(alpha_2=cc).name)
                         if cc == cc_critical:
                             to_print = 'Url;{};{};{};Detected {} {}'.format(
                                     PST.p_source, PST.p_date, PST.p_name,
                                     hostl, cc)
                             #publisher.warning(to_print)
-                            print to_print
+                            print(to_print)
                     else:
-                        print hostl, asn, cc
+                        print(hostl, asn, cc)
 
                 A_values = lib_refine.checking_A_record(r_serv2,
                                                         domains_list)
@@ -146,7 +146,7 @@ if __name__ == "__main__":
 
         else:
             publisher.debug("Script url is Idling 10s")
-            print 'Sleeping'
+            print('Sleeping')
             time.sleep(10)
 
         message = p.get_from_set()

bin/WebStats.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 """
@@ -33,7 +33,7 @@ def analyse(server, field_name, date, url_parsed):
         if field_name == "domain": #save domain in a set for the monthly plot
             domain_set_name = "domain_set_" + date[0:6]
             server.sadd(domain_set_name, field)
-            print "added in " + domain_set_name +": "+ field
+            print("added in " + domain_set_name +": "+ field)
 
 def get_date_range(num_day):
     curr_date = datetime.date.today()
@@ -145,24 +145,25 @@ if __name__ == '__main__':
                 year = today.year
                 month = today.month
 
-                print 'Building protocol graph'
+                print('Building protocol graph')
                 lib_words.create_curve_with_word_file(r_serv_trend, csv_path_proto,
                                                       protocolsfile_path, year,
                                                       month)
 
-                print 'Building tld graph'
+                print('Building tld graph')
                 lib_words.create_curve_with_word_file(r_serv_trend, csv_path_tld,
                                                       tldsfile_path, year,
                                                       month)
 
-                print 'Building domain graph'
+                print('Building domain graph')
                 lib_words.create_curve_from_redis_set(r_serv_trend, csv_path_domain,
                                                       "domain", year,
                                                       month)
-                print 'end building'
+                print('end building')
+
 
             publisher.debug("{} queue is empty, waiting".format(config_section))
-            print 'sleeping'
+            print('sleeping')
             time.sleep(5*60)
             continue
 
@@ -173,9 +174,13 @@ if __name__ == '__main__':
             faup.decode(url)
             url_parsed = faup.get()
 
-            analyse(r_serv_trend, 'scheme', date, url_parsed)	#Scheme analysis
+            # Scheme analysis
-            analyse(r_serv_trend, 'tld', date, url_parsed)	#Tld analysis
+            analyse(r_serv_trend, 'scheme', date, url_parsed)
-	    analyse(r_serv_trend, 'domain', date, url_parsed)	#Domain analysis
+            # Tld analysis
+            analyse(r_serv_trend, 'tld', date, url_parsed)
+            # Domain analysis
+	        analyse(r_serv_trend, 'domain', date, url_parsed)
+
             compute_progression(r_serv_trend, 'scheme', num_day_to_look, url_parsed)
             compute_progression(r_serv_trend, 'tld', num_day_to_look, url_parsed)
             compute_progression(r_serv_trend, 'domain', num_day_to_look, url_parsed)

bin/alertHandler.py

@@ -62,8 +62,9 @@ if __name__ == "__main__":
     while True:
             message = p.get_from_set()
             if message is not None:
-                message = message.decode('utf8') #decode because of pyhton3
+                #decode because of pyhton3
                 module_name, p_path = message.split(';')
+                print("new alert : {}".format(module_name))
                 #PST = Paste.Paste(p_path)
             else:
                 publisher.debug("Script Attribute is idling 10s")

bin/empty_queue.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 """
@@ -27,10 +27,9 @@ if __name__ == "__main__":
     config_section = ['Curve']
 
     for queue in config_section:
-        print 'dropping: ' + queue
+        print('dropping: ' + queue)
         p = Process(queue)
         while True:
             message = p.get_from_set()
             if message is None:
                 break
-

bin/feeder/pystemon-feeder.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3.5
 # -*- coding: utf-8 -*-
 #
 # This file is part of AIL framework - Analysis Information Leak framework
@@ -25,7 +25,7 @@ import time
 import redis
 import base64
 import os
-import ConfigParser
+import configparser
 
 configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
 if not os.path.exists(configfile):
@@ -33,7 +33,7 @@ if not os.path.exists(configfile):
         Did you set environment variables? \
         Or activate the virtualenv.')
 
-cfg = ConfigParser.ConfigParser()
+cfg = configparser.ConfigParser()
 cfg.read(configfile)
 
 if cfg.has_option("ZMQ_Global", "bind"):
@@ -59,6 +59,7 @@ while True:
     time.sleep(base_sleeptime + sleep_inc)
     topic = 101
     paste = r.lpop("pastes")
+    print(paste)
     if paste is None:
         continue
     socket.send("%d %s" % (topic, paste))

bin/feeder/test-zmq.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3.5
 # -*- coding: utf-8 -*-
 #
 # This file is part of AIL framework - Analysis Information Leak framework
@@ -19,14 +19,15 @@ socket.connect ("tcp://crf.circl.lu:%s" % port)
 # 101 Name of the pastes only
 # 102 Full pastes in raw base64(gz)
 
-topicfilter = "102"
+topicfilter = b"102"
 socket.setsockopt(zmq.SUBSCRIBE, topicfilter)
-
+print('b0')
 while True:
     message = socket.recv()
+    print('b1')
     print (message)
-    if topicfilter == "102":
+    if topicfilter == b"102":
         topic, paste, messagedata = message.split()
-        print paste, messagedata
+        print(paste, messagedata)
     else:
         print (message)

bin/import_dir.py

@@ -1,9 +1,9 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3.5
 # -*- coding: utf-8 -*-
 
 import zmq
 import base64
-import StringIO
+from io import StringIO
 import gzip
 import argparse
 import os
@@ -31,8 +31,7 @@ import mimetypes
 '
 '''
 
-import StringIO
+
-import gzip
 def is_hierachy_valid(path):
     var = path.split('/')
     try:
@@ -72,7 +71,12 @@ if __name__ == "__main__":
             wanted_path = wanted_path.split('/')
             wanted_path = '/'.join(wanted_path[-(4+args.hierarchy):])
 
-            messagedata = open(complete_path).read()
+            with gzip.open(complete_path, 'rb') as f:
+                messagedata = f.read()
+
+            #print(type(complete_path))
+            #file = open(complete_path)
+            #messagedata = file.read()
 
             #if paste do not have a 'date hierarchy' ignore it
             if not is_hierachy_valid(complete_path):
@@ -90,5 +94,8 @@ if __name__ == "__main__":
 
             print(args.name+'>'+wanted_path)
             path_to_send = args.name + '>' + wanted_path
-            socket.send('{} {} {}'.format(args.channel, path_to_send, base64.b64encode(messagedata)))
+            #s = b'{} {} {}'.format(args.channel, path_to_send, base64.b64encode(messagedata))
+            # use bytes object
+            s = b' '.join( [ args.channel.encode(), path_to_send.encode(), base64.b64encode(messagedata) ] )
+            socket.send(s)
             time.sleep(args.seconds)

bin/indexer_lookup.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3.5
 # -*- coding: utf-8 -*-
 #
 # This file is part of AIL framework - Analysis Information Leak framework
@@ -10,7 +10,7 @@
 #
 # Copyright (c) 2014 Alexandre Dulaunoy - a@foo.be
 
-import ConfigParser
+import configparser
 import argparse
 import gzip
 import os
@@ -23,7 +23,7 @@ def readdoc(path=None):
     return f.read()
 
 configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
-cfg = ConfigParser.ConfigParser()
+cfg = configparser.ConfigParser()
 cfg.read(configfile)
 
 # Indexer configuration - index dir and schema setup
@@ -51,7 +51,7 @@ ix = index.open_dir(indexpath)
 from whoosh.qparser import QueryParser
 
 if args.n:
-    print ix.doc_count_all()
+    print(ix.doc_count_all())
     exit(0)
 
 if args.l:

bin/launch_queues.py

@@ -1,8 +1,8 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 
-import ConfigParser
+import configparser
 import os
 import subprocess
 import time
@@ -23,21 +23,21 @@ if __name__ == '__main__':
         raise Exception('Unable to find the configuration file. \
                         Did you set environment variables? \
                         Or activate the virtualenv.')
-    config = ConfigParser.ConfigParser()
+    config = configparser.ConfigParser()
     config.read(configfile)
 
     modules = config.sections()
     pids = {}
     for module in modules:
-        pin = subprocess.Popen(["python", './QueueIn.py', '-c', module])
+        pin = subprocess.Popen(["python3", './QueueIn.py', '-c', module])
-        pout = subprocess.Popen(["python", './QueueOut.py', '-c', module])
+        pout = subprocess.Popen(["python3", './QueueOut.py', '-c', module])
         pids[module] = (pin, pout)
     is_running = True
     try:
         while is_running:
             time.sleep(5)
             is_running = False
-            for module, p in pids.iteritems():
+            for module, p in pids.items():
                 pin, pout = p
                 if pin is None:
                     # already dead
@@ -57,7 +57,7 @@ if __name__ == '__main__':
                     is_running = True
                 pids[module] = (pin, pout)
     except KeyboardInterrupt:
-        for module, p in pids.iteritems():
+        for module, p in pids.items():
             pin, pout = p
             if pin is not None:
                 pin.kill()

bin/packages/Paste.py

@@ -1,4 +1,4 @@
-#!/usr/bin/python2.7
+#!/usr/bin/python3.5
 
 """
 The ``Paste Class``
@@ -24,15 +24,8 @@ import operator
 import string
 import re
 import json
-try: # dirty to support python3
+import configparser
-    import ConfigParser
+from io import StringIO
-except:
-    import configparser
-    ConfigParser = configparser
-try: # dirty to support python3
-    import cStringIO
-except:
-    from io import StringIO as cStringIO
 import sys
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
 from Date import Date
@@ -71,7 +64,7 @@ class Paste(object):
                             Did you set environment variables? \
                             Or activate the virtualenv.')
 
-        cfg = ConfigParser.ConfigParser()
+        cfg = configparser.ConfigParser()
         cfg.read(configfile)
         self.cache = redis.StrictRedis(
             host=cfg.get("Redis_Queues", "host"),
@@ -85,11 +78,15 @@ class Paste(object):
         self.p_path = p_path
         self.p_name = os.path.basename(self.p_path)
         self.p_size = round(os.path.getsize(self.p_path)/1024.0, 2)
+        self.p_mime = magic.from_buffer("test", mime=True)
         self.p_mime = magic.from_buffer(self.get_p_content(), mime=True)
 
         # Assuming that the paste will alway be in a day folder which is itself
         # in a month folder which is itself in a year folder.
         # /year/month/day/paste.gz
+
+        # TODO use bytes ?
+
         var = self.p_path.split('/')
         self.p_date = Date(var[-4], var[-3], var[-2])
         self.p_source = var[-5]
@@ -117,17 +114,25 @@ class Paste(object):
         paste = self.cache.get(self.p_path)
         if paste is None:
             try:
+            #print('----------------------------------------------------------------')
+            #print(self.p_name)
+            #print('----------------------------------------------------------------')
                 with gzip.open(self.p_path, 'rb') as f:
                     paste = f.read()
                     self.cache.set(self.p_path, paste)
                     self.cache.expire(self.p_path, 300)
             except:
-                return ''
+                paste = b''
-                pass
+
-        return paste
+        return paste.decode('utf8')
 
     def get_p_content_as_file(self):
-        return cStringIO.StringIO(self.get_p_content())
+        try:
+            message = StringIO( (self.get_p_content()).decode('utf8') )
+        except AttributeError:
+            message = StringIO( (self.get_p_content()) )
+
+        return message
 
     def get_p_content_with_removed_lines(self, threshold):
         num_line_removed = 0
@@ -137,6 +142,7 @@ class Paste(object):
         line_id = 0
         for line_id, line in enumerate(f):
             length = len(line)
+
             if length < line_length_threshold:
                 string_content += line
             else:
@@ -202,8 +208,8 @@ class Paste(object):
         .. seealso:: _set_p_hash_kind("md5")
 
         """
-        for hash_name, the_hash in self.p_hash_kind.iteritems():
+        for hash_name, the_hash in self.p_hash_kind.items():
-            self.p_hash[hash_name] = the_hash.Calculate(self.get_p_content())
+            self.p_hash[hash_name] = the_hash.Calculate(self.get_p_content().encode('utf8'))
         return self.p_hash
 
     def _get_p_language(self):
@@ -342,7 +348,7 @@ class Paste(object):
         tokenizer = RegexpTokenizer('[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]+',
                                     gaps=True, discard_empty=True)
 
-        blob = TextBlob(clean(self.get_p_content()), tokenizer=tokenizer)
+        blob = TextBlob(clean( (self.get_p_content()) ), tokenizer=tokenizer)
 
         for word in blob.tokens:
             if word in words.keys():
@@ -351,7 +357,7 @@ class Paste(object):
                 num = 0
             words[word] = num + 1
         if sort:
-            var = sorted(words.iteritems(), key=operator.itemgetter(1), reverse=True)
+            var = sorted(words.items(), key=operator.itemgetter(1), reverse=True)
         else:
             var = words
 

bin/packages/lib_refine.py

@@ -76,7 +76,7 @@ def checking_MX_record(r_serv, adress_set):
                     r_serv.setex(MXdomain[1:], 1, timedelta(days=1))
 
                 except Exception as e:
-                    print e
+                    print(e)
 
     publisher.debug("emails before: {0} after: {1} (valid)".format(num, score))
     return (num, WalidMX)
@@ -125,7 +125,7 @@ def checking_A_record(r_serv, domains_set):
             publisher.debug('The Label is too long')
 
         except Exception as e:
-            print e
+            print(e)
 
     publisher.debug("URLs before: {0} after: {1} (valid)".format(num, score))
     return (num, WalidA)

bin/packages/lib_words.py

@@ -82,16 +82,16 @@ def create_curve_with_word_file(r_serv, csvfilename, feederfilename, year, month
 
     """
     threshold = 50
-    first_day = date(year, month, 01)
+    first_day = date(year, month, 1)
     last_day = date(year, month, calendar.monthrange(year, month)[1])
     words = []
 
-    with open(feederfilename, 'rb') as f:
+    with open(feederfilename, 'r') as f:
         # words of the files
         words = sorted([word.strip() for word in f if word.strip()[0:2]!='//' and word.strip()!='' ])
 
     headers = ['Date'] + words
-    with open(csvfilename+'.csv', 'wb') as f:
+    with open(csvfilename+'.csv', 'w') as f:
         writer = csv.writer(f)
         writer.writerow(headers)
 
@@ -103,11 +103,14 @@ def create_curve_with_word_file(r_serv, csvfilename, feederfilename, year, month
             # from the 1srt day to the last of the list
             for word in words:
                 value = r_serv.hget(word, curdate)
+
                 if value is None:
                     row.append(0)
                 else:
                     # if the word have a value for the day
                     # FIXME Due to performance issues (too many tlds, leads to more than 7s to perform this procedure), I added a threshold
+                    value = r_serv.hget(word, curdate)
+                    value = int(value.decode('utf8'))
                     if value >= threshold:
                         row.append(value)
             writer.writerow(row)
@@ -127,14 +130,14 @@ def create_curve_from_redis_set(server, csvfilename, set_to_plot, year, month):
 
     """
 
-    first_day = date(year, month, 01)
+    first_day = date(year, month, 1)
     last_day = date(year, month, calendar.monthrange(year, month)[1])
 
     redis_set_name = set_to_plot + "_set_" + str(year) + str(month).zfill(2)
     words = list(server.smembers(redis_set_name))
 
     headers = ['Date'] + words
-    with open(csvfilename+'.csv', 'wb') as f:
+    with open(csvfilename+'.csv', 'w') as f:
         writer = csv.writer(f)
         writer.writerow(headers)
 

bin/preProcessFeed.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 '''
@@ -48,7 +48,7 @@ if __name__ == '__main__':
         message = p.get_from_set()
         if message is None:
             publisher.debug("{} queue is empty, waiting".format(config_section))
-            print "queue empty"
+            print("queue empty")
             time.sleep(1)
             continue
 

bin/template.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 """
     Template for new modules

bin/tor_fetcher.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python2
+#!/usr/bin/env python3.5
 # -*-coding:UTF-8 -*
 
 import socks
@@ -63,5 +63,5 @@ if __name__ == "__main__":
     t, path = tempfile.mkstemp()
     with open(path,  'w') as f:
         f.write(to_write)
-    print path
+    print(path)
     exit(0)