From a900d5f08d6e827f80ae9b30cecccc07893efb8c Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 3 May 2018 16:21:33 +0200 Subject: [PATCH] add python 2 to 3 bash + fix Duplicate + clean --- bin/Base64.py | 2 +- bin/Categ.py | 1 - bin/Duplicates.py | 9 ++-- bin/ModuleStats.py | 1 - bin/packages/Paste.py | 4 -- bin/packages/config.cfg.sample | 3 ++ bin/packages/modules.cfg | 4 -- pip_packages_requirement.txt | 51 ------------------- .../browsepastes/Flask_browsepastes.py | 3 +- 9 files changed, 9 insertions(+), 69 deletions(-) delete mode 100644 pip_packages_requirement.txt diff --git a/bin/Base64.py b/bin/Base64.py index 7cfc98a7..fbb8a356 100755 --- a/bin/Base64.py +++ b/bin/Base64.py @@ -3,7 +3,7 @@ """ Base64 module - Dectect Base64, decode it and send to XXX for reprocess + Dectect Base64 and decode it """ import time import os diff --git a/bin/Categ.py b/bin/Categ.py index 25e8eaf7..019efc36 100755 --- a/bin/Categ.py +++ b/bin/Categ.py @@ -77,7 +77,6 @@ if __name__ == "__main__": tmp_dict[bname] = re.compile('|'.join(patterns), re.IGNORECASE) prec_filename = None - print(tmp_dict) while True: filename = p.get_from_set() diff --git a/bin/Duplicates.py b/bin/Duplicates.py index 5f7b3bb8..a85fa3e7 100755 --- a/bin/Duplicates.py +++ b/bin/Duplicates.py @@ -54,7 +54,6 @@ if __name__ == "__main__": dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis( host=p.config.get("Redis_Level_DB", "host"), port=year, db=month) - #print("dup: "+str(year)+str(month).zfill(2)+"\n") # FUNCTIONS # publisher.info("Script duplicate started") @@ -142,11 +141,12 @@ if __name__ == "__main__": paste_date = paste_date.decode('utf8') paste_date = paste_date if paste_date != None else "No date available" if paste_path != None: - hash_dico[dico_hash] = (hash_type, paste_path, percent, paste_date) + if paste_path != PST.p_path: + hash_dico[dico_hash] = (hash_type, paste_path, percent, paste_date) + + print('['+hash_type+'] '+'comparing: ' + str(PST.p_path[44:]) + ' and ' + str(paste_path[44:]) + ' percentage: ' + str(percent)) - print('['+hash_type+'] '+'comparing: ' + str(PST.p_path[44:]) + ' and ' + str(paste_path[44:]) + ' percentage: ' + str(percent)) except Exception: - #print(str(e)) print('hash not comparable, bad hash: '+dico_hash+' , current_hash: '+paste_hash) # Add paste in DB after checking to prevent its analysis twice @@ -181,7 +181,6 @@ if __name__ == "__main__": y = time.time() publisher.debug('{}Processed in {} sec'.format(to_print, y-x)) - #print '{}Processed in {} sec'.format(to_print, y-x) except IOError: to_print = 'Duplicate;{};{};{};'.format( diff --git a/bin/ModuleStats.py b/bin/ModuleStats.py index 13847685..9c25eb75 100755 --- a/bin/ModuleStats.py +++ b/bin/ModuleStats.py @@ -29,7 +29,6 @@ def get_date_range(num_day): def compute_most_posted(server, message): - print(message) module, num, keyword, paste_date = message.split(';') redis_progression_name_set = 'top_'+ module +'_set_' + paste_date diff --git a/bin/packages/Paste.py b/bin/packages/Paste.py index 87570aab..ce78e46b 100755 --- a/bin/packages/Paste.py +++ b/bin/packages/Paste.py @@ -112,9 +112,6 @@ class Paste(object): paste = self.cache.get(self.p_path) if paste is None: try: - #print('----------------------------------------------------------------') - #print(self.p_name) - #print('----------------------------------------------------------------') with gzip.open(self.p_path, 'rb') as f: paste = f.read() self.cache.set(self.p_path, paste) @@ -334,7 +331,6 @@ class Paste(object): Save a new duplicate on others pastes """ for hash_type, path, percent, date in list_value: - print(hash_type, path, percent, date) #get json json_duplicate = self.store.hget(path, attr_name) #json save on redis diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index dc8364c9..5d34f9b2 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -54,6 +54,9 @@ criticalNumberToAlert=8 #Will be considered as false positive if less that X matches from the top password list minTopPassList=5 +[Curve] +max_execution_time = 90 + [Base64] path = Base64/ max_execution_time = 60 diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 3d694066..b9e29506 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -74,10 +74,6 @@ subscribe = Redis_Url subscribe = Redis_Url publish = Redis_alertHandler,Redis_Duplicate -[Dox] -subscribe = Redis_Dox -publish = Redis_Duplicate,Redis_alertHandler - [ModuleStats] subscribe = Redis_ModuleStats diff --git a/pip_packages_requirement.txt b/pip_packages_requirement.txt deleted file mode 100644 index d6dd108d..00000000 --- a/pip_packages_requirement.txt +++ /dev/null @@ -1,51 +0,0 @@ -#Essential -redis -pyzmq -dnspython -logbook -pubsublogger -textblob - -#Graph -numpy -matplotlib -networkx -terminaltables -colorama -asciimatics - -#Tokeniser -nltk - -# Hashlib -crcmod -mmh3 -ssdeep -python-Levenshtein - -#Others -python-magic -pybloomfiltermmap -psutil -phonenumbers - -ipython -flask -texttable - -#DomainClassifier -DomainClassifier -#Indexer requirements -whoosh - -ipaddress -pycountry - -# To fetch Onion urls -PySocks - -#ASN lookup requirements -https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/adns-python/adns-python-1.2.1.tar.gz -https://github.com/trolldbois/python-cymru-services/archive/master.zip - -https://github.com/saffsd/langid.py/archive/master.zip diff --git a/var/www/modules/browsepastes/Flask_browsepastes.py b/var/www/modules/browsepastes/Flask_browsepastes.py index 3df2c49d..84a3149c 100644 --- a/var/www/modules/browsepastes/Flask_browsepastes.py +++ b/var/www/modules/browsepastes/Flask_browsepastes.py @@ -48,6 +48,7 @@ def getPastebyType(server, module_name): all_path = [] for path in server.smembers('WARNING_'+module_name): all_path.append(path) + return all_path @@ -95,8 +96,6 @@ def importantPasteByModule(): for path in allPastes[0:10]: path = path.decode('utf8') all_path.append(path) - #print(path) - #print(type(path)) paste = Paste.Paste(path) content = paste.get_p_content() content_range = max_preview_char if len(content)>max_preview_char else len(content)-1