diff --git a/bin/Credential.py b/bin/Credential.py index 3ac61faf..d81c9ff6 100755 --- a/bin/Credential.py +++ b/bin/Credential.py @@ -50,6 +50,8 @@ if __name__ == "__main__": if len(creds) > critical: print("========> Found more than 10 credentials in this file : {}".format(filepath)) publisher.warning(to_print) + #Send to duplicate + p.populate_set_out(filepath) if sites: print("=======> Probably on : {}".format(', '.join(sites))) else: diff --git a/bin/CreditCard.py b/bin/CreditCard.py index d4660c13..18703f4e 100755 --- a/bin/CreditCard.py +++ b/bin/CreditCard.py @@ -65,6 +65,8 @@ if __name__ == "__main__": if (len(creditcard_set) > 0): publisher.warning('{}Checked {} valid number(s)'.format( to_print, len(creditcard_set))) + #Send to duplicate + p.populate_set_out(filename) else: publisher.info('{}CreditCard related'.format(to_print)) else: diff --git a/bin/Duplicate_ssdeep_v2.py b/bin/Duplicate_ssdeep_v2.py index 35874371..f6aaca4f 100755 --- a/bin/Duplicate_ssdeep_v2.py +++ b/bin/Duplicate_ssdeep_v2.py @@ -6,8 +6,10 @@ The Duplicate module ==================== This huge module is, in short term, checking duplicates. +Its input comes from other modules, namely: + Credential, CreditCard, Keys, Mails and Phone -This one differ from v1 by only using redis and not json file on disk +This one differ from v1 by only using redis and not json file stored on disk Requirements: ------------- @@ -117,8 +119,7 @@ if __name__ == "__main__": if paste_path != None: hash_dico[dico_hash] = (paste_path, percent) - print 'comparing: ' + str(PST.p_path[44:]) + ' and ' + str(paste_path[44:]) + ' percentage: ' + str(percent) - #print ' '+ PST.p_path[44:] +', '+ paste_path[44:] + ', ' + str(percent) + #print 'comparing: ' + str(PST.p_path[44:]) + ' and ' + str(paste_path[44:]) + ' percentage: ' + str(percent) except: # ssdeep hash not comparable print 'ssdeep hash not comparable' @@ -146,7 +147,7 @@ if __name__ == "__main__": PST.__setattr__("p_duplicate", dupl) PST.save_attribute_redis("p_duplicate", dupl) publisher.info('{}Detected {}'.format(to_print, len(dupl))) - #print '{}Detected {}'.format(to_print, len(dupl)) + print '{}Detected {}'.format(to_print, len(dupl)) y = time.time() diff --git a/bin/Keys.py b/bin/Keys.py index 8058545b..9c44f60a 100755 --- a/bin/Keys.py +++ b/bin/Keys.py @@ -16,6 +16,8 @@ def search_gpg(message): content = paste.get_p_content() if '-----BEGIN PGP MESSAGE-----' in content: publisher.warning('{} has a PGP enc message'.format(paste.p_name)) + #Send to duplicate + p.populate_set_out(message) if __name__ == '__main__': diff --git a/bin/Mail.py b/bin/Mail.py index dd348ba6..964deb19 100755 --- a/bin/Mail.py +++ b/bin/Mail.py @@ -60,6 +60,8 @@ if __name__ == "__main__": MX_values[0]) if MX_values[0] > is_critical: publisher.warning(to_print) + #Send to duplicate + p.populate_set_out(filename) else: publisher.info(to_print) prec_filename = filename diff --git a/bin/Phone.py b/bin/Phone.py index 628f77c2..b53b079c 100755 --- a/bin/Phone.py +++ b/bin/Phone.py @@ -23,6 +23,8 @@ def search_phone(message): if len(results) > 4: print results publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name)) + #Send to duplicate + p.populate_set_out(message) if __name__ == '__main__': # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 9d8d6637..5f087427 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -3,7 +3,7 @@ subscribe = ZMQ_Global publish = Redis_Global [Duplicates] -subscribe = Redis_Global +subscribe = Redis_Duplicate [Indexer] subscribe = Redis_Global @@ -31,9 +31,11 @@ publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Re [CreditCards] subscribe = Redis_CreditCards +publish = Redis_Duplicate [Mail] subscribe = Redis_Mail +publish = Redis_Duplicate [Onion] subscribe = Redis_Onion @@ -55,15 +57,18 @@ subscribe = Redis_Global [Credential] subscribe = Redis_Credential +publish = Redis_Duplicate [Cve] subscribe = Redis_Cve [Phone] subscribe = Redis_Global +publish = Redis_Duplicate [SourceCode] subscribe = Redis_SourceCode [Keys] subscribe = Redis_Global +publish = Redis_Duplicate