Duplicate module takes its messages from other modules and no more from Global.

bin/Credential.py

@@ -50,6 +50,8 @@ if __name__ == "__main__":
         if len(creds) > critical:
             print("========> Found more than 10 credentials in this file : {}".format(filepath))
             publisher.warning(to_print)
+            #Send to duplicate
+            p.populate_set_out(filepath)
             if sites:
                 print("=======> Probably on : {}".format(', '.join(sites)))
         else:

bin/CreditCard.py

@@ -65,6 +65,8 @@ if __name__ == "__main__":
                 if (len(creditcard_set) > 0):
                     publisher.warning('{}Checked {} valid number(s)'.format(
                         to_print, len(creditcard_set)))
+                    #Send to duplicate
+                    p.populate_set_out(filename)
                 else:
                     publisher.info('{}CreditCard related'.format(to_print))
         else:

bin/Duplicate_ssdeep_v2.py

@@ -6,8 +6,10 @@ The Duplicate module
 ====================
 
 This huge module is, in short term, checking duplicates.
+Its input comes from other modules, namely:
+    Credential, CreditCard, Keys, Mails and Phone
 
-This one differ from v1 by only using redis and not json file on disk
+This one differ from v1 by only using redis and not json file stored on disk
 
 Requirements:
 -------------
@@ -117,8 +119,7 @@ if __name__ == "__main__":
                             if paste_path != None:
                                 hash_dico[dico_hash] = (paste_path, percent)
 
-                            print 'comparing: ' + str(PST.p_path[44:]) + '  and  ' + str(paste_path[44:]) + ' percentage: ' + str(percent)
+                            #print 'comparing: ' + str(PST.p_path[44:]) + '  and  ' + str(paste_path[44:]) + ' percentage: ' + str(percent)
-                            #print '   '+ PST.p_path[44:]  +', '+ paste_path[44:] + ', ' + str(percent)
                     except:
                         # ssdeep hash not comparable
                         print 'ssdeep hash not comparable' 
@@ -146,7 +147,7 @@ if __name__ == "__main__":
                     PST.__setattr__("p_duplicate", dupl)
                     PST.save_attribute_redis("p_duplicate", dupl)
                     publisher.info('{}Detected {}'.format(to_print, len(dupl)))
-                    #print '{}Detected {}'.format(to_print, len(dupl))
+                    print '{}Detected {}'.format(to_print, len(dupl))
 
                 y = time.time()
 

bin/Keys.py

@@ -16,6 +16,8 @@ def search_gpg(message):
     content = paste.get_p_content()
     if '-----BEGIN PGP MESSAGE-----' in content:
         publisher.warning('{} has a PGP enc message'.format(paste.p_name))
+        #Send to duplicate
+        p.populate_set_out(message)
 
 
 if __name__ == '__main__':

bin/Mail.py

@@ -60,6 +60,8 @@ if __name__ == "__main__":
                                MX_values[0])
                     if MX_values[0] > is_critical:
                         publisher.warning(to_print)
+                        #Send to duplicate
+                        p.populate_set_out(filename)
                     else:
                         publisher.info(to_print)
             prec_filename = filename

bin/Phone.py

@@ -23,6 +23,8 @@ def search_phone(message):
     if len(results) > 4:
         print results
         publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
+        #Send to duplicate
+        p.populate_set_out(message)
 
 if __name__ == '__main__':
     # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)

bin/packages/modules.cfg

@@ -3,7 +3,7 @@ subscribe = ZMQ_Global
 publish = Redis_Global
 
 [Duplicates]
-subscribe = Redis_Global
+subscribe = Redis_Duplicate
 
 [Indexer]
 subscribe = Redis_Global
@@ -31,9 +31,11 @@ publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Re
 
 [CreditCards]
 subscribe = Redis_CreditCards
+publish = Redis_Duplicate 
 
 [Mail]
 subscribe = Redis_Mail
+publish = Redis_Duplicate 
 
 [Onion]
 subscribe = Redis_Onion
@@ -55,15 +57,18 @@ subscribe = Redis_Global
 
 [Credential]
 subscribe = Redis_Credential
+publish = Redis_Duplicate 
 
 [Cve]
 subscribe = Redis_Cve
 
 [Phone]
 subscribe = Redis_Global
+publish = Redis_Duplicate 
 
 [SourceCode]
 subscribe = Redis_SourceCode
 
 [Keys]
 subscribe = Redis_Global
+publish = Redis_Duplicate