From 996c0e02dea69d49334a58c9a6fc1ae81058fc2d Mon Sep 17 00:00:00 2001
From: Mokaddem <mokaddem.sami@gmail.com>
Date: Mon, 18 Jul 2016 16:22:33 +0200
Subject: [PATCH] Duplicate module takes its messages from other modules and no
 more from Global.

---
 bin/Credential.py          | 2 ++
 bin/CreditCard.py          | 2 ++
 bin/Duplicate_ssdeep_v2.py | 9 +++++----
 bin/Keys.py                | 2 ++
 bin/Mail.py                | 2 ++
 bin/Phone.py               | 2 ++
 bin/packages/modules.cfg   | 7 ++++++-
 7 files changed, 21 insertions(+), 5 deletions(-)

diff --git a/bin/Credential.py b/bin/Credential.py
index 3ac61faf..d81c9ff6 100755
--- a/bin/Credential.py
+++ b/bin/Credential.py
@@ -50,6 +50,8 @@ if __name__ == "__main__":
         if len(creds) > critical:
             print("========> Found more than 10 credentials in this file : {}".format(filepath))
             publisher.warning(to_print)
+            #Send to duplicate
+            p.populate_set_out(filepath)
             if sites:
                 print("=======> Probably on : {}".format(', '.join(sites)))
         else:
diff --git a/bin/CreditCard.py b/bin/CreditCard.py
index d4660c13..18703f4e 100755
--- a/bin/CreditCard.py
+++ b/bin/CreditCard.py
@@ -65,6 +65,8 @@ if __name__ == "__main__":
                 if (len(creditcard_set) > 0):
                     publisher.warning('{}Checked {} valid number(s)'.format(
                         to_print, len(creditcard_set)))
+                    #Send to duplicate
+                    p.populate_set_out(filename)
                 else:
                     publisher.info('{}CreditCard related'.format(to_print))
         else:
diff --git a/bin/Duplicate_ssdeep_v2.py b/bin/Duplicate_ssdeep_v2.py
index 35874371..f6aaca4f 100755
--- a/bin/Duplicate_ssdeep_v2.py
+++ b/bin/Duplicate_ssdeep_v2.py
@@ -6,8 +6,10 @@ The Duplicate module
 ====================
 
 This huge module is, in short term, checking duplicates.
+Its input comes from other modules, namely:
+    Credential, CreditCard, Keys, Mails and Phone
 
-This one differ from v1 by only using redis and not json file on disk
+This one differ from v1 by only using redis and not json file stored on disk
 
 Requirements:
 -------------
@@ -117,8 +119,7 @@ if __name__ == "__main__":
                             if paste_path != None:
                                 hash_dico[dico_hash] = (paste_path, percent)
 
-                            print 'comparing: ' + str(PST.p_path[44:]) + '  and  ' + str(paste_path[44:]) + ' percentage: ' + str(percent)
-                            #print '   '+ PST.p_path[44:]  +', '+ paste_path[44:] + ', ' + str(percent)
+                            #print 'comparing: ' + str(PST.p_path[44:]) + '  and  ' + str(paste_path[44:]) + ' percentage: ' + str(percent)
                     except:
                         # ssdeep hash not comparable
                         print 'ssdeep hash not comparable' 
@@ -146,7 +147,7 @@ if __name__ == "__main__":
                     PST.__setattr__("p_duplicate", dupl)
                     PST.save_attribute_redis("p_duplicate", dupl)
                     publisher.info('{}Detected {}'.format(to_print, len(dupl)))
-                    #print '{}Detected {}'.format(to_print, len(dupl))
+                    print '{}Detected {}'.format(to_print, len(dupl))
 
                 y = time.time()
 
diff --git a/bin/Keys.py b/bin/Keys.py
index 8058545b..9c44f60a 100755
--- a/bin/Keys.py
+++ b/bin/Keys.py
@@ -16,6 +16,8 @@ def search_gpg(message):
     content = paste.get_p_content()
     if '-----BEGIN PGP MESSAGE-----' in content:
         publisher.warning('{} has a PGP enc message'.format(paste.p_name))
+        #Send to duplicate
+        p.populate_set_out(message)
 
 
 if __name__ == '__main__':
diff --git a/bin/Mail.py b/bin/Mail.py
index dd348ba6..964deb19 100755
--- a/bin/Mail.py
+++ b/bin/Mail.py
@@ -60,6 +60,8 @@ if __name__ == "__main__":
                                MX_values[0])
                     if MX_values[0] > is_critical:
                         publisher.warning(to_print)
+                        #Send to duplicate
+                        p.populate_set_out(filename)
                     else:
                         publisher.info(to_print)
             prec_filename = filename
diff --git a/bin/Phone.py b/bin/Phone.py
index 628f77c2..b53b079c 100755
--- a/bin/Phone.py
+++ b/bin/Phone.py
@@ -23,6 +23,8 @@ def search_phone(message):
     if len(results) > 4:
         print results
         publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
+        #Send to duplicate
+        p.populate_set_out(message)
 
 if __name__ == '__main__':
     # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg
index 9d8d6637..5f087427 100644
--- a/bin/packages/modules.cfg
+++ b/bin/packages/modules.cfg
@@ -3,7 +3,7 @@ subscribe = ZMQ_Global
 publish = Redis_Global
 
 [Duplicates]
-subscribe = Redis_Global
+subscribe = Redis_Duplicate
 
 [Indexer]
 subscribe = Redis_Global
@@ -31,9 +31,11 @@ publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Re
 
 [CreditCards]
 subscribe = Redis_CreditCards
+publish = Redis_Duplicate 
 
 [Mail]
 subscribe = Redis_Mail
+publish = Redis_Duplicate 
 
 [Onion]
 subscribe = Redis_Onion
@@ -55,15 +57,18 @@ subscribe = Redis_Global
 
 [Credential]
 subscribe = Redis_Credential
+publish = Redis_Duplicate 
 
 [Cve]
 subscribe = Redis_Cve
 
 [Phone]
 subscribe = Redis_Global
+publish = Redis_Duplicate 
 
 [SourceCode]
 subscribe = Redis_SourceCode
 
 [Keys]
 subscribe = Redis_Global
+publish = Redis_Duplicate