add apiKeys module

bin/ApiKey.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python3.5
+# -*-coding:UTF-8 -*
+
+"""
+The ApiKey Module
+======================
+
+This module is consuming the Redis-list created by the Categ module.
+
+It apply API_key regexes on paste content and warn if above a threshold.
+
+"""
+
+import redis
+import pprint
+import time
+import re
+
+from packages import Paste
+from packages import lib_refine
+from pubsublogger import publisher
+
+from Helper import Process
+
+
+def search_api_key(message):
+    filename, score = message.split()
+    paste = Paste.Paste(filename)
+    content = paste.get_p_content()
+
+    aws_access_key = regex_aws_access_key.findall(content)
+    aws_secret_key = regex_aws_secret_key.findall(content)
+    google_api_key = regex_google_api_key.findall(content)
+
+    print(aws_access_key)
+    print(aws_secret_key)
+    print(google_api_key)
+
+    if(len(aws_access_key) > 0 or len(aws_secret_key) > 0 or len(google_api_key) > 0):
+
+        print('-------------------------------')
+        print(aws_access_key)
+        print(aws_secret_key)
+        print(google_api_key)
+
+        to_print = 'ApiKey;{};{};{};'.format(
+            paste.p_source, paste.p_date, paste.p_name)
+        if(len(google_api_key) > 0):
+            print('found google api key')
+            print(to_print)
+            publisher.warning('{}Checked {} found Google API Key;{}'.format(
+                to_print, len(google_api_key), paste.p_path))
+
+        if(len(aws_access_key) > 0 or len(aws_secret_key) > 0):
+            print('found AWS key')
+            print(to_print)
+            total = len(aws_access_key) + len(aws_secret_key)
+            publisher.warning('{}Checked {} found AWS Key;{}'.format(
+                to_print, total, paste.p_path))
+
+
+        msg = 'apikey;{}'.format(filename)
+        p.populate_set_out(msg, 'alertHandler')
+        #Send to duplicate
+        p.populate_set_out(filename, 'Duplicate')
+
+if __name__ == "__main__":
+    publisher.port = 6380
+    publisher.channel = "Script"
+
+    config_section = 'ApiKey'
+
+    p = Process(config_section)
+
+    publisher.info("ApiKey started")
+
+    # REDIS #
+    r_serv2 = redis.StrictRedis(
+        host=p.config.get("Redis_Cache", "host"),
+        port=p.config.getint("Redis_Cache", "port"),
+        db=p.config.getint("Redis_Cache", "db"))
+
+    message = p.get_from_set()
+
+    # TODO improve REGEX
+    regex_aws_access_key = re.compile(r'(?<![A-Z0-9])=[A-Z0-9]{20}(?![A-Z0-9])')
+    regex_aws_secret_key = re.compile(r'(?<!=[A-Za-z0-9+])=[A-Za-z0-9+]{40}(?![A-Za-z0-9+])')
+
+    regex_google_api_key = re.compile(r'=AIza[0-9a-zA-Z-_]{35}')
+
+    while True:
+
+        message = p.get_from_set()
+
+        if message is not None:
+
+            search_api_key(message)
+
+
+        else:
+            publisher.debug("Script ApiKey is Idling 10s")
+            #print('Sleeping')
+            time.sleep(10)

bin/Categ.py

@@ -67,7 +67,7 @@ if __name__ == "__main__":
     # FUNCTIONS #
     publisher.info("Script Categ started")
 
-    categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve', 'Dox']
+    categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve', 'ApiKey']
     tmp_dict = {}
     for filename in categories:
         bname = os.path.basename(filename)

bin/Credential.py

@@ -118,7 +118,7 @@ if __name__ == "__main__":
             site_occurence = re.findall(regex_site_for_stats, content)
             for site in site_occurence:
                 site_domain = site[1:-1]
-                if site_domain in creds_sites.keys():
+                if site_domain.encode('utf8') in creds_sites.keys():
                     creds_sites[site_domain] += 1
                 else:
                     creds_sites[site_domain] = 1
@@ -132,6 +132,11 @@ if __name__ == "__main__":
                     creds_sites[domain] = 1
 
             for site, num in creds_sites.items(): # Send for each different site to moduleStats
+                try:
+                    site = site.decode('utf8')
+                except:
+                    pass
+
                 mssg = 'credential;{};{};{}'.format(num, site, paste.p_date)
                 print(mssg)
                 p.populate_set_out(mssg, 'ModuleStats')

bin/LAUNCH.sh

@@ -130,7 +130,7 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Lines" bash -c 'python3 Lines.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
+    #screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Categ" bash -c 'python3 Categ.py; read x'
     sleep 0.1
@@ -142,7 +142,7 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Mail" bash -c './Mail.py; read x'
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Dox" bash -c './Dox.py; read x'
+    screen -S "Script_AIL" -X screen -t "ApiKey" bash -c './ApiKey.py; read x'
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Web" bash -c './Web.py; read x'
     sleep 0.1

bin/packages/modules.cfg

@@ -45,7 +45,7 @@ subscribe = Redis_CurveManageTopSets
 
 [Categ]
 subscribe = Redis_Global
-publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve,Redis_Dox
+publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve,Redis_ApiKey
 
 [CreditCards]
 subscribe = Redis_CreditCards
@@ -76,7 +76,7 @@ publish = Redis_alertHandler,Redis_Duplicate
 
 [Dox]
 subscribe = Redis_Dox
-publish = Redis_Duplicate,Redis_alertHandler,Redis_ModuleStats
+publish = Redis_Duplicate,Redis_alertHandler
 
 [ModuleStats]
 subscribe = Redis_ModuleStats
@@ -110,6 +110,10 @@ publish = Redis_Duplicate,Redis_alertHandler
 subscribe = Redis_Global
 publish = Redis_Duplicate,Redis_alertHandler
 
+[ApiKey]
+subscribe = Redis_ApiKey
+publish = Redis_Duplicate,Redis_alertHandler
+
 [Base64]
 subscribe = Redis_Global
 publish = Redis_Duplicate,Redis_alertHandler

files/ApiKey

@@ -0,0 +1,5 @@
+amazon
+amazonaws
+amzn
+aws
+googleapis

var/www/modules/browsepastes/templates/browse_important_paste.html

@@ -87,6 +87,7 @@
         <li name='nav-pan'><a data-toggle="tab" href="#sqlinjection-tab" data-attribute-name="sqlinjection" data-panel="sqlinjection-panel">SQL injections</a></li>
         <li name='nav-pan'><a data-toggle="tab" href="#cve-tab" data-attribute-name="cve" data-panel="cve-panel">CVEs</a></li>
         <li name='nav-pan'><a data-toggle="tab" href="#keys-tab" data-attribute-name="keys" data-panel="keys-panel">Keys</a></li>
+        <li name='nav-pan'><a data-toggle="tab" href="#apikey-tab" data-attribute-name="apikey" data-panel="apikey-panel">API Keys</a></li>
         <li name='nav-pan'><a data-toggle="tab" href="#mail-tab" data-attribute-name="mail" data-panel="mail-panel">Mails</a></li>
         <li name='nav-pan'><a data-toggle="tab" href="#phone-tab" data-attribute-name="phone" data-panel="phone-panel">Phones</a></li>
         <li name='nav-pan'><a data-toggle="tab" href="#onion-tab" data-attribute-name="onion" data-panel="onion-panel">Onions</a></li>
@@ -112,6 +113,9 @@
         <div class="col-lg-12 tab-pane fade" id="keys-tab">
             <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
         </div>
+        <div class="col-lg-12 tab-pane fade" id="apikey-tab">
+            <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
+        </div>
         <div class="col-lg-12 tab-pane fade" id="mail-tab">
             <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" style="margin: 4px;">
         </div>