chg: [Tags alertHandler] remove alertHandler + add quick tag search sidebar + use tag last_seen to display tagged items

bin/ApiKey.py

@@ -57,8 +57,6 @@ def search_api_key(message):
         msg = 'infoleak:automatic-detection="api-key";{}'.format(filename)
         p.populate_set_out(msg, 'Tags')
 
-        msg = 'apikey;{}'.format(filename)
-        p.populate_set_out(msg, 'alertHandler')
         #Send to duplicate
         p.populate_set_out(filename, 'Duplicate')
 

bin/Bitcoin.py

@@ -62,8 +62,6 @@ def search_key(content, message, paste):
             to_print = 'Bitcoin found: {} address and {} private Keys'.format(len(bitcoin_address), len(bitcoin_private_key))
             print(to_print)
             publisher.warning(to_print)
-            msg = ('bitcoin;{}'.format(message))
-            p.populate_set_out( msg, 'alertHandler')
 
             msg = 'infoleak:automatic-detection="bitcoin-address";{}'.format(message)
             p.populate_set_out(msg, 'Tags')

bin/Credential.py

@@ -107,9 +107,6 @@ if __name__ == "__main__":
             publisher.warning(to_print)
             #Send to duplicate
             p.populate_set_out(filepath, 'Duplicate')
-            #Send to alertHandler
-            msg = 'credential;{}'.format(filepath)
-            p.populate_set_out(msg, 'alertHandler')
 
             msg = 'infoleak:automatic-detection="credential";{}'.format(filepath)
             p.populate_set_out(msg, 'Tags')

bin/CreditCards.py

@@ -82,9 +82,6 @@ if __name__ == "__main__":
                         to_print, len(creditcard_set), paste.p_path))
                     #Send to duplicate
                     p.populate_set_out(filename, 'Duplicate')
-                    #send to Browse_warning_paste
-                    msg = 'creditcard;{}'.format(filename)
-                    p.populate_set_out(msg, 'alertHandler')
 
                     msg = 'infoleak:automatic-detection="credit-card";{}'.format(filename)
                     p.populate_set_out(msg, 'Tags')

bin/Cve.py

@@ -31,10 +31,6 @@ def search_cve(message):
         print('{} contains CVEs'.format(paste.p_name))
         publisher.warning('{} contains CVEs'.format(paste.p_name))
 
-        #send to Browse_warning_paste
-        msg = 'cve;{}'.format(filepath)
-        p.populate_set_out(msg, 'alertHandler')
-
         msg = 'infoleak:automatic-detection="cve";{}'.format(filepath)
         p.populate_set_out(msg, 'Tags')
         #Send to duplicate

bin/Decoder.py

@@ -147,9 +147,6 @@ def set_out_paste(decoder_name, message):
     publisher.warning(decoder_name+' decoded')
     #Send to duplicate
     p.populate_set_out(message, 'Duplicate')
-    #send to Browse_warning_paste
-    msg = (decoder_name+';{}'.format(message))
-    p.populate_set_out( msg, 'alertHandler')
 
     msg = 'infoleak:automatic-detection="'+decoder_name+'";{}'.format(message)
     p.populate_set_out(msg, 'Tags')

bin/Keys.py

@@ -112,10 +112,7 @@ def search_key(paste):
 
         #Send to duplicate
         p.populate_set_out(message, 'Duplicate')
-        #send to Browse_warning_paste
-        msg = ('keys;{}'.format(message))
         print(message)
-        p.populate_set_out( msg, 'alertHandler')
 
 
 if __name__ == '__main__':

bin/LAUNCH.sh

@@ -206,8 +206,6 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./LibInjection.py; read x"
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "alertHandler" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./alertHandler.py; read x"
-    sleep 0.1
     screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tags.py; read x"

bin/LibInjection.py

@@ -51,8 +51,7 @@ def analyse(url, path):
         publisher.warning(to_print)
         #Send to duplicate
         p.populate_set_out(path, 'Duplicate')
-        #send to Browse_warning_paste
+
-        p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler')
         msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
         p.populate_set_out(msg, 'Tags')
 

bin/Mail.py

@@ -83,7 +83,6 @@ if __name__ == "__main__":
                         publisher.warning(to_print)
                         #Send to duplicate
                         p.populate_set_out(filename, 'Duplicate')
-                        p.populate_set_out('mail;{}'.format(filename), 'alertHandler')
 
                         msg = 'infoleak:automatic-detection="mail";{}'.format(filename)
                         p.populate_set_out(msg, 'Tags')

bin/Onion.py

@@ -235,7 +235,6 @@ if __name__ == "__main__":
                     else:
                         for url in fetch(p, r_cache, urls, domains_list, path):
                             publisher.info('{}Checked {};{}'.format(to_print, url, PST.p_path))
-                    #p.populate_set_out('onion;{}'.format(PST.p_path), 'alertHandler')
 
                     # TAG Item
                     msg = 'infoleak:automatic-detection="onion";{}'.format(PST.p_path)

bin/Phone.py

@@ -32,14 +32,11 @@ def search_phone(message):
     if len(results) > 4:
         print(results)
         publisher.warning('{} contains PID (phone numbers)'.format(paste.p_name))
-        #send to Browse_warning_paste
-        msg = 'phone;{}'.format(message)
-        p.populate_set_out(msg, 'alertHandler')
-        #Send to duplicate
 
         msg = 'infoleak:automatic-detection="phone-number";{}'.format(message)
         p.populate_set_out(msg, 'Tags')
 
+        #Send to duplicate
         p.populate_set_out(message, 'Duplicate')
         stats = {}
         for phone_number in results:

bin/SQLInjectionDetection.py

@@ -82,8 +82,6 @@ def analyse(url, path):
             publisher.warning(to_print)
             #Send to duplicate
             p.populate_set_out(path, 'Duplicate')
-            #send to Browse_warning_paste
-            p.populate_set_out('sqlinjection;{}'.format(path), 'alertHandler')
 
             msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
             p.populate_set_out(msg, 'Tags')

bin/alertHandler.py

@@ -1,63 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-
-"""
-The Browse_warning_paste module
-====================
-
-This module saved signaled paste (logged as 'warning') in redis for further usage
-like browsing by category
-
-Its input comes from other modules, namely:
-    Credential, CreditCard, SQLinjection, CVE, Keys, Mail and Phone
-
-"""
-
-import redis
-import time
-from datetime import datetime, timedelta
-from packages import Paste
-from pubsublogger import publisher
-from Helper import Process
-
-import sys
-sys.path.append('../')
-
-flag_misp = False
-
-if __name__ == "__main__":
-    publisher.port = 6380
-    publisher.channel = "Script"
-
-    config_section = 'alertHandler'
-
-    p = Process(config_section)
-
-    # port generated automatically depending on the date
-    curYear = datetime.now().year
-    server = redis.StrictRedis(
-                host=p.config.get("ARDB_DB", "host"),
-                port=p.config.get("ARDB_DB", "port"),
-                db=curYear,
-                decode_responses=True)
-
-    # FUNCTIONS #
-    publisher.info("Script duplicate started")
-
-    while True:
-            message = p.get_from_set()
-            if message is not None:
-                module_name, p_path = message.split(';')
-                print("new alert : {}".format(module_name))
-                #PST = Paste.Paste(p_path)
-            else:
-                publisher.debug("Script Attribute is idling 10s")
-                time.sleep(10)
-                continue
-
-            # Add in redis for browseWarningPaste
-            # Format in set: WARNING_moduleName -> p_path
-            key = "WARNING_" + module_name
-            server.sadd(key, p_path)
-
-            publisher.info('Saved warning paste {}'.format(p_path))

bin/launch_scripts.sh

@@ -1,77 +0,0 @@
-#!/bin/bash
-
-set -e
-set -x
-
-[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
-[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
-[ -z "$AIL_LEVELDB" ] && echo "Needs the env var AIL_LEVELDB. Run the script from the virtual environment." && exit 1;
-
-echo -e "\t* Checking configuration"
-bash -c "./Update-conf.py"
-exitStatus=$?
-if [ $exitStatus -ge 1 ]; then
-    echo -e $RED"\t* Configuration not up-to-date"$DEFAULT
-    exit
-fi
-echo -e $GREEN"\t* Configuration up-to-date"$DEFAULT
-
-screen -dmS "Script"
-sleep 0.1
-echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
-
-screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModulesInformationV2.py -k 0 -c 1; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "Mixer" bash -c './Mixer.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "Attributes" bash -c './Attributes.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "Lines" bash -c './Lines.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "CreditCards" bash -c './CreditCards.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "Web" bash -c './Web.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "RegexForTermsFrequency" bash -c './RegexForTermsFrequency.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "SetForTermsFrequency" bash -c './SetForTermsFrequency.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "alertHandler" bash -c './alertHandler.py; read x'
-sleep 0.1
-screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'

bin/packages/modules.cfg

@@ -51,7 +51,7 @@ publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Re
 
 [CreditCards]
 subscribe = Redis_CreditCards
-publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler,Redis_Tags
+publish = Redis_Duplicate,Redis_ModuleStats,Redis_Tags
 
 [BankAccount]
 subscribe = Redis_Global
@@ -59,12 +59,12 @@ publish = Redis_Duplicate,Redis_Tags
 
 [Mail]
 subscribe = Redis_Mail
-publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler,Redis_Tags
+publish = Redis_Duplicate,Redis_ModuleStats,Redis_Tags
 
 [Onion]
 subscribe = Redis_Onion
-publish = Redis_ValidOnion,ZMQ_FetchedOnion,Redis_alertHandler,Redis_Tags,Redis_Crawler
+publish = Redis_ValidOnion,ZMQ_FetchedOnion,Redis_Tags,Redis_Crawler
-#publish = Redis_Global,Redis_ValidOnion,ZMQ_FetchedOnion,Redis_alertHandler
+#publish = Redis_Global,Redis_ValidOnion,ZMQ_FetchedOnion
 
 [DumpValidOnion]
 subscribe = Redis_ValidOnion
@@ -78,18 +78,15 @@ subscribe = Redis_Url
 
 [LibInjection]
 subscribe = Redis_Url
-publish = Redis_alertHandler,Redis_Duplicate,Redis_Tags
+publish = Redis_Duplicate,Redis_Tags
 
 [SQLInjectionDetection]
 subscribe = Redis_Url
-publish = Redis_alertHandler,Redis_Duplicate,Redis_Tags
+publish = Redis_Duplicate,Redis_Tags
 
 [ModuleStats]
 subscribe = Redis_ModuleStats
 
-[alertHandler]
-subscribe = Redis_alertHandler
-
 [Tags]
 subscribe = Redis_Tags
 publish = Redis_Tags_feed
@@ -99,7 +96,7 @@ subscribe = Redis_Tags_feed
 
 #[send_to_queue]
 #subscribe = Redis_Cve
-#publish = Redis_alertHandler,Redis_Tags
+#publish = Redis_Tags
 
 [SentimentAnalysis]
 subscribe = Redis_Global
@@ -109,31 +106,31 @@ subscribe = Redis_Global
 
 [Credential]
 subscribe = Redis_Credential
-publish = Redis_Duplicate,Redis_ModuleStats,Redis_alertHandler,Redis_Tags
+publish = Redis_Duplicate,Redis_ModuleStats,Redis_Tags
 
 [Cve]
 subscribe = Redis_Cve
-publish = Redis_alertHandler,Redis_Duplicate,Redis_Tags
+publish = Redis_Duplicate,Redis_Tags
 
 [Phone]
 subscribe = Redis_Global
-publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags
+publish = Redis_Duplicate,Redis_Tags
 
 [Keys]
 subscribe = Redis_Global
-publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags
+publish = Redis_Duplicate,Redis_Tags
 
 [ApiKey]
 subscribe = Redis_ApiKey
-publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags
+publish = Redis_Duplicate,Redis_Tags
 
 [Decoder]
 subscribe = Redis_Global
-publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags
+publish = Redis_Duplicate,Redis_Tags
 
 [Bitcoin]
 subscribe = Redis_Global
-publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags
+publish = Redis_Duplicate,Redis_Tags
 
 [submit_paste]
 subscribe = Redis
@@ -142,4 +139,3 @@ publish = Redis_Mixer
 [Crawler]
 subscribe = Redis_Crawler
 publish = Redis_Mixer,Redis_Tags
-

var/www/modules/Tags/Flask_Tags.py

@@ -118,6 +118,16 @@ def get_all_dates_range(date_from, date_to):
     all_dates['date_range'] = date_range
     return all_dates
 
+def get_last_seen_from_tags_list(list_tags):
+    min_last_seen = 99999999
+    for tag in list_tags:
+        tag_last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
+        if tag_last_seen:
+            tag_last_seen = int(tag_last_seen)
+            if tag_last_seen < min_last_seen:
+                min_last_seen = tag_last_seen
+    return str(min_last_seen)
+
 def add_item_tag(tag, item_path):
     item_date = int(get_item_date(item_path))
 
@@ -211,23 +221,27 @@ def update_tag_last_seen(tag, tag_first_seen, tag_last_seen):
 def Tags_page():
     date_from = request.args.get('date_from')
     date_to = request.args.get('date_to')
-
-    dates = get_all_dates_range(date_from, date_to)
-
     tags = request.args.get('ltags')
+
     if tags is None:
+        dates = get_all_dates_range(date_from, date_to)
         return render_template("Tags.html", date_from=dates['date_from'], date_to=dates['date_to'])
 
-    else:
+    # unpack tags
-        tags = request.args.get('ltags')
-
     list_tags = tags.split(',')
     list_tag = []
     for tag in list_tags:
         list_tag.append(tag.replace('"','\"'))
 
+    #no search by date, use last_seen for  date_from/date_to
+    if date_from is None and date_to is None and tags is not None:
+        date_from = get_last_seen_from_tags_list(list_tags)
+        date_to = date_from
+
     # TODO verify input
 
+    dates = get_all_dates_range(date_from, date_to)
+
     if(type(list_tags) is list):
         # no tag
         if list_tags is False:
@@ -278,7 +292,7 @@ def Tags_page():
     start = max_tags_result*(page -1)
     stop = max_tags_result*page
 
-        for path in allPastes[start:stop]: ######################moduleName
+    for path in allPastes[start:stop]:
         all_path.append(path)
         paste = Paste.Paste(path)
         content = paste.get_p_content()
@@ -314,9 +328,15 @@ def Tags_page():
     else:
         finished = True
 
+    if len(list_tag) == 1:
+        tag_nav=tags.replace('"', '').replace('=', '').replace(':', '')
+    else:
+        tag_nav='empty'
+
     return render_template("Tags.html",
             all_path=all_path,
             tags=tags,
+            tag_nav=tag_nav,
             list_tag = list_tag,
             date_from=dates['date_from'],
             date_to=dates['date_to'],

var/www/modules/Tags/templates/Tags.html

@@ -217,7 +217,8 @@ var last_clicked_paste;
 var can_change_modal_content = true;
 
 $(document).ready(function(){
-	$("#page-Browse-Items").addClass("active");
+	$("#nav_quick_search").removeClass("text-muted");
+	$("#nav_tag_{{tag_nav}}").addClass("active");
 	search_table = $('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
 
 	// Use to bind the button with the new displayed data

var/www/templates/tags/menu_sidebar.html

@@ -34,5 +34,58 @@
               </a>
             </li>
           </ul>
+          <h5 class="d-flex text-muted w-100" id="nav_quick_search">
+      		  <span>Quick Search </span>
+      		</h5>
+          <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100">
+            <li class="nav-item">
+              <a class="nav-link" href='{{url_for('Tags.Tags_page')}}?ltags=infoleak:automatic-detection="credential"' id='nav_tag_infoleakautomatic-detectioncredential'>
+                <i class="fas fa-unlock-alt"></i>
+                Credentials
+              </a>
+            </li>
+            <li class="nav-item">
+              <a class="nav-link" href='{{url_for('Tags.Tags_page')}}?ltags=infoleak:automatic-detection="credit-card"' id='nav_tag_infoleakautomatic-detectioncredit-card'>
+                <i class="far fa-credit-card"></i>
+                Credit cards
+              </a>
+            </li>
+            <li class="nav-item">
+              <a class="nav-link" href='{{url_for('Tags.Tags_page')}}?ltags=infoleak:automatic-detection="mail"' id='nav_tag_infoleakautomatic-detectionmail'>
+                <i class="fas fa-envelope"></i>
+                Mails
+              </a>
+            </li>
+            <li class="nav-item">
+              <a class="nav-link" href='{{url_for('Tags.Tags_page')}}?ltags=infoleak:automatic-detection="cve"' id='nav_tag_infoleakautomatic-detectioncve'>
+                <i class="fas fa-bug"></i>
+                CVEs
+              </a>
+            </li>
+            <li class="nav-item">
+              <a class="nav-link" href='{{url_for('Tags.Tags_page')}}?ltags=infoleak:automatic-detection="onion"' id='nav_tag_infoleakautomatic-detectiononion'>
+                <i class="fas fa-user-secret"></i>
+                Onions
+              </a>
+            </li>
+            <li class="nav-item">
+              <a class="nav-link" href='{{url_for('Tags.Tags_page')}}?ltags=infoleak:automatic-detection="bitcoin-address"' id='nav_tag_infoleakautomatic-detectionbitcoin-address'>
+                <i class="fab fa-bitcoin"></i>
+                Bitcoin
+              </a>
+            </li>
+            <li class="nav-item">
+              <a class="nav-link" href='{{url_for('Tags.Tags_page')}}?ltags=infoleak:automatic-detection="base64"' id='nav_tag_infoleakautomatic-detectionbase64'>
+                <i class="fas fa-lock-open"></i>
+                Base64
+              </a>
+            </li>
+            <li class="nav-item">
+              <a class="nav-link" href='{{url_for('Tags.Tags_page')}}?ltags=infoleak:automatic-detection="phone-number"' id='nav_tag_infoleakautomatic-detectionphone-number'>
+                <i class="fas fa-phone"></i>
+                Phones
+              </a>
+            </li>
+          </ul>
       	</nav>
       </div>