chg: [submodule + YARA] add submodule auto update + update v3.1.1

.gitmodules

@@ -0,0 +1,3 @@
+[submodule "bin/trackers/yara/ail-yara-rules"]
+	path = bin/trackers/yara/ail-yara-rules
+	url = https://github.com/ail-project/ail-yara-rules.git

bin/LAUNCH.sh

@@ -184,6 +184,8 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x"
     sleep 0.1
+    screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
+    sleep 0.1
     screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x"
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x"

bin/Update.py

@@ -231,6 +231,15 @@ def get_git_upper_tags_remote(current_tag, is_fork):
             aborting_update()
             sys.exit(0)
 
+def update_submodules():
+    print('{}git submodule update:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
+    process = subprocess.run(['git', 'submodule', 'update'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+    if process.returncode == 0:
+        print(process.stdout.decode())
+        print()
+    else:
+        print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
+
 def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_fork):
     print('{}git checkout master:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
     process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -238,6 +247,9 @@ def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_for
     if process.returncode == 0:
         print(process.stdout.decode())
         print()
+
+        update_submodules()
+
         print('{}git pull:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
         process = subprocess.run(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
 

bin/trackers/Tracker_Yara.py

@@ -71,7 +71,6 @@ if __name__ == "__main__":
     # Regex Frequency
     while True:
         item_id = p.get_from_set()
-        item_id = 'archive/pastebin.com_pro/2020/03/04/AnwFX3w2.gz'
         if item_id is not None:
             item_content = item_basic.get_item_content(item_id)
             yara_match = rules.match(data=item_content, callback=yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60)

bin/trackers/yara/ail-yara-rules

@@ -0,0 +1 @@
+Subproject commit edc390c4a8d93a028e29938e92aacb399e270cc4

installing_deps.sh

@@ -39,6 +39,10 @@ sudo apt-get install build-essential libffi-dev automake autoconf libtool -qq
 # sflock, gz requirement
 sudo apt-get install p7zip-full -qq
 
+# SUBMODULES #
+git submodule init
+git submodule update
+
 # REDIS #
 test ! -d redis/ && git clone https://github.com/antirez/redis.git
 pushd redis/

update/v3.1.1/Update.py

@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import time
+import redis
+import argparse
+import datetime
+import configparser
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+new_version = 'v3.1.1'
+
+if __name__ == '__main__':
+
+    start_deb = time.time()
+
+    config_loader = ConfigLoader.ConfigLoader()
+    r_serv_db = config_loader.get_redis_conn("ARDB_DB")
+    config_loader = None
+
+    #### NEW EXPORTER
+    # remove old tags errors
+    #r_serv_db.delete('mess_not_saved_export')
+
+    # move solo tags to export in tags_db
+    #all_misp_tags = r_serv_db.smembers('whitelist_misp')
+    #all_hive_tags = r_serv_db.smembers('whitelist_hive')
+    # # TODO: save them in tags db
+    #### NEW EXPORTER
+
+    #Set current ail version
+    r_serv_db.set('ail:version', new_version)
+
+    #Set current ail version
+    r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))

update/v3.1.1/Update.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
+
+export PATH=$AIL_HOME:$PATH
+export PATH=$AIL_REDIS:$PATH
+export PATH=$AIL_ARDB:$PATH
+export PATH=$AIL_BIN:$PATH
+export PATH=$AIL_FLASK:$PATH
+
+GREEN="\\033[1;32m"
+DEFAULT="\\033[0;39m"
+
+echo -e $GREEN"Shutting down AIL ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+bash ${AIL_BIN}/LAUNCH.sh -ldbv &
+wait
+echo ""
+
+echo -e $GREEN"Installing YARA ..."$DEFAULT
+pip3 install yara-python
+bash ${AIL_BIN}/LAUNCH.sh -t
+
+# SUBMODULES #
+git submodule init
+git submodule update
+
+echo ""
+echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
+echo ""
+python ${AIL_HOME}/update/v3.1.1/Update.py
+wait
+echo ""
+echo ""
+
+
+echo ""
+echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+exit 0

var/www/modules/hunter/templates/Add_tracker.html

@@ -32,7 +32,7 @@
 								<h5 class="card-title">Create a new tracker</h5>
 							</div>
 						  <div class="card-body">
-						    <p class="card-text">Enter a domain and choose what kind of data you want.</p>
+						    <p class="card-text">Select a tracker type.</p>
 
 								<form action="{{ url_for('hunter.add_tracked_menu') }}" method='post'>
 
@@ -103,7 +103,7 @@
 										</div>
 
 									<div class="row" id="textarea">
-										<textarea class="form-control" id="text_input" name="yara_custom_rule" rows="5"></textarea>
+										<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5"></textarea>
 									</div>
 								</div>
 
@@ -136,6 +136,7 @@ $(document).ready(function(){
 	$("#tracker_desc").hide();
 	$("#term").hide();
 	$("#nb_word").hide();
+	$("#yara_rule").hide();
 
 	$('#tracker_type').on('change', function() {
 		var tracker_type = this.value;
@@ -144,16 +145,25 @@ $(document).ready(function(){
 			$("#tracker_desc").show();
 			$("#term").show();
 			$("#nb_word").hide();
+			$("#yara_rule").hide();
 		} else if (tracker_type=="set") {
 			$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
 			$("#tracker_desc").show();
 			$("#term").show();
 			$("#nb_word").show();
-		} else {
+			$("#yara_rule").hide();
+		} else if (tracker_type=="regex") {
 			$("#tracker_desc").text("Enter a valid Python regex");
 			$("#tracker_desc").show();
 			$("#term").show();
 			$("#nb_word").hide();
+			$("#yara_rule").hide();
+		} else if (tracker_type=="yara") {
+			$("#tracker_desc").text("Select a default yara rule or create your own rule:");
+			$("#tracker_desc").show();
+			$("#term").hide();
+			$("#nb_word").hide();
+			$("#yara_rule").show();
 		}
 	});
 

var/www/templates/hunter/menu_sidebar.html

@@ -22,19 +22,19 @@
       	    <li class="nav-item">
       	      <a class="nav-link" href="{{url_for('hunter.tracked_menu_word')}}" id="nav_tracker_word">
                 <i class="fas fa-font"></i>
-      	        <span>Tracked Words</span>
+      	        <span>Words</span>
       	      </a>
       	    </li>
             <li class="nav-item">
       	      <a class="nav-link" href="{{url_for('hunter.tracked_menu_set')}}" id="nav_tracker_set">
       	        <i class="fas fa-layer-group"></i>
-      	        <span>Tracked Set</span>
+      	        <span>Set</span>
       	      </a>
       	    </li>
             <li class="nav-item">
       	      <a class="nav-link" href="{{url_for('hunter.tracked_menu_regex')}}" id="nav_tracker_regex">
-      	        <i class="fas fa-ruler"></i>
+      	        <i class="fas fa-ruler-vertical"></i>
-                <span>Tracked Regex</span>
+                <span>Regex</span>
       	      </a>
       	    </li>
             <li class="nav-item">

var/www/update_thirdparty.sh

@@ -2,6 +2,9 @@
 
 set -e
 
+# submodules
+git submodule update
+
 wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
 
 SBADMIN_VERSION='3.3.7'