diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 00000000..2f7a1187
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "bin/trackers/yara/ail-yara-rules"]
+ path = bin/trackers/yara/ail-yara-rules
+ url = https://github.com/ail-project/ail-yara-rules.git
diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh
index 9c855668..1a117c1c 100755
--- a/bin/LAUNCH.sh
+++ b/bin/LAUNCH.sh
@@ -184,6 +184,8 @@ function launching_scripts {
sleep 0.1
screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x"
sleep 0.1
+ screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
+ sleep 0.1
screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x"
diff --git a/bin/Update.py b/bin/Update.py
index 49d9c5aa..5d65e83b 100755
--- a/bin/Update.py
+++ b/bin/Update.py
@@ -231,6 +231,15 @@ def get_git_upper_tags_remote(current_tag, is_fork):
aborting_update()
sys.exit(0)
+def update_submodules():
+ print('{}git submodule update:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
+ process = subprocess.run(['git', 'submodule', 'update'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+ if process.returncode == 0:
+ print(process.stdout.decode())
+ print()
+ else:
+ print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
+
def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_fork):
print('{}git checkout master:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@@ -238,6 +247,9 @@ def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_for
if process.returncode == 0:
print(process.stdout.decode())
print()
+
+ update_submodules()
+
print('{}git pull:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
process = subprocess.run(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py
index 1990556e..c2c25823 100755
--- a/bin/trackers/Tracker_Yara.py
+++ b/bin/trackers/Tracker_Yara.py
@@ -71,7 +71,6 @@ if __name__ == "__main__":
# Regex Frequency
while True:
item_id = p.get_from_set()
- item_id = 'archive/pastebin.com_pro/2020/03/04/AnwFX3w2.gz'
if item_id is not None:
item_content = item_basic.get_item_content(item_id)
yara_match = rules.match(data=item_content, callback=yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
diff --git a/bin/trackers/yara/ail-yara-rules b/bin/trackers/yara/ail-yara-rules
new file mode 160000
index 00000000..edc390c4
--- /dev/null
+++ b/bin/trackers/yara/ail-yara-rules
@@ -0,0 +1 @@
+Subproject commit edc390c4a8d93a028e29938e92aacb399e270cc4
diff --git a/installing_deps.sh b/installing_deps.sh
index 5ed4665b..d1ed2a1b 100755
--- a/installing_deps.sh
+++ b/installing_deps.sh
@@ -39,6 +39,10 @@ sudo apt-get install build-essential libffi-dev automake autoconf libtool -qq
# sflock, gz requirement
sudo apt-get install p7zip-full -qq
+# SUBMODULES #
+git submodule init
+git submodule update
+
# REDIS #
test ! -d redis/ && git clone https://github.com/antirez/redis.git
pushd redis/
diff --git a/update/v3.1.1/Update.py b/update/v3.1.1/Update.py
new file mode 100755
index 00000000..6b56e3da
--- /dev/null
+++ b/update/v3.1.1/Update.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import time
+import redis
+import argparse
+import datetime
+import configparser
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+new_version = 'v3.1.1'
+
+if __name__ == '__main__':
+
+ start_deb = time.time()
+
+ config_loader = ConfigLoader.ConfigLoader()
+ r_serv_db = config_loader.get_redis_conn("ARDB_DB")
+ config_loader = None
+
+ #### NEW EXPORTER
+ # remove old tags errors
+ #r_serv_db.delete('mess_not_saved_export')
+
+ # move solo tags to export in tags_db
+ #all_misp_tags = r_serv_db.smembers('whitelist_misp')
+ #all_hive_tags = r_serv_db.smembers('whitelist_hive')
+ # # TODO: save them in tags db
+ #### NEW EXPORTER
+
+ #Set current ail version
+ r_serv_db.set('ail:version', new_version)
+
+ #Set current ail version
+ r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
diff --git a/update/v3.1.1/Update.sh b/update/v3.1.1/Update.sh
new file mode 100755
index 00000000..0d3d860e
--- /dev/null
+++ b/update/v3.1.1/Update.sh
@@ -0,0 +1,48 @@
+#!/bin/bash
+
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
+
+export PATH=$AIL_HOME:$PATH
+export PATH=$AIL_REDIS:$PATH
+export PATH=$AIL_ARDB:$PATH
+export PATH=$AIL_BIN:$PATH
+export PATH=$AIL_FLASK:$PATH
+
+GREEN="\\033[1;32m"
+DEFAULT="\\033[0;39m"
+
+echo -e $GREEN"Shutting down AIL ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+bash ${AIL_BIN}/LAUNCH.sh -ldbv &
+wait
+echo ""
+
+echo -e $GREEN"Installing YARA ..."$DEFAULT
+pip3 install yara-python
+bash ${AIL_BIN}/LAUNCH.sh -t
+
+# SUBMODULES #
+git submodule init
+git submodule update
+
+echo ""
+echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
+echo ""
+python ${AIL_HOME}/update/v3.1.1/Update.py
+wait
+echo ""
+echo ""
+
+
+echo ""
+echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+exit 0
diff --git a/var/www/modules/hunter/templates/Add_tracker.html b/var/www/modules/hunter/templates/Add_tracker.html
index 19ba5feb..0653b834 100644
--- a/var/www/modules/hunter/templates/Add_tracker.html
+++ b/var/www/modules/hunter/templates/Add_tracker.html
@@ -32,7 +32,7 @@
Create a new tracker
-
Enter a domain and choose what kind of data you want.
+
Select a tracker type.
-
+
@@ -136,6 +136,7 @@ $(document).ready(function(){
$("#tracker_desc").hide();
$("#term").hide();
$("#nb_word").hide();
+ $("#yara_rule").hide();
$('#tracker_type').on('change', function() {
var tracker_type = this.value;
@@ -144,16 +145,25 @@ $(document).ready(function(){
$("#tracker_desc").show();
$("#term").show();
$("#nb_word").hide();
+ $("#yara_rule").hide();
} else if (tracker_type=="set") {
$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
$("#tracker_desc").show();
$("#term").show();
$("#nb_word").show();
- } else {
+ $("#yara_rule").hide();
+ } else if (tracker_type=="regex") {
$("#tracker_desc").text("Enter a valid Python regex");
$("#tracker_desc").show();
$("#term").show();
$("#nb_word").hide();
+ $("#yara_rule").hide();
+ } else if (tracker_type=="yara") {
+ $("#tracker_desc").text("Select a default yara rule or create your own rule:");
+ $("#tracker_desc").show();
+ $("#term").hide();
+ $("#nb_word").hide();
+ $("#yara_rule").show();
}
});
diff --git a/var/www/templates/hunter/menu_sidebar.html b/var/www/templates/hunter/menu_sidebar.html
index c209ada1..b0afe9ef 100644
--- a/var/www/templates/hunter/menu_sidebar.html
+++ b/var/www/templates/hunter/menu_sidebar.html
@@ -22,19 +22,19 @@