mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-12 17:48:22 +00:00
chg: [submodule + YARA] add submodule auto update + update v3.1.1
This commit is contained in:
parent
e70ae376c5
commit
8283488b6d
11 changed files with 129 additions and 8 deletions
3
.gitmodules
vendored
Normal file
3
.gitmodules
vendored
Normal file
|
@ -0,0 +1,3 @@
|
||||||
|
[submodule "bin/trackers/yara/ail-yara-rules"]
|
||||||
|
path = bin/trackers/yara/ail-yara-rules
|
||||||
|
url = https://github.com/ail-project/ail-yara-rules.git
|
|
@ -184,6 +184,8 @@ function launching_scripts {
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x"
|
screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x"
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
|
screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
|
||||||
|
sleep 0.1
|
||||||
screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x"
|
screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x"
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x"
|
screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x"
|
||||||
|
|
|
@ -231,6 +231,15 @@ def get_git_upper_tags_remote(current_tag, is_fork):
|
||||||
aborting_update()
|
aborting_update()
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
|
def update_submodules():
|
||||||
|
print('{}git submodule update:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
|
||||||
|
process = subprocess.run(['git', 'submodule', 'update'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
|
if process.returncode == 0:
|
||||||
|
print(process.stdout.decode())
|
||||||
|
print()
|
||||||
|
else:
|
||||||
|
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
|
||||||
|
|
||||||
def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_fork):
|
def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_fork):
|
||||||
print('{}git checkout master:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
|
print('{}git checkout master:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
|
||||||
process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
|
@ -238,6 +247,9 @@ def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_for
|
||||||
if process.returncode == 0:
|
if process.returncode == 0:
|
||||||
print(process.stdout.decode())
|
print(process.stdout.decode())
|
||||||
print()
|
print()
|
||||||
|
|
||||||
|
update_submodules()
|
||||||
|
|
||||||
print('{}git pull:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
|
print('{}git pull:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
|
||||||
process = subprocess.run(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
process = subprocess.run(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||||
|
|
||||||
|
|
|
@ -71,7 +71,6 @@ if __name__ == "__main__":
|
||||||
# Regex Frequency
|
# Regex Frequency
|
||||||
while True:
|
while True:
|
||||||
item_id = p.get_from_set()
|
item_id = p.get_from_set()
|
||||||
item_id = 'archive/pastebin.com_pro/2020/03/04/AnwFX3w2.gz'
|
|
||||||
if item_id is not None:
|
if item_id is not None:
|
||||||
item_content = item_basic.get_item_content(item_id)
|
item_content = item_basic.get_item_content(item_id)
|
||||||
yara_match = rules.match(data=item_content, callback=yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
|
yara_match = rules.match(data=item_content, callback=yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
|
||||||
|
|
1
bin/trackers/yara/ail-yara-rules
Submodule
1
bin/trackers/yara/ail-yara-rules
Submodule
|
@ -0,0 +1 @@
|
||||||
|
Subproject commit edc390c4a8d93a028e29938e92aacb399e270cc4
|
|
@ -39,6 +39,10 @@ sudo apt-get install build-essential libffi-dev automake autoconf libtool -qq
|
||||||
# sflock, gz requirement
|
# sflock, gz requirement
|
||||||
sudo apt-get install p7zip-full -qq
|
sudo apt-get install p7zip-full -qq
|
||||||
|
|
||||||
|
# SUBMODULES #
|
||||||
|
git submodule init
|
||||||
|
git submodule update
|
||||||
|
|
||||||
# REDIS #
|
# REDIS #
|
||||||
test ! -d redis/ && git clone https://github.com/antirez/redis.git
|
test ! -d redis/ && git clone https://github.com/antirez/redis.git
|
||||||
pushd redis/
|
pushd redis/
|
||||||
|
|
39
update/v3.1.1/Update.py
Executable file
39
update/v3.1.1/Update.py
Executable file
|
@ -0,0 +1,39 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import redis
|
||||||
|
import argparse
|
||||||
|
import datetime
|
||||||
|
import configparser
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
|
import ConfigLoader
|
||||||
|
|
||||||
|
new_version = 'v3.1.1'
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
|
||||||
|
start_deb = time.time()
|
||||||
|
|
||||||
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||||
|
config_loader = None
|
||||||
|
|
||||||
|
#### NEW EXPORTER
|
||||||
|
# remove old tags errors
|
||||||
|
#r_serv_db.delete('mess_not_saved_export')
|
||||||
|
|
||||||
|
# move solo tags to export in tags_db
|
||||||
|
#all_misp_tags = r_serv_db.smembers('whitelist_misp')
|
||||||
|
#all_hive_tags = r_serv_db.smembers('whitelist_hive')
|
||||||
|
# # TODO: save them in tags db
|
||||||
|
#### NEW EXPORTER
|
||||||
|
|
||||||
|
#Set current ail version
|
||||||
|
r_serv_db.set('ail:version', new_version)
|
||||||
|
|
||||||
|
#Set current ail version
|
||||||
|
r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
|
48
update/v3.1.1/Update.sh
Executable file
48
update/v3.1.1/Update.sh
Executable file
|
@ -0,0 +1,48 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||||
|
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||||
|
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||||
|
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||||
|
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||||
|
|
||||||
|
export PATH=$AIL_HOME:$PATH
|
||||||
|
export PATH=$AIL_REDIS:$PATH
|
||||||
|
export PATH=$AIL_ARDB:$PATH
|
||||||
|
export PATH=$AIL_BIN:$PATH
|
||||||
|
export PATH=$AIL_FLASK:$PATH
|
||||||
|
|
||||||
|
GREEN="\\033[1;32m"
|
||||||
|
DEFAULT="\\033[0;39m"
|
||||||
|
|
||||||
|
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||||
|
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||||
|
wait
|
||||||
|
|
||||||
|
bash ${AIL_BIN}/LAUNCH.sh -ldbv &
|
||||||
|
wait
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
echo -e $GREEN"Installing YARA ..."$DEFAULT
|
||||||
|
pip3 install yara-python
|
||||||
|
bash ${AIL_BIN}/LAUNCH.sh -t
|
||||||
|
|
||||||
|
# SUBMODULES #
|
||||||
|
git submodule init
|
||||||
|
git submodule update
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||||
|
echo ""
|
||||||
|
python ${AIL_HOME}/update/v3.1.1/Update.py
|
||||||
|
wait
|
||||||
|
echo ""
|
||||||
|
echo ""
|
||||||
|
|
||||||
|
|
||||||
|
echo ""
|
||||||
|
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||||
|
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||||
|
wait
|
||||||
|
|
||||||
|
exit 0
|
|
@ -32,7 +32,7 @@
|
||||||
<h5 class="card-title">Create a new tracker</h5>
|
<h5 class="card-title">Create a new tracker</h5>
|
||||||
</div>
|
</div>
|
||||||
<div class="card-body">
|
<div class="card-body">
|
||||||
<p class="card-text">Enter a domain and choose what kind of data you want.</p>
|
<p class="card-text">Select a tracker type.</p>
|
||||||
|
|
||||||
<form action="{{ url_for('hunter.add_tracked_menu') }}" method='post'>
|
<form action="{{ url_for('hunter.add_tracked_menu') }}" method='post'>
|
||||||
|
|
||||||
|
@ -103,7 +103,7 @@
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="row" id="textarea">
|
<div class="row" id="textarea">
|
||||||
<textarea class="form-control" id="text_input" name="yara_custom_rule" rows="5"></textarea>
|
<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5"></textarea>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
@ -136,6 +136,7 @@ $(document).ready(function(){
|
||||||
$("#tracker_desc").hide();
|
$("#tracker_desc").hide();
|
||||||
$("#term").hide();
|
$("#term").hide();
|
||||||
$("#nb_word").hide();
|
$("#nb_word").hide();
|
||||||
|
$("#yara_rule").hide();
|
||||||
|
|
||||||
$('#tracker_type').on('change', function() {
|
$('#tracker_type').on('change', function() {
|
||||||
var tracker_type = this.value;
|
var tracker_type = this.value;
|
||||||
|
@ -144,16 +145,25 @@ $(document).ready(function(){
|
||||||
$("#tracker_desc").show();
|
$("#tracker_desc").show();
|
||||||
$("#term").show();
|
$("#term").show();
|
||||||
$("#nb_word").hide();
|
$("#nb_word").hide();
|
||||||
|
$("#yara_rule").hide();
|
||||||
} else if (tracker_type=="set") {
|
} else if (tracker_type=="set") {
|
||||||
$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
|
$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
|
||||||
$("#tracker_desc").show();
|
$("#tracker_desc").show();
|
||||||
$("#term").show();
|
$("#term").show();
|
||||||
$("#nb_word").show();
|
$("#nb_word").show();
|
||||||
} else {
|
$("#yara_rule").hide();
|
||||||
|
} else if (tracker_type=="regex") {
|
||||||
$("#tracker_desc").text("Enter a valid Python regex");
|
$("#tracker_desc").text("Enter a valid Python regex");
|
||||||
$("#tracker_desc").show();
|
$("#tracker_desc").show();
|
||||||
$("#term").show();
|
$("#term").show();
|
||||||
$("#nb_word").hide();
|
$("#nb_word").hide();
|
||||||
|
$("#yara_rule").hide();
|
||||||
|
} else if (tracker_type=="yara") {
|
||||||
|
$("#tracker_desc").text("Select a default yara rule or create your own rule:");
|
||||||
|
$("#tracker_desc").show();
|
||||||
|
$("#term").hide();
|
||||||
|
$("#nb_word").hide();
|
||||||
|
$("#yara_rule").show();
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
|
|
|
@ -22,19 +22,19 @@
|
||||||
<li class="nav-item">
|
<li class="nav-item">
|
||||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu_word')}}" id="nav_tracker_word">
|
<a class="nav-link" href="{{url_for('hunter.tracked_menu_word')}}" id="nav_tracker_word">
|
||||||
<i class="fas fa-font"></i>
|
<i class="fas fa-font"></i>
|
||||||
<span>Tracked Words</span>
|
<span>Words</span>
|
||||||
</a>
|
</a>
|
||||||
</li>
|
</li>
|
||||||
<li class="nav-item">
|
<li class="nav-item">
|
||||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu_set')}}" id="nav_tracker_set">
|
<a class="nav-link" href="{{url_for('hunter.tracked_menu_set')}}" id="nav_tracker_set">
|
||||||
<i class="fas fa-layer-group"></i>
|
<i class="fas fa-layer-group"></i>
|
||||||
<span>Tracked Set</span>
|
<span>Set</span>
|
||||||
</a>
|
</a>
|
||||||
</li>
|
</li>
|
||||||
<li class="nav-item">
|
<li class="nav-item">
|
||||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu_regex')}}" id="nav_tracker_regex">
|
<a class="nav-link" href="{{url_for('hunter.tracked_menu_regex')}}" id="nav_tracker_regex">
|
||||||
<i class="fas fa-ruler"></i>
|
<i class="fas fa-ruler-vertical"></i>
|
||||||
<span>Tracked Regex</span>
|
<span>Regex</span>
|
||||||
</a>
|
</a>
|
||||||
</li>
|
</li>
|
||||||
<li class="nav-item">
|
<li class="nav-item">
|
||||||
|
|
|
@ -2,6 +2,9 @@
|
||||||
|
|
||||||
set -e
|
set -e
|
||||||
|
|
||||||
|
# submodules
|
||||||
|
git submodule update
|
||||||
|
|
||||||
wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
|
wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
|
||||||
|
|
||||||
SBADMIN_VERSION='3.3.7'
|
SBADMIN_VERSION='3.3.7'
|
||||||
|
|
Loading…
Reference in a new issue