AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-22 22:27:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [submodule + YARA] add submodule auto update + update v3.1.1

Browse source
This commit is contained in:
Terrtia 2020-08-13 15:15:08 +02:00
parent e70ae376c5
commit 8283488b6d
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
11 changed files with 129 additions and 8 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.gitmodules vendored Normal file
View file

@ -0,0 +1,3 @@
[submodule "bin/trackers/yara/ail-yara-rules"]
path = bin/trackers/yara/ail-yara-rules
url = https://github.com/ail-project/ail-yara-rules.git

2
bin/LAUNCH.sh
View file

@ -184,6 +184,8 @@ function launching_scripts {
sleep 0.1
screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x"

12
bin/Update.py
View file

@ -231,6 +231,15 @@ def get_git_upper_tags_remote(current_tag, is_fork):
aborting_update()
sys.exit(0)
def update_submodules():
print('{}git submodule update:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
process = subprocess.run(['git', 'submodule', 'update'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if process.returncode == 0:
print(process.stdout.decode())
print()
else:
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_fork):
print('{}git checkout master:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
@ -238,6 +247,9 @@ def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_for
if process.returncode == 0:
print(process.stdout.decode())
print()
update_submodules()
print('{}git pull:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
process = subprocess.run(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)

1
bin/trackers/Tracker_Yara.py
View file

@ -71,7 +71,6 @@ if __name__ == "__main__":
# Regex Frequency
while True:
item_id = p.get_from_set()
item_id = 'archive/pastebin.com_pro/2020/03/04/AnwFX3w2.gz'
if item_id is not None:
item_content = item_basic.get_item_content(item_id)
yara_match = rules.match(data=item_content, callback=yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60)

1
bin/trackers/yara/ail-yara-rules Submodule

@ -0,0 +1 @@
Subproject commit edc390c4a8d93a028e29938e92aacb399e270cc4

4
installing_deps.sh
View file

@ -39,6 +39,10 @@ sudo apt-get install build-essential libffi-dev automake autoconf libtool -qq
# sflock, gz requirement
sudo apt-get install p7zip-full -qq
# SUBMODULES #
git submodule init
git submodule update
# REDIS #
test ! -d redis/ && git clone https://github.com/antirez/redis.git
pushd redis/

39
update/v3.1.1/Update.py Executable file
View file

@ -0,0 +1,39 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import time
import redis
import argparse
import datetime
import configparser
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
new_version = 'v3.1.1'
if __name__ == '__main__':
start_deb = time.time()
config_loader = ConfigLoader.ConfigLoader()
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
config_loader = None
#### NEW EXPORTER
# remove old tags errors
#r_serv_db.delete('mess_not_saved_export')
# move solo tags to export in tags_db
#all_misp_tags = r_serv_db.smembers('whitelist_misp')
#all_hive_tags = r_serv_db.smembers('whitelist_hive')
# # TODO: save them in tags db
#### NEW EXPORTER
#Set current ail version
r_serv_db.set('ail:version', new_version)
#Set current ail version
r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))

48
update/v3.1.1/Update.sh Executable file
View file

@ -0,0 +1,48 @@
#!/bin/bash
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
export PATH=$AIL_HOME:$PATH
export PATH=$AIL_REDIS:$PATH
export PATH=$AIL_ARDB:$PATH
export PATH=$AIL_BIN:$PATH
export PATH=$AIL_FLASK:$PATH
GREEN="\\033[1;32m"
DEFAULT="\\033[0;39m"
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -ks
wait
bash ${AIL_BIN}/LAUNCH.sh -ldbv &
wait
echo ""
echo -e $GREEN"Installing YARA ..."$DEFAULT
pip3 install yara-python
bash ${AIL_BIN}/LAUNCH.sh -t
# SUBMODULES #
git submodule init
git submodule update
echo ""
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
echo ""
python ${AIL_HOME}/update/v3.1.1/Update.py
wait
echo ""
echo ""
echo ""
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -ks
wait
exit 0

16
var/www/modules/hunter/templates/Add_tracker.html
View file

@ -32,7 +32,7 @@
<h5 class="card-title">Create a new tracker</h5>
</div>
<div class="card-body">
<p class="card-text">Enter a domain and choose what kind of data you want.</p>
<p class="card-text">Select a tracker type.</p>
<form action="{{ url_for('hunter.add_tracked_menu') }}" method='post'>
@ -103,7 +103,7 @@
</div>
<div class="row" id="textarea">
<textarea class="form-control" id="text_input" name="yara_custom_rule" rows="5"></textarea>
<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5"></textarea>
</div>
</div>
@ -136,6 +136,7 @@ $(document).ready(function(){
$("#tracker_desc").hide();
$("#term").hide();
$("#nb_word").hide();
$("#yara_rule").hide();
$('#tracker_type').on('change', function() {
var tracker_type = this.value;
@ -144,16 +145,25 @@ $(document).ready(function(){
$("#tracker_desc").show();
$("#term").show();
$("#nb_word").hide();
$("#yara_rule").hide();
} else if (tracker_type=="set") {
$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
$("#tracker_desc").show();
$("#term").show();
$("#nb_word").show();
} else {
$("#yara_rule").hide();
} else if (tracker_type=="regex") {
$("#tracker_desc").text("Enter a valid Python regex");
$("#tracker_desc").show();
$("#term").show();
$("#nb_word").hide();
$("#yara_rule").hide();
} else if (tracker_type=="yara") {
$("#tracker_desc").text("Select a default yara rule or create your own rule:");
$("#tracker_desc").show();
$("#term").hide();
$("#nb_word").hide();
$("#yara_rule").show();
}
});

8
var/www/templates/hunter/menu_sidebar.html
View file

@ -22,19 +22,19 @@
<li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_word')}}" id="nav_tracker_word">
<i class="fas fa-font"></i>
<span>Tracked Words</span>
<span>Words</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_set')}}" id="nav_tracker_set">
<i class="fas fa-layer-group"></i>
<span>Tracked Set</span>
<span>Set</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_regex')}}" id="nav_tracker_regex">
<i class="fas fa-ruler"></i>
<span>Tracked Regex</span>
<i class="fas fa-ruler-vertical"></i>
<span>Regex</span>
</a>
</li>
<li class="nav-item">

3
var/www/update_thirdparty.sh
View file

@ -2,6 +2,9 @@
set -e
# submodules
git submodule update
wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
SBADMIN_VERSION='3.3.7'