mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 14:17:16 +00:00
chg: [submodule + YARA] add submodule auto update + update v3.1.1
This commit is contained in:
parent
e70ae376c5
commit
8283488b6d
11 changed files with 129 additions and 8 deletions
3
.gitmodules
vendored
Normal file
3
.gitmodules
vendored
Normal file
|
@ -0,0 +1,3 @@
|
|||
[submodule "bin/trackers/yara/ail-yara-rules"]
|
||||
path = bin/trackers/yara/ail-yara-rules
|
||||
url = https://github.com/ail-project/ail-yara-rules.git
|
|
@ -184,6 +184,8 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "RegexTracker" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexTracker.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x"
|
||||
|
|
|
@ -231,6 +231,15 @@ def get_git_upper_tags_remote(current_tag, is_fork):
|
|||
aborting_update()
|
||||
sys.exit(0)
|
||||
|
||||
def update_submodules():
|
||||
print('{}git submodule update:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
|
||||
process = subprocess.run(['git', 'submodule', 'update'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
if process.returncode == 0:
|
||||
print(process.stdout.decode())
|
||||
print()
|
||||
else:
|
||||
print('{}{}{}'.format(TERMINAL_RED, process.stderr.decode(), TERMINAL_DEFAULT))
|
||||
|
||||
def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_fork):
|
||||
print('{}git checkout master:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
|
||||
process = subprocess.run(['git', 'checkout', 'master'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
@ -238,6 +247,9 @@ def update_ail(current_tag, list_upper_tags_remote, current_version_path, is_for
|
|||
if process.returncode == 0:
|
||||
print(process.stdout.decode())
|
||||
print()
|
||||
|
||||
update_submodules()
|
||||
|
||||
print('{}git pull:{}'.format(TERMINAL_YELLOW, TERMINAL_DEFAULT))
|
||||
process = subprocess.run(['git', 'pull'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
|
||||
|
|
|
@ -71,7 +71,6 @@ if __name__ == "__main__":
|
|||
# Regex Frequency
|
||||
while True:
|
||||
item_id = p.get_from_set()
|
||||
item_id = 'archive/pastebin.com_pro/2020/03/04/AnwFX3w2.gz'
|
||||
if item_id is not None:
|
||||
item_content = item_basic.get_item_content(item_id)
|
||||
yara_match = rules.match(data=item_content, callback=yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
|
||||
|
|
1
bin/trackers/yara/ail-yara-rules
Submodule
1
bin/trackers/yara/ail-yara-rules
Submodule
|
@ -0,0 +1 @@
|
|||
Subproject commit edc390c4a8d93a028e29938e92aacb399e270cc4
|
|
@ -39,6 +39,10 @@ sudo apt-get install build-essential libffi-dev automake autoconf libtool -qq
|
|||
# sflock, gz requirement
|
||||
sudo apt-get install p7zip-full -qq
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule init
|
||||
git submodule update
|
||||
|
||||
# REDIS #
|
||||
test ! -d redis/ && git clone https://github.com/antirez/redis.git
|
||||
pushd redis/
|
||||
|
|
39
update/v3.1.1/Update.py
Executable file
39
update/v3.1.1/Update.py
Executable file
|
@ -0,0 +1,39 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import argparse
|
||||
import datetime
|
||||
import configparser
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
new_version = 'v3.1.1'
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
config_loader = None
|
||||
|
||||
#### NEW EXPORTER
|
||||
# remove old tags errors
|
||||
#r_serv_db.delete('mess_not_saved_export')
|
||||
|
||||
# move solo tags to export in tags_db
|
||||
#all_misp_tags = r_serv_db.smembers('whitelist_misp')
|
||||
#all_hive_tags = r_serv_db.smembers('whitelist_hive')
|
||||
# # TODO: save them in tags db
|
||||
#### NEW EXPORTER
|
||||
|
||||
#Set current ail version
|
||||
r_serv_db.set('ail:version', new_version)
|
||||
|
||||
#Set current ail version
|
||||
r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))
|
48
update/v3.1.1/Update.sh
Executable file
48
update/v3.1.1/Update.sh
Executable file
|
@ -0,0 +1,48 @@
|
|||
#!/bin/bash
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
export PATH=$AIL_BIN:$PATH
|
||||
export PATH=$AIL_FLASK:$PATH
|
||||
|
||||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ldbv &
|
||||
wait
|
||||
echo ""
|
||||
|
||||
echo -e $GREEN"Installing YARA ..."$DEFAULT
|
||||
pip3 install yara-python
|
||||
bash ${AIL_BIN}/LAUNCH.sh -t
|
||||
|
||||
# SUBMODULES #
|
||||
git submodule init
|
||||
git submodule update
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
|
||||
echo ""
|
||||
python ${AIL_HOME}/update/v3.1.1/Update.py
|
||||
wait
|
||||
echo ""
|
||||
echo ""
|
||||
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
exit 0
|
|
@ -32,7 +32,7 @@
|
|||
<h5 class="card-title">Create a new tracker</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<p class="card-text">Enter a domain and choose what kind of data you want.</p>
|
||||
<p class="card-text">Select a tracker type.</p>
|
||||
|
||||
<form action="{{ url_for('hunter.add_tracked_menu') }}" method='post'>
|
||||
|
||||
|
@ -103,7 +103,7 @@
|
|||
</div>
|
||||
|
||||
<div class="row" id="textarea">
|
||||
<textarea class="form-control" id="text_input" name="yara_custom_rule" rows="5"></textarea>
|
||||
<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5"></textarea>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
@ -136,6 +136,7 @@ $(document).ready(function(){
|
|||
$("#tracker_desc").hide();
|
||||
$("#term").hide();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").hide();
|
||||
|
||||
$('#tracker_type').on('change', function() {
|
||||
var tracker_type = this.value;
|
||||
|
@ -144,16 +145,25 @@ $(document).ready(function(){
|
|||
$("#tracker_desc").show();
|
||||
$("#term").show();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").hide();
|
||||
} else if (tracker_type=="set") {
|
||||
$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
|
||||
$("#tracker_desc").show();
|
||||
$("#term").show();
|
||||
$("#nb_word").show();
|
||||
} else {
|
||||
$("#yara_rule").hide();
|
||||
} else if (tracker_type=="regex") {
|
||||
$("#tracker_desc").text("Enter a valid Python regex");
|
||||
$("#tracker_desc").show();
|
||||
$("#term").show();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").hide();
|
||||
} else if (tracker_type=="yara") {
|
||||
$("#tracker_desc").text("Select a default yara rule or create your own rule:");
|
||||
$("#tracker_desc").show();
|
||||
$("#term").hide();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").show();
|
||||
}
|
||||
});
|
||||
|
||||
|
|
|
@ -22,19 +22,19 @@
|
|||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu_word')}}" id="nav_tracker_word">
|
||||
<i class="fas fa-font"></i>
|
||||
<span>Tracked Words</span>
|
||||
<span>Words</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu_set')}}" id="nav_tracker_set">
|
||||
<i class="fas fa-layer-group"></i>
|
||||
<span>Tracked Set</span>
|
||||
<span>Set</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu_regex')}}" id="nav_tracker_regex">
|
||||
<i class="fas fa-ruler"></i>
|
||||
<span>Tracked Regex</span>
|
||||
<i class="fas fa-ruler-vertical"></i>
|
||||
<span>Regex</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
|
|
|
@ -2,6 +2,9 @@
|
|||
|
||||
set -e
|
||||
|
||||
# submodules
|
||||
git submodule update
|
||||
|
||||
wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
|
||||
|
||||
SBADMIN_VERSION='3.3.7'
|
||||
|
|
Loading…
Reference in a new issue