From 85f2573023ad1fb90a0992589beeaf65f46eb920 Mon Sep 17 00:00:00 2001 From: Pietro Mazzini Date: Sat, 25 Jul 2020 11:00:34 +0200 Subject: [PATCH 01/17] LAUNCH update function fail because the folder isn't a git repo, comment it --- bin/LAUNCH.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 9c855668..6f7786e6 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -497,7 +497,7 @@ function reset_password() { function launch_all { checking_configuration; - update; + #update; launch_redis; launch_ardb; launch_logs; From d4745e835beb72a1ba133d4dd868606e357e96dc Mon Sep 17 00:00:00 2001 From: Pietro Mazzini Date: Sat, 25 Jul 2020 11:01:53 +0200 Subject: [PATCH 02/17] Comment git repo tag in installing_deps, git repo not recognised --- installing_deps.sh | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/installing_deps.sh b/installing_deps.sh index 5ed4665b..4d0cddf4 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -111,13 +111,13 @@ $AIL_HOME/doc/generate_modules_data_flow_graph.sh #### DB SETUP #### # init update version -pushd ${AIL_HOME} +#pushd ${AIL_HOME} # shallow clone -git fetch --depth=500 --tags --prune -git describe --abbrev=0 --tags | tr -d '\n' > ${AIL_HOME}/update/current_version -echo "AIL current version:" -git describe --abbrev=0 --tags -popd +#git fetch --depth=500 --tags --prune +#git describe --abbrev=0 --tags | tr -d '\n' > ${AIL_HOME}/update/current_version +#echo "AIL current version:" +#git describe --abbrev=0 --tags +#popd # LAUNCH ARDB bash ${AIL_BIN}/LAUNCH.sh -lav & From a52ae937df2615dc81fd06a2ae5d9b94f22a77c8 Mon Sep 17 00:00:00 2001 From: Pietro Mazzini Date: Sat, 25 Jul 2020 11:04:16 +0200 Subject: [PATCH 03/17] Update Dockerfile: use ubuntu 18, remove redundant update_thirdparty, remove old crawler requirements pip install --- other_installers/docker/Dockerfile | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/other_installers/docker/Dockerfile b/other_installers/docker/Dockerfile index 533c44c4..b95159c6 100644 --- a/other_installers/docker/Dockerfile +++ b/other_installers/docker/Dockerfile @@ -1,4 +1,6 @@ -FROM ubuntu:16.04 +FROM ubuntu:18.04 +ENV TZ=Europe/Rome +RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone # Make sure that all updates are in place RUN apt-get clean && apt-get update -y && apt-get upgrade -y \ @@ -19,12 +21,6 @@ WORKDIR /opt/AIL RUN ./installing_deps.sh WORKDIR /opt/AIL -# Installing Web dependencies, -# remove all the parts below if you dont need the Web UI -WORKDIR /opt/AIL/var/www -RUN ./update_thirdparty.sh -WORKDIR /opt/AIL - # Default to UTF-8 file.encoding ENV LANG C.UTF-8 ENV AIL_HOME /opt/AIL @@ -38,7 +34,6 @@ ENV PATH ${AIL_VENV}/bin:${AIL_HOME}:${AIL_REDIS}:${AIL_ARDB}:${AIL_BIN}:${AIL_F RUN ./pystemon/install.sh RUN pip install -r /opt/pystemon/requirements.txt -RUN pip install -r /opt/AIL/crawler_requirements.txt COPY docker_start.sh /docker_start.sh ENTRYPOINT ["/bin/bash", "docker_start.sh"] From 44e4d6802ec9605357a5892ae6bffb2c73de76dc Mon Sep 17 00:00:00 2001 From: Pietro Mazzini Date: Sat, 25 Jul 2020 11:05:10 +0200 Subject: [PATCH 04/17] Add to docker start crawlers launch --- other_installers/docker/docker_start.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/other_installers/docker/docker_start.sh b/other_installers/docker/docker_start.sh index c8f06010..5b0e1b06 100755 --- a/other_installers/docker/docker_start.sh +++ b/other_installers/docker/docker_start.sh @@ -13,6 +13,7 @@ signalListener() { source ./AILENV/bin/activate cd bin ./LAUNCH.sh -l +./LAUNCH.sh -c ./LAUNCH.sh -f signalListener tail -f /dev/null $@ From bc8064f9bb52dea65b50cda9b6c0068c24d89b97 Mon Sep 17 00:00:00 2001 From: Pietro Mazzini Date: Sat, 25 Jul 2020 11:05:40 +0200 Subject: [PATCH 05/17] Fix canvas folder name --- var/www/update_thirdparty.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh index c33454ca..cda4cc55 100755 --- a/var/www/update_thirdparty.sh +++ b/var/www/update_thirdparty.sh @@ -90,6 +90,7 @@ wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb61 wget -q http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js wget -q http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip unzip -qq temp/canvasjs.zip -d temp/ +cp -r temp/canvasjs-2.3.2/Chart\ 2.3.2\ GA\ -\ Stable/* temp/ mv temp/canvasjs-2.3.2/Chart\ 2.3.2\ GA\ -\ Stable/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js wget -q https://jqueryui.com/resources/download/jquery-ui-1.12.1.zip -O temp/jquery-ui.zip From 61b5bbaf9d3f73f9417c45514376dcdf25d7f046 Mon Sep 17 00:00:00 2001 From: Pietro Mazzini Date: Sat, 25 Jul 2020 11:06:49 +0200 Subject: [PATCH 06/17] Update docker readme --- other_installers/docker/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/other_installers/docker/README.md b/other_installers/docker/README.md index 265defab..6faf7123 100644 --- a/other_installers/docker/README.md +++ b/other_installers/docker/README.md @@ -1,4 +1,4 @@ -Docker Quick Start (Ubuntu 16.04 LTS) +Docker Quick Start (Ubuntu 18.04 LTS) ------------ :warning: From 738e96e9d6420631e9b6ddad509b8bc26710b97b Mon Sep 17 00:00:00 2001 From: Pietro Mazzini Date: Sat, 25 Jul 2020 11:08:45 +0200 Subject: [PATCH 07/17] Update docker readme (cp) --- other_installers/docker/README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/other_installers/docker/README.md b/other_installers/docker/README.md index 6faf7123..6d09123b 100644 --- a/other_installers/docker/README.md +++ b/other_installers/docker/README.md @@ -19,6 +19,7 @@ curl https://get.docker.com | /bin/bash ```bash git clone https://github.com/CIRCL/AIL-framework.git cd AIL-framework +cp -r ./other_installers/docker/Dockerfile ./other_installers/docker/docker_start.sh ./other_installers/docker/pystemon ./ docker build -t ail-framework . ``` 3. To start AIL on port 7000, type the following command below: From 7ee8f4ad286028c7abfa9e141c498e57c8db9215 Mon Sep 17 00:00:00 2001 From: Pietro Mazzini Date: Sat, 25 Jul 2020 14:48:17 +0200 Subject: [PATCH 08/17] Add tz as argument --- other_installers/docker/Dockerfile | 3 ++- other_installers/docker/README.md | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/other_installers/docker/Dockerfile b/other_installers/docker/Dockerfile index b95159c6..782571ab 100644 --- a/other_installers/docker/Dockerfile +++ b/other_installers/docker/Dockerfile @@ -1,5 +1,6 @@ FROM ubuntu:18.04 -ENV TZ=Europe/Rome +ARG tz_buildtime=Europe/Rome +ENV TZ=$tz_buildtime RUN ln -snf /usr/share/zoneinfo/$TZ /etc/localtime && echo $TZ > /etc/timezone # Make sure that all updates are in place diff --git a/other_installers/docker/README.md b/other_installers/docker/README.md index 6d09123b..6e601b39 100644 --- a/other_installers/docker/README.md +++ b/other_installers/docker/README.md @@ -20,7 +20,7 @@ curl https://get.docker.com | /bin/bash git clone https://github.com/CIRCL/AIL-framework.git cd AIL-framework cp -r ./other_installers/docker/Dockerfile ./other_installers/docker/docker_start.sh ./other_installers/docker/pystemon ./ -docker build -t ail-framework . +docker build --build-arg tz_buildtime=YOUR_GEO_AREA/YOUR_CITY -t ail-framework . ``` 3. To start AIL on port 7000, type the following command below: ``` From 97949396b4b4ac6da1c3f0ab74740e246b02ad31 Mon Sep 17 00:00:00 2001 From: Pietro Mazzini Date: Mon, 3 Aug 2020 10:53:32 +0200 Subject: [PATCH 09/17] Uncomment update and explain in docker readme --- bin/LAUNCH.sh | 2 +- other_installers/docker/README.md | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 6f7786e6..9c855668 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -497,7 +497,7 @@ function reset_password() { function launch_all { checking_configuration; - #update; + update; launch_redis; launch_ardb; launch_logs; diff --git a/other_installers/docker/README.md b/other_installers/docker/README.md index 6e601b39..c7e6d4ff 100644 --- a/other_installers/docker/README.md +++ b/other_installers/docker/README.md @@ -20,6 +20,8 @@ curl https://get.docker.com | /bin/bash git clone https://github.com/CIRCL/AIL-framework.git cd AIL-framework cp -r ./other_installers/docker/Dockerfile ./other_installers/docker/docker_start.sh ./other_installers/docker/pystemon ./ +cp ./configs/update.cfg.sample ./configs/update.cfg +vim/nano ./configs/update.cfg (set auto_update to False) docker build --build-arg tz_buildtime=YOUR_GEO_AREA/YOUR_CITY -t ail-framework . ``` 3. To start AIL on port 7000, type the following command below: From 57973455484a19e0d06c879f4a7cac35627914a1 Mon Sep 17 00:00:00 2001 From: Pietro Mazzini Date: Mon, 3 Aug 2020 12:07:09 +0200 Subject: [PATCH 10/17] Remove useless copy --- var/www/update_thirdparty.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh index cda4cc55..c33454ca 100755 --- a/var/www/update_thirdparty.sh +++ b/var/www/update_thirdparty.sh @@ -90,7 +90,6 @@ wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb61 wget -q http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js wget -q http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip unzip -qq temp/canvasjs.zip -d temp/ -cp -r temp/canvasjs-2.3.2/Chart\ 2.3.2\ GA\ -\ Stable/* temp/ mv temp/canvasjs-2.3.2/Chart\ 2.3.2\ GA\ -\ Stable/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js wget -q https://jqueryui.com/resources/download/jquery-ui-1.12.1.zip -O temp/jquery-ui.zip From 7f9df2d08367c9a97d2cfcdec597e1e4ced28e0e Mon Sep 17 00:00:00 2001 From: Pietro Mazzini Date: Mon, 10 Aug 2020 20:22:12 +0200 Subject: [PATCH 11/17] Update update_thirdparty.sh --- var/www/update_thirdparty.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh index c33454ca..3b18c740 100755 --- a/var/www/update_thirdparty.sh +++ b/var/www/update_thirdparty.sh @@ -90,7 +90,7 @@ wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb61 wget -q http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js wget -q http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip unzip -qq temp/canvasjs.zip -d temp/ -mv temp/canvasjs-2.3.2/Chart\ 2.3.2\ GA\ -\ Stable/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js +mv temp/Chart\ 2.3.2\ GA\ -\ Stable/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js wget -q https://jqueryui.com/resources/download/jquery-ui-1.12.1.zip -O temp/jquery-ui.zip unzip -qq temp/jquery-ui.zip -d temp/ From 0b52465f633b97d9fff124a4c21cb44fab7e498f Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 18 Aug 2020 20:51:15 +0200 Subject: [PATCH 12/17] fix: [travis] --- installing_deps.sh | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/installing_deps.sh b/installing_deps.sh index a3275648..839527c9 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -115,13 +115,14 @@ $AIL_HOME/doc/generate_modules_data_flow_graph.sh #### DB SETUP #### # init update version -#pushd ${AIL_HOME} +pushd ${AIL_HOME} # shallow clone -#git fetch --depth=500 --tags --prune -#git describe --abbrev=0 --tags | tr -d '\n' > ${AIL_HOME}/update/current_version -#echo "AIL current version:" -#git describe --abbrev=0 --tags -#popd +git fetch --depth=500 --tags --prune +git fetch --unshallow +git describe --abbrev=0 --tags | tr -d '\n' > ${AIL_HOME}/update/current_version +echo "AIL current version:" +git describe --abbrev=0 --tags +popd # LAUNCH ARDB bash ${AIL_BIN}/LAUNCH.sh -lav & From a2e4e911fba424cb662e4ddecb85f3535329f09b Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 18 Aug 2020 21:21:27 +0200 Subject: [PATCH 13/17] fix: [update v3.1.1] init submodule --- update/v3.1.1/Update.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/update/v3.1.1/Update.sh b/update/v3.1.1/Update.sh index 0d3d860e..f43591c0 100755 --- a/update/v3.1.1/Update.sh +++ b/update/v3.1.1/Update.sh @@ -23,6 +23,10 @@ bash ${AIL_BIN}/LAUNCH.sh -ldbv & wait echo "" +# SUBMODULES # +git submodule init +git submodule update + echo -e $GREEN"Installing YARA ..."$DEFAULT pip3 install yara-python bash ${AIL_BIN}/LAUNCH.sh -t From 5a1e75e0f5e08ad0c1080ef2bfb43de8a10302f6 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 18 Aug 2020 21:32:09 +0200 Subject: [PATCH 14/17] fix: [updater] tag subversion --- bin/Update.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/Update.py b/bin/Update.py index 5d65e83b..88573d50 100755 --- a/bin/Update.py +++ b/bin/Update.py @@ -221,7 +221,7 @@ def get_git_upper_tags_remote(current_tag, is_fork): # add tag with last commit if float(tag_val) >= float(current_tag_val): - dict_tags_commit[tag_val] = commit + dict_tags_commit[tag] = commit list_upper_tags = [('v{}'.format(key), dict_tags_commit[key]) for key in dict_tags_commit] # force update order list_upper_tags.sort() From 6e8f76da614e2af6ebf0c0e71b34749f16581735 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 18 Aug 2020 21:42:30 +0200 Subject: [PATCH 15/17] fix: [trackers] typo --- bin/lib/Tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 16dd2a73..638a9822 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -41,7 +41,7 @@ def get_email_subject(tracker_uuid): return 'AIL framework: {}'.format(tracker_description) def get_tracker_last_updated_by_type(tracker_type): - epoch_update = r_serv_tracker.get('tracker:refresh:{}'.format(term_type)) + epoch_update = r_serv_tracker.get('tracker:refresh:{}'.format(tracker_type)) if not epoch_update: epoch_update = 0 return float(epoch_update) From 0c466c51302a13eea9bd971e8886aab0a3c1ece1 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 18 Aug 2020 21:54:26 +0200 Subject: [PATCH 16/17] fix: [trackers yara] remove sleep time --- bin/trackers/Tracker_Yara.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py index c2c25823..25666304 100755 --- a/bin/trackers/Tracker_Yara.py +++ b/bin/trackers/Tracker_Yara.py @@ -77,8 +77,6 @@ if __name__ == "__main__": if yara_match: print(f'{item_id}: {yara_match}') - time.sleep(30) - else: time.sleep(5) From b7d37e81d4bf94bcd96f57936ebaf3b2588e4678 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 19 Aug 2020 11:37:51 +0200 Subject: [PATCH 17/17] chg: [tracker yara] show rule content --- bin/lib/Tracker.py | 14 ++++++++++++++ var/www/modules/hunter/Flask_hunter.py | 9 ++++++++- var/www/modules/hunter/templates/showTracker.html | 5 +++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index 638a9822..b09a1d3e 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -123,6 +123,20 @@ def save_yara_rule(yara_rule_type, yara_rule, tracker_uuid=None): if yara_rule_type == 'yara_default': filename = os.path.join('ail-yara-rules', 'rules', yara_rule) return filename + +def get_yara_rule_content(yara_rule): + yara_dir = get_yara_rules_dir() + filename = os.path.join(yara_dir, yara_rule) + filename = os.path.realpath(filename) + + # incorrect filename + if not os.path.commonprefix([filename, yara_dir]) == yara_dir: + return '' # # TODO: throw exception + + with open(filename, 'r') as f: + rule_content = f.read() + return rule_content + ##-- YARA --## diff --git a/var/www/modules/hunter/Flask_hunter.py b/var/www/modules/hunter/Flask_hunter.py index 1fac2866..48530574 100644 --- a/var/www/modules/hunter/Flask_hunter.py +++ b/var/www/modules/hunter/Flask_hunter.py @@ -153,6 +153,11 @@ def show_tracker(): tracker_metadata = Term.get_term_metedata(term_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sparkline=True) + if tracker_metadata['type'] == 'yara': + yara_rule_content = Tracker.get_yara_rule_content(tracker_metadata['term']) + else: + yara_rule_content = None + if date_from: res = Term.parse_get_tracker_term_item({'uuid': term_uuid, 'date_from': date_from, 'date_to': date_to}, user_id) if res[1] !=200: @@ -165,7 +170,9 @@ def show_tracker(): tracker_metadata['date_from'] = '' tracker_metadata['date_to'] = '' - return render_template("showTracker.html", tracker_metadata=tracker_metadata, bootstrap_label=bootstrap_label) + return render_template("showTracker.html", tracker_metadata=tracker_metadata, + yara_rule_content=yara_rule_content, + bootstrap_label=bootstrap_label) @hunter.route("/tracker/update_tracker_description", methods=['POST']) @login_required diff --git a/var/www/modules/hunter/templates/showTracker.html b/var/www/modules/hunter/templates/showTracker.html index 29e8b103..d55c4c6b 100644 --- a/var/www/modules/hunter/templates/showTracker.html +++ b/var/www/modules/hunter/templates/showTracker.html @@ -175,6 +175,11 @@ + + {%if yara_rule_content%} + 



{{ yara_rule_content }}

+ {%endif%} +