From e331ff25cb83e17577acf4d04133aa9462aadc64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thirion=20Aur=C3=A9lien?= Date: Tue, 25 Jun 2019 09:38:40 +0200 Subject: [PATCH 1/7] Update README.md --- README.md | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/README.md b/README.md index 9f9d8d4d..eea6510f 100644 --- a/README.md +++ b/README.md @@ -102,6 +102,13 @@ To upgrade from an existing AIL installation, you have to launch [python3_upgrad Docker Quick Start (Ubuntu 16.04 LTS) ------------ +:warning: +Not maintained at the moment. +If you are interested to get this running, please: + +Fork -> Branch -> PR + + 1. Install Docker ```bash sudo su From 9a883aef2300cb1502725c44baad536f85f8931b Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 25 Jun 2019 14:11:30 +0200 Subject: [PATCH 2/7] chg: [UI submit items] bootstrap 4 migration --- .../modules/PasteSubmit/Flask_PasteSubmit.py | 6 +- .../PasteSubmit/templates/PasteSubmit.html | 203 ---------- .../PasteSubmit/templates/submit_items.html | 358 +++++++++++++++++ .../PasteSubmit/templates/submiting.html | 361 ------------------ .../templates/item_submit/menu_sidebar.html | 13 + 5 files changed, 374 insertions(+), 567 deletions(-) delete mode 100644 var/www/modules/PasteSubmit/templates/PasteSubmit.html create mode 100644 var/www/modules/PasteSubmit/templates/submit_items.html delete mode 100644 var/www/modules/PasteSubmit/templates/submiting.html create mode 100644 var/www/templates/item_submit/menu_sidebar.html diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 6b7f5360..68f0952d 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -280,7 +280,7 @@ def PasteSubmit_page(): #active galaxies active_galaxies = r_serv_tags.smembers('active_galaxies') - return render_template("PasteSubmit.html", + return render_template("submiting.html", active_taxonomies = active_taxonomies, active_galaxies = active_galaxies) @@ -353,7 +353,7 @@ def submit(): launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password ,True) - return render_template("submiting.html", + return render_template("submit_items.html", active_taxonomies = active_taxonomies, active_galaxies = active_galaxies, UUID = UUID) @@ -376,7 +376,7 @@ def submit(): launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password) - return render_template("submiting.html", + return render_template("submit_items.html", active_taxonomies = active_taxonomies, active_galaxies = active_galaxies, UUID = UUID) diff --git a/var/www/modules/PasteSubmit/templates/PasteSubmit.html b/var/www/modules/PasteSubmit/templates/PasteSubmit.html deleted file mode 100644 index 44058214..00000000 --- a/var/www/modules/PasteSubmit/templates/PasteSubmit.html +++ /dev/null @@ -1,203 +0,0 @@ - - - - - - - - Submit Paste - AIL - - - - - - - - - - - - - - - - - - - - - {% include 'navbar.html' %} - -
-
- - - - -
- - - - -
-
Files submission -
-
- -
- - -
- -
- - -
-
-
- -
-
Tags : -
    -
  • - -
    - -
    - -
    - - -
    - -
    • -
  • - -
    - -
    - -
    - - -
    - -
    • -
-
-
- -
- -
- -
- -
- -
- - - - - -
- -
- - - - - - - - - - - - diff --git a/var/www/modules/PasteSubmit/templates/submit_items.html b/var/www/modules/PasteSubmit/templates/submit_items.html new file mode 100644 index 00000000..e419e599 --- /dev/null +++ b/var/www/modules/PasteSubmit/templates/submit_items.html @@ -0,0 +1,358 @@ + + + + + AIL-Framework + + + + + + + + + + + + + + + + + {% if UUID %} + + {% endif %} + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'item_submit/menu_sidebar.html' %} + +
+ +
+ + + + +
+
+ +
+
+ Files submission +
+
+ +
+ + +
+ +
+ + +
+
+
+ +
+
+ +
+
+ Tags : +
+
+ +
+ +
+ + +
+
Taxonomie Tags
+ +
+ {% for taxo in active_taxonomies %} + + {% endfor %} +
+ +
+ +
+ + +
+
Galaxy Tags
+ +
+ {% for galaxy in active_galaxies %} + + {% endfor %} +
+ +
+
+ +
+
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+
+ + + + + + + + + + + + + + + + + + + diff --git a/var/www/modules/PasteSubmit/templates/submiting.html b/var/www/modules/PasteSubmit/templates/submiting.html deleted file mode 100644 index f43a7b3d..00000000 --- a/var/www/modules/PasteSubmit/templates/submiting.html +++ /dev/null @@ -1,361 +0,0 @@ - - - - - - - - Submit Paste - AIL - - - - - - - - - - - - - - - - - - - - - - - {% include 'navbar.html' %} - -
-
- - -
- - - - -
-
Files submission -
-
- -
- - -
- -
- - -
-
-
- -
-
Tags : -
    -
  • - -
    - -
    - -
    - - -
    - -
    • -
  • - -
    - -
    - -
    - - -
    - -
    • -
-
-
- -
- -
- -
- -
- -
- - - - - -
- -
- - - - - - - - - - - - - - - - - - - - diff --git a/var/www/templates/item_submit/menu_sidebar.html b/var/www/templates/item_submit/menu_sidebar.html new file mode 100644 index 00000000..51d36093 --- /dev/null +++ b/var/www/templates/item_submit/menu_sidebar.html @@ -0,0 +1,13 @@ +
+ + + + +
From 3357b855ded464170692fd582e03e7b0359d3ce1 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 25 Jun 2019 14:18:39 +0200 Subject: [PATCH 3/7] fix: [UI items_submit] add active tag + fix template name --- .../modules/PasteSubmit/Flask_PasteSubmit.py | 2 +- .../PasteSubmit/templates/submit_items.html | 24 +++++++++---------- var/www/templates/nav_bar.html | 2 +- 3 files changed, 13 insertions(+), 15 deletions(-) diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 68f0952d..3b8ea8bd 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -280,7 +280,7 @@ def PasteSubmit_page(): #active galaxies active_galaxies = r_serv_tags.smembers('active_galaxies') - return render_template("submiting.html", + return render_template("submit_items.html", active_taxonomies = active_taxonomies, active_galaxies = active_galaxies) diff --git a/var/www/modules/PasteSubmit/templates/submit_items.html b/var/www/modules/PasteSubmit/templates/submit_items.html index e419e599..5c2081a5 100644 --- a/var/www/modules/PasteSubmit/templates/submit_items.html +++ b/var/www/modules/PasteSubmit/templates/submit_items.html @@ -169,7 +169,7 @@
- +
@@ -187,6 +187,7 @@ var ltags var ltagsgalaxies $(document).ready(function(){ + $("#page-items-submit").addClass("active"); $.getJSON("{{ url_for('Tags.get_all_tags_taxonomies') }}", function(data) { @@ -198,20 +199,17 @@ $(document).ready(function(){ }); }); - $.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}", - function(data) { + $.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}", + function(data) { - ltagsgalaxies = $('#ltagsgalaxies').tagSuggest({ - data: data, - maxDropHeight: 200, - name: 'ltagsgalaxies' - }); - }); + ltagsgalaxies = $('#ltagsgalaxies').tagSuggest({ + data: data, + maxDropHeight: 200, + name: 'ltagsgalaxies' + }); + }); - activePage = "page-PasteSubmit" - $("#"+activePage).addClass("active"); - - $('#modalsubmit').modal({backdrop: 'static'}) + $('#modalsubmit').modal({backdrop: 'static'}) }); function toggle_sidebar(){ diff --git a/var/www/templates/nav_bar.html b/var/www/templates/nav_bar.html index 462c375f..2908b984 100644 --- a/var/www/templates/nav_bar.html +++ b/var/www/templates/nav_bar.html @@ -13,7 +13,7 @@ Home
  • - Submit + Submit
  • Browse Items From d26b55973ca1acf3b60b5a0f9de7b1907fe10389 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 25 Jun 2019 14:55:03 +0200 Subject: [PATCH 4/7] fix: [MISP export] fix event creation --- var/www/modules/PasteSubmit/Flask_PasteSubmit.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 3b8ea8bd..f8f55f83 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -211,7 +211,7 @@ def misp_create_event(distribution, threat_level_id, analysis, info, l_tags, pub leak_obj.add_attribute('duplicate_number', value=p_duplicate_number, type='counter') try: - templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == obj_name][0] + templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list()['response'] if x['ObjectTemplate']['name'] == obj_name][0] except IndexError: valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()]) print ("Template for type {} not found! Valid types are: {%s}".format(obj_name, valid_types)) From 5377369032082370826935507152892830a1e368 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 26 Jun 2019 11:29:28 +0200 Subject: [PATCH 5/7] chg: [helper] dump crawler history by daterange --- bin/helper/dump_crawler_history.py | 62 ++++++++++++++++++++++++++++++ 1 file changed, 62 insertions(+) create mode 100755 bin/helper/dump_crawler_history.py diff --git a/bin/helper/dump_crawler_history.py b/bin/helper/dump_crawler_history.py new file mode 100755 index 00000000..5bd07cfe --- /dev/null +++ b/bin/helper/dump_crawler_history.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import gzip +import datetime +import redis +import json +import time + +import shutil + +sys.path.append(os.environ['AIL_BIN']) +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) +from HiddenServices import HiddenServices +from Helper import Process + +def substract_date(date_from, date_to): + date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8])) + date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8])) + delta = date_to - date_from # timedelta + l_date = [] + for i in range(delta.days + 1): + date = date_from + datetime.timedelta(i) + l_date.append( date.strftime('%Y%m%d') ) + return l_date + +config_section = 'Keys' +p = Process(config_section) + +r_serv_onion = redis.StrictRedis( + host=p.config.get("ARDB_Onion", "host"), + port=p.config.getint("ARDB_Onion", "port"), + db=p.config.getint("ARDB_Onion", "db"), + decode_responses=True) + +date_from = '20190614' +date_to = '20190615' +service_type = 'onion' +date_range = substract_date(date_from, date_to) + +dir_path = os.path.join(os.environ['AIL_HOME'], 'temp') + +for date in date_range: + domains_up = list(r_serv_onion.smembers('{}_up:{}'.format(service_type, date))) + if domains_up: + save_path = os.path.join(dir_path, date[0:4], date[4:6], date[6:8]) + try: + os.makedirs(save_path) + except FileExistsError: + pass + for domain in domains_up: + print(domain) + h = HiddenServices(domain, 'onion') + item_core = h.get_domain_crawled_core_item() + l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item']) + res = h.create_domain_basic_archive(l_pastes) + filename = os.path.join(save_path, '{}'.format(domain)) + with open(filename, 'wb') as f: + shutil.copyfileobj(res, f) + print('done') From 06ab66ff572b51cb4ac6eafcbef6b357cffa82fe Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 26 Jun 2019 11:42:19 +0200 Subject: [PATCH 6/7] fix: [helper dump_crawler] fix empty dict --- bin/helper/dump_crawler_history.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/bin/helper/dump_crawler_history.py b/bin/helper/dump_crawler_history.py index 5bd07cfe..ea3ef54e 100755 --- a/bin/helper/dump_crawler_history.py +++ b/bin/helper/dump_crawler_history.py @@ -54,9 +54,10 @@ for date in date_range: print(domain) h = HiddenServices(domain, 'onion') item_core = h.get_domain_crawled_core_item() - l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item']) - res = h.create_domain_basic_archive(l_pastes) - filename = os.path.join(save_path, '{}'.format(domain)) - with open(filename, 'wb') as f: - shutil.copyfileobj(res, f) - print('done') + if 'root_item' in item_core: + l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item']) + res = h.create_domain_basic_archive(l_pastes) + filename = os.path.join(save_path, '{}'.format(domain)) + with open(filename, 'wb') as f: + shutil.copyfileobj(res, f) + print('done') From bd0da210e7d8b501545004eef693e5966a7c707b Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 26 Jun 2019 11:51:26 +0200 Subject: [PATCH 7/7] fix: [helper dump_crawler] fix files not found --- bin/helper/dump_crawler_history.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/bin/helper/dump_crawler_history.py b/bin/helper/dump_crawler_history.py index ea3ef54e..1ecbf0e3 100755 --- a/bin/helper/dump_crawler_history.py +++ b/bin/helper/dump_crawler_history.py @@ -42,6 +42,8 @@ date_range = substract_date(date_from, date_to) dir_path = os.path.join(os.environ['AIL_HOME'], 'temp') +domain_skipped = [] + for date in date_range: domains_up = list(r_serv_onion.smembers('{}_up:{}'.format(service_type, date))) if domains_up: @@ -56,8 +58,19 @@ for date in date_range: item_core = h.get_domain_crawled_core_item() if 'root_item' in item_core: l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item']) - res = h.create_domain_basic_archive(l_pastes) - filename = os.path.join(save_path, '{}'.format(domain)) - with open(filename, 'wb') as f: - shutil.copyfileobj(res, f) - print('done') + try: + res = h.create_domain_basic_archive(l_pastes) + filename = os.path.join(save_path, '{}'.format(domain)) + with open(filename, 'wb') as f: + shutil.copyfileobj(res, f) + print('done') + except Exception as e: + print('skipped') + domain_skipped.append(domain) + pass + +print() +print() +print('DOMAINS SKIPPED: ') +for domain in domain_skipped: + print(domain)