diff --git a/README.md b/README.md index e894b163..65ed67f6 100644 --- a/README.md +++ b/README.md @@ -101,6 +101,13 @@ To upgrade from an existing AIL installation, you have to launch [python3_upgrad Docker Quick Start (Ubuntu 16.04 LTS) ------------ +:warning: +Not maintained at the moment. +If you are interested to get this running, please: + +Fork -> Branch -> PR + + 1. Install Docker ```bash sudo su diff --git a/bin/helper/dump_crawler_history.py b/bin/helper/dump_crawler_history.py new file mode 100755 index 00000000..1ecbf0e3 --- /dev/null +++ b/bin/helper/dump_crawler_history.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import gzip +import datetime +import redis +import json +import time + +import shutil + +sys.path.append(os.environ['AIL_BIN']) +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) +from HiddenServices import HiddenServices +from Helper import Process + +def substract_date(date_from, date_to): + date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8])) + date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8])) + delta = date_to - date_from # timedelta + l_date = [] + for i in range(delta.days + 1): + date = date_from + datetime.timedelta(i) + l_date.append( date.strftime('%Y%m%d') ) + return l_date + +config_section = 'Keys' +p = Process(config_section) + +r_serv_onion = redis.StrictRedis( + host=p.config.get("ARDB_Onion", "host"), + port=p.config.getint("ARDB_Onion", "port"), + db=p.config.getint("ARDB_Onion", "db"), + decode_responses=True) + +date_from = '20190614' +date_to = '20190615' +service_type = 'onion' +date_range = substract_date(date_from, date_to) + +dir_path = os.path.join(os.environ['AIL_HOME'], 'temp') + +domain_skipped = [] + +for date in date_range: + domains_up = list(r_serv_onion.smembers('{}_up:{}'.format(service_type, date))) + if domains_up: + save_path = os.path.join(dir_path, date[0:4], date[4:6], date[6:8]) + try: + os.makedirs(save_path) + except FileExistsError: + pass + for domain in domains_up: + print(domain) + h = HiddenServices(domain, 'onion') + item_core = h.get_domain_crawled_core_item() + if 'root_item' in item_core: + l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item']) + try: + res = h.create_domain_basic_archive(l_pastes) + filename = os.path.join(save_path, '{}'.format(domain)) + with open(filename, 'wb') as f: + shutil.copyfileobj(res, f) + print('done') + except Exception as e: + print('skipped') + domain_skipped.append(domain) + pass + +print() +print() +print('DOMAINS SKIPPED: ') +for domain in domain_skipped: + print(domain) diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 85ca57e2..efd0650e 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -214,7 +214,7 @@ def misp_create_event(distribution, threat_level_id, analysis, info, l_tags, pub leak_obj.add_attribute('duplicate_number', value=p_duplicate_number, type='counter') try: - templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == obj_name][0] + templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list()['response'] if x['ObjectTemplate']['name'] == obj_name][0] except IndexError: valid_types = ", ".join([x['ObjectTemplate']['name'] for x in pymisp.get_object_templates_list()]) print ("Template for type {} not found! Valid types are: {%s}".format(obj_name, valid_types)) @@ -285,7 +285,7 @@ def PasteSubmit_page(): #active galaxies active_galaxies = r_serv_tags.smembers('active_galaxies') - return render_template("PasteSubmit.html", + return render_template("submit_items.html", active_taxonomies = active_taxonomies, active_galaxies = active_galaxies) @@ -360,7 +360,7 @@ def submit(): launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password ,True) - return render_template("submiting.html", + return render_template("submit_items.html", active_taxonomies = active_taxonomies, active_galaxies = active_galaxies, UUID = UUID) @@ -383,7 +383,7 @@ def submit(): launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password) - return render_template("submiting.html", + return render_template("submit_items.html", active_taxonomies = active_taxonomies, active_galaxies = active_galaxies, UUID = UUID) diff --git a/var/www/modules/PasteSubmit/templates/PasteSubmit.html b/var/www/modules/PasteSubmit/templates/PasteSubmit.html deleted file mode 100644 index 44058214..00000000 --- a/var/www/modules/PasteSubmit/templates/PasteSubmit.html +++ /dev/null @@ -1,203 +0,0 @@ - - - - - - - - Submit Paste - AIL - - - - - - - - - - - - - - - - - - - - - {% include 'navbar.html' %} - -
-
- - - - -
- - - - -
-
Files submission -
-
- -
- - -
- -
- - -
-
-
- -
-
Tags : -
    -
  • - -
    - -
    - -
    - - -
    - -
    • -
  • - -
    - -
    - -
    - - -
    - -
    • -
-
-
- -
- -
- -
- -
- -
- - - - - -
- -
- - - - - - - - - - - - diff --git a/var/www/modules/PasteSubmit/templates/submit_items.html b/var/www/modules/PasteSubmit/templates/submit_items.html new file mode 100644 index 00000000..5c2081a5 --- /dev/null +++ b/var/www/modules/PasteSubmit/templates/submit_items.html @@ -0,0 +1,356 @@ + + + + + AIL-Framework + + + + + + + + + + + + + + + + + {% if UUID %} + + {% endif %} + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'item_submit/menu_sidebar.html' %} + +
+ +
+ + + + +
+
+ +
+
+ Files submission +
+
+ +
+ + +
+ +
+ + +
+
+
+ +
+
+ +
+
+ Tags : +
+
+ +
+ +
+ + +
+
Taxonomie Tags
+ +
+ {% for taxo in active_taxonomies %} + + {% endfor %} +
+ +
+ +
+ + +
+
Galaxy Tags
+ +
+ {% for galaxy in active_galaxies %} + + {% endfor %} +
+ +
+
+ +
+
+ +
+ +
+ +
+ +
+ +
+ +
+ +
+
+ + + + + + + + + + + + + + + + + + + diff --git a/var/www/modules/PasteSubmit/templates/submiting.html b/var/www/modules/PasteSubmit/templates/submiting.html deleted file mode 100644 index f43a7b3d..00000000 --- a/var/www/modules/PasteSubmit/templates/submiting.html +++ /dev/null @@ -1,361 +0,0 @@ - - - - - - - - Submit Paste - AIL - - - - - - - - - - - - - - - - - - - - - - - {% include 'navbar.html' %} - -
-
- - -
- - - - -
-
Files submission -
-
- -
- - -
- -
- - -
-
-
- -
-
Tags : -
    -
  • - -
    - -
    - -
    - - -
    - -
    • -
  • - -
    - -
    - -
    - - -
    - -
    • -
-
-
- -
- -
- -
- -
- -
- - - - - -
- -
- - - - - - - - - - - - - - - - - - - - diff --git a/var/www/templates/item_submit/menu_sidebar.html b/var/www/templates/item_submit/menu_sidebar.html new file mode 100644 index 00000000..51d36093 --- /dev/null +++ b/var/www/templates/item_submit/menu_sidebar.html @@ -0,0 +1,13 @@ +
+ + + + +
diff --git a/var/www/templates/nav_bar.html b/var/www/templates/nav_bar.html index c1cb18ba..7a813b42 100644 --- a/var/www/templates/nav_bar.html +++ b/var/www/templates/nav_bar.html @@ -13,7 +13,7 @@ Home
  • - Submit + Submit
  • Browse Items