diff --git a/bin/RegexTracker.py b/bin/RegexTracker.py index 2de211b9..904be623 100755 --- a/bin/RegexTracker.py +++ b/bin/RegexTracker.py @@ -23,7 +23,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) import Tracker import regex_helper -full_item_url = "/showsavedpaste/?paste=" +full_item_url = "/object/item?id=" mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}" dict_regex_tracked = Term.get_regex_tracked_words_dict() diff --git a/bin/TermTrackerMod.py b/bin/TermTrackerMod.py index 2b4241c0..ee64ab67 100755 --- a/bin/TermTrackerMod.py +++ b/bin/TermTrackerMod.py @@ -20,7 +20,7 @@ from packages import Term from lib import Tracker -full_item_url = "/showsavedpaste/?paste=" +full_item_url = "/object/item/?id=" mail_body_template = "AIL Framework,\nNew occurrence for term tracked term: {}\nitem id: {}\nurl: {}{}" diff --git a/bin/export/Export.py b/bin/export/Export.py index 67d631f2..90de4570 100755 --- a/bin/export/Export.py +++ b/bin/export/Export.py @@ -9,6 +9,24 @@ from uuid import uuid4 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) import ConfigLoader +sys.path.append('../../configs/keys') +try: + from thehive4py.api import TheHiveApi + import thehive4py.exceptions + from theHiveKEYS import the_hive_url, the_hive_key, the_hive_verifycert + if the_hive_url == '': + is_hive_connected = False + else: + is_hive_connected = TheHiveApi(the_hive_url, the_hive_key, cert=the_hive_verifycert) +except: + is_hive_connected = False +if is_hive_connected != False: + try: + is_hive_connected.get_alert(0) + is_hive_connected = True + except thehive4py.exceptions.AlertException: + is_hive_connected = False + ## LOAD CONFIG ## config_loader = ConfigLoader.ConfigLoader() r_serv_cache = config_loader.get_redis_conn("Redis_Cache") @@ -37,6 +55,16 @@ def load_tags_to_export_in_cache(): # save combinaison of tags in cache pass +def is_hive_connected(): # # TODO: REFRACTOR, put in cache (with retry) + return is_hive_connected + +def get_item_hive_cases(item_id): + hive_case = r_serv_metadata.get('hive_cases:{}'.format(item_id)) + if hive_case: + hive_case = the_hive_url + '/index.html#/case/{}/details'.format(hive_case) + return hive_case + + ########################################################### # # set default # if r_serv_db.get('hive:auto-alerts') is None: diff --git a/bin/lib/Correlate_object.py b/bin/lib/Correlate_object.py index c0d959f5..1138949a 100755 --- a/bin/lib/Correlate_object.py +++ b/bin/lib/Correlate_object.py @@ -223,8 +223,8 @@ def get_item_url(correlation_name, value, correlation_type=None): endpoint = 'crawler_splash.showDomain' url = url_for(endpoint, domain=value) elif correlation_name == 'item': - endpoint = 'showsavedpastes.showsavedpaste' - url = url_for(endpoint, paste=value) + endpoint = 'objects_item.showItem' + url = url_for(endpoint, id=value) elif correlation_name == 'paste': ### # TODO: remove me endpoint = 'showsavedpastes.showsavedpaste' url = url_for(endpoint, paste=value) diff --git a/bin/lib/item_basic.py b/bin/lib/item_basic.py index c1005b49..9ad6161d 100755 --- a/bin/lib/item_basic.py +++ b/bin/lib/item_basic.py @@ -113,9 +113,6 @@ def get_item_parent(item_id): def get_item_children(item_id): return list(r_serv_metadata.smembers('paste_children:{}'.format(item_id))) -def add_item_parent(item_parent, item_id): - return item_basic.add_item_parent(item_parent, item_id) - # # TODO: handle domain last origin in domain lib def _delete_node(item_id): # only if item isn't deleted diff --git a/bin/packages/Item.py b/bin/packages/Item.py index d898d1e0..15993d7a 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -4,6 +4,7 @@ import os import sys import redis +import html2text from io import BytesIO @@ -59,6 +60,9 @@ def get_item_basename(item_id): def get_item_size(item_id): return round(os.path.getsize(os.path.join(PASTES_FOLDER, item_id))/1024.0, 2) +def get_item_encoding(item_id): + return None + def get_lines_info(item_id, item_content=None): if not item_content: item_content = get_item_content(item_id) @@ -73,9 +77,37 @@ def get_lines_info(item_id, item_content=None): return {'nb': nb_line, 'max_length': max_length} +def get_item_metadata(item_id, item_content=None): + ## TODO: FIXME ##performance + # encoding + # language + # lines info + + item_metadata = {} + item_metadata['date'] = get_item_date(item_id, add_separator=True) + item_metadata['source'] = get_source(item_id) + item_metadata['size'] = get_item_size(item_id) + item_metadata['encoding'] = get_item_encoding(item_id) + item_metadata['lines'] = get_lines_info(item_id, item_content=item_content) + + return item_metadata + +def get_item_parent(item_id): + return item_basic.get_item_parent(item_id) + +def add_item_parent(item_parent, item_id): + return item_basic.add_item_parent(item_parent, item_id) + def get_item_content(item_id): return item_basic.get_item_content(item_id) +def get_item_content_html2text(item_id, item_content=None): + if not item_content: + item_content = get_item_content(item_id) + h = html2text.HTML2Text() + h.ignore_links = False + return h.handle(item_content) + # API def get_item(request_dict): if not request_dict: @@ -257,6 +289,18 @@ def get_item_list_desc(list_item_id): def is_crawled(item_id): return item_basic.is_crawled(item_id) +def get_crawler_matadata(item_id, ltags=None): + dict_crawler = {} + if is_crawled(item_id): + dict_crawler['domain'] = get_item_domain(item_id) + if not ltags: + ltags = Tag.get_obj_tag(item_id) + dict_crawler['is_tags_safe'] = Tag.is_tags_safe(ltags) + dict_crawler['url'] = get_item_link(item_id) + dict_crawler['screenshot'] = get_item_screenshot(item_id) + dict_crawler['har'] = get_item_har_name(item_id) + return dict_crawler + def is_onion(item_id): is_onion = False if len(is_onion) > 62: @@ -293,7 +337,7 @@ def get_item_screenshot(item_id): return '' def get_item_har_name(item_id): - os.path.join(screenshot_directory, item_id) + '.json' + har_path = os.path.join(screenshot_directory, item_id) + '.json' if os.path.isfile(har_path): return har_path else: @@ -322,6 +366,24 @@ def get_item_duplicate(item_id, r_list=True): return [] return res +def get_item_nb_duplicates(item_id): + return r_serv_metadata.scard('dup:{}'.format(item_id)) + +def get_item_duplicates_dict(item_id): + dict_duplicates = {} + for duplicate in get_item_duplicate(item_id): + duplicate = duplicate[1:-1].replace('\'', '').replace(' ', '').split(',') + duplicate_id = duplicate[1] + if not duplicate_id in dict_duplicates: + dict_duplicates[duplicate_id] = {'date': get_item_date(duplicate_id, add_separator=True), 'algo': {}} + algo = duplicate[0] + if algo == 'tlsh': + similarity = 100 - int(duplicate[2]) + else: + similarity = int(duplicate[2]) + dict_duplicates[duplicate_id]['algo'][algo] = similarity + return dict_duplicates + def add_item_duplicate(item_id, l_dup): for item_dup in l_dup: r_serv_metadata.sadd('dup:{}'.format(item_dup), item_id) diff --git a/bin/trackers/Tracker_Yara.py b/bin/trackers/Tracker_Yara.py index a141f352..55e5a00b 100755 --- a/bin/trackers/Tracker_Yara.py +++ b/bin/trackers/Tracker_Yara.py @@ -24,7 +24,7 @@ import Tracker import item_basic -full_item_url = "/showsavedpaste/?paste=" +full_item_url = "/object/item?id=" mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}" last_refresh = time.time() diff --git a/requirements.txt b/requirements.txt index 33542f8b..73d06144 100644 --- a/requirements.txt +++ b/requirements.txt @@ -22,6 +22,7 @@ textblob #Tokeniser nltk +html2text yara-python #Crawler diff --git a/update/v3.3/Update.py b/update/v3.3/Update.py new file mode 100755 index 00000000..39d1371b --- /dev/null +++ b/update/v3.3/Update.py @@ -0,0 +1,29 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import time +import redis +import argparse +import datetime +import configparser + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + +new_version = 'v3.3' + +if __name__ == '__main__': + + start_deb = time.time() + + config_loader = ConfigLoader.ConfigLoader() + r_serv_db = config_loader.get_redis_conn("ARDB_DB") + config_loader = None + + #Set current ail version + r_serv_db.set('ail:version', new_version) + + #Set current ail version + r_serv_db.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d")) diff --git a/update/v3.3/Update.sh b/update/v3.3/Update.sh new file mode 100755 index 00000000..86289dba --- /dev/null +++ b/update/v3.3/Update.sh @@ -0,0 +1,54 @@ +#!/bin/bash + +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; +[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; + +export PATH=$AIL_HOME:$PATH +export PATH=$AIL_REDIS:$PATH +export PATH=$AIL_ARDB:$PATH +export PATH=$AIL_BIN:$PATH +export PATH=$AIL_FLASK:$PATH + +GREEN="\\033[1;32m" +DEFAULT="\\033[0;39m" + +echo -e $GREEN"Shutting down AIL ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -ks +wait + +bash ${AIL_BIN}/LAUNCH.sh -ldbv & +wait +echo "" + +# SUBMODULES # +git submodule update + +# echo "" +# echo -e $GREEN"installing KVORCKS ..."$DEFAULT +# cd ${AIL_HOME} +# test ! -d kvrocks/ && git clone https://github.com/bitleak/kvrocks.git +# pushd kvrocks/ +# make -j4 +# popd + +echo -e $GREEN"Installing html2text ..."$DEFAULT +pip3 install html2text + +echo "" +echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT +echo "" +python ${AIL_HOME}/update/v3.3/Update.py +wait +echo "" +echo "" + + +echo "" +echo -e $GREEN"Shutting down ARDB ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -ks +wait + +exit 0 diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 5f1f22d4..bce50bb3 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -42,6 +42,8 @@ from blueprints.crawler_splash import crawler_splash from blueprints.correlation import correlation from blueprints.tags_ui import tags_ui from blueprints.import_export import import_export +from blueprints.objects_item import objects_item +from blueprints.old_endpoints import old_endpoints Flask_dir = os.environ['AIL_FLASK'] @@ -97,6 +99,8 @@ app.register_blueprint(crawler_splash, url_prefix=baseUrl) app.register_blueprint(correlation, url_prefix=baseUrl) app.register_blueprint(tags_ui, url_prefix=baseUrl) app.register_blueprint(import_export, url_prefix=baseUrl) +app.register_blueprint(objects_item, url_prefix=baseUrl) +app.register_blueprint(old_endpoints, url_prefix=baseUrl) # ========= =========# # ========= Cookie name ======== diff --git a/var/www/blueprints/objects_item.py b/var/www/blueprints/objects_item.py new file mode 100644 index 00000000..2b951353 --- /dev/null +++ b/var/www/blueprints/objects_item.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... +''' + +import os +import sys +import json + +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file +from flask_login import login_required, current_user + +# Import Role_Manager +from Role_Manager import login_admin, login_analyst, login_read_only + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) +import Item +import Tag + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'export')) +import Export + +# ============ BLUEPRINT ============ +objects_item = Blueprint('objects_item', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/item')) + +# ============ VARIABLES ============ +bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] + + +# ============ FUNCTIONS ============ + + +# ============= ROUTES ============== +@objects_item.route("/object/item") #completely shows the paste in a new tab +@login_required +@login_read_only +def showItem(): # # TODO: support post + item_id = request.args.get('id') + if not item_id or not Item.exist_item(item_id): + abort(404) + + dict_item = {} + dict_item['id'] = item_id + dict_item['name'] = dict_item['id'].replace('/', ' / ') + dict_item['father'] = Item.get_item_parent(item_id) + dict_item['content'] = Item.get_item_content(item_id) + dict_item['metadata'] = Item.get_item_metadata(item_id, item_content=dict_item['content']) + dict_item['tags'] = Tag.get_obj_tag(item_id) + #dict_item['duplicates'] = Item.get_item_nb_duplicates(item_id) + dict_item['duplicates'] = Item.get_item_duplicates_dict(item_id) + dict_item['crawler'] = Item.get_crawler_matadata(item_id, ltags=dict_item['tags']) + + ## EXPORT SECTION + # # TODO: ADD in Export SECTION + dict_item['hive_case'] = Export.get_item_hive_cases(item_id) + + return render_template("show_item.html", bootstrap_label=bootstrap_label, + modal_add_tags=Tag.get_modal_add_tags(dict_item['id'], object_type='item'), + is_hive_connected=Export.get_item_hive_cases(item_id), + dict_item=dict_item) + + # kvrocks data + + # # TODO: dynamic load: + ## duplicates + ## correlations + + ## Dynamic Path FIX + +@objects_item.route("/object/item/html2text") +@login_required +@login_read_only +def html2text(): # # TODO: support post + item_id = request.args.get('id') + if not item_id or not Item.exist_item(item_id): + abort(404) + return Item.get_item_content_html2text(item_id) + +@objects_item.route("/object/item/raw_content") +@login_required +@login_read_only +def item_raw_content(): # # TODO: support post + item_id = request.args.get('id') + if not item_id or not Item.exist_item(item_id): + abort(404) + return Response(Item.get_item_content(item_id), mimetype='text/plain') + +@objects_item.route("/object/item/download") +@login_required +@login_read_only +def item_download(): # # TODO: support post + item_id = request.args.get('id') + if not item_id or not Item.exist_item(item_id): + abort(404) + return send_file(Item.get_raw_content(item_id), attachment_filename=item_id, as_attachment=True) diff --git a/var/www/blueprints/old_endpoints.py b/var/www/blueprints/old_endpoints.py new file mode 100644 index 00000000..09f6bfaa --- /dev/null +++ b/var/www/blueprints/old_endpoints.py @@ -0,0 +1,34 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... +''' + +import os +import sys +import json + +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response +from flask_login import login_required, current_user + +# Import Role_Manager +from Role_Manager import login_admin, login_analyst, login_read_only + +# ============ BLUEPRINT ============ +old_endpoints = Blueprint('old_endpoints', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates')) + +# ============ VARIABLES ============ + + + +# ============ FUNCTIONS ============ + + +# ============= ROUTES ============== +@old_endpoints.route("/showsavedpaste/") +@login_required +@login_read_only +def showsavedpaste(): + item_id = request.args.get('paste', '') + return redirect(url_for('objects_item.showItem', id=item_id)) diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 134753e7..0a10c251 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -358,7 +358,7 @@ def submit_status(): link = '' if paste_submit_link: for paste in paste_submit_link: - url = url_for('showsavedpastes.showsavedpaste') + '?paste=' + paste + url = url_for('objects_item.showItem') + '?id=' + paste link += '' + paste +'' if nb_total == '-1': diff --git a/var/www/modules/Tags/Flask_Tags.py b/var/www/modules/Tags/Flask_Tags.py index fa8873ba..010610e6 100644 --- a/var/www/modules/Tags/Flask_Tags.py +++ b/var/www/modules/Tags/Flask_Tags.py @@ -377,7 +377,7 @@ def remove_tag(): #TODO remove me , used by showpaste res = Tag.api_delete_obj_tags(tags=[tag], object_id=path, object_type="item") if res[1] != 200: return str(res[0]) - return redirect(url_for('showsavedpastes.showsavedpaste', paste=path)) + return redirect(url_for('objects_item.showItem', id=path)) @Tags.route("/Tags/confirm_tag") @login_required @@ -395,7 +395,7 @@ def confirm_tag(): #add analyst tag Tag.add_tag('item', tag, path) - return redirect(url_for('showsavedpastes.showsavedpaste', paste=path)) + return redirect(url_for('objects_item.showItem', id=path)) return 'incompatible tag' @@ -417,7 +417,7 @@ def tag_validation(): r_serv_statistics.sadd('fp:'+tag, path) r_serv_statistics.srem('tp:'+tag, path) - return redirect(url_for('showsavedpastes.showsavedpaste', paste=path)) + return redirect(url_for('objects_item.showItem', id=path)) else: return 'input error' diff --git a/var/www/modules/Tags/templates/Tags.html b/var/www/modules/Tags/templates/Tags.html index b4fb85c8..83ebfb3c 100644 --- a/var/www/modules/Tags/templates/Tags.html +++ b/var/www/modules/Tags/templates/Tags.html @@ -119,7 +119,7 @@ {% for path in all_path %} {{ paste_date[loop.index0] }} - +
{{ path }}
@@ -375,7 +375,7 @@ function toggle_sidebar(){ button.tooltip(button); $("#container-show-more").append(button); - $("#button_show_path").attr('href', '{{ url_for('showsavedpastes.showsavedpaste') }}?paste=' + $(modal).attr('data-path')); + $("#button_show_path").attr('href', '{{ url_for('objects_item.showItem') }}?id=' + $(modal).attr('data-path')); $("#button_show_path").show('fast'); $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed diff --git a/var/www/modules/Tags/templates/tagged.html b/var/www/modules/Tags/templates/tagged.html index 75be817f..4177e6cc 100644 --- a/var/www/modules/Tags/templates/tagged.html +++ b/var/www/modules/Tags/templates/tagged.html @@ -103,7 +103,7 @@ {% for path in all_path %} {{ loop.index0 }} - {{ path }} + {{ path }}
{% for tag in paste_tags[loop.index0] %} @@ -114,7 +114,7 @@ {{ paste_date[loop.index0] }} {{ paste_linenum[loop.index0] }} -

+

{% endfor %} diff --git a/var/www/modules/dashboard/Flask_dashboard.py b/var/www/modules/dashboard/Flask_dashboard.py index 0091df5a..d57c4e67 100644 --- a/var/www/modules/dashboard/Flask_dashboard.py +++ b/var/www/modules/dashboard/Flask_dashboard.py @@ -76,7 +76,7 @@ def dashboard_alert(log): log = log[46:].split(';') if len(log) == 6: time = datetime_from_utc_to_local(utc_str) - path = url_for('showsavedpastes.showsavedpaste',paste=log[5]) + path = url_for('objects_item.showItem',id=log[5]) res = {'date': date, 'time': time, 'script': log[0], 'domain': log[1], 'date_paste': log[2], 'paste': log[3], 'message': log[4], 'path': path} diff --git a/var/www/modules/dashboard/templates/index.html b/var/www/modules/dashboard/templates/index.html index 812e1ea0..0cd858d0 100644 --- a/var/www/modules/dashboard/templates/index.html +++ b/var/www/modules/dashboard/templates/index.html @@ -165,7 +165,7 @@ - + diff --git a/var/www/templates/modals/show_min_item.html b/var/www/templates/modals/show_min_item.html index 69ff1239..8613d9c6 100644 --- a/var/www/templates/modals/show_min_item.html +++ b/var/www/templates/modals/show_min_item.html @@ -86,7 +86,7 @@ function get_html_and_update_modal(event, truemodal) { button.tooltip(button); $("#container-show-more").append(button); - $("#modal_show_min_item_button_show_item").attr('href', '{{ url_for('showsavedpastes.showsavedpaste') }}?paste=' + $(modal).attr('data-path')); + $("#modal_show_min_item_button_show_item").attr('href', '{{ url_for('objects_item.showItem') }}?id=' + $(modal).attr('data-path')); $("#modal_show_min_item_button_show_item").show('fast'); $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed diff --git a/var/www/templates/objects/item/show_item.html b/var/www/templates/objects/item/show_item.html new file mode 100644 index 00000000..084b3477 --- /dev/null +++ b/var/www/templates/objects/item/show_item.html @@ -0,0 +1,446 @@ + + + + Show Item Object - AIL + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+

{{ dict_item['name'] }}

+
+
+ + + + + + + + + + + + + + + + + + + + + +
DateSourceEncodingSize (Kb)Number of linesMax line length
{{ dict_item['metadata']['date'] }}{{ dict_item['metadata']['source'] }}{{ dict_item['metadata']['encoding'] }}{{ dict_item['metadata']['size'] }}{{ dict_item['metadata']['lines']['nb'] }}{{ dict_item['metadata']['lines']['max_length'] }}
+ +
+
+
+ {% include 'modals/edit_tag.html' %} + {% for tag in dict_item['tags'] %} + + + {% endfor %} +
+ {% include 'modals/add_tags.html' %} + +
+
+
+ + {% if dict_item['father'] %} +
+ Father: {{dict_item['father']}} +
+ {% endif %} + +
+ +
+ {% with obj_type='item', obj_id=dict_item['id'], obj_lvl=0%} + {% include 'import_export/block_add_user_object_to_export.html' %} + {% endwith %} +
+ {% if is_hive_connected %} +
+ {% include 'modals/create_hive_case.html' %} + +
+ {% endif %} +
+ +
+
+ + {% if misp_eventid %} +
+
  • MISP Events already Created
    • + {{ misp_url }} +
    + {% endif %} + + {% if dict_item['hive_case'] %} +
    +
  • The Hive Case already Created
    • + {{ hive_url }} +
    + {% endif %} + + {% if dict_item['duplicates'] != 0 %} +
    +
    +
    +
    +
    +
    + duplicates   +
    {{dict_item['duplicates']|length}}
    +
    +
    +
    + +
    +
    +
    + +
    +
    + + + + + + + + + + + + {% for duplicate_id in dict_item['duplicates'] %} + + + + + + + {% endfor %} + +
    DateSimilarityItemDiff
    {{dict_item['duplicates'][duplicate_id]['date']}} + + + {%for algo in dict_item['duplicates'][duplicate_id]['algo']|sort()%} + + + + + {%endfor%} + +
    {{algo}} +
    +
    + {{dict_item['duplicates'][duplicate_id]['algo'][algo]}}% +
    +
    +
    +     		+ + {{duplicate_id}} + + + +
    + +
    +
    + +
    +
    + {% endif %} + + + {% if l_64|length != 0 %} +
    +
    +
    +
    +
    +
    + Decoded Files   +
    {{l_64|length}}
    +
    +
    +
    + +
    +
    +
    + +
    +
    + + + + + + + + + + {% for b64 in l_64 %} + + + + + {% endfor %} + +
    estimated typehash
      {{ b64[1] }} {{b64[2]}} ({{ b64[4] }})
    + +
    +
    +
    +
    + {% endif %} + + + {% if dict_item['crawler'] %} +
    +
    +
    +
    +
    +
    + Crawler +
    +
    +
    + +
    +
    +
    + +
    + +
    + +
    +
    + + + + + + + + + + + + + + + + + + +
    Last Origin:
    + {{ dict_item['father'] }} +
    + {{ dict_item['crawler']['domain'] }} +
    url + {{ dict_item['crawler']['url'] }} +
    +
    +
    + +
    +
    +
    +
    +
    + +
    +
    + +
    +
    +
    +
    + +
    +
    + +
    +
    +
    + {% endif %} + + + + +
    +
    + + + + +
    +
    + 

    {{ dict_item['content'] }}

    +
    +
    + 

    
+        
    
+      
    
+    
    
+  
    
+
+  
+
+{% if dict_item['crawler'] %}
+  
+{% endif %}
+
+
+
+
+
diff --git a/var/www/templates/tags/search_obj_by_tags.html b/var/www/templates/tags/search_obj_by_tags.html
index 3e9400ba..93ac9579 100644
--- a/var/www/templates/tags/search_obj_by_tags.html
+++ b/var/www/templates/tags/search_obj_by_tags.html
@@ -129,7 +129,7 @@
                     
                       {{ dict_obj['date'] }}
                       
-                        
+                        
                           
    {{ dict_obj['id'] }}