From 65e6eae00d6c5afed719461d71c6e504a53a782c Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 19 Mar 2019 13:34:41 +0100 Subject: [PATCH] chg: [UI preview items] b4 migration + new template + better display + decrease item loading time + add basic animation/transition --- bin/packages/Paste.py | 10 + var/www/modules/Tags/templates/Tags.html | 172 +++++----- var/www/modules/showpaste/Flask_showpaste.py | 129 +++++++- .../templates/show_saved_item_min.html | 300 ++++++++++++++++++ 4 files changed, 532 insertions(+), 79 deletions(-) create mode 100644 var/www/modules/showpaste/templates/show_saved_item_min.html diff --git a/bin/packages/Paste.py b/bin/packages/Paste.py index 48bd710b..0ab0286c 100755 --- a/bin/packages/Paste.py +++ b/bin/packages/Paste.py @@ -245,6 +245,12 @@ class Paste(object): def get_p_date(self): return self.p_date + def get_item_source(self): + return self.p_source + + def get_item_size(self): + return self.p_size + def _get_p_size(self): return self.p_size @@ -298,6 +304,10 @@ class Paste(object): else: return '[]' + def get_nb_duplicate(self): + # # TODO: FIXME use relative path + return self.store_metadata.scard('dup:'+self.p_path) + self.store_metadata.scard('dup:'+self.p_rel_path) + def _get_p_tags(self): self.p_tags = self.store_metadata.smembers('tag:'+path, tag) if self.self.p_tags is not None: diff --git a/var/www/modules/Tags/templates/Tags.html b/var/www/modules/Tags/templates/Tags.html index 2efc3439..ea6d170b 100644 --- a/var/www/modules/Tags/templates/Tags.html +++ b/var/www/modules/Tags/templates/Tags.html @@ -22,6 +22,20 @@ + + @@ -119,7 +133,7 @@ {{ paste_linenum[loop.index0] }}

-

@@ -256,94 +270,96 @@ function toggle_sidebar(){ diff --git a/var/www/modules/showpaste/Flask_showpaste.py b/var/www/modules/showpaste/Flask_showpaste.py index 188af759..ade62f08 100644 --- a/var/www/modules/showpaste/Flask_showpaste.py +++ b/var/www/modules/showpaste/Flask_showpaste.py @@ -223,6 +223,128 @@ def showpaste(content_range, requested_path): crawler_metadata=crawler_metadata, l_64=l_64, vt_enabled=vt_enabled, misp=misp, hive=hive, misp_eventid=misp_eventid, misp_url=misp_url, hive_caseid=hive_caseid, hive_url=hive_url) +def get_item_basic_info(item): + item_basic_info = {} + item_basic_info['date'] = str(item.get_p_date()) + item_basic_info['date'] = '{}/{}/{}'.format(item_basic_info['date'][0:4], item_basic_info['date'][4:6], item_basic_info['date'][6:8]) + item_basic_info['source'] = item.get_item_source() + item_basic_info['size'] = item.get_item_size() + + ## TODO: FIXME ##performance + item_basic_info['encoding'] = item._get_p_encoding() + ## TODO: FIXME ##performance + #item_basic_info['language'] = item._get_p_language() + ## TODO: FIXME ##performance + info_line = item.get_lines_info() + item_basic_info['nb_lines'] = info_line[0] + item_basic_info['max_length_line'] = info_line[1] + + return item_basic_info + +def show_item_min(requested_path , content_range=0): + relative_path = None + if PASTES_FOLDER not in requested_path: + relative_path = requested_path + requested_path = os.path.join(PASTES_FOLDER, requested_path) + else: + relative_path = requested_path.replace(PASTES_FOLDER, '', 1)[1:] + # remove old full path + #requested_path = requested_path.replace(PASTES_FOLDER, '') + # escape directory transversal + if os.path.commonprefix((os.path.realpath(requested_path),PASTES_FOLDER)) != PASTES_FOLDER: + return 'path transversal detected' + + item_info ={} + + paste = Paste.Paste(requested_path) + item_basic_info = get_item_basic_info(paste) + item_info['nb_duplictates'] = paste.get_nb_duplicate() + + ## TODO: use this for fix ? + item_content = paste.get_p_content() + char_to_display = len(item_content) + if content_range != 0: + item_content = item_content[0:content_range] + + vt_enabled = Flask_config.vt_enabled + + + p_hashtype_list = [] + + l_tags = r_serv_metadata.smembers('tag:'+requested_path) + if relative_path is not None: + l_tags.union( r_serv_metadata.smembers('tag:'+relative_path) ) + item_info['tags'] = l_tags + item_info['name'] = relative_path.replace('/', ' / ') + + + l_64 = [] + # load hash files + if r_serv_metadata.scard('hash_paste:'+requested_path) > 0: + set_b64 = r_serv_metadata.smembers('hash_paste:'+requested_path) + for hash in set_b64: + nb_in_file = int(r_serv_metadata.zscore('nb_seen_hash:'+hash, requested_path)) + estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type') + file_type = estimated_type.split('/')[0] + # set file icon + if file_type == 'application': + file_icon = 'fa-file ' + elif file_type == 'audio': + file_icon = 'fa-file-video ' + elif file_type == 'image': + file_icon = 'fa-file-image' + elif file_type == 'text': + file_icon = 'fa-file-alt' + else: + file_icon = 'fa-file' + saved_path = r_serv_metadata.hget('metadata_hash:'+hash, 'saved_path') + if r_serv_metadata.hexists('metadata_hash:'+hash, 'vt_link'): + b64_vt = True + b64_vt_link = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_link') + b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report') + else: + b64_vt = False + b64_vt_link = '' + b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report') + # hash never refreshed + if b64_vt_report is None: + b64_vt_report = '' + + l_64.append( (file_icon, estimated_type, hash, saved_path, nb_in_file, b64_vt, b64_vt_link, b64_vt_report) ) + + crawler_metadata = {} + if 'infoleak:submission="crawler"' in l_tags: + crawler_metadata['get_metadata'] = True + crawler_metadata['domain'] = r_serv_metadata.hget('paste_metadata:'+requested_path, 'domain') + crawler_metadata['paste_father'] = r_serv_metadata.hget('paste_metadata:'+requested_path, 'father') + crawler_metadata['real_link'] = r_serv_metadata.hget('paste_metadata:'+requested_path,'real_link') + crawler_metadata['screenshot'] = paste.get_p_rel_path() + else: + crawler_metadata['get_metadata'] = False + + misp_event = r_serv_metadata.get('misp_events:' + requested_path) + if misp_event is None: + misp_eventid = False + misp_url = '' + else: + misp_eventid = True + misp_url = misp_event_url + misp_event + + hive_case = r_serv_metadata.get('hive_cases:' + requested_path) + if hive_case is None: + hive_caseid = False + hive_url = '' + else: + hive_caseid = True + hive_url = hive_case_url.replace('id_here', hive_case) + + return render_template("show_saved_item_min.html", bootstrap_label=bootstrap_label, content=item_content, + item_basic_info=item_basic_info, item_info=item_info, + initsize=len(item_content), + hashtype_list = p_hashtype_list, + crawler_metadata=crawler_metadata, + l_64=l_64, vt_enabled=vt_enabled, misp_eventid=misp_eventid, misp_url=misp_url, hive_caseid=hive_caseid, hive_url=hive_url) + # ============ ROUTES ============ @showsavedpastes.route("/showsavedpaste/") #completely shows the paste in a new tab @@ -230,6 +352,11 @@ def showsavedpaste(): requested_path = request.args.get('paste', '') return showpaste(0, requested_path) +@showsavedpastes.route("/showsaveditem_min/") #completely shows the paste in a new tab +def showsaveditem_min(): + requested_path = request.args.get('paste', '') + return show_item_min(requested_path) + @showsavedpastes.route("/showsavedrawpaste/") #shows raw def showsavedrawpaste(): requested_path = request.args.get('paste', '') @@ -241,7 +368,7 @@ def showsavedrawpaste(): def showpreviewpaste(): num = request.args.get('num', '') requested_path = request.args.get('paste', '') - return "|num|"+num+"|num|"+showpaste(max_preview_modal, requested_path) + return "|num|"+num+"|num|"+show_item_min(requested_path, content_range=max_preview_modal) @showsavedpastes.route("/getmoredata/") diff --git a/var/www/modules/showpaste/templates/show_saved_item_min.html b/var/www/modules/showpaste/templates/show_saved_item_min.html new file mode 100644 index 00000000..3c050001 --- /dev/null +++ b/var/www/modules/showpaste/templates/show_saved_item_min.html @@ -0,0 +1,300 @@ + + + + Paste information - AIL + + + + + + + + + + + + + + + + + + +
+
+

{{ item_info['name'] }}

+
+
+ + + + + + + + + + + + + + + + + + + + + +
DateSourceEncodingSize (Kb)Number of linesMax line length
{{ item_basic_info['date'] }}{{ item_basic_info['source'] }}{{ item_basic_info['encoding'] }}{{ item_basic_info['size'] }}{{ item_basic_info['nb_lines'] }}{{ item_basic_info['max_length_line'] }}
+ +
+
+ {% for tag in item_info['tags'] %} + {{ tag }} + {% endfor %} +
+
+ +
+
+ + {% if misp_eventid %} +
+
  • MISP Events already Created
    • + {{ misp_url }} +
    + {% endif %} + + {% if hive_caseid %} +
    +
  • The Hive Case already Created
    • + {{ hive_url }} +
    + {% endif %} + + {% if item_info['nb_duplictates'] != 0 %} +
    +
    +
    +
    + duplicates   +
    {{item_info['nb_duplictates']}}
    +
    +
    +
    +
    + {% endif %} + + + {% if l_64|length != 0 %} +
    +
    +
    +
    +
    +
    + Decoded Files   +
    {{l_64|length}}
    +
    +
    +
    + +
    +
    +
    + +
    +
    + + + + + + + + + + {% for b64 in l_64 %} + + + + + {% endfor %} + +
    estimated typehash
      {{ b64[1] }}{{ b64[2] }} ({{ b64[4] }})
    + +
    +
    +
    +
    + {% endif %} + + {% if crawler_metadata['get_metadata'] %} +
    +
    +
    +
    +
    +
    + Crawled Item +
    +
    +
    + +
    +
    +
    + +
    +
    + + + + + + + + + + + + + + + + +
    Domain{{ crawler_metadata['domain'] }}
    Father{{ crawler_metadata['paste_father'] }}
    Url{{ crawler_metadata['real_link'] }}
    + +
    +
    +
    +
    + +
    +
    + +
    +
    +
    +
    + + +
    +
    +
    +
    + {% endif %} + +
    +
    + +
    +
    +

    Content:

    +
    +
    +
    + [Raw content] +
    +
    +
    + +
    +
    + + + 

    {{ content }}

    + + + + + +
    + +
    + + + + + + +