From 7e8ba8cdaf8a6fec9a423c679b23c4d026373f10 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 17 May 2019 17:51:07 +0200 Subject: [PATCH] chg: [decoded UI] add PgpDump UI + fix hashdecoded js --- .../modules/hashDecoded/Flask_hashDecoded.py | 342 +++++++++ .../hashDecoded/templates/PgpDecoded.html | 657 ++++++++++++++++++ .../hashDecoded/templates/hashDecoded.html | 21 +- .../hashDecoded/templates/showHash.html | 47 +- .../hashDecoded/templates/showPgpDump.html | 569 +++++++++++++++ var/www/templates/decoded/menu_sidebar.html | 14 +- 6 files changed, 1620 insertions(+), 30 deletions(-) create mode 100644 var/www/modules/hashDecoded/templates/PgpDecoded.html create mode 100644 var/www/modules/hashDecoded/templates/showPgpDump.html diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py index 1c5f8e89..f775d058 100644 --- a/var/www/modules/hashDecoded/Flask_hashDecoded.py +++ b/var/www/modules/hashDecoded/Flask_hashDecoded.py @@ -60,6 +60,15 @@ def list_sparkline_values(date_range_sparkline, hash): sparklines_value.append(int(nb_seen_this_day)) return sparklines_value +def list_sparkline_pgp_values(date_range_sparkline, type_id, key_id): + sparklines_value = [] + for date_day in date_range_sparkline: + nb_seen_this_day = r_serv_metadata.hget('pgp:{}:{}'.format(type_id, date_day), key_id) + if nb_seen_this_day is None: + nb_seen_this_day = 0 + sparklines_value.append(int(nb_seen_this_day)) + return sparklines_value + def get_file_icon(estimated_type): file_type = estimated_type.split('/')[0] # set file icon @@ -92,6 +101,48 @@ def get_file_icon_text(estimated_type): return file_icon_text +def get_pgp_id_icon_text(type_id): + # set file icon + if type_id == 'key': + file_icon_text = '\uf084' + elif type_id == 'name': + file_icon_text = '\uf507' + elif type_id == 'mail': + file_icon_text = '\uf1fa' + else: + file_icon_text = '\uf249' + return file_icon_text + +def verify_pgp_type_id(type_id): + if type_id in ['key', 'name', 'mail']: + return True + else: + return False + +def get_key_id_metadata(type_id, key_id): + key_id_metadata = {} + if r_serv_metadata.exists('pgp_metadata_{}:{}'.format(type_id, key_id)): + key_id_metadata['first_seen'] = r_serv_metadata.hget('pgp_metadata_{}:{}'.format(type_id, key_id), 'first_seen') + key_id_metadata['first_seen'] = '{}/{}/{}'.format(key_id_metadata['first_seen'][0:4], key_id_metadata['first_seen'][4:6], key_id_metadata['first_seen'][6:8]) + key_id_metadata['last_seen'] = r_serv_metadata.hget('pgp_metadata_{}:{}'.format(type_id, key_id), 'last_seen') + key_id_metadata['last_seen'] = '{}/{}/{}'.format(key_id_metadata['last_seen'][0:4], key_id_metadata['last_seen'][4:6], key_id_metadata['last_seen'][6:8]) + key_id_metadata['nb_seen'] = r_serv_metadata.scard('pgp_{}:{}'.format(type_id, key_id)) + return key_id_metadata + +def get_all_pgp_from_item(item_path): + all_pgp_dump = set() + if item_path is not None: + res = r_serv_metadata.smembers('item_pgp_key:{}'.format(item_path)) + for pgp_dump in res: + all_pgp_dump.add( (pgp_dump, 'key') ) + res = r_serv_metadata.smembers('item_pgp_name:{}'.format(item_path)) + for pgp_dump in res: + all_pgp_dump.add( (pgp_dump, 'name') ) + res = r_serv_metadata.smembers('item_pgp_mail:{}'.format(item_path)) + for pgp_dump in res: + all_pgp_dump.add( (pgp_dump, 'mail') ) + return all_pgp_dump + def one(): return 1 @@ -105,6 +156,14 @@ def all_hash_search(): show_decoded_files = request.form.get('show_decoded_files') return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type, encoding=encoding, show_decoded_files=show_decoded_files)) +@hashDecoded.route("/hashDecoded/all_pgp_dump_search", methods=['POST']) +def all_pgp_dump_search(): + date_from = request.form.get('date_from') + date_to = request.form.get('date_to') + type_id = request.form.get('type') + show_decoded_files = request.form.get('show_decoded_files') + return redirect(url_for('hashDecoded.pgpdump_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files)) + @hashDecoded.route("/hashDecoded/", methods=['GET']) def hashDecoded_page(): @@ -706,5 +765,288 @@ def update_vt_result(): # TODO FIXME make json response return jsonify() +## PGPDump ## + +@hashDecoded.route("/decoded/pgpdump", methods=['GET']) +def pgpdump_page(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + type_id = request.args.get('type_id') + + show_decoded_files = request.args.get('show_decoded_files') + + if type_id == 'All types': + type_id = None + + # verify type input + if type_id is not None: + #retrieve char + type_id = type_id.replace(' ', '') + if not verify_pgp_type_id(type_id): + type_id = None + + date_range = [] + if date_from is not None and date_to is not None: + #change format + try: + if len(date_from) != 8: + date_from = date_from[0:4] + date_from[5:7] + date_from[8:10] + date_to = date_to[0:4] + date_to[5:7] + date_to[8:10] + date_range = substract_date(date_from, date_to) + except: + pass + + if not date_range: + date_range.append(datetime.date.today().strftime("%Y%m%d")) + date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8] + date_to = date_from + + else: + date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8] + date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8] + + # display day type bar chart + if len(date_range) == 1 and type is None: + daily_type_chart = True + daily_date = date_range[0] + else: + daily_type_chart = False + daily_date = None + + if type_id is None: + all_type_id = ['key', 'name', 'mail'] + else: + all_type_id = type_id + + l_pgp_dump = set() + if show_decoded_files: + for date in date_range: + if isinstance(all_type_id, str): + l_dump = r_serv_metadata.hkeys('pgp:{}:{}'.format(all_type_id, date)) + if l_dump: + for dump in l_dump: + l_pgp_dump.add( (dump, all_type_id) ) + else: + for typ_id in all_type_id: + l_dump = r_serv_metadata.hkeys('pgp:{}:{}'.format(typ_id, date)) + if l_dump: + for dump in l_dump: + l_pgp_dump.add( (dump, typ_id) ) + + + num_day_sparkline = 6 + date_range_sparkline = get_date_range(num_day_sparkline) + + sparkline_id = 0 + pgp_metadata = {} + for dump_res in l_pgp_dump: + dump_id, typ_id = dump_res + + pgp_metadata[dump_id] = get_key_id_metadata(typ_id, dump_id) + + if pgp_metadata[dump_id]: + pgp_metadata[dump_id]['type_id'] = typ_id + #file_icon = get_file_icon(estimated_type) + + pgp_metadata[dump_id]['sparklines_data'] = list_sparkline_pgp_values(date_range_sparkline, typ_id, dump_id) + pgp_metadata[dump_id]['sparklines_id'] = sparkline_id + sparkline_id += 1 + + l_type = ['key', 'name', 'mail'] + + return render_template("PgpDecoded.html", l_pgpdump=pgp_metadata, + l_type=l_type, type_id=type_id, daily_type_chart=daily_type_chart, daily_date=daily_date, + date_from=date_from, date_to=date_to, show_decoded_files=show_decoded_files) + + + + + +@hashDecoded.route('/decoded/show_pgpdump') +def show_pgpdump(): + type_id = request.args.get('type_id') + key_id = request.args.get('key_id') + + if verify_pgp_type_id(type_id): + key_id_metadata = get_key_id_metadata(type_id, key_id) + if key_id_metadata: + + num_day_sparkline = 6 + date_range_sparkline = get_date_range(num_day_sparkline) + + sparkline_values = list_sparkline_pgp_values(date_range_sparkline, type_id, key_id) + return render_template('showPgpDump.html', key_id=key_id, type_id=type_id, + key_id_metadata=key_id_metadata, + sparkline_values=sparkline_values) + else: + return '404' + else: + return 'error' + +@hashDecoded.route('/decoded/pgp_dump_graph_node_json') +def pgp_dump_graph_node_json(): + type_id = request.args.get('type_id') + key_id = request.args.get('key_id') + + if key_id is not None and verify_pgp_type_id(type_id): + + nodes_set_pgp_dump = set() + nodes_set_paste = set() + links_set = set() + + pgp_id_metadata = get_key_id_metadata(type_id, key_id) + + #key_id = '' + + nodes_set_pgp_dump.add((key_id, 1, type_id, pgp_id_metadata['first_seen'], pgp_id_metadata['last_seen'], pgp_id_metadata['nb_seen'])) + + #get related paste + l_pastes = r_serv_metadata.smembers('pgp_{}:{}'.format(type_id, key_id)) + for paste in l_pastes: + nodes_set_paste.add((paste, 2)) + links_set.add((key_id, paste)) + + l_pgpdump = get_all_pgp_from_item(paste) + for pgp_dump_with_type in l_pgpdump: + pgp_dump, pgp_type_id = pgp_dump_with_type + if pgp_dump != key_id: + + pgp_id_metadata = get_key_id_metadata(pgp_type_id, pgp_dump) + + nodes_set_pgp_dump.add((pgp_dump, 3, pgp_type_id, pgp_id_metadata['first_seen'], pgp_id_metadata['last_seen'], pgp_id_metadata['nb_seen'])) + links_set.add((pgp_dump, paste)) + + nodes = [] + for node in nodes_set_pgp_dump: + nodes.append({"id": node[0], "group": node[1], "first_seen": node[3], "last_seen": node[4], "nb_seen_in_paste": node[5], 'icon': get_pgp_id_icon_text(node[2]),"url": url_for('hashDecoded.showHash', hash=node[0]), 'hash': True}) + for node in nodes_set_paste: + nodes.append({"id": node[0], "group": node[1],"url": url_for('showsavedpastes.showsavedpaste', paste=node[0]), 'hash': False}) + links = [] + for link in links_set: + links.append({"source": link[0], "target": link[1]}) + json = {"nodes": nodes, "links": links} + return jsonify(json) + + else: + return jsonify({}) + +@hashDecoded.route('/hashDecoded/pgp_graph_line_json') +def pgp_graph_line_json(): + type_id = request.args.get('type_id') + key_id = request.args.get('key_id') + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + + # verify input + if key_id is not None and verify_pgp_type_id(type_id) and r_serv_metadata.exists('pgp_metadata_{}:{}'.format(type_id, key_id)): + + if date_from is None or date_to is None: + nb_days_seen_in_pastes = 30 + else: + # # TODO: # FIXME: + nb_days_seen_in_pastes = 30 + + date_range_seen_in_pastes = get_date_range(nb_days_seen_in_pastes) + + json_seen_in_paste = [] + for date in date_range_seen_in_pastes: + nb_seen_this_day = r_serv_metadata.zscore('hash_date:'+date, hash) + if nb_seen_this_day is None: + nb_seen_this_day = 0 + date = date[0:4] + '-' + date[4:6] + '-' + date[6:8] + json_seen_in_paste.append({'date': date, 'value': int(nb_seen_this_day)}) + + return jsonify(json_seen_in_paste) + else: + return jsonify() + +@hashDecoded.route('/decoded/pgp_range_type_json') +def pgp_range_type_json(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + + date_range = [] + if date_from is not None and date_to is not None: + #change format + if len(date_from) != 8: + date_from = date_from[0:4] + date_from[5:7] + date_from[8:10] + date_to = date_to[0:4] + date_to[5:7] + date_to[8:10] + date_range = substract_date(date_from, date_to) + + if not date_range: + date_range.append(datetime.date.today().strftime("%Y%m%d")) + + range_type = [] + + list_type_id = ['key', 'name', 'mail'] + + + if len(date_range) == 1: + for type_id in list_type_id: + day_type = {} + day_type['key'] = 0 + day_type['name'] = 0 + day_type['mail'] = 0 + day_type['date'] = type_id + num_day_type_id = 0 + all_keys = r_serv_metadata.hvals('pgp:{}:{}'.format(type_id, date_range[0])) + if all_keys: + for val in all_keys: + num_day_type_id += int(val) + day_type[type_id]= num_day_type_id + + #if day_type[type_id] != 0: + range_type.append(day_type) + + else: + # display type_id + for date in date_range: + day_type = {} + day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8] + for type_id in list_type_id: + num_day_type_id = 0 + all_keys = r_serv_metadata.hvals('pgp:{}:{}'.format(type_id, date)) + if all_keys: + for val in all_keys: + num_day_type_id += int(val) + day_type[type_id]= num_day_type_id + range_type.append(day_type) + + return jsonify(range_type) + +@hashDecoded.route('/decoded/pgp_by_type_json') #################################### +def pgp_by_type_json(): + type_id = request.args.get('type_id') + + #retrieve + char + type_id = type_id.replace(' ', '+') + + num_day_type = 30 + date_range = get_date_range(num_day_type) + + #verify input + if verify_pgp_type_id(type_id): + + r_serv_metadata.smembers('hash_all_type'): + type_value = [] + all_decoder = r_serv_metadata.smembers('all_decoder') + + range_decoder = [] + for date in date_range: + day_decoder = {} + day_decoder['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8] + for decoder in all_decoder: + num_day_decoder = r_serv_metadata.zscore(decoder+'_type:'+type, date) + if num_day_decoder is None: + num_day_decoder = 0 + day_decoder[decoder]= num_day_decoder + range_decoder.append(day_decoder) + + + + return jsonify(range_decoder) + else: + return jsonify() + # ========= REGISTRATION ========= app.register_blueprint(hashDecoded, url_prefix=baseUrl) diff --git a/var/www/modules/hashDecoded/templates/PgpDecoded.html b/var/www/modules/hashDecoded/templates/PgpDecoded.html new file mode 100644 index 00000000..da568936 --- /dev/null +++ b/var/www/modules/hashDecoded/templates/PgpDecoded.html @@ -0,0 +1,657 @@ + + + + + Decoded - AIL + + + + + + + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'decoded/menu_sidebar.html' %} + +
+ +
+
+
+
+
+ +
+ +
+
+
Select a date range :
+
+
+
+ +
+
+
+ +
+
Type ID :
+ +
+ + +
+ + +
+
+ +
+
+
+
+
+
+ + {% if l_pgpdump|length != 0 %} + {% if date_from|string == date_to|string %} +

{{ date_from }} Dumped Keys:

+ {% else %} +

{{ date_from }} to {{ date_to }} Dumped Keys:

+ {% endif %} + + + + + + + + + + + + + {% for pgp_dump in l_pgpdump %} + + + + + + + + + {% endfor %} + +
type idkey idfirst seenlast seennb pasteSparkline
  {{ l_pgpdump[pgp_dump]['type_id'] }}{{ pgp_dump }}{{ l_pgpdump[pgp_dump]['first_seen'] }}{{ l_pgpdump[pgp_dump]['last_seen'] }}{{ l_pgpdump[pgp_dump]['nb_seen'] }}
+ {% else %} + {% if show_decoded_files %} + {% if date_from|string == date_to|string %} +

{{ date_from }}, No Dumped Keys

+ {% else %} +

{{ date_from }} to {{ date_to }}, No Dumped Keys

+ {% endif %} + {% endif %} + {% endif %} +
+ +
+
+ + + + + + + + + + + + + + + + + + diff --git a/var/www/modules/hashDecoded/templates/hashDecoded.html b/var/www/modules/hashDecoded/templates/hashDecoded.html index bf0a12bf..e031530a 100644 --- a/var/www/modules/hashDecoded/templates/hashDecoded.html +++ b/var/www/modules/hashDecoded/templates/hashDecoded.html @@ -493,16 +493,19 @@ function removePopovers () { function showPopover (d) { $(this).popover({ - title: d.name, + title: "", placement: 'top', container: 'body', trigger: 'manual', html : true, content: function() { - return d.label + - "
num: " + d3.format(",")(d.value ? d.value: d.y1 - d.y0); } + return "" + + "
num: "; } }); - $(this).popover('show') + $(this).popover('show'); + $("#tooltip-id-name-bar").text(d.name); + $("#tooltip-id-label").text(d.label); + $("#tooltip-id-value-bar").text(d3.format(",")(d.value ? d.value: d.y1 - d.y0)); } chart.onResize = function () { @@ -571,13 +574,16 @@ function draw_pie_chart(id, url_json, pie_on_click_url) { function mouseovered_pie(d) { + //remove old content + $("#tooltip-id-name").remove(); + $("#tooltip-id-value").remove(); // tooltip var content; - content = ""+d.data.name+""+"
"+ + content = "
"+ "
"+ - "Decoded: "+d.data.value+"
" + "Decoded:
" div_pie.transition() .duration(200) @@ -585,6 +591,9 @@ function draw_pie_chart(id, url_json, pie_on_click_url) { div_pie.html(content) .style("left", (d3.event.pageX) + "px") .style("top", (d3.event.pageY - 28) + "px"); + + $("#tooltip-id-name").text(d.data.name); + $("#tooltip-id-value").text(d.data.value); } function mouseouted_pie() { diff --git a/var/www/modules/hashDecoded/templates/showHash.html b/var/www/modules/hashDecoded/templates/showHash.html index 45faf5c8..c7cac4d8 100644 --- a/var/www/modules/hashDecoded/templates/showHash.html +++ b/var/www/modules/hashDecoded/templates/showHash.html @@ -171,8 +171,8 @@
{% for encoding in list_hash_decoder %} - {% endfor %}
@@ -398,8 +398,7 @@ d3.json(url) node.append('text') .attr('text-anchor', 'middle') .attr('dominant-baseline', 'central') - .attr("class", "graph_node_icon") - .attr('font-family', 'FontAwesome') + .attr("class", "graph_node_icon fa") .attr('font-size', '8px' ) .attr('pointer-events', 'none') .text(function(d) { @@ -475,24 +474,36 @@ function mouseovered(d) { var content; if(d.hash == true){ - content = ""+d.id+""+"
"+ + content = "
"+ "
"+ - "First seen: "+d.first_seen+"
"+ - "Last seen: "+d.last_seen+"
"+ - "nb_seen_in_paste: "+d.nb_seen_in_paste+"
"+ - "Size (kb): "+d.size+"
"+ - "
"+ - "Estimated type: "+d.estimated_type; -} else { - content = ""+d.id+""+"
"; -} + "First seen:
"+ + "Last seen:
"+ + "nb_seen: "; div.transition() - .duration(200) - .style("opacity", .9); + .duration(200) + .style("opacity", .9); div.html(content) - .style("left", (d3.event.pageX) + "px") - .style("top", (d3.event.pageY - 28) + "px"); + .style("left", (d3.event.pageX) + "px") + .style("top", (d3.event.pageY - 28) + "px"); + + $("#tooltip-id-name").text(d.id); + $("#tooltip-id-first_seen").text(d.first_seen); + $("#tooltip-id-last_seen").text(d.last_seen); + $("#tooltip-id-nb_seen").text(d.nb_seen_in_paste); + +} else { + content = "
"; + + div.transition() + .duration(200) + .style("opacity", .9); + div.html(content) + .style("left", (d3.event.pageX) + "px") + .style("top", (d3.event.pageY - 28) + "px"); + + $("#tooltip-id-name").text(d.id); +} //links /*link.style("stroke-opacity", function(o) { diff --git a/var/www/modules/hashDecoded/templates/showPgpDump.html b/var/www/modules/hashDecoded/templates/showPgpDump.html new file mode 100644 index 00000000..8f7b05aa --- /dev/null +++ b/var/www/modules/hashDecoded/templates/showPgpDump.html @@ -0,0 +1,569 @@ + + + + + + + AIL - framework + + + + + + + + + + + + + + + + + {% include 'nav_bar.html' %} + +
+
+ + {% include 'decoded/menu_sidebar.html' %} + +
+ +
+
+

{{ key_id }} :

+
    +
  • +
    +
    + + + + + + + + + + + + + + + + + +
    typeFirst_seenLast_seenNb seen
      {{ type_id }}{{ key_id_metadata['first_seen'] }}{{ key_id_metadata['last_seen'] }}{{ key_id_metadata['nb_seen'] }}
    +
    +
    +
    +
    +
    +
    • +
+
+
+ +
+
+ +
+
+ Graph +
+
+
+
+
+
+
+ +
+ +
+
+ Graph +
+
+ + + +
    +
    • +
  • +

    Double click on a node to open Hash/Paste

    + + Current Hash
    + + Hashes
    + + Pastes +

    +
    • +
  • + Hash Types: +
    • +
  • + Key
    + Name
    + Mail
    +
    • +
+
+
+
+
+ +
+
+ Graph +
+
+
+
+
+
+ +
+
+
+ + + + + + + + + + + + + diff --git a/var/www/templates/decoded/menu_sidebar.html b/var/www/templates/decoded/menu_sidebar.html index 3cb5868b..48fefd56 100644 --- a/var/www/templates/decoded/menu_sidebar.html +++ b/var/www/templates/decoded/menu_sidebar.html @@ -8,9 +8,6 @@