diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py index 1c5f8e89..f775d058 100644 --- a/var/www/modules/hashDecoded/Flask_hashDecoded.py +++ b/var/www/modules/hashDecoded/Flask_hashDecoded.py @@ -60,6 +60,15 @@ def list_sparkline_values(date_range_sparkline, hash): sparklines_value.append(int(nb_seen_this_day)) return sparklines_value +def list_sparkline_pgp_values(date_range_sparkline, type_id, key_id): + sparklines_value = [] + for date_day in date_range_sparkline: + nb_seen_this_day = r_serv_metadata.hget('pgp:{}:{}'.format(type_id, date_day), key_id) + if nb_seen_this_day is None: + nb_seen_this_day = 0 + sparklines_value.append(int(nb_seen_this_day)) + return sparklines_value + def get_file_icon(estimated_type): file_type = estimated_type.split('/')[0] # set file icon @@ -92,6 +101,48 @@ def get_file_icon_text(estimated_type): return file_icon_text +def get_pgp_id_icon_text(type_id): + # set file icon + if type_id == 'key': + file_icon_text = '\uf084' + elif type_id == 'name': + file_icon_text = '\uf507' + elif type_id == 'mail': + file_icon_text = '\uf1fa' + else: + file_icon_text = '\uf249' + return file_icon_text + +def verify_pgp_type_id(type_id): + if type_id in ['key', 'name', 'mail']: + return True + else: + return False + +def get_key_id_metadata(type_id, key_id): + key_id_metadata = {} + if r_serv_metadata.exists('pgp_metadata_{}:{}'.format(type_id, key_id)): + key_id_metadata['first_seen'] = r_serv_metadata.hget('pgp_metadata_{}:{}'.format(type_id, key_id), 'first_seen') + key_id_metadata['first_seen'] = '{}/{}/{}'.format(key_id_metadata['first_seen'][0:4], key_id_metadata['first_seen'][4:6], key_id_metadata['first_seen'][6:8]) + key_id_metadata['last_seen'] = r_serv_metadata.hget('pgp_metadata_{}:{}'.format(type_id, key_id), 'last_seen') + key_id_metadata['last_seen'] = '{}/{}/{}'.format(key_id_metadata['last_seen'][0:4], key_id_metadata['last_seen'][4:6], key_id_metadata['last_seen'][6:8]) + key_id_metadata['nb_seen'] = r_serv_metadata.scard('pgp_{}:{}'.format(type_id, key_id)) + return key_id_metadata + +def get_all_pgp_from_item(item_path): + all_pgp_dump = set() + if item_path is not None: + res = r_serv_metadata.smembers('item_pgp_key:{}'.format(item_path)) + for pgp_dump in res: + all_pgp_dump.add( (pgp_dump, 'key') ) + res = r_serv_metadata.smembers('item_pgp_name:{}'.format(item_path)) + for pgp_dump in res: + all_pgp_dump.add( (pgp_dump, 'name') ) + res = r_serv_metadata.smembers('item_pgp_mail:{}'.format(item_path)) + for pgp_dump in res: + all_pgp_dump.add( (pgp_dump, 'mail') ) + return all_pgp_dump + def one(): return 1 @@ -105,6 +156,14 @@ def all_hash_search(): show_decoded_files = request.form.get('show_decoded_files') return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type, encoding=encoding, show_decoded_files=show_decoded_files)) +@hashDecoded.route("/hashDecoded/all_pgp_dump_search", methods=['POST']) +def all_pgp_dump_search(): + date_from = request.form.get('date_from') + date_to = request.form.get('date_to') + type_id = request.form.get('type') + show_decoded_files = request.form.get('show_decoded_files') + return redirect(url_for('hashDecoded.pgpdump_page', date_from=date_from, date_to=date_to, type_id=type_id, show_decoded_files=show_decoded_files)) + @hashDecoded.route("/hashDecoded/", methods=['GET']) def hashDecoded_page(): @@ -706,5 +765,288 @@ def update_vt_result(): # TODO FIXME make json response return jsonify() +## PGPDump ## + +@hashDecoded.route("/decoded/pgpdump", methods=['GET']) +def pgpdump_page(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + type_id = request.args.get('type_id') + + show_decoded_files = request.args.get('show_decoded_files') + + if type_id == 'All types': + type_id = None + + # verify type input + if type_id is not None: + #retrieve char + type_id = type_id.replace(' ', '') + if not verify_pgp_type_id(type_id): + type_id = None + + date_range = [] + if date_from is not None and date_to is not None: + #change format + try: + if len(date_from) != 8: + date_from = date_from[0:4] + date_from[5:7] + date_from[8:10] + date_to = date_to[0:4] + date_to[5:7] + date_to[8:10] + date_range = substract_date(date_from, date_to) + except: + pass + + if not date_range: + date_range.append(datetime.date.today().strftime("%Y%m%d")) + date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8] + date_to = date_from + + else: + date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8] + date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8] + + # display day type bar chart + if len(date_range) == 1 and type is None: + daily_type_chart = True + daily_date = date_range[0] + else: + daily_type_chart = False + daily_date = None + + if type_id is None: + all_type_id = ['key', 'name', 'mail'] + else: + all_type_id = type_id + + l_pgp_dump = set() + if show_decoded_files: + for date in date_range: + if isinstance(all_type_id, str): + l_dump = r_serv_metadata.hkeys('pgp:{}:{}'.format(all_type_id, date)) + if l_dump: + for dump in l_dump: + l_pgp_dump.add( (dump, all_type_id) ) + else: + for typ_id in all_type_id: + l_dump = r_serv_metadata.hkeys('pgp:{}:{}'.format(typ_id, date)) + if l_dump: + for dump in l_dump: + l_pgp_dump.add( (dump, typ_id) ) + + + num_day_sparkline = 6 + date_range_sparkline = get_date_range(num_day_sparkline) + + sparkline_id = 0 + pgp_metadata = {} + for dump_res in l_pgp_dump: + dump_id, typ_id = dump_res + + pgp_metadata[dump_id] = get_key_id_metadata(typ_id, dump_id) + + if pgp_metadata[dump_id]: + pgp_metadata[dump_id]['type_id'] = typ_id + #file_icon = get_file_icon(estimated_type) + + pgp_metadata[dump_id]['sparklines_data'] = list_sparkline_pgp_values(date_range_sparkline, typ_id, dump_id) + pgp_metadata[dump_id]['sparklines_id'] = sparkline_id + sparkline_id += 1 + + l_type = ['key', 'name', 'mail'] + + return render_template("PgpDecoded.html", l_pgpdump=pgp_metadata, + l_type=l_type, type_id=type_id, daily_type_chart=daily_type_chart, daily_date=daily_date, + date_from=date_from, date_to=date_to, show_decoded_files=show_decoded_files) + + + + + +@hashDecoded.route('/decoded/show_pgpdump') +def show_pgpdump(): + type_id = request.args.get('type_id') + key_id = request.args.get('key_id') + + if verify_pgp_type_id(type_id): + key_id_metadata = get_key_id_metadata(type_id, key_id) + if key_id_metadata: + + num_day_sparkline = 6 + date_range_sparkline = get_date_range(num_day_sparkline) + + sparkline_values = list_sparkline_pgp_values(date_range_sparkline, type_id, key_id) + return render_template('showPgpDump.html', key_id=key_id, type_id=type_id, + key_id_metadata=key_id_metadata, + sparkline_values=sparkline_values) + else: + return '404' + else: + return 'error' + +@hashDecoded.route('/decoded/pgp_dump_graph_node_json') +def pgp_dump_graph_node_json(): + type_id = request.args.get('type_id') + key_id = request.args.get('key_id') + + if key_id is not None and verify_pgp_type_id(type_id): + + nodes_set_pgp_dump = set() + nodes_set_paste = set() + links_set = set() + + pgp_id_metadata = get_key_id_metadata(type_id, key_id) + + #key_id = '' + + nodes_set_pgp_dump.add((key_id, 1, type_id, pgp_id_metadata['first_seen'], pgp_id_metadata['last_seen'], pgp_id_metadata['nb_seen'])) + + #get related paste + l_pastes = r_serv_metadata.smembers('pgp_{}:{}'.format(type_id, key_id)) + for paste in l_pastes: + nodes_set_paste.add((paste, 2)) + links_set.add((key_id, paste)) + + l_pgpdump = get_all_pgp_from_item(paste) + for pgp_dump_with_type in l_pgpdump: + pgp_dump, pgp_type_id = pgp_dump_with_type + if pgp_dump != key_id: + + pgp_id_metadata = get_key_id_metadata(pgp_type_id, pgp_dump) + + nodes_set_pgp_dump.add((pgp_dump, 3, pgp_type_id, pgp_id_metadata['first_seen'], pgp_id_metadata['last_seen'], pgp_id_metadata['nb_seen'])) + links_set.add((pgp_dump, paste)) + + nodes = [] + for node in nodes_set_pgp_dump: + nodes.append({"id": node[0], "group": node[1], "first_seen": node[3], "last_seen": node[4], "nb_seen_in_paste": node[5], 'icon': get_pgp_id_icon_text(node[2]),"url": url_for('hashDecoded.showHash', hash=node[0]), 'hash': True}) + for node in nodes_set_paste: + nodes.append({"id": node[0], "group": node[1],"url": url_for('showsavedpastes.showsavedpaste', paste=node[0]), 'hash': False}) + links = [] + for link in links_set: + links.append({"source": link[0], "target": link[1]}) + json = {"nodes": nodes, "links": links} + return jsonify(json) + + else: + return jsonify({}) + +@hashDecoded.route('/hashDecoded/pgp_graph_line_json') +def pgp_graph_line_json(): + type_id = request.args.get('type_id') + key_id = request.args.get('key_id') + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + + # verify input + if key_id is not None and verify_pgp_type_id(type_id) and r_serv_metadata.exists('pgp_metadata_{}:{}'.format(type_id, key_id)): + + if date_from is None or date_to is None: + nb_days_seen_in_pastes = 30 + else: + # # TODO: # FIXME: + nb_days_seen_in_pastes = 30 + + date_range_seen_in_pastes = get_date_range(nb_days_seen_in_pastes) + + json_seen_in_paste = [] + for date in date_range_seen_in_pastes: + nb_seen_this_day = r_serv_metadata.zscore('hash_date:'+date, hash) + if nb_seen_this_day is None: + nb_seen_this_day = 0 + date = date[0:4] + '-' + date[4:6] + '-' + date[6:8] + json_seen_in_paste.append({'date': date, 'value': int(nb_seen_this_day)}) + + return jsonify(json_seen_in_paste) + else: + return jsonify() + +@hashDecoded.route('/decoded/pgp_range_type_json') +def pgp_range_type_json(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + + date_range = [] + if date_from is not None and date_to is not None: + #change format + if len(date_from) != 8: + date_from = date_from[0:4] + date_from[5:7] + date_from[8:10] + date_to = date_to[0:4] + date_to[5:7] + date_to[8:10] + date_range = substract_date(date_from, date_to) + + if not date_range: + date_range.append(datetime.date.today().strftime("%Y%m%d")) + + range_type = [] + + list_type_id = ['key', 'name', 'mail'] + + + if len(date_range) == 1: + for type_id in list_type_id: + day_type = {} + day_type['key'] = 0 + day_type['name'] = 0 + day_type['mail'] = 0 + day_type['date'] = type_id + num_day_type_id = 0 + all_keys = r_serv_metadata.hvals('pgp:{}:{}'.format(type_id, date_range[0])) + if all_keys: + for val in all_keys: + num_day_type_id += int(val) + day_type[type_id]= num_day_type_id + + #if day_type[type_id] != 0: + range_type.append(day_type) + + else: + # display type_id + for date in date_range: + day_type = {} + day_type['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8] + for type_id in list_type_id: + num_day_type_id = 0 + all_keys = r_serv_metadata.hvals('pgp:{}:{}'.format(type_id, date)) + if all_keys: + for val in all_keys: + num_day_type_id += int(val) + day_type[type_id]= num_day_type_id + range_type.append(day_type) + + return jsonify(range_type) + +@hashDecoded.route('/decoded/pgp_by_type_json') #################################### +def pgp_by_type_json(): + type_id = request.args.get('type_id') + + #retrieve + char + type_id = type_id.replace(' ', '+') + + num_day_type = 30 + date_range = get_date_range(num_day_type) + + #verify input + if verify_pgp_type_id(type_id): + + r_serv_metadata.smembers('hash_all_type'): + type_value = [] + all_decoder = r_serv_metadata.smembers('all_decoder') + + range_decoder = [] + for date in date_range: + day_decoder = {} + day_decoder['date']= date[0:4] + '-' + date[4:6] + '-' + date[6:8] + for decoder in all_decoder: + num_day_decoder = r_serv_metadata.zscore(decoder+'_type:'+type, date) + if num_day_decoder is None: + num_day_decoder = 0 + day_decoder[decoder]= num_day_decoder + range_decoder.append(day_decoder) + + + + return jsonify(range_decoder) + else: + return jsonify() + # ========= REGISTRATION ========= app.register_blueprint(hashDecoded, url_prefix=baseUrl) diff --git a/var/www/modules/hashDecoded/templates/PgpDecoded.html b/var/www/modules/hashDecoded/templates/PgpDecoded.html new file mode 100644 index 00000000..da568936 --- /dev/null +++ b/var/www/modules/hashDecoded/templates/PgpDecoded.html @@ -0,0 +1,657 @@ + + + +
+type id | +key id | +first seen | +last seen | +nb paste | +Sparkline | +
---|---|---|---|---|---|
{{ l_pgpdump[pgp_dump]['type_id'] }} | +{{ pgp_dump }} | +{{ l_pgpdump[pgp_dump]['first_seen'] }} | +{{ l_pgpdump[pgp_dump]['last_seen'] }} | +{{ l_pgpdump[pgp_dump]['nb_seen'] }} | ++ |
type | +First_seen | +Last_seen | +Nb seen | +
---|---|---|---|
{{ type_id }} | +{{ key_id_metadata['first_seen'] }} | +{{ key_id_metadata['last_seen'] }} | +{{ key_id_metadata['nb_seen'] }} | +
Double click on a node to open Hash/Paste
+
+ Current Hash
+
+ Hashes
+
+ Pastes
+