diff --git a/bin/MISP_The_Hive_feeder.py b/bin/MISP_The_Hive_feeder.py index 07c121c9..2bc33431 100755 --- a/bin/MISP_The_Hive_feeder.py +++ b/bin/MISP_The_Hive_feeder.py @@ -55,21 +55,22 @@ from thehive4py.models import Case, CaseTask, CustomFieldHelper def create_the_hive_alert(source, path, tag): - tags = list(r_serv_metadata.smembers('tag:'+path)) + # # TODO: check items status (processed by all modules) + # # TODO: add item metadata: decoded content, link to auto crawled content, pgp correlation, cryptocurrency correlation... + # # # TODO: description, add AIL link:show items ? + tags = list( r_serv_metadata.smembers('tag:{}'.format(path)) ) artifacts = [ AlertArtifact( dataType='uuid-ail', data=r_serv_db.get('ail:uuid') ), AlertArtifact( dataType='file', data=path, tags=tags ) ] - l_tags = tag.split(',') - # Prepare the sample Alert sourceRef = str(uuid.uuid4())[0:6] alert = Alert(title='AIL Leak', tlp=3, - tags=l_tags, - description='infoleak', + tags=tags, + description='AIL Leak, triggered by {}'.format(tag), type='ail', source=source, sourceRef=sourceRef, diff --git a/bin/PgpDump.py b/bin/PgpDump.py index 21ffd263..4b7ec629 100755 --- a/bin/PgpDump.py +++ b/bin/PgpDump.py @@ -82,7 +82,12 @@ def get_pgp_packet(message, save_path): process1 = subprocess.Popen([ 'echo', '-e', save_path], stdout=subprocess.PIPE) process2 = subprocess.Popen([ 'pgpdump'], stdin=process1.stdout, stdout=subprocess.PIPE) process1.stdout.close() - output = process2.communicate()[0].decode() + output = process2.communicate()[0] + try: + output = output.decode() + except UnicodeDecodeError: + publisher.error('Error PgpDump UnicodeDecodeError: {}'.format(message)) + output = '' return output def get_pgp_packet_file(file): diff --git a/bin/packages/HiddenServices.py b/bin/packages/HiddenServices.py index f1ed0767..b60c6d99 100755 --- a/bin/packages/HiddenServices.py +++ b/bin/packages/HiddenServices.py @@ -17,10 +17,14 @@ Conditions to fulfill to be able to use this class correctly: """ import os +import time import gzip import redis import random +from io import BytesIO +import zipfile + import configparser import sys sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) @@ -71,6 +75,7 @@ class HiddenServices(object): self.paste_crawled_directory = os.path.join(self.paste_directory, cfg.get("Directories", "crawled")) self.paste_crawled_directory_name = cfg.get("Directories", "crawled") self.screenshot_directory = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot")) + self.screenshot_directory_screenshot = os.path.join(self.screenshot_directory, 'screenshot') elif type == 'i2p': self.paste_directory = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot")) self.screenshot_directory = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot")) @@ -120,6 +125,26 @@ class HiddenServices(object): for tag in p_tags: self.tags[tag] = self.tags.get(tag, 0) + 1 + def extract_epoch_from_history(self, crawled_history): + epoch_list = [] + for res, epoch_val in crawled_history: + epoch_val = int(epoch_val) # force int + try: + # domain down + if int(res) == epoch_val: + status = False + # domain up + else: + status = True + except ValueError: + status = True + epoch_val = int(epoch_val) # force int + epoch_list.append((epoch_val, time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(epoch_val)), status)) + return epoch_list + + def get_domain_crawled_history(self): + return self.r_serv_onion.zrange('crawler_history_{}:{}:{}'.format(self.type, self.domain, self.port), 0, -1, withscores=True) + def get_first_crawled(self): res = self.r_serv_onion.zrange('crawler_history_{}:{}:{}'.format(self.type, self.domain, self.port), 0, 0, withscores=True) if res: @@ -230,6 +255,13 @@ class HiddenServices(object): return l_crawled_pastes ''' + def get_item_screenshot(self, item): + screenshot = self.r_serv_metadata.hget('paste_metadata:{}'.format(item), 'screenshot') + if screenshot: + screenshot = os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:]) + return screenshot + return '' + def get_domain_random_screenshot(self, l_crawled_pastes, num_screenshot = 1): l_screenshot_paste = [] for paste in l_crawled_pastes: @@ -237,9 +269,8 @@ class HiddenServices(object): origin_paste = paste paste= paste.replace(self.paste_directory+'/', '') - screenshot = self.r_serv_metadata.hget('paste_metadata:{}'.format(paste), 'screenshot') + screenshot = self.get_item_screenshot(paste) if screenshot: - screenshot = os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:]) l_screenshot_paste.append({'screenshot': screenshot, 'item': origin_paste}) if len(l_screenshot_paste) > num_screenshot: @@ -250,6 +281,35 @@ class HiddenServices(object): else: return l_screenshot_paste + def get_all_domain_screenshot(self, l_crawled_pastes, filename=False): + l_screenshot_paste = [] + for paste in l_crawled_pastes: + ## FIXME: # TODO: remove me + origin_paste = paste + paste= paste.replace(self.paste_directory+'/', '') + + screenshot = self.get_item_screenshot(paste) + if screenshot: + screenshot = screenshot + '.png' + screenshot_full_path = os.path.join(self.screenshot_directory_screenshot, screenshot) + if filename: + screen_file_name = os.path.basename(paste) + '.png' + l_screenshot_paste.append( (screenshot_full_path, screen_file_name) ) + else: + l_screenshot_paste.append(screenshot_full_path) + return l_screenshot_paste + + def get_all_item_full_path(self, l_items, filename=False): + l_full_items = [] + for item in l_items: + item = os.path.join(self.PASTES_FOLDER, item) + if filename: + file_name = os.path.basename(item) + '.gz' + l_full_items.append( (item, file_name) ) + else: + l_full_items.append(item) + return l_full_items + def get_crawled_pastes_by_date(self, date): pastes_path = os.path.join(self.paste_crawled_directory, date[0:4], date[4:6], date[6:8]) @@ -258,6 +318,63 @@ class HiddenServices(object): l_crawled_pastes = [] return l_crawled_pastes + def get_all_har(self, l_pastes, filename=False): + all_har = [] + for item in l_pastes: + if filename: + all_har.append( (self.get_item_har(item), os.path.basename(item) + '.json') ) + else: + all_har.append(self.get_item_har(item)) + return all_har + + + def get_item_har(self, item_path): + item_path = item_path.replace('{}/'.format(self.paste_crawled_directory_name), '', 1) + har_path = os.path.join(self.screenshot_directory, item_path) + '.json' + return har_path + + def create_domain_basic_archive(self, l_pastes): + all_har = self.get_all_har(l_pastes, filename=True) + all_screenshot = self.get_all_domain_screenshot(l_pastes, filename=True) + all_items = self.get_all_item_full_path(l_pastes, filename=True) + + # try: + + # zip buffer + zip_buffer = BytesIO() + + with zipfile.ZipFile(zip_buffer, "a") as zf: + + #print(all_har) + self.write_in_zip_buffer(zf, all_har) + self.write_in_zip_buffer(zf, all_screenshot) + self.write_in_zip_buffer(zf, all_items) + + # write map url + map_file_content = self.get_metadata_file(l_pastes).encode() + zf.writestr( '_URL_MAP_', BytesIO(map_file_content).getvalue()) + + zip_buffer.seek(0) + return zip_buffer + + # except Exception as e: + # print(e) + # return 'Server Error' + + def write_in_zip_buffer(self, zf, list_file): + for file_path, file_name in list_file: + with open(file_path, "rb") as f: + har_content = f.read() + zf.writestr( file_name, BytesIO(har_content).getvalue()) + + def get_metadata_file(self, list_items): + file_content = '' + dict_url = self.get_all_links(list_items) + for key in dict_url: + file_content = '{}\n{} : {}'.format(file_content, os.path.basename(key), dict_url[key]) + return file_content + + ''' def get_last_crawled_pastes_fileSearch(self): diff --git a/var/www/modules/dashboard/templates/index.html b/var/www/modules/dashboard/templates/index.html index 49f3e59b..812e1ea0 100644 --- a/var/www/modules/dashboard/templates/index.html +++ b/var/www/modules/dashboard/templates/index.html @@ -2,23 +2,40 @@ - - - Analysis Information Leak framework Dashboard - - - + + - + + + + + + + - -
- -
-
-
-
-
-
+ +{% include 'nav_bar.html' %} + +
+
+ + {% include 'dashboard/menu_sidebar.html' %} + +
{%if update_in_progress%} -
- × + {%endif%} -
- × - Bootstrap 4 migration! Some pages are still in bootstrap 3. You can check the migration progress Here. + -
-
-
-
- Feeder(s) Monitor: -
-
- - Processed pastes -
-
- Filtered duplicates -
- -
- -
- -
- -
-
-
- Queues Monitor -
-
-
-
-
-
-
- -
- -
- -
- -
-
-
- Logs -
- - INFO - WARNING - CRITICAL +
+
+
+
+ Feeder(s) Monitor: +
+
+ Processed pastes +
+
+ Filtered duplicates +
+
+
+ +
+
+
+ Queues Monitor +
+
+
+
+
+
+ +
+
+
+ + +
+
+ Logs +
+ + INFO + WARNING + CRITICAL + +
+
+ +
+ + + + + + + + + + + + + + + + +
TimeChannelLevelScript NameSourceDatePaste nameMessageActions
+
+
+ +
-
-
-
- - - - - - - - - - - - - - - - -
TimeChannelLevelScript NameSourceDatePaste nameMessageActions
-
-
-
- -
- + +
+
- - + + + - - - - - +function toggle_sidebar(){ + if($('#nav_menu').is(':visible')){ + $('#nav_menu').hide(); + $('#side-bard-dashboard-content').hide(); + $('#side_menu').removeClass('border-right') + $('#side_menu').removeClass('col-lg-2') + $('#core_content').removeClass('col-lg-10') + }else{ + $('#nav_menu').show(); + $('#side-bard-dashboard-content').show(); + $('#side_menu').addClass('border-right') + $('#side_menu').addClass('col-lg-2') + $('#core_content').addClass('col-lg-10') + } +} + diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py index 067e5a0e..09d1df1f 100644 --- a/var/www/modules/hiddenServices/Flask_hiddenServices.py +++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py @@ -11,7 +11,7 @@ import os import time import json from pyfaup.faup import Faup -from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for +from flask import Flask, render_template, jsonify, request, send_file, Blueprint, redirect, url_for from flask_login import login_required from Date import Date @@ -764,6 +764,7 @@ def show_domain(): if first_seen is None: first_seen = '********' first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8]) + ports = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'ports') origin_paste = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'paste_parent') h = HiddenServices(domain, type, port=port) @@ -793,13 +794,67 @@ def show_domain(): p_tags = r_serv_metadata.smembers('tag:'+path) paste_tags.append(unpack_paste_tags(p_tags)) + domain_history = h.extract_epoch_from_history(h.get_domain_crawled_history()) + return render_template("showDomain.html", domain=domain, last_check=last_check, first_seen=first_seen, l_pastes=l_pastes, paste_tags=paste_tags, bootstrap_label=bootstrap_label, - dict_links=dict_links, + dict_links=dict_links, port=port, epoch=epoch, + ports=ports, domain_history=domain_history, origin_paste_tags=origin_paste_tags, status=status, origin_paste=origin_paste, origin_paste_name=origin_paste_name, domain_tags=domain_tags, screenshot=screenshot) +@hiddenServices.route("/crawlers/download_domain", methods=['GET']) +def download_domain(): + domain = request.args.get('domain') + epoch = request.args.get('epoch') + try: + epoch = int(epoch) + except: + epoch = None + port = request.args.get('port') + faup.decode(domain) + unpack_url = faup.get() + + ## TODO: # FIXME: remove me + try: + domain = unpack_url['domain'].decode() + except: + domain = unpack_url['domain'] + + if not port: + if unpack_url['port']: + try: + port = unpack_url['port'].decode() + except: + port = unpack_url['port'] + else: + port = 80 + try: + port = int(port) + except: + port = 80 + type = get_type_domain(domain) + if domain is None or not r_serv_onion.exists('{}_metadata:{}'.format(type, domain)): + return '404' + # # TODO: FIXME return 404 + + origin_paste = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'paste_parent') + + h = HiddenServices(domain, type, port=port) + item_core = h.get_domain_crawled_core_item(epoch=epoch) + if item_core: + l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item']) + else: + l_pastes = [] + #dict_links = h.get_all_links(l_pastes) + + zip_file = h.create_domain_basic_archive(l_pastes) + filename = domain + '.zip' + + return send_file(zip_file, attachment_filename=filename, as_attachment=True) + + @hiddenServices.route("/hiddenServices/onion_son", methods=['GET']) @login_required def onion_son(): diff --git a/var/www/modules/hiddenServices/templates/showDomain.html b/var/www/modules/hiddenServices/templates/showDomain.html index 22224fd0..4230ec5a 100644 --- a/var/www/modules/hiddenServices/templates/showDomain.html +++ b/var/www/modules/hiddenServices/templates/showDomain.html @@ -53,12 +53,14 @@ First Seen Last Check + Ports {{ first_seen }} {{ last_check }} + {{ ports }} @@ -119,6 +121,42 @@ {%endif%} + +
+ + + +
+ + {% if domain_history %} + + + + + + + + {% for epoch_item in domain_history %} + + + {% endfor %} + +
Domain History
+ +
+
{{domain}}
+ {% if epoch_item[2] %} +
UP
+ {% else %} +
DOWN
+ {% endif %} +
{{ epoch_item[1] }}
+
+ +
+ {%endif%} +
diff --git a/var/www/static/js/indexjavascript.js b/var/www/static/js/indexjavascript.js index aead7181..86d00cbb 100644 --- a/var/www/static/js/indexjavascript.js +++ b/var/www/static/js/indexjavascript.js @@ -201,92 +201,94 @@ function create_log_table(obj_json) { return; } - if( chansplit[1] == "INFO" ){ - tr.className = "info"; - } - else if ( chansplit[1] == "WARNING" ){ - tr.className = "warning"; - } - else if ( chansplit[1] == "CRITICAL"){ - tr.className = "danger" - } + if( parsedmess.length>2 ){ + if( chansplit[1] == "INFO" ){ + tr.className = "table-disabled"; + } + else if ( chansplit[1] == "WARNING" ){ + tr.className = "table-log-warning"; + } + else if ( chansplit[1] == "CRITICAL"){ + tr.className = "table-danger" + } - source_link = document.createElement("A"); - if (parsedmess[1] == "slexy.org"){ - source_url = "http://"+parsedmess[1]+"/view/"+parsedmess[3].split(".")[0]; - } - else{ - source_url = "http://"+parsedmess[1]+"/"+parsedmess[3].split(".")[0]; - } - source_link.setAttribute("HREF",source_url); - source_link.setAttribute("TARGET", "_blank"); - source_link.appendChild(document.createTextNode(parsedmess[1])); + source_link = document.createElement("A"); + if (parsedmess[1] == "slexy.org"){ + source_url = "http://"+parsedmess[1]+"/view/"+parsedmess[3].split(".")[0]; + } + else{ + source_url = "http://"+parsedmess[1]+"/"+parsedmess[3].split(".")[0]; + } + source_link.setAttribute("HREF",source_url); + source_link.setAttribute("TARGET", "_blank"); + source_link.appendChild(document.createTextNode(parsedmess[1])); - src.appendChild(source_link); + src.appendChild(source_link); - var now = new Date(); - var timepaste = pad_2(now.getHours()) + ":" + pad_2(now.getMinutes()) + ":" + pad_2(now.getSeconds()); + var now = new Date(); + var timepaste = pad_2(now.getHours()) + ":" + pad_2(now.getMinutes()) + ":" + pad_2(now.getSeconds()); - time.appendChild(document.createTextNode(timepaste)); - chan.appendChild(document.createTextNode(chansplit[0])); - level.appendChild(document.createTextNode(chansplit[1])); + time.appendChild(document.createTextNode(timepaste)); + chan.appendChild(document.createTextNode(chansplit[0])); + level.appendChild(document.createTextNode(chansplit[1])); - scrpt.appendChild(document.createTextNode(parsedmess[0])); - pdate.appendChild(document.createTextNode(parsedmess[2])); - nam.appendChild(document.createTextNode(parsedmess[3])); + scrpt.appendChild(document.createTextNode(parsedmess[0])); + pdate.appendChild(document.createTextNode(parsedmess[2])); + nam.appendChild(document.createTextNode(parsedmess[3])); - var iconspan = document.createElement('SPAN'); - if (parsedmess[4].split(" ")[0] == "Detected"){ - iconspan.className = "glyphicon glyphicon-eye-open"; - } - else if (parsedmess[4].split(" ")[0] == "Checked"){ - iconspan.className = "glyphicon glyphicon-thumbs-up"; - } - iconspan.innerHTML = " "; - msage.appendChild(iconspan); - var message = parsedmess[4].split(" "); - message.shift(); + var iconspan = document.createElement('SPAN'); + if (parsedmess[4].split(" ")[0] == "Detected"){ + iconspan.className = "fas fa-eye"; + } + else if (parsedmess[4].split(" ")[0] == "Checked"){ + iconspan.className = "far fa-thumbs-up"; + } + iconspan.innerHTML = " "; + msage.appendChild(iconspan); + var message = parsedmess[4].split(" "); + message.shift(); - msage.appendChild(document.createTextNode(message.join(" "))); + msage.appendChild(document.createTextNode(message.join(" "))); - var paste_path = parsedmess[5]; - var url_to_saved_paste = url_showSavedPath+"?paste="+paste_path+"&num="+parsedmess[0]; + var paste_path = parsedmess[5]; + var url_to_saved_paste = url_showSavedPath+"?paste="+paste_path+"&num="+parsedmess[0]; - var action_icon_a = document.createElement("A"); - action_icon_a.setAttribute("TARGET", "_blank"); - action_icon_a.setAttribute("HREF", url_to_saved_paste); - var action_icon_span = document.createElement('SPAN'); - action_icon_span.className = "fa fa-search-plus"; - action_icon_a.appendChild(action_icon_span); + var action_icon_a = document.createElement("A"); + action_icon_a.setAttribute("TARGET", "_blank"); + action_icon_a.setAttribute("HREF", url_to_saved_paste); + var action_icon_span = document.createElement('SPAN'); + action_icon_span.className = "fas fa-search-plus"; + action_icon_a.appendChild(action_icon_span); - inspect.appendChild(action_icon_a); - inspect.setAttribute("style", "text-align:center;"); + inspect.appendChild(action_icon_a); + inspect.setAttribute("style", "text-align:center;"); - tr.appendChild(time) - tr.appendChild(chan); - tr.appendChild(level); - tr.appendChild(scrpt); - tr.appendChild(src); - tr.appendChild(pdate); - tr.appendChild(nam); - tr.appendChild(msage); - tr.appendChild(inspect); + tr.appendChild(time) + tr.appendChild(chan); + tr.appendChild(level); + tr.appendChild(scrpt); + tr.appendChild(src); + tr.appendChild(pdate); + tr.appendChild(nam); + tr.appendChild(msage); + tr.appendChild(inspect); - if (tr.className == document.getElementById("checkbox_log_info").value && document.getElementById("checkbox_log_info").checked == true) { - tableBody.appendChild(tr); - } - if (tr.className == document.getElementById("checkbox_log_warning").value && document.getElementById("checkbox_log_warning").checked == true) { - tableBody.appendChild(tr); - } - if (tr.className == document.getElementById("checkbox_log_critical").value && document.getElementById("checkbox_log_critical").checked == true) { - tableBody.appendChild(tr); - }; + if (chansplit[1] == document.getElementById("checkbox_log_info").value && document.getElementById("checkbox_log_info").checked == true) { + tableBody.appendChild(tr); + } + if (chansplit[1] == document.getElementById("checkbox_log_warning").value && document.getElementById("checkbox_log_warning").checked == true) { + tableBody.appendChild(tr); + } + if (chansplit[1] == document.getElementById("checkbox_log_critical").value && document.getElementById("checkbox_log_critical").checked == true) { + tableBody.appendChild(tr); + }; - var sel = document.getElementById("log_select") - if (tableBody.rows.length > sel.options[sel.options.selectedIndex].value) { - while (tableBody.rows.length != sel.options[sel.options.selectedIndex].value){ - tableBody.deleteRow(0); - } + var sel = document.getElementById("log_select") + if (tableBody.rows.length > sel.options[sel.options.selectedIndex].value) { + while (tableBody.rows.length != sel.options[sel.options.selectedIndex].value){ + tableBody.deleteRow(0); + } + } } } @@ -294,7 +296,7 @@ function create_queue_table() { document.getElementById("queueing").innerHTML = ""; var Tablediv = document.getElementById("queueing") var table = document.createElement('TABLE') - table.className = "table table-bordered table-hover table-striped tableQueue"; + table.className = "table table-bordered table-hover tableQueue"; var tableHead = document.createElement('THEAD') var tableBody = document.createElement('TBODY') @@ -323,8 +325,8 @@ function create_queue_table() { var td2 = document.createElement('TD'); td.appendChild(document.createTextNode("No running queues")); td2.appendChild(document.createTextNode("Or no feed")); - td.className += " danger"; - td2.className += " danger"; + td.className += " table-danger"; + td2.className += " table-danger"; tr.appendChild(td); tr.appendChild(td2); tableBody.appendChild(tr); @@ -345,11 +347,11 @@ function create_queue_table() { // - j=2: LastProcessedPasteTime // - j=3: Number of the module belonging in the same category if (parseInt(glob_tabvar.row1[i][2]) > window.threshold_stucked_module && parseInt(glob_tabvar.row1[i][1]) > 2) - tr.className += " danger"; + tr.className += " table-danger"; else if (parseInt(glob_tabvar.row1[i][1]) == 0) - tr.className += " warning"; + tr.className += " table-disabled"; else - tr.className += " success"; + tr.className += " table-success"; tableBody.appendChild(tr); } } @@ -474,29 +476,12 @@ function load_queues() { g.updateOptions( { 'file': data } ); g2.updateOptions( { 'file': data2 } ); - - // TagCanvas.Reload('myCanvas'); - }, interval); } }); }; refresh(); - - try { - var options = { - weight:true, - weightMode:"both", - noMouse:true, - textColour: '#2E9AFE' - } - TagCanvas.Start('myCanvas','',options); - TagCanvas.SetSpeed('myCanvas', [0.05, -0.15]); - } catch(e) { - // something went wrong, hide the canvas container - document.getElementById('myCanvasContainer').style.display = 'none'; - } } function manage_undefined() { diff --git a/var/www/templates/dashboard/menu_sidebar.html b/var/www/templates/dashboard/menu_sidebar.html new file mode 100644 index 00000000..b1abd954 --- /dev/null +++ b/var/www/templates/dashboard/menu_sidebar.html @@ -0,0 +1,41 @@ +
+ + + + + + +
+
+ Total pastes since {{ default_minute }} min +
+
+
+
+
+ +
+ + +
+
+ +
+
+ + + + + + +
Idling queues
 Working queues
 Stuck queues
+
+
+
+ +
diff --git a/var/www/templates/nav_bar.html b/var/www/templates/nav_bar.html index 77d595b3..c1cb18ba 100644 --- a/var/www/templates/nav_bar.html +++ b/var/www/templates/nav_bar.html @@ -10,7 +10,7 @@