From aee1b50ef1397a8c8cacadf66708564b60fdfe78 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 15 Jun 2018 17:25:43 +0200 Subject: [PATCH] fix duplicate export attribute + tag export whitelist --- bin/MISP_The_Hive_feeder.py | 59 ++-- bin/ailleakObject.py | 22 +- var/www/Flask_server.py | 13 + var/www/modules/Flask_config.py | 15 +- .../modules/PasteSubmit/Flask_PasteSubmit.py | 139 ++++++++- .../templates/edit_tag_export.html | 264 ++++++++++++++++++ var/www/modules/search/Flask_search.py | 1 - var/www/modules/search/templates/search.html | 2 +- var/www/modules/showpaste/Flask_showpaste.py | 13 +- .../showpaste/templates/show_saved_paste.html | 4 + 10 files changed, 483 insertions(+), 49 deletions(-) create mode 100644 var/www/modules/PasteSubmit/templates/edit_tag_export.html diff --git a/bin/MISP_The_Hive_feeder.py b/bin/MISP_The_Hive_feeder.py index 9cdb562e..38b1e8c4 100755 --- a/bin/MISP_The_Hive_feeder.py +++ b/bin/MISP_The_Hive_feeder.py @@ -38,7 +38,10 @@ except: # import The Hive Keys try: from theHiveKEYS import the_hive_url, the_hive_key - flag_the_hive = True + if the_hive_url == '': + flag_the_hive = False + else: + flag_the_hive = True except: print('The HIVE keys not present') flag_the_hive = False @@ -73,16 +76,18 @@ def create_the_hive_alert(source, path, content, tag): # Create the Alert id = None - response = HiveApi.create_alert(alert) - if response.status_code == 201: - #print(json.dumps(response.json(), indent=4, sort_keys=True)) - print('Alert Created') - print('') - id = response.json()['id'] - else: - print('ko: {}/{}'.format(response.status_code, response.text)) - return 0 - + try: + response = HiveApi.create_alert(alert) + if response.status_code == 201: + #print(json.dumps(response.json(), indent=4, sort_keys=True)) + print('Alert Created') + print('') + id = response.json()['id'] + else: + print('ko: {}/{}'.format(response.status_code, response.text)) + return 0 + except: + print('hive connection error') if __name__ == "__main__": @@ -121,14 +126,14 @@ if __name__ == "__main__": p = Process(config_section) # create MISP connection if flag_misp: - #try: - pymisp = PyMISP(misp_url, misp_key, misp_verifycert) - misp_wrapper = ailleakObject.ObjectWrapper(pymisp) - r_serv_db.set('ail:misp', True) - print('Connected to MISP:', misp_url) - #except: - #flag_misp = False - #print('Not connected to MISP') + try: + pymisp = PyMISP(misp_url, misp_key, misp_verifycert) + misp_wrapper = ailleakObject.ObjectWrapper(pymisp) + r_serv_db.set('ail:misp', True) + print('Connected to MISP:', misp_url) + except: + flag_misp = False + print('Not connected to MISP') # create The HIVE connection if flag_the_hive: @@ -158,8 +163,18 @@ if __name__ == "__main__": full_path = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes"), path) - if HiveApi != False: - create_the_hive_alert(source, path, full_path, tag) + if HiveApi != False: + if int(r_serv_db.get('hive:auto-alerts')) == 1: + whitelist_hive = r_serv_db.scard('whitelist_hive') + if r_serv_db.scard('whitelist_hive') == 0 or r_serv_db.sismember('whitelist_hive', tag): + create_the_hive_alert(source, path, full_path, tag) + + else: + print('hive, auto alerts creation disable') if flag_misp: - misp_wrapper.pushToMISP(uuid_ail, path, tag) + if int(r_serv_db.get('misp:auto-events')) == 1: + if r_serv_db.scard('whitelist_misp') == 0 or r_serv_db.sismember('whitelist_misp', tag): + misp_wrapper.pushToMISP(uuid_ail, path, tag) + else: + print('misp, auto events creation disable') diff --git a/bin/ailleakObject.py b/bin/ailleakObject.py index d7158edb..e2ff2850 100755 --- a/bin/ailleakObject.py +++ b/bin/ailleakObject.py @@ -51,29 +51,29 @@ class ObjectWrapper: self.p_content = self.paste.get_p_content() self.p_tag = tag - '''print(path) temp = self.paste._get_p_duplicate() #beautifier if not temp: temp = '' - temp = json.dumps(temp) - print(temp) - self.p_duplicate_number = len(temp) if len(temp) >= 0 else 0 + p_duplicate_number = len(temp) if len(temp) >= 0 else 0 + to_ret = "" - for dup in temp[:self.maxDuplicateToPushToMISP]: - print(dup) + for dup in temp[:10]: + dup = dup.replace('\'','\"').replace('(','[').replace(')',']') + dup = json.loads(dup) algo = dup[0] path = dup[1].split('/')[-6:] path = '/'.join(path)[:-3] # -3 removes .gz - perc = dup[2] + if algo == 'tlsh': + perc = 100 - int(dup[2]) + else: + perc = dup[2] to_ret += "{}: {} [{}%]\n".format(path, algo, perc) - self.p_duplicate = to_ret''' - self.p_duplicate = "" - self.p_duplicate_number = 0 + p_duplicate = to_ret - self.mispObject = AilLeakObject(self.uuid_ail, self.p_source, self.p_date, self.p_content, self.p_duplicate, self.p_duplicate_number) + self.mispObject = AilLeakObject(self.uuid_ail, self.p_source, self.p_date, self.p_content, p_duplicate, p_duplicate_number) def date_to_str(self, date): return "{0}-{1}-{2}".format(date.year, date.month, date.day) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 85353a7d..8c97761e 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -135,6 +135,19 @@ for tag in taxonomies.get('gdpr').machinetags(): for tag in taxonomies.get('fpf').machinetags(): r_serv_tags.sadd('active_tag_fpf', tag) +# ========== INITIAL tags auto export ============ +r_serv_db = redis.StrictRedis( + host=cfg.get("ARDB_DB", "host"), + port=cfg.getint("ARDB_DB", "port"), + db=cfg.getint("ARDB_DB", "db"), + decode_responses=True) +infoleak_tags = taxonomies.get('infoleak').machinetags() +infoleak_automatic_tags = [] +for tag in taxonomies.get('infoleak').machinetags(): + if tag.split('=')[0][:] == 'infoleak:automatic-detection': + r_serv_db.sadd('list_export_tags', tag) + +r_serv_db.sadd('list_export_tags', 'submitted') # ============ MAIN ============ if __name__ == "__main__": diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index eb8a542e..b377cbc5 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -101,19 +101,24 @@ try: print('Misp connected') except: print('Misp not connected') - pymisp = None + pymisp = False misp_event_url = '#' # The Hive # from thehive4py.api import TheHiveApi import thehive4py.exceptions try: from theHiveKEYS import the_hive_url, the_hive_key - HiveApi = TheHiveApi(the_hive_url, the_hive_key) - hive_case_url = the_hive_url+'/index.html#/case/id_here/details' - print('The Hive connected') + if the_hive_url == '': + HiveApi = False + hive_case_url = '#' + print('The HIVE not connected') + else: + HiveApi = TheHiveApi(the_hive_url, the_hive_key) + hive_case_url = the_hive_url+'/index.html#/case/id_here/details' + print('The Hive connected') except: print('The HIVE not connected') - HiveApi = None + HiveApi = False hive_case_url = '#' # VARIABLES # diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index f5fe00b5..dab642ac 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -5,7 +5,7 @@ Flask functions and routes for the trending modules page ''' import redis -from flask import Flask, render_template, jsonify, request, Blueprint, url_for +from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect import unicodedata import string @@ -16,6 +16,7 @@ import datetime import uuid from io import BytesIO from Date import Date +import json import Paste @@ -140,6 +141,28 @@ def misp_create_event(distribution, threat_level_id, analysis, info, l_tags, pat ail_uuid = r_serv_db.get('ail:uuid') pseudofile = BytesIO(paste.get_p_content().encode()) + temp = paste._get_p_duplicate() + + #beautifier + if not temp: + temp = '' + + p_duplicate_number = len(temp) if len(temp) >= 0 else 0 + + to_ret = "" + for dup in temp[:10]: + dup = dup.replace('\'','\"').replace('(','[').replace(')',']') + dup = json.loads(dup) + algo = dup[0] + path = dup[1].split('/')[-6:] + path = '/'.join(path)[:-3] # -3 removes .gz + if algo == 'tlsh': + perc = 100 - int(dup[2]) + else: + perc = dup[2] + to_ret += "{}: {} [{}%]\n".format(path, algo, perc) + p_duplicate = to_ret + today = datetime.date.today() # [0-3] published = False @@ -167,6 +190,10 @@ def misp_create_event(distribution, threat_level_id, analysis, info, l_tags, pat # FIXME TODO: delete this leak_obj.add_attribute('type', value='Onion', type='text') + if p_duplicate_number > 0: + leak_obj.add_attribute('duplicate', value=p_duplicate, type='text') + leak_obj.add_attribute('duplicate_number', value=p_duplicate_number, type='counter') + try: templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == obj_name][0] except IndexError: @@ -176,11 +203,8 @@ def misp_create_event(distribution, threat_level_id, analysis, info, l_tags, pat if 'errors' in r: return False else: - #if self._p_duplicate_number > 0: - #event.add_attribute('duplicate', value=self._p_duplicate, type='text') - #event.add_attribute('duplicate_number', value=self._p_duplicate_number, type='counter') event_url = misp_event_url + eventid - return eventid + return event_url def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, l_tags, path): @@ -403,8 +427,11 @@ def create_misp_event(): l_tags = list(r_serv_metadata.smembers('tag:'+path)) event = misp_create_event(distribution, threat_level_id, analysis, info, l_tags, path) - - return event + if event != False: + return redirect(event) + else: + return 'error' + return 'error' @PasteSubmit.route("/PasteSubmit/create_hive_case", methods=['POST']) def create_hive_case(): @@ -421,8 +448,104 @@ def create_hive_case(): l_tags = list(r_serv_metadata.smembers('tag:'+path)) case = hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, l_tags, path) + if case != False: + return redirect(case) + else: + return 'error' - return case + return 'error' + +@PasteSubmit.route("/PasteSubmit/edit_tag_export") +def edit_tag_export(): + misp_auto_events = r_serv_db.get('misp:auto-events') + hive_auto_alerts = r_serv_db.get('hive:auto-alerts') + + whitelist_misp = r_serv_db.scard('whitelist_misp') + whitelist_hive = r_serv_db.scard('whitelist_hive') + + list_export_tags = list(r_serv_db.smembers('list_export_tags')) + status_misp = [] + status_hive = [] + + # empty whitelist + if whitelist_misp == 0: + for tag in list_export_tags: + status_misp.append(True) + else: + for tag in list_export_tags: + if r_serv_db.sismember('whitelist_misp', tag): + status_misp.append(True) + else: + status_misp.append(False) + + # empty whitelist + if whitelist_hive == 0: + for tag in list_export_tags: + status_hive.append(True) + else: + for tag in list_export_tags: + if r_serv_db.sismember('whitelist_hive', tag): + status_hive.append(True) + else: + status_hive.append(False) + + if int(misp_auto_events) == 1: + misp_active = True + else: + misp_active = False + if int(hive_auto_alerts) == 1: + hive_active = True + else: + hive_active = False + return render_template("edit_tag_export.html", + misp_active=misp_active, + hive_active=hive_active, + list_export_tags=list_export_tags, + status_misp=status_misp, + status_hive=status_hive) + +@PasteSubmit.route("/PasteSubmit/tag_export_edited", methods=['POST']) +def tag_export_edited(): + tag_enabled_misp = request.form.getlist('tag_enabled_misp') + tag_enabled_hive = request.form.getlist('tag_enabled_hive') + + list_export_tags = list(r_serv_db.smembers('list_export_tags')) + + r_serv_db.delete('whitelist_misp') + r_serv_db.delete('whitelist_hive') + + for tag in tag_enabled_misp: + if r_serv_db.sismember('list_export_tags', tag): + r_serv_db.sadd('whitelist_misp', tag) + else: + return 'invalid input' + + for tag in tag_enabled_hive: + if r_serv_db.sismember('list_export_tags', tag): + r_serv_db.sadd('whitelist_hive', tag) + else: + return 'invalid input' + return redirect(url_for('PasteSubmit.edit_tag_export')) + +@PasteSubmit.route("/PasteSubmit/enable_misp_auto_event") +def enable_misp_auto_event(): + r_serv_db.set('misp:auto-events', 1) + return edit_tag_export() + +@PasteSubmit.route("/PasteSubmit/disable_misp_auto_event") +def disable_misp_auto_event(): + r_serv_db.set('misp:auto-events', 0) + return edit_tag_export() + +@PasteSubmit.route("/PasteSubmit/enable_hive_auto_alert") +def enable_hive_auto_alert(): + r_serv_db.set('hive:auto-alerts', 1) + return edit_tag_export() + +@PasteSubmit.route("/PasteSubmit/disable_hive_auto_alert") +def disable_hive_auto_alert(): + r_serv_db.set('hive:auto-alerts', 0) + return edit_tag_export() # ========= REGISTRATION ========= app.register_blueprint(PasteSubmit) diff --git a/var/www/modules/PasteSubmit/templates/edit_tag_export.html b/var/www/modules/PasteSubmit/templates/edit_tag_export.html new file mode 100644 index 00000000..8f0ddb35 --- /dev/null +++ b/var/www/modules/PasteSubmit/templates/edit_tag_export.html @@ -0,0 +1,264 @@ + + + + + + + + Analysis Information Leak framework + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + +
+
+
+
+ +
+ + +
+
+
+ +
+
MISP Auto Event Creation + {% if misp_active %} + Enabled + {% endif %} + {% if not misp_active %} + Disabled + {% endif %} +
+
+ +
+ +
+
+ + {% if misp_active %} + + Disable Event Creation + + {% endif %} + {% if not misp_active %} + + Enable Event Creation + + {% endif %} +
+
+
+ +
+
+
The hive auto export + {% if hive_active %} + Enabled +     + {{ badge }} + {% endif %} + {% if not hive_active %} + Disabled + {% endif %} +
+
+ +
+ +
+ + {% if hive_active %} + + Disable Alert Creation + + {% endif %} + {% if not hive_active %} + + Enable Alert Creation + + {% endif %} +
+
+
+ +
+
+ +
+ + +
+
+ +
+ +
+
+ Metadata : +
  • + + + + + + + + + + {% for tag in list_export_tags %} + + + + + {% endfor %} + + +
    WhitelistTag
    + {% if status_misp[loop.index0] %} +
    Enabled
    + + {% endif %} + {% if not status_misp[loop.index0] %} +
    Disabled
    + + {% endif %} +     		{{ tag }}
    +
+
+
+ +
+ +
+
+ Metadata : +
  • + + + + + + + + + + + {% for tag in list_export_tags %} + + + + + {% endfor %} + + +
    WhitelistTag
    + {% if status_hive[loop.index0] %} +
    Enabled
    + + {% endif %} + {% if not status_hive[loop.index0] %} +
    Disabled
    + + {% endif %} +     		{{ tag }}
    +
+
+
+ +
+
+ + +
+ +
+ +
+ +
+ + + + + + + + + + diff --git a/var/www/modules/search/Flask_search.py b/var/www/modules/search/Flask_search.py index 818e3e61..9ff9343d 100644 --- a/var/www/modules/search/Flask_search.py +++ b/var/www/modules/search/Flask_search.py @@ -122,7 +122,6 @@ def search(): # Search full line schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) - print(selected_index) ix = index.open_dir(selected_index) with ix.searcher() as searcher: query = QueryParser("content", ix.schema).parse(" ".join(q)) diff --git a/var/www/modules/search/templates/search.html b/var/www/modules/search/templates/search.html index bed826dd..3c7e3472 100644 --- a/var/www/modules/search/templates/search.html +++ b/var/www/modules/search/templates/search.html @@ -66,7 +66,7 @@
- {{ r|length }} Results for "{{ query }} + {{ r|length }} Results for "{{ query }}"
diff --git a/var/www/modules/showpaste/Flask_showpaste.py b/var/www/modules/showpaste/Flask_showpaste.py index c54c9f19..4a6e87d3 100644 --- a/var/www/modules/showpaste/Flask_showpaste.py +++ b/var/www/modules/showpaste/Flask_showpaste.py @@ -114,7 +114,18 @@ def showpaste(content_range): else: list_tags.append( (tag, False) ) - return render_template("show_saved_paste.html", date=p_date, bootstrap_label=bootstrap_label, active_taxonomies=active_taxonomies, active_galaxies=active_galaxies, list_tags=list_tags, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list) + if Flask_config.pymisp is False: + misp = False + else: + misp = True + + if Flask_config.HiveApi is False: + hive = False + else: + hive = True + + return render_template("show_saved_paste.html", date=p_date, bootstrap_label=bootstrap_label, active_taxonomies=active_taxonomies, active_galaxies=active_galaxies, list_tags=list_tags, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list, + misp=misp, hive=hive) # ============ ROUTES ============ diff --git a/var/www/modules/showpaste/templates/show_saved_paste.html b/var/www/modules/showpaste/templates/show_saved_paste.html index 513572db..85d7c7aa 100644 --- a/var/www/modules/showpaste/templates/show_saved_paste.html +++ b/var/www/modules/showpaste/templates/show_saved_paste.html @@ -171,6 +171,7 @@
+ {% if misp %}
+ {% endif %} + {% if hive %}
+ {% endif %}