fix duplicate export attribute + tag export whitelist

bin/MISP_The_Hive_feeder.py

@@ -38,6 +38,9 @@ except:
 # import The Hive Keys
 try:
     from theHiveKEYS import the_hive_url, the_hive_key
+    if the_hive_url == '':
+        flag_the_hive = False
+    else:
         flag_the_hive = True
 except:
     print('The HIVE keys not present')
@@ -73,6 +76,7 @@ def create_the_hive_alert(source, path, content, tag):
 
     # Create the Alert
     id = None
+    try:
         response = HiveApi.create_alert(alert)
         if response.status_code == 201:
             #print(json.dumps(response.json(), indent=4, sort_keys=True))
@@ -82,7 +86,8 @@ def create_the_hive_alert(source, path, content, tag):
         else:
             print('ko: {}/{}'.format(response.status_code, response.text))
             return 0
-
+    except:
+        print('hive connection error')
 
 if __name__ == "__main__":
 
@@ -121,14 +126,14 @@ if __name__ == "__main__":
     p = Process(config_section)
     # create MISP connection
     if flag_misp:
-        #try:
+        try:
             pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
             misp_wrapper = ailleakObject.ObjectWrapper(pymisp)
             r_serv_db.set('ail:misp', True)
             print('Connected to MISP:', misp_url)
-        #except:
+        except:
-            #flag_misp = False
+            flag_misp = False
-            #print('Not connected to MISP')
+            print('Not connected to MISP')
 
     # create The HIVE connection
     if flag_the_hive:
@@ -158,8 +163,18 @@ if __name__ == "__main__":
                 full_path = os.path.join(os.environ['AIL_HOME'],
                                         p.config.get("Directories", "pastes"), path)
 
+
                 if HiveApi != False:
+                    if int(r_serv_db.get('hive:auto-alerts')) == 1:
+                        whitelist_hive = r_serv_db.scard('whitelist_hive')
+                        if r_serv_db.scard('whitelist_hive') == 0 or r_serv_db.sismember('whitelist_hive', tag):
                             create_the_hive_alert(source, path, full_path, tag)
 
+                    else:
+                        print('hive, auto alerts creation disable')
                 if flag_misp:
+                    if int(r_serv_db.get('misp:auto-events')) == 1:
+                        if r_serv_db.scard('whitelist_misp') == 0 or r_serv_db.sismember('whitelist_misp', tag):
                             misp_wrapper.pushToMISP(uuid_ail, path, tag)
+                    else:
+                        print('misp, auto events creation disable')

bin/ailleakObject.py

@@ -51,29 +51,29 @@ class ObjectWrapper:
         self.p_content = self.paste.get_p_content()
         self.p_tag = tag
 
-        '''print(path)
         temp = self.paste._get_p_duplicate()
 
         #beautifier
         if not temp:
             temp = ''
 
-        temp = json.dumps(temp)
+        p_duplicate_number = len(temp) if len(temp) >= 0 else 0
-        print(temp)
+
-        self.p_duplicate_number = len(temp) if len(temp) >= 0 else 0
         to_ret = ""
-        for dup in temp[:self.maxDuplicateToPushToMISP]:
+        for dup in temp[:10]:
-            print(dup)
+            dup = dup.replace('\'','\"').replace('(','[').replace(')',']')
+            dup = json.loads(dup)
             algo = dup[0]
             path = dup[1].split('/')[-6:]
             path = '/'.join(path)[:-3] # -3 removes .gz
+            if algo == 'tlsh':
+                perc = 100 - int(dup[2])
+            else:
                 perc = dup[2]
             to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
-        self.p_duplicate = to_ret'''
+        p_duplicate = to_ret
-        self.p_duplicate = ""
-        self.p_duplicate_number = 0
 
-        self.mispObject = AilLeakObject(self.uuid_ail, self.p_source, self.p_date, self.p_content, self.p_duplicate, self.p_duplicate_number)
+        self.mispObject = AilLeakObject(self.uuid_ail, self.p_source, self.p_date, self.p_content, p_duplicate, p_duplicate_number)
 
     def date_to_str(self, date):
         return "{0}-{1}-{2}".format(date.year, date.month, date.day)

var/www/Flask_server.py

@@ -135,6 +135,19 @@ for tag in taxonomies.get('gdpr').machinetags():
 for tag in taxonomies.get('fpf').machinetags():
     r_serv_tags.sadd('active_tag_fpf', tag)
 
+# ========== INITIAL tags auto export ============
+r_serv_db = redis.StrictRedis(
+    host=cfg.get("ARDB_DB", "host"),
+    port=cfg.getint("ARDB_DB", "port"),
+    db=cfg.getint("ARDB_DB", "db"),
+    decode_responses=True)
+infoleak_tags = taxonomies.get('infoleak').machinetags()
+infoleak_automatic_tags = []
+for tag in taxonomies.get('infoleak').machinetags():
+    if tag.split('=')[0][:] == 'infoleak:automatic-detection':
+        r_serv_db.sadd('list_export_tags', tag)
+
+r_serv_db.sadd('list_export_tags', 'submitted')
 # ============ MAIN ============
 
 if __name__ == "__main__":

var/www/modules/Flask_config.py

@@ -101,19 +101,24 @@ try:
     print('Misp connected')
 except:
     print('Misp not connected')
-    pymisp = None
+    pymisp = False
     misp_event_url = '#'
 # The Hive #
 from thehive4py.api import TheHiveApi
 import thehive4py.exceptions
 try:
     from theHiveKEYS import the_hive_url, the_hive_key
+    if the_hive_url == '':
+        HiveApi = False
+        hive_case_url = '#'
+        print('The HIVE not connected')
+    else:
         HiveApi = TheHiveApi(the_hive_url, the_hive_key)
         hive_case_url = the_hive_url+'/index.html#/case/id_here/details'
         print('The Hive connected')
 except:
     print('The HIVE not connected')
-    HiveApi = None
+    HiveApi = False
     hive_case_url = '#'
 
 # VARIABLES #

var/www/modules/PasteSubmit/Flask_PasteSubmit.py

@@ -5,7 +5,7 @@
     Flask functions and routes for the trending modules page
 '''
 import redis
-from flask import Flask, render_template, jsonify, request, Blueprint, url_for
+from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect
 
 import unicodedata
 import string
@@ -16,6 +16,7 @@ import datetime
 import uuid
 from io import BytesIO
 from Date import Date
+import json
 
 import Paste
 
@@ -140,6 +141,28 @@ def misp_create_event(distribution, threat_level_id, analysis, info, l_tags, pat
     ail_uuid = r_serv_db.get('ail:uuid')
     pseudofile = BytesIO(paste.get_p_content().encode())
 
+    temp = paste._get_p_duplicate()
+
+    #beautifier
+    if not temp:
+        temp = ''
+
+    p_duplicate_number = len(temp) if len(temp) >= 0 else 0
+
+    to_ret = ""
+    for dup in temp[:10]:
+        dup = dup.replace('\'','\"').replace('(','[').replace(')',']')
+        dup = json.loads(dup)
+        algo = dup[0]
+        path = dup[1].split('/')[-6:]
+        path = '/'.join(path)[:-3] # -3 removes .gz
+        if algo == 'tlsh':
+            perc = 100 - int(dup[2])
+        else:
+            perc = dup[2]
+        to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
+    p_duplicate = to_ret
+
     today = datetime.date.today()
     # [0-3]
     published = False
@@ -167,6 +190,10 @@ def misp_create_event(distribution, threat_level_id, analysis, info, l_tags, pat
     # FIXME TODO: delete this
     leak_obj.add_attribute('type', value='Onion', type='text')
 
+    if p_duplicate_number > 0:
+        leak_obj.add_attribute('duplicate', value=p_duplicate, type='text')
+        leak_obj.add_attribute('duplicate_number', value=p_duplicate_number, type='counter')
+
     try:
         templateID = [x['ObjectTemplate']['id'] for x in pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == obj_name][0]
     except IndexError:
@@ -176,11 +203,8 @@ def misp_create_event(distribution, threat_level_id, analysis, info, l_tags, pat
     if 'errors' in r:
         return False
     else:
-    #if self._p_duplicate_number > 0:
-        #event.add_attribute('duplicate', value=self._p_duplicate, type='text')
-        #event.add_attribute('duplicate_number', value=self._p_duplicate_number, type='counter')
         event_url = misp_event_url + eventid
-        return eventid
+        return event_url
 
 def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, l_tags, path):
 
@@ -403,8 +427,11 @@ def create_misp_event():
         l_tags = list(r_serv_metadata.smembers('tag:'+path))
         event = misp_create_event(distribution, threat_level_id, analysis, info, l_tags, path)
 
-
+        if event != False:
-    return event
+            return redirect(event)
+        else:
+            return 'error'
+    return 'error'
 
 @PasteSubmit.route("/PasteSubmit/create_hive_case", methods=['POST'])
 def create_hive_case():
@@ -421,8 +448,104 @@ def create_hive_case():
         l_tags = list(r_serv_metadata.smembers('tag:'+path))
         case = hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title, l_tags, path)
 
+        if case != False:
+            return redirect(case)
+        else:
+            return 'error'
 
-    return case
+    return 'error'
+
+@PasteSubmit.route("/PasteSubmit/edit_tag_export")
+def edit_tag_export():
+    misp_auto_events = r_serv_db.get('misp:auto-events')
+    hive_auto_alerts = r_serv_db.get('hive:auto-alerts')
+
+    whitelist_misp = r_serv_db.scard('whitelist_misp')
+    whitelist_hive = r_serv_db.scard('whitelist_hive')
+
+    list_export_tags = list(r_serv_db.smembers('list_export_tags'))
+    status_misp = []
+    status_hive = []
+
+    # empty whitelist
+    if whitelist_misp == 0:
+        for tag in list_export_tags:
+            status_misp.append(True)
+    else:
+        for tag in list_export_tags:
+            if r_serv_db.sismember('whitelist_misp', tag):
+                status_misp.append(True)
+            else:
+                status_misp.append(False)
+
+    # empty whitelist
+    if whitelist_hive == 0:
+        for tag in list_export_tags:
+            status_hive.append(True)
+    else:
+        for tag in list_export_tags:
+            if r_serv_db.sismember('whitelist_hive', tag):
+                status_hive.append(True)
+            else:
+                status_hive.append(False)
+
+    if int(misp_auto_events) == 1:
+        misp_active = True
+    else:
+        misp_active = False
+    if int(hive_auto_alerts)  == 1:
+        hive_active = True
+    else:
+        hive_active = False
+    return render_template("edit_tag_export.html",
+                            misp_active=misp_active,
+                            hive_active=hive_active,
+                            list_export_tags=list_export_tags,
+                            status_misp=status_misp,
+                            status_hive=status_hive)
+
+@PasteSubmit.route("/PasteSubmit/tag_export_edited", methods=['POST'])
+def tag_export_edited():
+    tag_enabled_misp = request.form.getlist('tag_enabled_misp')
+    tag_enabled_hive = request.form.getlist('tag_enabled_hive')
+
+    list_export_tags = list(r_serv_db.smembers('list_export_tags'))
+
+    r_serv_db.delete('whitelist_misp')
+    r_serv_db.delete('whitelist_hive')
+
+    for tag in tag_enabled_misp:
+        if r_serv_db.sismember('list_export_tags', tag):
+            r_serv_db.sadd('whitelist_misp', tag)
+        else:
+            return 'invalid input'
+
+    for tag in tag_enabled_hive:
+        if r_serv_db.sismember('list_export_tags', tag):
+            r_serv_db.sadd('whitelist_hive', tag)
+        else:
+            return 'invalid input'
+    return redirect(url_for('PasteSubmit.edit_tag_export'))
+
+@PasteSubmit.route("/PasteSubmit/enable_misp_auto_event")
+def enable_misp_auto_event():
+    r_serv_db.set('misp:auto-events', 1)
+    return edit_tag_export()
+
+@PasteSubmit.route("/PasteSubmit/disable_misp_auto_event")
+def disable_misp_auto_event():
+    r_serv_db.set('misp:auto-events', 0)
+    return edit_tag_export()
+
+@PasteSubmit.route("/PasteSubmit/enable_hive_auto_alert")
+def enable_hive_auto_alert():
+    r_serv_db.set('hive:auto-alerts', 1)
+    return edit_tag_export()
+
+@PasteSubmit.route("/PasteSubmit/disable_hive_auto_alert")
+def disable_hive_auto_alert():
+    r_serv_db.set('hive:auto-alerts', 0)
+    return edit_tag_export()
 
 # ========= REGISTRATION =========
 app.register_blueprint(PasteSubmit)

var/www/modules/PasteSubmit/templates/edit_tag_export.html

@@ -0,0 +1,264 @@
+<!DOCTYPE html>
+<html>
+
+	<head>
+	  <meta charset="utf-8">
+	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
+
+	  <title>Analysis Information Leak framework</title>
+
+	  <!-- Core CSS -->
+	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+	  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+	  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
+	 <link href="/static//css/dataTables.bootstrap.css" rel="stylesheet" type="text/css" />
+	  <!-- JS -->
+	  <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
+	  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+		<script src="/static//js/bootstrap.min.js"></script>
+	  <script src="/static//js/jquery.dataTables.min.js"></script>
+	  <script src="/static//js/dataTables.bootstrap.js"></script>
+
+		<style>
+			.tooltip-inner {
+				text-align: left;
+				height: 200%;
+				width: 200%;
+				max-width: 500px;
+				max-height: 500px;
+				font-size: 13px;
+			}
+			xmp {
+				white-space:pre-wrap;
+				word-wrap:break-word;
+			}
+			.test thead{
+				background: #d91f2d;
+				color: #fff;
+			}
+		</style>
+
+	</head>
+	<body>
+
+            {% include 'navbar.html' %}
+
+    <div id="page-wrapper">
+			<div class="row">
+				<div class="col-lg-12">
+				</div>
+				<!-- /.col-lg-12 -->
+			</div>
+			<!-- /.row -->
+
+			<div class="container">
+  			<div class="row">
+						<div class="col-md-6">
+
+				      <div class="panel panel-primary">
+				        <div class="panel-heading"> MISP Auto Event Creation
+				          {% if misp_active %}
+				            <span class="label label-success pull-right"> Enabled</span>
+				          {% endif %}
+				          {% if not misp_active %}
+				            <span class="label label-danger pull-right"> Disabled</span>
+				          {% endif %}
+				        </div>
+				        <div class="panel-body">
+
+									<div class="text-center">
+										<img id="misp-logo" src="{{url_for('static', filename='image/misp-logo.png') }}" class="text-center" >
+									</div>
+									<br>
+
+				          {% if misp_active %}
+				          <a href="{{ url_for('PasteSubmit.disable_misp_auto_event') }}" class="btn btn-danger pull-right">
+				            <i class="fa fa-times fa"></i> Disable Event Creation
+				          </a>
+				          {% endif %}
+				          {% if not misp_active %}
+				          <a href="{{ url_for('PasteSubmit.enable_misp_auto_event') }}" class="btn btn-success pull-right">
+				            <i class="fa fa-check-square-o fa"></i> Enable Event Creation
+				          </a>
+				          {% endif %}
+				        </div>
+				      </div>
+						</div>
+
+					<div class="col-md-6">
+						<div class="panel panel-primary" style="min-width: 500px;">
+				      <div class="panel-heading">The hive auto export
+				        {% if hive_active %}
+				          <span class="label label-success pull-right"> Enabled</span>
+									<span class="pull-right">&nbsp;&nbsp;&nbsp;</span>
+									<span class="badge badge-light pull-right">{{ badge }}</span>
+				        {% endif %}
+				        {% if not hive_active %}
+				          <span class="label label-danger pull-right"> Disabled</span>
+				        {% endif %}
+				      </div>
+				      <div class="panel-body">
+
+								<div class="text-center">
+									<img id="misp-logo" src="{{url_for('static', filename='image/thehive-logo.png') }}" width="500">
+								</div>
+
+				        {% if hive_active %}
+				        <a href="{{ url_for('PasteSubmit.disable_hive_auto_alert') }}" class="btn btn-danger pull-right">
+				          <i class="fa fa-times fa"></i> Disable Alert Creation
+				        </a>
+				        {% endif %}
+				        {% if not hive_active %}
+				        <a href="{{ url_for('PasteSubmit.enable_hive_auto_alert') }}" class="btn btn-success pull-right">
+				          <i class="fa fa-check-square-o fa"></i> Enable Alert Creation
+				        </a>
+				        {% endif %}
+				      </div>
+				    </div>
+					</div>
+
+				</div>
+			</div>
+
+      <form action="/PasteSubmit/tag_export_edited" id="checkboxForm" method='post'>
+
+
+				<div class="container">
+	  			<div class="row">
+
+							<div class="col-md-6">
+
+								<div class="panel panel-info">
+									<div class="panel-heading">
+								Metadata :
+							 	<ul class="list-group"><li class="list-group-item">
+									<table class="test table table-striped table-bordered table-hover table-responsive " id="myTable_misp">
+										<thead>
+											<tr>
+												<th>Whitelist</th>
+												<th>Tag</th>
+											</tr>
+										</thead>
+										<tbody>
+
+											{% for tag in list_export_tags %}
+												<tr>
+														<td>
+						                  {% if status_misp[loop.index0] %}
+																<div style="display:none;">Enabled</div>
+						                    <input type="checkbox" value="{{ tag }}" name="tag_enabled_misp" checked>
+															{% endif %}
+						                  {% if not status_misp[loop.index0] %}
+																<div style="display:none;">Disabled</div>
+																<input type="checkbox" value="{{ tag }}" name="tag_enabled_misp" >
+															{% endif %}
+						                </td>
+														<td>{{ tag }}</td>
+												</tr>
+											{%  endfor %}
+
+										</tbody>
+									</table>
+								</li></ul>
+								</div></div>
+							</div>
+
+							<div class="col-md-6">
+
+								<div class="panel panel-info">
+									<div class="panel-heading">
+								Metadata :
+							 	<ul class="list-group"><li class="list-group-item">
+
+									<table class="test table table-striped table-bordered table-hover table-responsive " id="myTable_hive">
+										<thead>
+											<tr>
+												<th>Whitelist</th>
+												<th>Tag</th>
+											</tr>
+										</thead>
+										<tbody>
+
+											{% for tag in list_export_tags %}
+												<tr>
+														<td>
+						                  {% if status_hive[loop.index0] %}
+																<div style="display:none;">Enabled</div>
+						                    <input type="checkbox" value="{{ tag }}" name="tag_enabled_hive" checked>
+															{% endif %}
+						                  {% if not status_hive[loop.index0] %}
+																<div style="display:none;">Disabled</div>
+																<input type="checkbox" value="{{ tag }}" name="tag_enabled_hive" >
+															{% endif %}
+						                </td>
+														<td>{{ tag }}</td>
+												</tr>
+											{%  endfor %}
+
+										</tbody>
+									</table>
+								</li></ul>
+								</div></div>
+							</div>
+
+						</div>
+					</div>
+
+
+	      <div class="input-group-btn text-center">
+	          <button class="btn btn-primary btn-lg" onclick="submitActiveTags()">
+	              <i class="fa fa-check-square-o fa"></i>
+	              Update Tags
+	          </button>
+	      </div>
+
+      </form>
+
+      </div>
+            <!-- /#page-wrapper -->
+
+
+</body>
+
+<script>
+var table_misp
+var table_hive
+$(document).ready(function(){
+
+		table_misp = $('#myTable_misp').DataTable(
+			{
+				"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
+				"iDisplayLength": 10,
+				"order": [[ 1, "asc" ]]
+			}
+		);
+
+		table_hive = $('#myTable_hive').DataTable(
+			{
+				"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
+				"iDisplayLength": 10,
+				"order": [[ 1, "asc" ]]
+			}
+		);
+
+});
+</script>
+
+<script>
+  function submitActiveTags(){
+		table_misp.destroy()
+		table_misp = $('#myTable_misp').DataTable(
+			{
+				"iDisplayLength": -1,
+			}
+		);
+		table_hive.destroy()
+		table_hive = $('#myTable_hive').DataTable(
+			{
+				"iDisplayLength": -1,
+			}
+		);
+  }
+</script>
+
+</html>

var/www/modules/search/Flask_search.py

@@ -122,7 +122,6 @@ def search():
     # Search full line
     schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
 
-    print(selected_index)
     ix = index.open_dir(selected_index)
     with ix.searcher() as searcher:
         query = QueryParser("content", ix.schema).parse(" ".join(q))

var/www/modules/search/templates/search.html

@@ -66,7 +66,7 @@
                     </br>
                     <div class="panel panel-default">
                         <div class="panel-heading">
-                            <i class="glyphicon glyphicon-search"></i> <b id="numberOfRes">{{ r|length }}</b> Results for "<strong>{{ query }}</strong>
+                            <i class="glyphicon glyphicon-search"></i> <b id="numberOfRes">{{ r|length }}</b> Results for "<strong>{{ query }}</strong>"
                             <div class="pull-right">
 
                             </div>

var/www/modules/showpaste/Flask_showpaste.py

@@ -114,7 +114,18 @@ def showpaste(content_range):
         else:
             list_tags.append( (tag, False) )
 
-    return render_template("show_saved_paste.html", date=p_date, bootstrap_label=bootstrap_label, active_taxonomies=active_taxonomies, active_galaxies=active_galaxies, list_tags=list_tags, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list)
+    if Flask_config.pymisp is False:
+        misp = False
+    else:
+        misp = True
+
+    if Flask_config.HiveApi is False:
+        hive = False
+    else:
+        hive = True
+
+    return render_template("show_saved_paste.html", date=p_date, bootstrap_label=bootstrap_label, active_taxonomies=active_taxonomies, active_galaxies=active_galaxies, list_tags=list_tags, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list,
+                            misp=misp, hive=hive)
 
 # ============ ROUTES ============
 

var/www/modules/showpaste/templates/show_saved_paste.html

@@ -171,6 +171,7 @@
         </table>
 
         <div>
+          {% if misp %}
           <button class="btn btn-default btn-lg" data-toggle="modal" data-target="#myModal_MISP">
             Create
             <img id="misp-logo" src="{{url_for('static', filename='image/misp-logo.png') }}" height="32">
@@ -234,7 +235,9 @@
 
             </div>
           </div>
+          {% endif %}
 
+          {% if hive %}
           <button class="btn btn-primary btn-lg" data-toggle="modal" data-target="#myModal_hive">
             Create
             <img id="thehive-icon" src="{{url_for('static', filename='image/thehive_icon.png') }}">
@@ -294,6 +297,7 @@
 
             </div>
           </div>
+          {% endif %}
         </div>
 
       </div>