chg: [UI MISP export] add object export picker by user

bin/Cryptocurrencies.py

@@ -66,7 +66,7 @@ def search_crytocurrency(item_id, item_content):
                     is_valid_crypto_addr = True
                     print('{} address found : {}'.format(crypto_name, address))
                     # build bitcoin correlation
-                    Cryptocurrency.save_cryptocurrency_data(crypto_name, Item.get_item_date(item_id), item_id, address)
+                    Cryptocurrency.cryptocurrency.save_item_correlation(crypto_name, address, item_id, Item.get_item_date(item_id))
 
             # At least one valid cryptocurrency address was found
             if(is_valid_crypto_addr):

bin/Decoder.py

@@ -50,24 +50,30 @@ def decode_string(content, item_id, item_date, encoded_list, decoder_name, encod
             find = True
 
             sha1_string = sha1(decoded_file).hexdigest()
-            mimetype = Decoded.get_file_mimetype(file_content)
+            mimetype = Decoded.get_file_mimetype(decoded_file)
+            if not mimetype:
+                print(item_id)
+                print(sha1_string)
+                raise Exception('Invalid mimetype')
             Decoded.save_decoded_file_content(sha1_string, decoded_file, item_date, mimetype=mimetype)
             Decoded.save_item_relationship(sha1_string, item_id)
             Decoded.create_decoder_matadata(sha1_string, item_id, decoder_name)
 
             #remove encoded from item content
             content = content.replace(encoded, '', 1)
+
+            print('{} : {} - {}'.format(item_id, decoder_name, mimetype))
     if(find):
-        set_out_item(decoder_name, message)
+        set_out_item(decoder_name, item_id)
 
     return content
 
-def set_out_item(decoder_name, message):
+def set_out_item(decoder_name, item_id):
     publisher.warning(decoder_name+' decoded')
     #Send to duplicate
-    p.populate_set_out(message, 'Duplicate')
+    p.populate_set_out(item_id, 'Duplicate')
 
-    msg = 'infoleak:automatic-detection="'+decoder_name+'";{}'.format(message)
+    msg = 'infoleak:automatic-detection="'+decoder_name+'";{}'.format(item_id)
     p.populate_set_out(msg, 'Tags')
 
 

bin/Duplicates.py

@@ -38,6 +38,8 @@ if __name__ == "__main__":
 
     p = Process(config_section)
 
+    PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes"))
+
     maximum_month_range = int(p.config.get("Modules_Duplicates", "maximum_month_range"))
     threshold_duplicate_ssdeep = int(p.config.get("Modules_Duplicates", "threshold_duplicate_ssdeep"))
     threshold_duplicate_tlsh = int(p.config.get("Modules_Duplicates", "threshold_duplicate_tlsh"))
@@ -142,6 +144,7 @@ if __name__ == "__main__":
                                 paste_date = paste_date
                                 paste_date = paste_date if paste_date != None else "No date available"
                                 if paste_path != None:
+                                    paste_path = paste_path.replace(PASTES_FOLDER+'/', '', 1)
                                     if paste_path != PST.p_rel_path:
                                         hash_dico[dico_hash] = (hash_type, paste_path, percent, paste_date)
 

bin/MISP_The_Hive_feeder.py

@@ -165,7 +165,7 @@ if __name__ == "__main__":
                 misp_wrapper = ailleakObject.ObjectWrapper(pymisp)
                 r_serv_db.set('ail:misp', True)
                 print('Connected to MISP:', misp_url)
-            except e:
+            except Exception as e:
                 flag_misp = False
                 r_serv_db.set('ail:misp', False)
                 print(e)

bin/PgpDump.py

@@ -19,6 +19,7 @@ from pubsublogger import publisher
 from bs4 import BeautifulSoup
 
 from Helper import Process
+from packages import Item
 from packages import Paste
 
 from packages import Pgp
@@ -214,14 +215,16 @@ if __name__ == '__main__':
             extract_all_id(message, content, regex_pgp_signature)
             extract_all_id(message, content, regex_pgp_message)
 
+        item_date = Item.get_item_date(message)
+
         for key_id in set_key:
             print(key_id)
-            Pgp.save_pgp_data('key', date, message, key_id)
+            Pgp.pgp.save_item_correlation('key', key_id, message, item_date)
 
         for name_id in set_name:
             print(name_id)
-            Pgp.save_pgp_data('name', date, message, name_id)
+            Pgp.pgp.save_item_correlation('name', key_id, message, item_date)
 
         for mail_id in set_mail:
             print(mail_id)
-            Pgp.save_pgp_data('mail', date, message, mail_id)
+            Pgp.pgp.save_item_correlation('mail', key_id, message, item_date)

bin/export/AILObjects.py

@@ -9,6 +9,7 @@ import redis
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
 import ConfigLoader
+import Correlate_object
 
 config_loader = ConfigLoader.ConfigLoader()
 r_serv_objects = config_loader.get_redis_conn("ARDB_Objects")
@@ -19,3 +20,30 @@ def create_map_obj_uuid_golbal_id(obj_uuid, global_id):
     r_serv_objects.sadd('all_object:global_id', global_id)
     r_serv_objects.sadd('object:map:uuid_id:{}'.format(obj_uuid), global_id)
     r_serv_objects.sadd('object:map:id_uuid:{}'.format(global_id), obj_uuid)
+
+def get_user_list_of_obj_to_export(user_id, add_uuid=False):
+    set_objs_to_export = []
+    res = r_serv_objects.hgetall('user:all_objs_to_export:{}'.format(user_id))
+    for global_id in res:
+        dict_obj = Correlate_object.get_global_id_from_id(global_id)
+        dict_obj['lvl'] = int(res[global_id])
+        if add_uuid:
+            obj_dict['uuid'] = str(uuid.uuid4())
+        set_objs_to_export.append(dict_obj)
+    return set_objs_to_export
+
+def add_user_object_to_export(user_id, obj_type, obj_id, lvl, obj_subtype=None):
+    ## TODO: check if user exist
+    # # TODO: check if valid object
+    # # TODO: check lvl
+    global_id = Correlate_object.get_obj_global_id(obj_type, obj_id, obj_sub_type=obj_subtype)
+    return r_serv_objects.hset('user:all_objs_to_export:{}'.format(user_id), global_id, lvl)
+
+def delete_user_object_to_export(user_id, obj_type, obj_id, obj_subtype=None):
+    ## TODO: check if user exist
+    global_id = Correlate_object.get_obj_global_id(obj_type, obj_id, obj_sub_type=obj_subtype)
+    r_serv_objects.hdel('user:all_objs_to_export:{}'.format(user_id), global_id)
+
+def delete_all_user_object_to_export(user_id):
+    ## TODO: check if user exist
+    r_serv_objects.delete('user:all_objs_to_export:{}'.format(user_id))

bin/export/MispExport.py

@@ -42,7 +42,7 @@ def sanitize_obj_export_lvl(lvl):
     return lvl
 
 def get_export_filename(json_content):
-    return 'ail_export.json'
+    return 'ail_export{}.json'.format(str(uuid.uuid4()))
 
 def create_in_memory_file(json_content):
     return io.BytesIO(json_content.encode())

bin/lib/Correlate_object.py

@@ -149,6 +149,8 @@ def get_correlation_node_icon(correlation_name, correlation_type=None, value=Non
 
     elif correlation_name == 'decoded':
         node_color = '#88CCEE'
+        print(Decoded.get_decoded_item_type(value))
+        print(value)
         correlation_type = Decoded.get_decoded_item_type(value).split('/')[0]
         if correlation_type == 'application':
             icon_text = '\uf15b'

bin/packages/Paste.py

@@ -377,7 +377,6 @@ class Paste(object):
         Save an attribute as a field
         """
         for tuple in value:
-            tuple = tuple.replace(PASTES_FOLDER, '', 1)
             self.store_metadata.sadd('dup:'+self.p_path, tuple)
 
     def save_others_pastes_attribute_duplicate(self, list_value):
@@ -385,8 +384,8 @@ class Paste(object):
         Save a new duplicate on others pastes
         """
         for hash_type, path, percent, date in list_value:
-            path = path.replace(PASTES_FOLDER, '', 1)
+            path = path.replace(self.PASTES_FOLDER, '', 1)
-            to_add = (hash_type, self.p_path, percent, date)
+            to_add = (hash_type, self.p_rel_path, percent, date)
             self.store_metadata.sadd('dup:'+path,to_add)
 
     def _get_from_redis(self, r_serv):

bin/packages/Pgp.py

@@ -6,8 +6,8 @@ import sys
 import redis
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
-import Correlation
 import Item
+import Correlation
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
 import ConfigLoader
@@ -20,10 +20,10 @@ pgp = Correlation.Correlation('pgpdump', ['key', 'mail', 'name'])
 
 def get_pgp(request_dict, pgp_type):
     # basic verification
-    res = pgpdump.verify_correlation_field_request(request_dict, pgp_type)
+    res = pgp.verify_correlation_field_request(request_dict, pgp_type)
     if res:
         return res
     # cerify address
     field_name = request_dict.get(pgp_type)
 
-    return pgpdump.get_correlation(request_dict, pgp_type, field_name)
+    return pgp.get_correlation(request_dict, pgp_type, field_name)

var/www/blueprints/import_export.py

@@ -28,7 +28,7 @@ import MispExport
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
 import Correlate_object
 
-bootstrap_label = Flask_config.bootstrap_label
+import AILObjects
 
 # ============ BLUEPRINT ============
 import_export = Blueprint('import_export', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export'))
@@ -82,18 +82,27 @@ def import_object_file():
 @login_required
 @login_analyst
 def export_object():
-    return render_template("export_object.html")
+    user_id = current_user.get_id()
+    l_obj_to_export = set()
+
+    # get user saved obj to export
+    l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id)
+
+    return render_template("export_object.html", l_obj_to_export=l_obj_to_export)
 
 @import_export.route("/import_export/export_file", methods=['POST'])
 @login_required
 @login_analyst
 def export_object_file():
+    user_id = current_user.get_id()
+
     l_obj_to_export = []
     l_obj_invalid = []
 
     export_to_misp = False
     dict_misp_event_export = {}
 
+    # Get new added Object
     for obj_tuple in list(request.form):
         l_input = request.form.getlist(obj_tuple)
         if len(l_input) == 3:
@@ -115,6 +124,7 @@ def export_object_file():
 
             if MispExport.is_valid_obj_to_export(obj_type, obj_subtype, obj_id):
                 l_obj_to_export.append(obj_dict)
+                AILObjects.add_user_object_to_export(user_id, obj_dict['type'], obj_dict['id'], obj_dict['lvl'], obj_subtype=obj_dict.get('subtype', None))
             else:
                 if obj_id:
                     l_obj_invalid.append(obj_dict)
@@ -130,6 +140,10 @@ def export_object_file():
         for obj_dict in l_obj_to_export:
             obj_dict['uuid'] = str(uuid.uuid4())
             obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(obj_dict['type'], obj_dict.get('subtype', None))
+
+        # get user saved obj to export # # TODO: # performance
+        l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id)
+
         for obj_dict in l_obj_invalid:
             obj_dict['uuid'] = str(uuid.uuid4())
             obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(obj_dict['type'], obj_dict.get('subtype', None))
@@ -144,11 +158,44 @@ def export_object_file():
                                         publish=dict_misp_event_export.get('misp_publish', None),
                                         analysis=dict_misp_event_export.get('misp_event_analysis', None),
                                         event_info=dict_misp_event_export.get('misp_event_info', None))
+
+            AILObjects.delete_all_user_object_to_export(user_id)
             return render_template("export_object.html", l_obj_to_export=l_obj_to_export,
                                     event_metadata=event_metadata,
                                     l_obj_invalid=[], dict_misp_event_export=[])
         else:
+            # get user saved obj to export # # TODO: # performance
+            l_obj_to_export = AILObjects.get_user_list_of_obj_to_export(user_id)
             json_export = MispExport.create_list_of_objs_to_export(l_obj_to_export)
             export_filename = MispExport.get_export_filename(json_export)
             json_export = MispExport.create_in_memory_file(json_export)
+            AILObjects.delete_all_user_object_to_export(user_id)
             return send_file(json_export, as_attachment=True, attachment_filename=export_filename)
+
+
+@import_export.route("/import_export/add_object_id_to_export", methods=['GET'])
+@login_required
+@login_analyst
+def add_object_id_to_export():
+    user_id = current_user.get_id()
+    user_id = current_user.get_id()
+    obj_type = request.args.get('obj_type')
+    obj_id = request.args.get('obj_id')
+    obj_subtype = request.args.get('obj_subtype')
+    obj_lvl = request.args.get('obj_lvl')
+    AILObjects.add_user_object_to_export(user_id, obj_type, obj_id, obj_lvl, obj_subtype=obj_subtype)
+    # redirect
+    return redirect(url_for('import_export.export_object'))
+
+# @import_export.route("/import_export/delete_object_id_to_export", methods=['GET'])
+# @login_required
+# @login_analyst
+# def delete_object_id_to_export():
+#     user_id = current_user.get_id()
+#     obj_type = request.args.get('obj_type')
+#     obj_id = request.args.get('obj_id')
+#     obj_subtype = request.args.get('obj_subtype')
+#     obj_lvl = request.args.get('obj_lvl')
+#     AILObjects.delete_user_object_to_export(user_id, object_type, object_id, obj_lvl, obj_subtype=obj_subtype)
+#     # redirect
+#     return 'ok'

var/www/modules/hashDecoded/Flask_hashDecoded.py

@@ -910,7 +910,8 @@ def range_type_json():
         if l_hash:
             for hash in l_hash:
                 estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
-                all_type.add(estimated_type)
+                if estimated_type:
+                    all_type.add(estimated_type)
 
     range_type = []
 

var/www/modules/showpaste/templates/show_saved_paste.html

@@ -199,82 +199,9 @@
         </table>
 
         <div>
-          {% if misp %}
+          {% with obj_type='item', obj_id=request.args.get('paste'), obj_lvl=0%}
-          <button class="btn btn-default btn-lg" data-toggle="modal" data-target="#myModal_MISP">
+            {% include 'import_export/block_add_user_object_to_export.html' %}
-            Create
+          {% endwith %}
-            <img id="misp-logo" src="{{url_for('static', filename='image/misp-logo.png') }}" height="32">
-            Event
-          </button>
-          <!-- Modal MISP-->
-          <div class="modal fade" id="myModal_MISP" role="dialog">
-            <div class="modal-dialog">
-
-              <div class="modal-content">
-                <div class="modal-header text-center">
-                  <img id="misp-logo" src="{{url_for('static', filename='image/misp-logo.png') }}" >
-                </div>
-                <div class="modal-body">
-
-                  <form method="post" action="{{ url_for('PasteSubmit.create_misp_event') }}" target="_blank">
-
-                  <div class="input select required">
-                    <label for="EventDistribution">Distribution</label>
-                    <select name="misp_data[Event][distribution]" id="EventDistribution" required="required">
-                      <option value="0" selected="selected">Your organisation only</option>
-                      <option value="1">This community only</option>
-                      <option value="2">Connected communities</option>
-                      <option value="3">All communities</option>
-                    </select>
-                  </div>
-                  <div class="input clear required">
-                    <label for="EventThreatLevelId">Threat Level</label>
-                    <select name="misp_data[Event][threat_level_id]" id="EventThreatLevelId" required="required">
-                      <option value="1">High</option>
-                      <option value="2" selected="selected">Medium</option>
-                      <option value="3">Low</option>
-                      <option value="4">Undefined</option>
-                    </select>
-                  </div>
-                  <div class="input select required">
-                    <label for="EventAnalysis">Analysis</label>
-                    <select name="misp_data[Event][analysis]" id="EventAnalysis" required="required">
-                      <option value="0">Initial</option>
-                      <option value="1">Ongoing</option>
-                      <option value="2">Completed</option>
-                    </select>
-                  </div>
-                  <div class="clear required">
-                    <label for="EventInfo">Event Info</label>
-                    <input name="misp_data[Event][info]" class="form-control span6" placeholder="Quick Event Description or Tracking Info" type="text" id="EventInfo" required="required"/>
-                  </div>
-                  <div class="clear">
-                    <label for="EventInfo">Publish Event</label>
-                    <input type="checkbox" value="True" id="misp_publish" name="misp_publish" >
-                  </div>
-                  <input type="hidden" id="paste" name="paste" value="{{ request.args.get('paste') }}">
-
-                  {% if misp_eventid %}
-                  <br>
-                  <div class="list-group" id="misp_event">
-                    <li class="list-group-item active">MISP Events already Created</li>
-                      <a target="_blank" href="{{ misp_url }}" class="list-group-item">{{ misp_url }}</a></div>
-                  {% endif %}
-
-                </div>
-                <div class="modal-footer center">
-                  <button class="btn btn-primary">
-                    <span class="glyphicon glyphicon-ok "></span> Create Event
-                  </button>
-
-                </form>
-                  <button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
-
-                </div>
-              </div>
-
-            </div>
-          </div>
-          {% endif %}
 
           {% if hive %}
           <button class="btn btn-primary btn-lg" data-toggle="modal" data-target="#myModal_hive">

var/www/templates/correlation/show_correlation.html

@@ -111,6 +111,11 @@
 							<div class="card mb-3">
 								<div class="card-header">
 									<i class="fas fa-project-diagram"></i> Graph
+									<span class="float-right">
+										{% with obj_type=dict_object["object_type"], obj_id=dict_object["correlation_id"], obj_subtype=dict_object["metadata"]["type_id"],obj_lvl=1%}
+				              {% include 'import_export/block_add_user_object_to_export.html' %}
+				            {% endwith %}
+									</span>
 									<span class="float-right">
 										<button class="btn btn-primary py-1" onclick="resize_graph();">
 											<i class="fas fa-sync"></i>&nbsp;Resize Graph

var/www/templates/crawler/crawler_splash/showDomain.html

@@ -114,6 +114,12 @@
                   {%endif%}
                 {%endif%}
 
+                <div class="mt-2">
+                  {% with obj_type='domain', obj_id=dict_domain['domain'], obj_lvl=0%}
+				              {% include 'import_export/block_add_user_object_to_export.html' %}
+				          {% endwith %}
+                </div>
+
 							</div>
 						</div>
 

var/www/templates/import_export/block_add_user_object_to_export.html

@@ -0,0 +1,5 @@
+<a class="btn btn-lg btn-outline-dark" target="_blank" href="{{ url_for('import_export.add_object_id_to_export')}}?obj_type={{obj_type}}&obj_id={{obj_id}}&obj_lvl={{obj_lvl}}{%if obj_subtype%}&obj_subtype={{obj_subtype}}{%endif%}">
+  Add to
+  <img id="misp-logo" src="{{ url_for('static', filename='image/misp-logo.png')}}" height="32">
+  Export
+</a>

var/www/templates/import_export/export_object.html

@@ -165,7 +165,9 @@ $('.add-field').click(function() {
 });
 
 $('.fields').on('click', '.delete-field', function(){
+	console.log($(this).parent());
   $(this).parent().remove();
+	//$.get( "#")
 });