chg: [UI MISP import] add MISP import

bin/export/MispExport.py

@@ -8,11 +8,11 @@ import redis
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
+import Item
 import Cryptocurrency
 import Pgp
 import Decoded
 import Domain
-import Item
 import Screenshot
 
 import Correlate_object
@@ -221,8 +221,8 @@ def create_list_of_objs_to_export(l_obj, mode='union'):
 
     print(event.to_json())
 
-    #misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False)
+    misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False)
-    #misp.add_event(event, pythonify=True)
+    misp.add_event(event, pythonify=True)
 
 
 def create_all_misp_obj(all_obj_to_export, set_relationship):

bin/export/MispImport.py

@@ -44,6 +44,21 @@ def get_global_id_from_id(global_id):
         obj_meta['id'] = global_id[1]
     return obj_meta
 
+def get_import_dir():
+    return os.path.join(os.environ['AIL_HOME'], 'temp/import')
+
+def sanitize_import_file_path(filename):
+    IMPORT_FOLDER = get_import_dir()
+    filename = os.path.join(IMPORT_FOLDER, filename)
+    filename = os.path.realpath(filename)
+    # path traversal
+    if not os.path.commonprefix([filename, IMPORT_FOLDER]) == IMPORT_FOLDER:
+        return os.path.join(IMPORT_FOLDER, str(uuid.uuid4()) + '.json')
+    # check if file already exist
+    if os.path.isfile(filename):
+        return os.path.join(IMPORT_FOLDER, str(uuid.uuid4()) + '.json')
+    return filename
+
 def get_misp_obj_tag(misp_obj):
     if misp_obj.attributes:
         misp_tags = misp_obj.attributes[0].tags
@@ -204,19 +219,19 @@ def import_objs_from_file(filepath):
 
     for misp_obj in event_to_import.objects:
         create_obj_relationships(map_uuid_global_id, misp_obj)
-
     #print(map_uuid_global_id)
+    return map_uuid_global_id
 
 
 if __name__ == '__main__':
 
     # misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False)
 
-    #import_objs_from_file('test_import_item.json')
+    import_objs_from_file('test_import_item.json')
 
     #Screenshot.delete_screenshot('a92d459f70c4dea8a14688f585a5e2364be8b91fbf924290ead361d9b909dcf1')
     #Decoded.delete_decoded('d59a110ab233fe87cefaa0cf5603b047b432ee07')
     #Pgp.pgp.delete_correlation('key', '0xA4BB02A75E6AF448')
 
     #Item.delete_item('submitted/2020/02/10/b2485894-4325-469b-bc8f-6ad1c2dbb202.gz')
-    Item.delete_item('archive/pastebin.com_pro/2020/02/10/K2cerjP4.gz')
+    #Item.delete_item('archive/pastebin.com_pro/2020/02/10/K2cerjP4.gz')

var/www/Flask_server.py

@@ -40,6 +40,7 @@ from blueprints.root import root
 from blueprints.crawler_splash import crawler_splash
 from blueprints.correlation import correlation
 from blueprints.tags_ui import tags_ui
+from blueprints.import_export import import_export
 
 
 Flask_dir = os.environ['AIL_FLASK']
@@ -93,6 +94,7 @@ app.register_blueprint(root, url_prefix=baseUrl)
 app.register_blueprint(crawler_splash, url_prefix=baseUrl)
 app.register_blueprint(correlation, url_prefix=baseUrl)
 app.register_blueprint(tags_ui, url_prefix=baseUrl)
+app.register_blueprint(import_export, url_prefix=baseUrl)
 # =========       =========#
 
 # ========= Cookie name ========

var/www/blueprints/import_export.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+'''
+    Blueprint Flask: MISP format import export
+'''
+
+import os
+import sys
+import json
+import random
+
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
+from flask_login import login_required, current_user, login_user, logout_user
+
+sys.path.append('modules')
+import Flask_config
+
+# Import Role_Manager
+from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
+from Role_Manager import login_admin, login_analyst, login_read_only
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'export'))
+import MispImport
+import MispExport
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
+import Correlate_object
+
+bootstrap_label = Flask_config.bootstrap_label
+
+# ============ BLUEPRINT ============
+import_export = Blueprint('import_export', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export'))
+
+# ============ VARIABLES ============
+
+
+
+# ============ FUNCTIONS ============
+
+
+# ============= ROUTES ==============
+@import_export.route('/import_export/import')
+@login_required
+@login_analyst
+def import_object():
+    tags = request.args.get('tags')
+    return render_template("import_object.html", bootstrap_label=bootstrap_label)
+
+@import_export.route("/import_export/import_file", methods=['POST'])
+@login_required
+@login_analyst
+def import_object_file():
+
+    is_file = False
+    if 'file' in request.files:
+        file = request.files['file']
+        if file:
+            if file.filename:
+                is_file = True
+
+    if is_file:
+        filename = MispImport.sanitize_import_file_path(file.filename)
+        file.save(filename)
+        map_uuid_global_id = MispImport.import_objs_from_file(filename)
+        os.remove(filename)
+
+    return render_template("import_object.html", bootstrap_label=bootstrap_label)
+
+@import_export.route('/import_export/export')
+@login_required
+@login_analyst
+def export_object():
+    object_type = request.args.get('object_type')
+    return render_template("export_object.html", bootstrap_label=bootstrap_label)

var/www/templates/decoded/menu_sidebar.html

@@ -6,32 +6,22 @@
       	</button>
 
       	<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
-      		<h5 class="d-flex text-muted w-100">
+          <h5 class="d-flex text-muted w-100">
-      		  <span>Items Decoded </span>
+      		  <span>Objects</span>
       		</h5>
-      	  <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
+      	  <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100 mb-4">
       	    <li class="nav-item">
       	      <a class="nav-link" href="{{url_for('hashDecoded.hashDecoded_page')}}" id="nav_dashboard">
-      	        <i class="fas fa-search"></i>
+      	        <i class="fas fa-lock-open"></i>
-      	        <span>Dashboard</span>
+      	        <span>Decoded</span>
       	      </a>
       	    </li>
-          </ul>
-          <h5 class="d-flex text-muted w-100">
-      		  <span>PGP Dumps</span>
-      		</h5>
-      	  <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
       	    <li class="nav-item">
       	      <a class="nav-link" href="{{url_for('hashDecoded.pgpdump_page')}}" id="nav_dashboard_pgpdump">
       	        <i class="fas fa-key"></i>
       	        <span>PGP Dumps</span>
       	      </a>
       	    </li>
-          </ul>
-          <h5 class="d-flex text-muted w-100">
-      		  <span>Cryptocurrency</span>
-      		</h5>
-      	  <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
       	    <li class="nav-item">
       	      <a class="nav-link" href="{{url_for('hashDecoded.cryptocurrency_page')}}" id="nav_dashboard_cryptocurrency">
       	        <i class="fas fa-coins"></i>
@@ -39,5 +29,26 @@
       	      </a>
       	    </li>
           </ul>
+
+
+
+          <h5 class="d-flex text-muted w-100">
+      		  <span>
+              <img src="{{ url_for('static', filename='image/misp-logo.png')}}" alt="MISP" style="width:80px;">
+               Format
+             </span>
+      		</h5>
+      	  <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100">
+            <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('import_export.import_object')}}" id="nav_show_item_by_id">
+                <b>Import</b>
+      	      </a>
+      	    </li>
+            <li class="nav-item">
+              <a class="nav-link" href="{{url_for('import_export.export_object')}}" id="nav_show_item_by_id">
+                <b>Export</b>
+              </a>
+            </li>
+          </ul>
       	</nav>
       </div>

var/www/templates/import_export/export_object.html

@@ -0,0 +1,62 @@
+<!DOCTYPE html>
+<html>
+
+	<head>
+	  <meta charset="utf-8">
+
+	  <title>AIL - framework</title>
+		<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+	  <!-- Core CSS -->
+		<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+		<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+
+	  <!-- JS -->
+		<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+		<script src="{{ url_for('static', filename='js/helper.js')}}"></script>
+		<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+		<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+		<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
+
+	</head>
+	<body>
+
+		{% include 'nav_bar.html' %}
+
+		<div class="container-fluid">
+			<div class="row">
+
+				{% include 'decoded/menu_sidebar.html' %}
+
+				<div class="col-12 col-lg-10" id="core_content">
+
+					MISP exporter
+
+				</div>
+			</div>
+		</div>
+
+<script>
+$(document).ready(function(){
+		$("#page-Decoded").addClass("active");
+});
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+
+</body>
+
+</html>

var/www/templates/import_export/import_object.html

@@ -0,0 +1,81 @@
+<!DOCTYPE html>
+<html>
+
+	<head>
+	  <meta charset="utf-8">
+
+	  <title>AIL - framework</title>
+		<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+	  <!-- Core CSS -->
+		<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+		<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+
+	  <!-- JS -->
+		<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+		<script src="{{ url_for('static', filename='js/helper.js')}}"></script>
+		<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+		<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+		<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
+
+	</head>
+	<body>
+
+		{% include 'nav_bar.html' %}
+
+		<div class="container-fluid">
+			<div class="row">
+
+				{% include 'decoded/menu_sidebar.html' %}
+
+				<div class="col-12 col-lg-10" id="core_content">
+
+					<div class="card mb-3 mt-1">
+						<div class="card-header text-white bg-dark">
+							<h5 class="card-title"><img src="{{ url_for('static', filename='image/misp-logo.png')}}" alt="MISP" style="width:100px;"> Import from MISP Export File</h5>
+						</div>
+							<div class="card-body">
+
+								<form action="{{ url_for('import_export.import_object_file') }}" method="post" enctype=multipart/form-data onsubmit="submitPaste()">
+
+									<div class="form-group">
+										<label for="file">Select a <b>JSON File</b> to import:</label>
+										<input type="file" class="form-control-file btn btn-outline-secondary" id="file" name="file">
+									</div>
+									<div class="form-group">
+										<button class="btn btn-info" name="submit" type="submit">Import Objects</button>
+									</div>
+
+								</form>
+
+
+						</div>
+
+				</div>
+			</div>
+		</div>
+
+<script>
+$(document).ready(function(){
+		$("#page-Decoded").addClass("active");
+});
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+
+</body>
+
+</html>

var/www/templates/nav_bar.html

@@ -25,7 +25,7 @@
         <a class="nav-link" id="page-Crawler" href="{{ url_for('hiddenServices.dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>
       </li>
       <li class="nav-item mr-3">
-        <a class="nav-link" id="page-Decoded" href="{{ url_for('hashDecoded.hashDecoded_page') }}" aria-disabled="true"><i class="fas fa-lock-open"></i> Decoded</a>
+        <a class="nav-link" id="page-Decoded" href="{{ url_for('hashDecoded.hashDecoded_page') }}" aria-disabled="true"><i class="fas fa-cube"></i> Objects</a>
       </li>
       <li class="nav-item mr-3">
         <a class="nav-link" href="{{ url_for('trendingmodules.moduletrending') }}" aria-disabled="true"><i class="fas fa-chart-bar"></i> Statistics</a>