chg: [API + import] add API format + item_import refractor

OVERVIEW.md

@@ -38,6 +38,21 @@ Redis and ARDB overview
 | failed_login_ip:**ip**           | **nb login failed** | TTL
 | failed_login_user_id:**user_id** | **nb login failed** | TTL
 
+##### Item Import:
+
+| Key | Value |
+| ------ | ------ |
+| **uuid**:nb_total   | **nb total**      | TTL *(if imported)*
+| **uuid**:nb_end     | **nb**            | TTL *(if imported)*
+| **uuid**:nb_sucess  | **nb success**    | TTL *(if imported)*
+| **uuid**:end        | **0 (in progress) or (item imported)** | TTL *(if imported)*
+| **uuid**:processing | **process status: 0 or 1**             | TTL *(if imported)*
+| **uuid**:error      | **error message** | TTL *(if imported)*
+
+| Set Key | Value |
+| ------ | ------ |
+| **uuid**:paste_submit_link | **item_path** | TTL *(if imported)*
+
 ## DB0 - Core:
 
 ##### Update keys:

bin/packages/Import_helper.py

@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import redis
+
+import Flask_config
+
+r_serv_db = Flask_config.r_serv_db
+r_serv_log = Flask_config.r_serv_log
+
+def create_import_queue(ltags, ltagsgalaxies, paste_content, UUID,  password, isfile = False):
+
+    # save temp value on disk
+    r_serv_db.set(UUID + ':ltags', ltags)
+    r_serv_db.set(UUID + ':ltagsgalaxies', ltagsgalaxies)
+    r_serv_db.set(UUID + ':paste_content', paste_content)
+    r_serv_db.set(UUID + ':password', password)
+    r_serv_db.set(UUID + ':isfile', isfile)
+
+    r_serv_log.set(UUID + ':end', 0)
+    r_serv_log.set(UUID + ':processing', 0)
+    r_serv_log.set(UUID + ':nb_total', -1)
+    r_serv_log.set(UUID + ':nb_end', 0)
+    r_serv_log.set(UUID + ':nb_sucess', 0)
+
+    # save UUID on disk
+    r_serv_db.sadd('submitted:uuid', UUID)
+    return UUID
+
+def import_text_item():
+    res = r_serv_db.smembers('submitted:uuid')
+    print(res)
+    return res

bin/packages/Tags.py

@@ -0,0 +1,61 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import redis
+
+import Flask_config
+
+from pytaxonomies import Taxonomies
+from pymispgalaxies import Galaxies, Clusters
+
+r_serv_tags = Flask_config.r_serv_tags
+
+def get_taxonomie_from_tag(tag):
+    return tag.split(':')[0]
+
+def get_galaxy_from_tag(tag):
+    galaxy = tag.split(':')[1]
+    galaxy = galaxy.split('=')[0]
+    return galaxy
+
+def get_active_taxonomies():
+    return r_serv_tags.smembers('active_taxonomies')
+
+def get_active_galaxies():
+    return r_serv_tags.smembers('active_galaxies')
+
+def is_taxonomie_tag_enabled(taxonomie, tag):
+    if tag in r_serv_tags.smembers('active_tag_' + taxonomie):
+        return True
+    else:
+        return False
+
+def is_galaxy_tag_enabled(taxonomie, galaxy):
+    if tag in r_serv_tags.smembers('active_tag_galaxies_' + galaxy):
+        return True
+    else:
+        return False
+
+# Check if tags are enabled in AIL
+def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
+    if list_tags:
+        active_taxonomies = Tags.get_active_taxonomies()
+
+        for tag in list_tags:
+            taxonomie = get_taxonomie_from_tag(tag)
+            if taxonomie not in active_taxonomies:
+                return False
+            if not is_taxonomie_tag_enabled(taxonomie, tag):
+                return False
+
+    if list_tags_galaxy:
+        active_galaxies = Tags.get_active_galaxies()
+
+        for tag in list_tags_galaxy:
+            galaxy = get_galaxy_from_tag(tag)
+            if galaxy not in active_galaxies:
+                return False
+            if not is_galaxy_tag_enabled(galaxy, tag):
+                return False
+    return True

bin/submit_paste.py

@@ -92,7 +92,6 @@ def remove_submit_uuid(uuid):
     r_serv_log_submit.expire(uuid + ':nb_sucess', expire_time)
     r_serv_log_submit.expire(uuid + ':nb_end', expire_time)
     r_serv_log_submit.expire(uuid + ':error', expire_time)
-    r_serv_log_submit.srem(uuid + ':paste_submit_link', '')
     r_serv_log_submit.expire(uuid + ':paste_submit_link', expire_time)
 
     # delete uuid
@@ -230,8 +229,6 @@ if __name__ == "__main__":
             r_serv_log_submit.set(uuid + ':nb_total', -1)
             r_serv_log_submit.set(uuid + ':nb_end', 0)
             r_serv_log_submit.set(uuid + ':nb_sucess', 0)
-            r_serv_log_submit.set(uuid + ':error', 'error:')
-            r_serv_log_submit.sadd(uuid + ':paste_submit_link', '')
 
 
             r_serv_log_submit.set(uuid + ':processing', 1)

var/www/Flask_server.py

@@ -67,15 +67,15 @@ log_dir = os.path.join(os.environ['AIL_HOME'], 'logs')
 if not os.path.isdir(log_dir):
     os.makedirs(logs_dir)
 
-log_filename = os.path.join(log_dir, 'flask_server.logs')
+# log_filename = os.path.join(log_dir, 'flask_server.logs')
-logger = logging.getLogger()
+# logger = logging.getLogger()
-formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
+# formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
-handler_log = logging.handlers.TimedRotatingFileHandler(log_filename, when="midnight", interval=1)
+# handler_log = logging.handlers.TimedRotatingFileHandler(log_filename, when="midnight", interval=1)
-handler_log.suffix = '%Y-%m-%d.log'
+# handler_log.suffix = '%Y-%m-%d.log'
-handler_log.setFormatter(formatter)
+# handler_log.setFormatter(formatter)
-handler_log.setLevel(30)
+# handler_log.setLevel(30)
-logger.addHandler(handler_log)
+# logger.addHandler(handler_log)
-logger.setLevel(30)
+# logger.setLevel(30)
 
 # =========       =========#
 
@@ -226,7 +226,7 @@ def login():
             # login failed
             else:
                 # set brute force protection
-                logger.warning("Login failed, ip={}, username={}".format(current_ip, username))
+                #logger.warning("Login failed, ip={}, username={}".format(current_ip, username))
                 r_cache.incr('failed_login_ip:{}'.format(current_ip))
                 r_cache.expire('failed_login_ip:{}'.format(current_ip), 300)
                 r_cache.incr('failed_login_user_id:{}'.format(username))

var/www/modules/Flask_config.py

@@ -12,7 +12,6 @@ import sys
 
 # FLASK #
 app = None
-#secret_key = 'ail-super-secret_key01C'
 
 # CONFIG #
 configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
@@ -146,7 +145,7 @@ if HiveApi != False:
         HiveApi = False
         print('The Hive not connected')
 
-# VARIABLES #
+#### VARIABLES ####
 baseUrl = cfg.get("Flask", "baseurl")
 baseUrl = baseUrl.replace('/', '')
 if baseUrl != '':

var/www/modules/PasteSubmit/Flask_PasteSubmit.py

@@ -23,6 +23,9 @@ import json
 
 import Paste
 
+import Import_helper
+import Tags
+
 from pytaxonomies import Taxonomies
 from pymispgalaxies import Galaxies, Clusters
 
@@ -108,44 +111,6 @@ def launch_submit(ltags, ltagsgalaxies, paste_content, UUID,  password, isfile =
     # save UUID on disk
     r_serv_db.sadd('submitted:uuid', UUID)
 
-
-def addTagsVerification(tags, tagsgalaxies):
-
-    list_tag = tags.split(',')
-    list_tag_galaxies = tagsgalaxies.split(',')
-
-    taxonomies = Taxonomies()
-    active_taxonomies = r_serv_tags.smembers('active_taxonomies')
-
-    active_galaxies = r_serv_tags.smembers('active_galaxies')
-
-    if list_tag != ['']:
-        for tag in list_tag:
-            # verify input
-            tax = tag.split(':')[0]
-            if tax in active_taxonomies:
-                if tag in r_serv_tags.smembers('active_tag_' + tax):
-                    pass
-                else:
-                    return False
-            else:
-                return False
-
-    if list_tag_galaxies != ['']:
-        for tag in list_tag_galaxies:
-            # verify input
-            gal = tag.split(':')[1]
-            gal = gal.split('=')[0]
-
-            if gal in active_galaxies:
-                if tag in r_serv_tags.smembers('active_tag_galaxies_' + gal):
-                    pass
-                else:
-                    return False
-            else:
-                return False
-    return True
-
 def date_to_str(date):
     return "{0}-{1}-{2}".format(date.year, date.month, date.day)
 
@@ -279,11 +244,9 @@ def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title,
 @login_required
 @login_analyst
 def PasteSubmit_page():
-    #active taxonomies
+    # Get all active tags/galaxy
-    active_taxonomies = r_serv_tags.smembers('active_taxonomies')
+    active_taxonomies = Tags.get_active_taxonomies()
-
+    active_galaxies = Tags.get_active_galaxies()
-    #active galaxies
-    active_galaxies = r_serv_tags.smembers('active_galaxies')
 
     return render_template("submit_items.html",
                             active_taxonomies = active_taxonomies,
@@ -301,6 +264,9 @@ def submit():
     ltagsgalaxies = request.form['tags_galaxies']
     paste_content = request.form['paste_content']
 
+    print(ltags)
+    print(ltagsgalaxies)
+
     is_file = False
     if 'file' in request.files:
         file = request.files['file']
@@ -311,12 +277,16 @@ def submit():
     submitted_tag = 'infoleak:submission="manual"'
 
     #active taxonomies
-    active_taxonomies = r_serv_tags.smembers('active_taxonomies')
+    active_taxonomies = Tags.get_active_taxonomies()
     #active galaxies
-    active_galaxies = r_serv_tags.smembers('active_galaxies')
+    active_galaxies = Tags.get_active_galaxies()
 
     if ltags or ltagsgalaxies:
-        if not addTagsVerification(ltags, ltagsgalaxies):
+
+        list_tag = tags.split(',')
+        list_tag_galaxies = tagsgalaxies.split(',')
+
+        if not Tags.is_valid_tags_taxonomies_galaxy(ltags, ltagsgalaxies):
             content = 'INVALID TAGS'
             print(content)
             return content, 400
@@ -358,7 +328,7 @@ def submit():
 
                 paste_content = full_path
 
-                launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password ,True)
+                Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password ,True)
 
                 return render_template("submit_items.html",
                                             active_taxonomies = active_taxonomies,
@@ -381,7 +351,7 @@ def submit():
                 # clean file name
                 #id = clean_filename(paste_name)
 
-            launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password)
+            Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password)
 
             return render_template("submit_items.html",
                                         active_taxonomies = active_taxonomies,
@@ -433,10 +403,10 @@ def submit_status():
             else:
                 prog = 0
 
-            if error == 'error:':
+            if error:
-                isError = False
-            else:
                 isError = True
+            else:
+                isError = False
 
             if end == '0':
                 end = False

var/www/modules/restApi/Flask_restApi.py

@@ -8,10 +8,13 @@
 import os
 import re
 import sys
+import uuid
 import json
 import redis
 import datetime
 
+import Import_helper
+
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
 from flask_login import login_required
 
@@ -20,6 +23,7 @@ from functools import wraps
 # ============ VARIABLES ============
 import Flask_config
 
+
 app = Flask_config.app
 cfg = Flask_config.cfg
 baseUrl = Flask_config.baseUrl
@@ -108,8 +112,20 @@ def authErrors(user_role):
     else:
         return None
 
+# ============ API CORE =============
+
+
+
 # ============ FUNCTIONS ============
 
+def is_valid_uuid_v4(header_uuid):
+    try:
+        header_uuid=header_uuid.replace('-', '')
+        uuid_test = uuid.UUID(hex=header_uuid, version=4)
+        return uuid_test.hex == header_uuid
+    except:
+        return False
+
 def one():
     return 1
 
@@ -127,5 +143,57 @@ def items():
 
     return Response(json.dumps({'test': 2}), mimetype='application/json')
 
+
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+#
+# POST JSON FORMAT
+#
+# {
+#   "type": "text",         (default value)
+#   "tags": [],             (default value)
+#   "default_ags": True,    (default value)
+#   "galaxy" [],            (default value)
+#   "text": "",             mandatory if type = text
+# }
+#
+# response: {"uuid": "uuid"}
+#
+# # # #
+# GET
+#
+# {
+#   "uuid": "uuid",      mandatory
+# }
+#
+# response: {"uuid": "uuid"}
+#
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+
+@restApi.route("api/import/item", methods=['POST'])
+@token_required('admin')
+def import_item():
+    data = request.get_json()
+    if not data:
+        return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400
+
+    # TODO: add submitted tag
+
+    UUID = 'uuuuuuu'
+
+    return Response(json.dumps({'uuid': UUID}, indent=2, sort_keys=True), mimetype='application/json')
+
+@restApi.route("api/import/item/<UUID>", methods=['GET'])
+@token_required('admin')
+def import_item_uuid(UUID):
+
+    # Verify uuid
+    if not is_valid_uuid_v4(UUID):
+        Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400
+
+
+
+
+    return Response(json.dumps({'item_id': 4}), mimetype='application/json')
+
 # ========= REGISTRATION =========
 app.register_blueprint(restApi, url_prefix=baseUrl)