chg: [API] import item (text)

OVERVIEW.md

@@ -97,6 +97,18 @@ Redis and ARDB overview
 | ------ | ------ | ------ |
 | ail:all_role | **role** | **int, role priority (1=admin)** |
 
+##### Item Import:
+| Key | Value |
+| ------ | ------ |
+| **uuid**:isfile   | **boolean** |
+| **uuid**:paste_content | **item_content** |
+
+| Set Key | Value |
+| ------ | ------ |
+| submitted:uuid | **uuid** |
+| **uuid**:ltags | **tag** |
+| **uuid**:ltagsgalaxies | **tag** |
+
 ## DB2 - TermFreq:
 
 ##### Set:

bin/LAUNCH.sh

@@ -66,8 +66,8 @@ function helptext {
     "$DEFAULT"
     This script launch:
     "$CYAN"
-    - All the ZMQ queuing modules.
+    - All the queuing modules.
-    - All the ZMQ processing modules.
+    - All the processing modules.
     - All Redis in memory servers.
     - All ARDB on disk servers.
     "$DEFAULT"
@@ -143,7 +143,7 @@ function launching_scripts {
 
     screen -dmS "Script_AIL"
     sleep 0.1
-    echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
+    echo -e $GREEN"\t* Launching scripts"$DEFAULT
 
     screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"
     sleep 0.1

bin/packages/Import_helper.py

@@ -2,33 +2,75 @@
 # -*-coding:UTF-8 -*
 
 import os
+import uuid
 import redis
 
 import Flask_config
 
 r_serv_db = Flask_config.r_serv_db
-r_serv_log = Flask_config.r_serv_log
+r_serv_log_submit = Flask_config.r_serv_log_submit
 
-def create_import_queue(ltags, ltagsgalaxies, paste_content, UUID,  password, isfile = False):
+def is_valid_uuid_v4(UUID):
+    UUID = UUID.replace('-', '')
+    try:
+        uuid_test = uuid.UUID(hex=UUID, version=4)
+        return uuid_test.hex == UUID
+    except:
+        return False
+
+def create_import_queue(tags, galaxy, paste_content, UUID,  password=None, isfile = False):
 
     # save temp value on disk
-    r_serv_db.set(UUID + ':ltags', ltags)
+    for tag in tags:
-    r_serv_db.set(UUID + ':ltagsgalaxies', ltagsgalaxies)
+        r_serv_db.sadd(UUID + ':ltags', tag)
+    for tag in galaxy:
+        r_serv_db.sadd(UUID + ':ltagsgalaxies', tag)
+
     r_serv_db.set(UUID + ':paste_content', paste_content)
+
+    if password:
         r_serv_db.set(UUID + ':password', password)
+
     r_serv_db.set(UUID + ':isfile', isfile)
 
-    r_serv_log.set(UUID + ':end', 0)
+    r_serv_log_submit.set(UUID + ':end', 0)
-    r_serv_log.set(UUID + ':processing', 0)
+    r_serv_log_submit.set(UUID + ':processing', 0)
-    r_serv_log.set(UUID + ':nb_total', -1)
+    r_serv_log_submit.set(UUID + ':nb_total', -1)
-    r_serv_log.set(UUID + ':nb_end', 0)
+    r_serv_log_submit.set(UUID + ':nb_end', 0)
-    r_serv_log.set(UUID + ':nb_sucess', 0)
+    r_serv_log_submit.set(UUID + ':nb_sucess', 0)
 
     # save UUID on disk
     r_serv_db.sadd('submitted:uuid', UUID)
     return UUID
 
-def import_text_item():
+def check_import_status(UUID):
-    res = r_serv_db.smembers('submitted:uuid')
+    if not is_valid_uuid_v4(UUID):
-    print(res)
+        return ({'status': 'error', 'reason': 'Invalid uuid'}, 400)
-    return res
+
+    processing = r_serv_log_submit.get(UUID + ':processing')
+    if not processing:
+        return ({'status': 'error', 'reason': 'Unknow uuid'}, 400)
+
+    # nb_total = r_serv_log_submit.get(UUID + ':nb_total')
+    # nb_sucess = r_serv_log_submit.get(UUID + ':nb_sucess')
+    # nb_end = r_serv_log_submit.get(UUID + ':nb_end')
+    items_id = list(r_serv_log_submit.smembers(UUID + ':paste_submit_link'))
+    error = r_serv_log_submit.get(UUID + ':error')
+    end = r_serv_log_submit.get(UUID + ':end')
+
+    dict_import_status = {}
+    if items_id:
+        dict_import_status['items'] = items_id
+    if error:
+        dict_import_status['error'] = error
+
+    if processing == '0':
+        status = 'in queue'
+    else:
+        if end == '0':
+            status = 'in progress'
+        else:
+            status = 'imported'
+    dict_import_status['status'] = status
+
+    return (dict_import_status, 200)

bin/packages/Tags.py

@@ -31,7 +31,7 @@ def is_taxonomie_tag_enabled(taxonomie, tag):
     else:
         return False
 
-def is_galaxy_tag_enabled(taxonomie, galaxy):
+def is_galaxy_tag_enabled(galaxy, tag):
     if tag in r_serv_tags.smembers('active_tag_galaxies_' + galaxy):
         return True
     else:
@@ -39,8 +39,10 @@ def is_galaxy_tag_enabled(taxonomie, galaxy):
 
 # Check if tags are enabled in AIL
 def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
+    print(list_tags)
+    print(list_tags_galaxy)
     if list_tags:
-        active_taxonomies = Tags.get_active_taxonomies()
+        active_taxonomies = get_active_taxonomies()
 
         for tag in list_tags:
             taxonomie = get_taxonomie_from_tag(tag)
@@ -50,7 +52,7 @@ def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
                 return False
 
     if list_tags_galaxy:
-        active_galaxies = Tags.get_active_galaxies()
+        active_galaxies = get_active_galaxies()
 
         for tag in list_tags_galaxy:
             galaxy = get_galaxy_from_tag(tag)

bin/submit_paste.py

@@ -47,7 +47,11 @@ def create_paste(uuid, paste_content, ltags, ltagsgalaxies, name):
     r_serv_log_submit.hincrby("mixer_cache:list_feeder", "submitted", 1)
 
     # add tags
-    add_tags(ltags, ltagsgalaxies, rel_item_path)
+    for tag in ltags:
+        add_item_tag(tag, rel_item_path)
+
+    for tag in ltagsgalaxies:
+        add_item_tag(tag, rel_item_path)
 
     r_serv_log_submit.incr(uuid + ':nb_end')
     r_serv_log_submit.incr(uuid + ':nb_sucess')
@@ -133,18 +137,6 @@ def add_item_tag(tag, item_path):
     if item_date > tag_last_seen:
         r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', item_date)
 
-def add_tags(tags, tagsgalaxies, path):
-    list_tag = tags.split(',')
-    list_tag_galaxies = tagsgalaxies.split(',')
-
-    if list_tag != ['']:
-        for tag in list_tag:
-            add_item_tag(tag, path)
-
-    if list_tag_galaxies != ['']:
-        for tag in list_tag_galaxies:
-            add_item_tag(tag, path)
-
 def verify_extention_filename(filename):
     if not '.' in filename:
         return True
@@ -217,8 +209,8 @@ if __name__ == "__main__":
             uuid = r_serv_db.srandmember('submitted:uuid')
 
             # get temp value save on disk
-            ltags = r_serv_db.get(uuid + ':ltags')
+            ltags = r_serv_db.smembers(uuid + ':ltags')
-            ltagsgalaxies = r_serv_db.get(uuid + ':ltagsgalaxies')
+            ltagsgalaxies = r_serv_db.smembers(uuid + ':ltagsgalaxies')
             paste_content = r_serv_db.get(uuid + ':paste_content')
             isfile = r_serv_db.get(uuid + ':isfile')
             password = r_serv_db.get(uuid + ':password')
@@ -272,7 +264,7 @@ if __name__ == "__main__":
                         else:
                             #decompress file
                             try:
-                                if password == '':
+                                if password == None:
                                     files = unpack(file_full_path.encode())
                                     #print(files.children)
                                 else:

var/www/modules/Flask_config.py

@@ -178,6 +178,8 @@ crawler_enabled = cfg.getboolean("Crawler", "activate_crawler")
 email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
 email_regex = re.compile(email_regex)
 
+IMPORT_MAX_TEXT_SIZE = 900000 # size in bytes
+
 # VT
 try:
     from virusTotalKEYS import vt_key

var/www/modules/PasteSubmit/Flask_PasteSubmit.py

@@ -90,27 +90,6 @@ def clean_filename(filename, whitelist=valid_filename_chars, replace=' '):
     # keep only whitelisted chars
     return ''.join(c for c in cleaned_filename if c in whitelist)
 
-def launch_submit(ltags, ltagsgalaxies, paste_content, UUID,  password, isfile = False):
-
-    # save temp value on disk
-    r_serv_db.set(UUID + ':ltags', ltags)
-    r_serv_db.set(UUID + ':ltagsgalaxies', ltagsgalaxies)
-    r_serv_db.set(UUID + ':paste_content', paste_content)
-    r_serv_db.set(UUID + ':password', password)
-    r_serv_db.set(UUID + ':isfile', isfile)
-
-    r_serv_log_submit.set(UUID + ':end', 0)
-    r_serv_log_submit.set(UUID + ':processing', 0)
-    r_serv_log_submit.set(UUID + ':nb_total', -1)
-    r_serv_log_submit.set(UUID + ':nb_end', 0)
-    r_serv_log_submit.set(UUID + ':nb_sucess', 0)
-    r_serv_log_submit.set(UUID + ':error', 'error:')
-    r_serv_log_submit.sadd(UUID + ':paste_submit_link', '')
-
-
-    # save UUID on disk
-    r_serv_db.sadd('submitted:uuid', UUID)
-
 def date_to_str(date):
     return "{0}-{1}-{2}".format(date.year, date.month, date.day)
 
@@ -264,9 +243,6 @@ def submit():
     ltagsgalaxies = request.form['tags_galaxies']
     paste_content = request.form['paste_content']
 
-    print(ltags)
-    print(ltagsgalaxies)
-
     is_file = False
     if 'file' in request.files:
         file = request.files['file']
@@ -283,8 +259,11 @@ def submit():
 
     if ltags or ltagsgalaxies:
 
-        list_tag = tags.split(',')
+        ltags = ltags.split(',')
-        list_tag_galaxies = tagsgalaxies.split(',')
+        ltagsgalaxies = ltagsgalaxies.split(',')
+
+        print(ltags)
+        print(ltagsgalaxies)
 
         if not Tags.is_valid_tags_taxonomies_galaxy(ltags, ltagsgalaxies):
             content = 'INVALID TAGS'
@@ -292,10 +271,9 @@ def submit():
             return content, 400
 
     # add submitted tags
-    if(ltags != ''):
+    if not ltags:
-        ltags = ltags + ',' + submitted_tag
+        ltags = []
-    else:
+    ltags.append(submitted_tag)
-        ltags = submitted_tag
 
     if is_file:
         if file:
@@ -346,11 +324,6 @@ def submit():
 
             # get id
             UUID = str(uuid.uuid4())
-
-            #if paste_name:
-                # clean file name
-                #id = clean_filename(paste_name)
-
             Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password)
 
             return render_template("submit_items.html",
@@ -385,7 +358,7 @@ def submit_status():
         nb_sucess = r_serv_log_submit.get(UUID + ':nb_sucess')
         paste_submit_link = list(r_serv_log_submit.smembers(UUID + ':paste_submit_link'))
 
-        if (end != None) and (nb_total != None) and (nb_end != None) and (error != None) and (processing != None) and (paste_submit_link != None):
+        if (end != None) and (nb_total != None) and (nb_end != None) and (processing != None):
 
             link = ''
             if paste_submit_link:

var/www/modules/restApi/Flask_restApi.py

@@ -14,6 +14,7 @@ import redis
 import datetime
 
 import Import_helper
+import Tags
 
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
 from flask_login import login_required
@@ -151,24 +152,14 @@ def items():
 # {
 #   "type": "text",         (default value)
 #   "tags": [],             (default value)
-#   "default_ags": True,    (default value)
+#   "default_tags": True,    (default value)
 #   "galaxy" [],            (default value)
 #   "text": "",             mandatory if type = text
 # }
 #
 # response: {"uuid": "uuid"}
 #
-# # # #
-# GET
-#
-# {
-#   "uuid": "uuid",      mandatory
-# }
-#
-# response: {"uuid": "uuid"}
-#
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
-
 @restApi.route("api/import/item", methods=['POST'])
 @token_required('admin')
 def import_item():
@@ -176,24 +167,59 @@ def import_item():
     if not data:
         return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400
 
-    # TODO: add submitted tag
+    # unpack json
+    text_to_import = data.get('text', None)
+    if not text_to_import:
+        return Response(json.dumps({'status': 'error', 'reason': 'No text supplied'}, indent=2, sort_keys=True), mimetype='application/json'), 400
 
-    UUID = 'uuuuuuu'
+    tags = data.get('tags', [])
+    if not type(tags) is list:
+        tags = []
+    galaxy = data.get('galaxy', [])
+    if not type(galaxy) is list:
+        galaxy = []
+
+    if not Tags.is_valid_tags_taxonomies_galaxy(tags, galaxy):
+        return Response(json.dumps({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, indent=2, sort_keys=True), mimetype='application/json'), 400
+
+    default_tags = data.get('default_tags', True)
+    if default_tags:
+        tags.append('infoleak:submission="manual"')
+
+    if sys.getsizeof(text_to_import) > 900000:
+        return Response(json.dumps({'status': 'error', 'reason': 'Size exceeds default'}, indent=2, sort_keys=True), mimetype='application/json'), 400
+
+    UUID = str(uuid.uuid4())
+    Import_helper.create_import_queue(tags, galaxy, text_to_import, UUID)
 
     return Response(json.dumps({'uuid': UUID}, indent=2, sort_keys=True), mimetype='application/json')
 
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
+# GET
+#
+# {
+#   "uuid": "uuid",      mandatory
+# }
+#
+# response: {
+#               "status": "in queue"/"in progress"/"imported",
+#               "items": [all item id]
+#           }
+#
+# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 @restApi.route("api/import/item/<UUID>", methods=['GET'])
 @token_required('admin')
 def import_item_uuid(UUID):
 
     # Verify uuid
     if not is_valid_uuid_v4(UUID):
-        Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400
+        return Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400
 
+    data = Import_helper.check_import_status(UUID)
+    if data:
+        return Response(json.dumps(data[0]), mimetype='application/json'), data[1]
 
-
+    return Response(json.dumps({'status': 'error', 'reason': 'Invalid response'}), mimetype='application/json'), 400
-
-    return Response(json.dumps({'item_id': 4}), mimetype='application/json')
 
 # ========= REGISTRATION =========
 app.register_blueprint(restApi, url_prefix=baseUrl)