From 0a756294fe7b74aaa5c3167a7de320f85248dd74 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 26 Jul 2019 14:28:02 +0200 Subject: [PATCH] chg: [API] import item (text) --- OVERVIEW.md | 12 ++++ bin/LAUNCH.sh | 6 +- bin/packages/Import_helper.py | 70 +++++++++++++++---- bin/packages/Tags.py | 8 ++- bin/submit_paste.py | 24 +++---- var/www/modules/Flask_config.py | 2 + .../modules/PasteSubmit/Flask_PasteSubmit.py | 45 +++--------- var/www/modules/restApi/Flask_restApi.py | 60 +++++++++++----- 8 files changed, 138 insertions(+), 89 deletions(-) diff --git a/OVERVIEW.md b/OVERVIEW.md index ee553848..f4ee12ec 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -97,6 +97,18 @@ Redis and ARDB overview | ------ | ------ | ------ | | ail:all_role | **role** | **int, role priority (1=admin)** | +##### Item Import: +| Key | Value | +| ------ | ------ | +| **uuid**:isfile | **boolean** | +| **uuid**:paste_content | **item_content** | + +| Set Key | Value | +| ------ | ------ | +| submitted:uuid | **uuid** | +| **uuid**:ltags | **tag** | +| **uuid**:ltagsgalaxies | **tag** | + ## DB2 - TermFreq: ##### Set: diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index e4175b90..98645165 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -66,8 +66,8 @@ function helptext { "$DEFAULT" This script launch: "$CYAN" - - All the ZMQ queuing modules. - - All the ZMQ processing modules. + - All the queuing modules. + - All the processing modules. - All Redis in memory servers. - All ARDB on disk servers. "$DEFAULT" @@ -143,7 +143,7 @@ function launching_scripts { screen -dmS "Script_AIL" sleep 0.1 - echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT + echo -e $GREEN"\t* Launching scripts"$DEFAULT screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x" sleep 0.1 diff --git a/bin/packages/Import_helper.py b/bin/packages/Import_helper.py index 85a8b0d5..3ce4406f 100755 --- a/bin/packages/Import_helper.py +++ b/bin/packages/Import_helper.py @@ -2,33 +2,75 @@ # -*-coding:UTF-8 -* import os +import uuid import redis import Flask_config r_serv_db = Flask_config.r_serv_db -r_serv_log = Flask_config.r_serv_log +r_serv_log_submit = Flask_config.r_serv_log_submit -def create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password, isfile = False): +def is_valid_uuid_v4(UUID): + UUID = UUID.replace('-', '') + try: + uuid_test = uuid.UUID(hex=UUID, version=4) + return uuid_test.hex == UUID + except: + return False + +def create_import_queue(tags, galaxy, paste_content, UUID, password=None, isfile = False): # save temp value on disk - r_serv_db.set(UUID + ':ltags', ltags) - r_serv_db.set(UUID + ':ltagsgalaxies', ltagsgalaxies) + for tag in tags: + r_serv_db.sadd(UUID + ':ltags', tag) + for tag in galaxy: + r_serv_db.sadd(UUID + ':ltagsgalaxies', tag) + r_serv_db.set(UUID + ':paste_content', paste_content) - r_serv_db.set(UUID + ':password', password) + + if password: + r_serv_db.set(UUID + ':password', password) + r_serv_db.set(UUID + ':isfile', isfile) - r_serv_log.set(UUID + ':end', 0) - r_serv_log.set(UUID + ':processing', 0) - r_serv_log.set(UUID + ':nb_total', -1) - r_serv_log.set(UUID + ':nb_end', 0) - r_serv_log.set(UUID + ':nb_sucess', 0) + r_serv_log_submit.set(UUID + ':end', 0) + r_serv_log_submit.set(UUID + ':processing', 0) + r_serv_log_submit.set(UUID + ':nb_total', -1) + r_serv_log_submit.set(UUID + ':nb_end', 0) + r_serv_log_submit.set(UUID + ':nb_sucess', 0) # save UUID on disk r_serv_db.sadd('submitted:uuid', UUID) return UUID -def import_text_item(): - res = r_serv_db.smembers('submitted:uuid') - print(res) - return res +def check_import_status(UUID): + if not is_valid_uuid_v4(UUID): + return ({'status': 'error', 'reason': 'Invalid uuid'}, 400) + + processing = r_serv_log_submit.get(UUID + ':processing') + if not processing: + return ({'status': 'error', 'reason': 'Unknow uuid'}, 400) + + # nb_total = r_serv_log_submit.get(UUID + ':nb_total') + # nb_sucess = r_serv_log_submit.get(UUID + ':nb_sucess') + # nb_end = r_serv_log_submit.get(UUID + ':nb_end') + items_id = list(r_serv_log_submit.smembers(UUID + ':paste_submit_link')) + error = r_serv_log_submit.get(UUID + ':error') + end = r_serv_log_submit.get(UUID + ':end') + + dict_import_status = {} + if items_id: + dict_import_status['items'] = items_id + if error: + dict_import_status['error'] = error + + if processing == '0': + status = 'in queue' + else: + if end == '0': + status = 'in progress' + else: + status = 'imported' + dict_import_status['status'] = status + + return (dict_import_status, 200) diff --git a/bin/packages/Tags.py b/bin/packages/Tags.py index d916a29d..88963732 100755 --- a/bin/packages/Tags.py +++ b/bin/packages/Tags.py @@ -31,7 +31,7 @@ def is_taxonomie_tag_enabled(taxonomie, tag): else: return False -def is_galaxy_tag_enabled(taxonomie, galaxy): +def is_galaxy_tag_enabled(galaxy, tag): if tag in r_serv_tags.smembers('active_tag_galaxies_' + galaxy): return True else: @@ -39,8 +39,10 @@ def is_galaxy_tag_enabled(taxonomie, galaxy): # Check if tags are enabled in AIL def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy): + print(list_tags) + print(list_tags_galaxy) if list_tags: - active_taxonomies = Tags.get_active_taxonomies() + active_taxonomies = get_active_taxonomies() for tag in list_tags: taxonomie = get_taxonomie_from_tag(tag) @@ -50,7 +52,7 @@ def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy): return False if list_tags_galaxy: - active_galaxies = Tags.get_active_galaxies() + active_galaxies = get_active_galaxies() for tag in list_tags_galaxy: galaxy = get_galaxy_from_tag(tag) diff --git a/bin/submit_paste.py b/bin/submit_paste.py index e6875b3b..0609f581 100755 --- a/bin/submit_paste.py +++ b/bin/submit_paste.py @@ -47,7 +47,11 @@ def create_paste(uuid, paste_content, ltags, ltagsgalaxies, name): r_serv_log_submit.hincrby("mixer_cache:list_feeder", "submitted", 1) # add tags - add_tags(ltags, ltagsgalaxies, rel_item_path) + for tag in ltags: + add_item_tag(tag, rel_item_path) + + for tag in ltagsgalaxies: + add_item_tag(tag, rel_item_path) r_serv_log_submit.incr(uuid + ':nb_end') r_serv_log_submit.incr(uuid + ':nb_sucess') @@ -133,18 +137,6 @@ def add_item_tag(tag, item_path): if item_date > tag_last_seen: r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', item_date) -def add_tags(tags, tagsgalaxies, path): - list_tag = tags.split(',') - list_tag_galaxies = tagsgalaxies.split(',') - - if list_tag != ['']: - for tag in list_tag: - add_item_tag(tag, path) - - if list_tag_galaxies != ['']: - for tag in list_tag_galaxies: - add_item_tag(tag, path) - def verify_extention_filename(filename): if not '.' in filename: return True @@ -217,8 +209,8 @@ if __name__ == "__main__": uuid = r_serv_db.srandmember('submitted:uuid') # get temp value save on disk - ltags = r_serv_db.get(uuid + ':ltags') - ltagsgalaxies = r_serv_db.get(uuid + ':ltagsgalaxies') + ltags = r_serv_db.smembers(uuid + ':ltags') + ltagsgalaxies = r_serv_db.smembers(uuid + ':ltagsgalaxies') paste_content = r_serv_db.get(uuid + ':paste_content') isfile = r_serv_db.get(uuid + ':isfile') password = r_serv_db.get(uuid + ':password') @@ -272,7 +264,7 @@ if __name__ == "__main__": else: #decompress file try: - if password == '': + if password == None: files = unpack(file_full_path.encode()) #print(files.children) else: diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index 0e0d0e8b..0e3852e7 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -178,6 +178,8 @@ crawler_enabled = cfg.getboolean("Crawler", "activate_crawler") email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}' email_regex = re.compile(email_regex) +IMPORT_MAX_TEXT_SIZE = 900000 # size in bytes + # VT try: from virusTotalKEYS import vt_key diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 11e405a7..71d16de2 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -90,27 +90,6 @@ def clean_filename(filename, whitelist=valid_filename_chars, replace=' '): # keep only whitelisted chars return ''.join(c for c in cleaned_filename if c in whitelist) -def launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password, isfile = False): - - # save temp value on disk - r_serv_db.set(UUID + ':ltags', ltags) - r_serv_db.set(UUID + ':ltagsgalaxies', ltagsgalaxies) - r_serv_db.set(UUID + ':paste_content', paste_content) - r_serv_db.set(UUID + ':password', password) - r_serv_db.set(UUID + ':isfile', isfile) - - r_serv_log_submit.set(UUID + ':end', 0) - r_serv_log_submit.set(UUID + ':processing', 0) - r_serv_log_submit.set(UUID + ':nb_total', -1) - r_serv_log_submit.set(UUID + ':nb_end', 0) - r_serv_log_submit.set(UUID + ':nb_sucess', 0) - r_serv_log_submit.set(UUID + ':error', 'error:') - r_serv_log_submit.sadd(UUID + ':paste_submit_link', '') - - - # save UUID on disk - r_serv_db.sadd('submitted:uuid', UUID) - def date_to_str(date): return "{0}-{1}-{2}".format(date.year, date.month, date.day) @@ -264,9 +243,6 @@ def submit(): ltagsgalaxies = request.form['tags_galaxies'] paste_content = request.form['paste_content'] - print(ltags) - print(ltagsgalaxies) - is_file = False if 'file' in request.files: file = request.files['file'] @@ -283,8 +259,11 @@ def submit(): if ltags or ltagsgalaxies: - list_tag = tags.split(',') - list_tag_galaxies = tagsgalaxies.split(',') + ltags = ltags.split(',') + ltagsgalaxies = ltagsgalaxies.split(',') + + print(ltags) + print(ltagsgalaxies) if not Tags.is_valid_tags_taxonomies_galaxy(ltags, ltagsgalaxies): content = 'INVALID TAGS' @@ -292,10 +271,9 @@ def submit(): return content, 400 # add submitted tags - if(ltags != ''): - ltags = ltags + ',' + submitted_tag - else: - ltags = submitted_tag + if not ltags: + ltags = [] + ltags.append(submitted_tag) if is_file: if file: @@ -346,11 +324,6 @@ def submit(): # get id UUID = str(uuid.uuid4()) - - #if paste_name: - # clean file name - #id = clean_filename(paste_name) - Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password) return render_template("submit_items.html", @@ -385,7 +358,7 @@ def submit_status(): nb_sucess = r_serv_log_submit.get(UUID + ':nb_sucess') paste_submit_link = list(r_serv_log_submit.smembers(UUID + ':paste_submit_link')) - if (end != None) and (nb_total != None) and (nb_end != None) and (error != None) and (processing != None) and (paste_submit_link != None): + if (end != None) and (nb_total != None) and (nb_end != None) and (processing != None): link = '' if paste_submit_link: diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 07e3240f..ae3f0375 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -14,6 +14,7 @@ import redis import datetime import Import_helper +import Tags from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response from flask_login import login_required @@ -151,24 +152,14 @@ def items(): # { # "type": "text", (default value) # "tags": [], (default value) -# "default_ags": True, (default value) +# "default_tags": True, (default value) # "galaxy" [], (default value) # "text": "", mandatory if type = text # } # # response: {"uuid": "uuid"} # -# # # # -# GET -# -# { -# "uuid": "uuid", mandatory -# } -# -# response: {"uuid": "uuid"} -# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # - @restApi.route("api/import/item", methods=['POST']) @token_required('admin') def import_item(): @@ -176,24 +167,59 @@ def import_item(): if not data: return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400 - # TODO: add submitted tag + # unpack json + text_to_import = data.get('text', None) + if not text_to_import: + return Response(json.dumps({'status': 'error', 'reason': 'No text supplied'}, indent=2, sort_keys=True), mimetype='application/json'), 400 - UUID = 'uuuuuuu' + tags = data.get('tags', []) + if not type(tags) is list: + tags = [] + galaxy = data.get('galaxy', []) + if not type(galaxy) is list: + galaxy = [] + + if not Tags.is_valid_tags_taxonomies_galaxy(tags, galaxy): + return Response(json.dumps({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, indent=2, sort_keys=True), mimetype='application/json'), 400 + + default_tags = data.get('default_tags', True) + if default_tags: + tags.append('infoleak:submission="manual"') + + if sys.getsizeof(text_to_import) > 900000: + return Response(json.dumps({'status': 'error', 'reason': 'Size exceeds default'}, indent=2, sort_keys=True), mimetype='application/json'), 400 + + UUID = str(uuid.uuid4()) + Import_helper.create_import_queue(tags, galaxy, text_to_import, UUID) return Response(json.dumps({'uuid': UUID}, indent=2, sort_keys=True), mimetype='application/json') +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # +# GET +# +# { +# "uuid": "uuid", mandatory +# } +# +# response: { +# "status": "in queue"/"in progress"/"imported", +# "items": [all item id] +# } +# +# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @restApi.route("api/import/item/", methods=['GET']) @token_required('admin') def import_item_uuid(UUID): # Verify uuid if not is_valid_uuid_v4(UUID): - Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400 + return Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400 + data = Import_helper.check_import_status(UUID) + if data: + return Response(json.dumps(data[0]), mimetype='application/json'), data[1] - - - return Response(json.dumps({'item_id': 4}), mimetype='application/json') + return Response(json.dumps({'status': 'error', 'reason': 'Invalid response'}), mimetype='application/json'), 400 # ========= REGISTRATION ========= app.register_blueprint(restApi, url_prefix=baseUrl)