mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 22:27:17 +00:00
chg: [API] import item (text)
This commit is contained in:
parent
3a8531cafa
commit
0a756294fe
8 changed files with 138 additions and 89 deletions
12
OVERVIEW.md
12
OVERVIEW.md
|
@ -97,6 +97,18 @@ Redis and ARDB overview
|
|||
| ------ | ------ | ------ |
|
||||
| ail:all_role | **role** | **int, role priority (1=admin)** |
|
||||
|
||||
##### Item Import:
|
||||
| Key | Value |
|
||||
| ------ | ------ |
|
||||
| **uuid**:isfile | **boolean** |
|
||||
| **uuid**:paste_content | **item_content** |
|
||||
|
||||
| Set Key | Value |
|
||||
| ------ | ------ |
|
||||
| submitted:uuid | **uuid** |
|
||||
| **uuid**:ltags | **tag** |
|
||||
| **uuid**:ltagsgalaxies | **tag** |
|
||||
|
||||
## DB2 - TermFreq:
|
||||
|
||||
##### Set:
|
||||
|
|
|
@ -66,8 +66,8 @@ function helptext {
|
|||
"$DEFAULT"
|
||||
This script launch:
|
||||
"$CYAN"
|
||||
- All the ZMQ queuing modules.
|
||||
- All the ZMQ processing modules.
|
||||
- All the queuing modules.
|
||||
- All the processing modules.
|
||||
- All Redis in memory servers.
|
||||
- All ARDB on disk servers.
|
||||
"$DEFAULT"
|
||||
|
@ -143,7 +143,7 @@ function launching_scripts {
|
|||
|
||||
screen -dmS "Script_AIL"
|
||||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
|
||||
echo -e $GREEN"\t* Launching scripts"$DEFAULT
|
||||
|
||||
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"
|
||||
sleep 0.1
|
||||
|
|
|
@ -2,33 +2,75 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import uuid
|
||||
import redis
|
||||
|
||||
import Flask_config
|
||||
|
||||
r_serv_db = Flask_config.r_serv_db
|
||||
r_serv_log = Flask_config.r_serv_log
|
||||
r_serv_log_submit = Flask_config.r_serv_log_submit
|
||||
|
||||
def create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password, isfile = False):
|
||||
def is_valid_uuid_v4(UUID):
|
||||
UUID = UUID.replace('-', '')
|
||||
try:
|
||||
uuid_test = uuid.UUID(hex=UUID, version=4)
|
||||
return uuid_test.hex == UUID
|
||||
except:
|
||||
return False
|
||||
|
||||
def create_import_queue(tags, galaxy, paste_content, UUID, password=None, isfile = False):
|
||||
|
||||
# save temp value on disk
|
||||
r_serv_db.set(UUID + ':ltags', ltags)
|
||||
r_serv_db.set(UUID + ':ltagsgalaxies', ltagsgalaxies)
|
||||
for tag in tags:
|
||||
r_serv_db.sadd(UUID + ':ltags', tag)
|
||||
for tag in galaxy:
|
||||
r_serv_db.sadd(UUID + ':ltagsgalaxies', tag)
|
||||
|
||||
r_serv_db.set(UUID + ':paste_content', paste_content)
|
||||
r_serv_db.set(UUID + ':password', password)
|
||||
|
||||
if password:
|
||||
r_serv_db.set(UUID + ':password', password)
|
||||
|
||||
r_serv_db.set(UUID + ':isfile', isfile)
|
||||
|
||||
r_serv_log.set(UUID + ':end', 0)
|
||||
r_serv_log.set(UUID + ':processing', 0)
|
||||
r_serv_log.set(UUID + ':nb_total', -1)
|
||||
r_serv_log.set(UUID + ':nb_end', 0)
|
||||
r_serv_log.set(UUID + ':nb_sucess', 0)
|
||||
r_serv_log_submit.set(UUID + ':end', 0)
|
||||
r_serv_log_submit.set(UUID + ':processing', 0)
|
||||
r_serv_log_submit.set(UUID + ':nb_total', -1)
|
||||
r_serv_log_submit.set(UUID + ':nb_end', 0)
|
||||
r_serv_log_submit.set(UUID + ':nb_sucess', 0)
|
||||
|
||||
# save UUID on disk
|
||||
r_serv_db.sadd('submitted:uuid', UUID)
|
||||
return UUID
|
||||
|
||||
def import_text_item():
|
||||
res = r_serv_db.smembers('submitted:uuid')
|
||||
print(res)
|
||||
return res
|
||||
def check_import_status(UUID):
|
||||
if not is_valid_uuid_v4(UUID):
|
||||
return ({'status': 'error', 'reason': 'Invalid uuid'}, 400)
|
||||
|
||||
processing = r_serv_log_submit.get(UUID + ':processing')
|
||||
if not processing:
|
||||
return ({'status': 'error', 'reason': 'Unknow uuid'}, 400)
|
||||
|
||||
# nb_total = r_serv_log_submit.get(UUID + ':nb_total')
|
||||
# nb_sucess = r_serv_log_submit.get(UUID + ':nb_sucess')
|
||||
# nb_end = r_serv_log_submit.get(UUID + ':nb_end')
|
||||
items_id = list(r_serv_log_submit.smembers(UUID + ':paste_submit_link'))
|
||||
error = r_serv_log_submit.get(UUID + ':error')
|
||||
end = r_serv_log_submit.get(UUID + ':end')
|
||||
|
||||
dict_import_status = {}
|
||||
if items_id:
|
||||
dict_import_status['items'] = items_id
|
||||
if error:
|
||||
dict_import_status['error'] = error
|
||||
|
||||
if processing == '0':
|
||||
status = 'in queue'
|
||||
else:
|
||||
if end == '0':
|
||||
status = 'in progress'
|
||||
else:
|
||||
status = 'imported'
|
||||
dict_import_status['status'] = status
|
||||
|
||||
return (dict_import_status, 200)
|
||||
|
|
|
@ -31,7 +31,7 @@ def is_taxonomie_tag_enabled(taxonomie, tag):
|
|||
else:
|
||||
return False
|
||||
|
||||
def is_galaxy_tag_enabled(taxonomie, galaxy):
|
||||
def is_galaxy_tag_enabled(galaxy, tag):
|
||||
if tag in r_serv_tags.smembers('active_tag_galaxies_' + galaxy):
|
||||
return True
|
||||
else:
|
||||
|
@ -39,8 +39,10 @@ def is_galaxy_tag_enabled(taxonomie, galaxy):
|
|||
|
||||
# Check if tags are enabled in AIL
|
||||
def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
|
||||
print(list_tags)
|
||||
print(list_tags_galaxy)
|
||||
if list_tags:
|
||||
active_taxonomies = Tags.get_active_taxonomies()
|
||||
active_taxonomies = get_active_taxonomies()
|
||||
|
||||
for tag in list_tags:
|
||||
taxonomie = get_taxonomie_from_tag(tag)
|
||||
|
@ -50,7 +52,7 @@ def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
|
|||
return False
|
||||
|
||||
if list_tags_galaxy:
|
||||
active_galaxies = Tags.get_active_galaxies()
|
||||
active_galaxies = get_active_galaxies()
|
||||
|
||||
for tag in list_tags_galaxy:
|
||||
galaxy = get_galaxy_from_tag(tag)
|
||||
|
|
|
@ -47,7 +47,11 @@ def create_paste(uuid, paste_content, ltags, ltagsgalaxies, name):
|
|||
r_serv_log_submit.hincrby("mixer_cache:list_feeder", "submitted", 1)
|
||||
|
||||
# add tags
|
||||
add_tags(ltags, ltagsgalaxies, rel_item_path)
|
||||
for tag in ltags:
|
||||
add_item_tag(tag, rel_item_path)
|
||||
|
||||
for tag in ltagsgalaxies:
|
||||
add_item_tag(tag, rel_item_path)
|
||||
|
||||
r_serv_log_submit.incr(uuid + ':nb_end')
|
||||
r_serv_log_submit.incr(uuid + ':nb_sucess')
|
||||
|
@ -133,18 +137,6 @@ def add_item_tag(tag, item_path):
|
|||
if item_date > tag_last_seen:
|
||||
r_serv_tags.hset('tag_metadata:{}'.format(tag), 'last_seen', item_date)
|
||||
|
||||
def add_tags(tags, tagsgalaxies, path):
|
||||
list_tag = tags.split(',')
|
||||
list_tag_galaxies = tagsgalaxies.split(',')
|
||||
|
||||
if list_tag != ['']:
|
||||
for tag in list_tag:
|
||||
add_item_tag(tag, path)
|
||||
|
||||
if list_tag_galaxies != ['']:
|
||||
for tag in list_tag_galaxies:
|
||||
add_item_tag(tag, path)
|
||||
|
||||
def verify_extention_filename(filename):
|
||||
if not '.' in filename:
|
||||
return True
|
||||
|
@ -217,8 +209,8 @@ if __name__ == "__main__":
|
|||
uuid = r_serv_db.srandmember('submitted:uuid')
|
||||
|
||||
# get temp value save on disk
|
||||
ltags = r_serv_db.get(uuid + ':ltags')
|
||||
ltagsgalaxies = r_serv_db.get(uuid + ':ltagsgalaxies')
|
||||
ltags = r_serv_db.smembers(uuid + ':ltags')
|
||||
ltagsgalaxies = r_serv_db.smembers(uuid + ':ltagsgalaxies')
|
||||
paste_content = r_serv_db.get(uuid + ':paste_content')
|
||||
isfile = r_serv_db.get(uuid + ':isfile')
|
||||
password = r_serv_db.get(uuid + ':password')
|
||||
|
@ -272,7 +264,7 @@ if __name__ == "__main__":
|
|||
else:
|
||||
#decompress file
|
||||
try:
|
||||
if password == '':
|
||||
if password == None:
|
||||
files = unpack(file_full_path.encode())
|
||||
#print(files.children)
|
||||
else:
|
||||
|
|
|
@ -178,6 +178,8 @@ crawler_enabled = cfg.getboolean("Crawler", "activate_crawler")
|
|||
email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
|
||||
email_regex = re.compile(email_regex)
|
||||
|
||||
IMPORT_MAX_TEXT_SIZE = 900000 # size in bytes
|
||||
|
||||
# VT
|
||||
try:
|
||||
from virusTotalKEYS import vt_key
|
||||
|
|
|
@ -90,27 +90,6 @@ def clean_filename(filename, whitelist=valid_filename_chars, replace=' '):
|
|||
# keep only whitelisted chars
|
||||
return ''.join(c for c in cleaned_filename if c in whitelist)
|
||||
|
||||
def launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password, isfile = False):
|
||||
|
||||
# save temp value on disk
|
||||
r_serv_db.set(UUID + ':ltags', ltags)
|
||||
r_serv_db.set(UUID + ':ltagsgalaxies', ltagsgalaxies)
|
||||
r_serv_db.set(UUID + ':paste_content', paste_content)
|
||||
r_serv_db.set(UUID + ':password', password)
|
||||
r_serv_db.set(UUID + ':isfile', isfile)
|
||||
|
||||
r_serv_log_submit.set(UUID + ':end', 0)
|
||||
r_serv_log_submit.set(UUID + ':processing', 0)
|
||||
r_serv_log_submit.set(UUID + ':nb_total', -1)
|
||||
r_serv_log_submit.set(UUID + ':nb_end', 0)
|
||||
r_serv_log_submit.set(UUID + ':nb_sucess', 0)
|
||||
r_serv_log_submit.set(UUID + ':error', 'error:')
|
||||
r_serv_log_submit.sadd(UUID + ':paste_submit_link', '')
|
||||
|
||||
|
||||
# save UUID on disk
|
||||
r_serv_db.sadd('submitted:uuid', UUID)
|
||||
|
||||
def date_to_str(date):
|
||||
return "{0}-{1}-{2}".format(date.year, date.month, date.day)
|
||||
|
||||
|
@ -264,9 +243,6 @@ def submit():
|
|||
ltagsgalaxies = request.form['tags_galaxies']
|
||||
paste_content = request.form['paste_content']
|
||||
|
||||
print(ltags)
|
||||
print(ltagsgalaxies)
|
||||
|
||||
is_file = False
|
||||
if 'file' in request.files:
|
||||
file = request.files['file']
|
||||
|
@ -283,8 +259,11 @@ def submit():
|
|||
|
||||
if ltags or ltagsgalaxies:
|
||||
|
||||
list_tag = tags.split(',')
|
||||
list_tag_galaxies = tagsgalaxies.split(',')
|
||||
ltags = ltags.split(',')
|
||||
ltagsgalaxies = ltagsgalaxies.split(',')
|
||||
|
||||
print(ltags)
|
||||
print(ltagsgalaxies)
|
||||
|
||||
if not Tags.is_valid_tags_taxonomies_galaxy(ltags, ltagsgalaxies):
|
||||
content = 'INVALID TAGS'
|
||||
|
@ -292,10 +271,9 @@ def submit():
|
|||
return content, 400
|
||||
|
||||
# add submitted tags
|
||||
if(ltags != ''):
|
||||
ltags = ltags + ',' + submitted_tag
|
||||
else:
|
||||
ltags = submitted_tag
|
||||
if not ltags:
|
||||
ltags = []
|
||||
ltags.append(submitted_tag)
|
||||
|
||||
if is_file:
|
||||
if file:
|
||||
|
@ -346,11 +324,6 @@ def submit():
|
|||
|
||||
# get id
|
||||
UUID = str(uuid.uuid4())
|
||||
|
||||
#if paste_name:
|
||||
# clean file name
|
||||
#id = clean_filename(paste_name)
|
||||
|
||||
Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password)
|
||||
|
||||
return render_template("submit_items.html",
|
||||
|
@ -385,7 +358,7 @@ def submit_status():
|
|||
nb_sucess = r_serv_log_submit.get(UUID + ':nb_sucess')
|
||||
paste_submit_link = list(r_serv_log_submit.smembers(UUID + ':paste_submit_link'))
|
||||
|
||||
if (end != None) and (nb_total != None) and (nb_end != None) and (error != None) and (processing != None) and (paste_submit_link != None):
|
||||
if (end != None) and (nb_total != None) and (nb_end != None) and (processing != None):
|
||||
|
||||
link = ''
|
||||
if paste_submit_link:
|
||||
|
|
|
@ -14,6 +14,7 @@ import redis
|
|||
import datetime
|
||||
|
||||
import Import_helper
|
||||
import Tags
|
||||
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
|
||||
from flask_login import login_required
|
||||
|
@ -151,24 +152,14 @@ def items():
|
|||
# {
|
||||
# "type": "text", (default value)
|
||||
# "tags": [], (default value)
|
||||
# "default_ags": True, (default value)
|
||||
# "default_tags": True, (default value)
|
||||
# "galaxy" [], (default value)
|
||||
# "text": "", mandatory if type = text
|
||||
# }
|
||||
#
|
||||
# response: {"uuid": "uuid"}
|
||||
#
|
||||
# # # #
|
||||
# GET
|
||||
#
|
||||
# {
|
||||
# "uuid": "uuid", mandatory
|
||||
# }
|
||||
#
|
||||
# response: {"uuid": "uuid"}
|
||||
#
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
|
||||
@restApi.route("api/import/item", methods=['POST'])
|
||||
@token_required('admin')
|
||||
def import_item():
|
||||
|
@ -176,24 +167,59 @@ def import_item():
|
|||
if not data:
|
||||
return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400
|
||||
|
||||
# TODO: add submitted tag
|
||||
# unpack json
|
||||
text_to_import = data.get('text', None)
|
||||
if not text_to_import:
|
||||
return Response(json.dumps({'status': 'error', 'reason': 'No text supplied'}, indent=2, sort_keys=True), mimetype='application/json'), 400
|
||||
|
||||
UUID = 'uuuuuuu'
|
||||
tags = data.get('tags', [])
|
||||
if not type(tags) is list:
|
||||
tags = []
|
||||
galaxy = data.get('galaxy', [])
|
||||
if not type(galaxy) is list:
|
||||
galaxy = []
|
||||
|
||||
if not Tags.is_valid_tags_taxonomies_galaxy(tags, galaxy):
|
||||
return Response(json.dumps({'status': 'error', 'reason': 'Tags or Galaxy not enabled'}, indent=2, sort_keys=True), mimetype='application/json'), 400
|
||||
|
||||
default_tags = data.get('default_tags', True)
|
||||
if default_tags:
|
||||
tags.append('infoleak:submission="manual"')
|
||||
|
||||
if sys.getsizeof(text_to_import) > 900000:
|
||||
return Response(json.dumps({'status': 'error', 'reason': 'Size exceeds default'}, indent=2, sort_keys=True), mimetype='application/json'), 400
|
||||
|
||||
UUID = str(uuid.uuid4())
|
||||
Import_helper.create_import_queue(tags, galaxy, text_to_import, UUID)
|
||||
|
||||
return Response(json.dumps({'uuid': UUID}, indent=2, sort_keys=True), mimetype='application/json')
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
# GET
|
||||
#
|
||||
# {
|
||||
# "uuid": "uuid", mandatory
|
||||
# }
|
||||
#
|
||||
# response: {
|
||||
# "status": "in queue"/"in progress"/"imported",
|
||||
# "items": [all item id]
|
||||
# }
|
||||
#
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
@restApi.route("api/import/item/<UUID>", methods=['GET'])
|
||||
@token_required('admin')
|
||||
def import_item_uuid(UUID):
|
||||
|
||||
# Verify uuid
|
||||
if not is_valid_uuid_v4(UUID):
|
||||
Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400
|
||||
return Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400
|
||||
|
||||
data = Import_helper.check_import_status(UUID)
|
||||
if data:
|
||||
return Response(json.dumps(data[0]), mimetype='application/json'), data[1]
|
||||
|
||||
|
||||
|
||||
return Response(json.dumps({'item_id': 4}), mimetype='application/json')
|
||||
return Response(json.dumps({'status': 'error', 'reason': 'Invalid response'}), mimetype='application/json'), 400
|
||||
|
||||
# ========= REGISTRATION =========
|
||||
app.register_blueprint(restApi, url_prefix=baseUrl)
|
||||
|
|
Loading…
Reference in a new issue