mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-27 00:07:16 +00:00
chg: [API + import] add API format + item_import refractor
This commit is contained in:
parent
44d6eb8570
commit
3a8531cafa
8 changed files with 210 additions and 66 deletions
15
OVERVIEW.md
15
OVERVIEW.md
|
@ -38,6 +38,21 @@ Redis and ARDB overview
|
||||||
| failed_login_ip:**ip** | **nb login failed** | TTL
|
| failed_login_ip:**ip** | **nb login failed** | TTL
|
||||||
| failed_login_user_id:**user_id** | **nb login failed** | TTL
|
| failed_login_user_id:**user_id** | **nb login failed** | TTL
|
||||||
|
|
||||||
|
##### Item Import:
|
||||||
|
|
||||||
|
| Key | Value |
|
||||||
|
| ------ | ------ |
|
||||||
|
| **uuid**:nb_total | **nb total** | TTL *(if imported)*
|
||||||
|
| **uuid**:nb_end | **nb** | TTL *(if imported)*
|
||||||
|
| **uuid**:nb_sucess | **nb success** | TTL *(if imported)*
|
||||||
|
| **uuid**:end | **0 (in progress) or (item imported)** | TTL *(if imported)*
|
||||||
|
| **uuid**:processing | **process status: 0 or 1** | TTL *(if imported)*
|
||||||
|
| **uuid**:error | **error message** | TTL *(if imported)*
|
||||||
|
|
||||||
|
| Set Key | Value |
|
||||||
|
| ------ | ------ |
|
||||||
|
| **uuid**:paste_submit_link | **item_path** | TTL *(if imported)*
|
||||||
|
|
||||||
## DB0 - Core:
|
## DB0 - Core:
|
||||||
|
|
||||||
##### Update keys:
|
##### Update keys:
|
||||||
|
|
34
bin/packages/Import_helper.py
Executable file
34
bin/packages/Import_helper.py
Executable file
|
@ -0,0 +1,34 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import redis
|
||||||
|
|
||||||
|
import Flask_config
|
||||||
|
|
||||||
|
r_serv_db = Flask_config.r_serv_db
|
||||||
|
r_serv_log = Flask_config.r_serv_log
|
||||||
|
|
||||||
|
def create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password, isfile = False):
|
||||||
|
|
||||||
|
# save temp value on disk
|
||||||
|
r_serv_db.set(UUID + ':ltags', ltags)
|
||||||
|
r_serv_db.set(UUID + ':ltagsgalaxies', ltagsgalaxies)
|
||||||
|
r_serv_db.set(UUID + ':paste_content', paste_content)
|
||||||
|
r_serv_db.set(UUID + ':password', password)
|
||||||
|
r_serv_db.set(UUID + ':isfile', isfile)
|
||||||
|
|
||||||
|
r_serv_log.set(UUID + ':end', 0)
|
||||||
|
r_serv_log.set(UUID + ':processing', 0)
|
||||||
|
r_serv_log.set(UUID + ':nb_total', -1)
|
||||||
|
r_serv_log.set(UUID + ':nb_end', 0)
|
||||||
|
r_serv_log.set(UUID + ':nb_sucess', 0)
|
||||||
|
|
||||||
|
# save UUID on disk
|
||||||
|
r_serv_db.sadd('submitted:uuid', UUID)
|
||||||
|
return UUID
|
||||||
|
|
||||||
|
def import_text_item():
|
||||||
|
res = r_serv_db.smembers('submitted:uuid')
|
||||||
|
print(res)
|
||||||
|
return res
|
61
bin/packages/Tags.py
Executable file
61
bin/packages/Tags.py
Executable file
|
@ -0,0 +1,61 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import redis
|
||||||
|
|
||||||
|
import Flask_config
|
||||||
|
|
||||||
|
from pytaxonomies import Taxonomies
|
||||||
|
from pymispgalaxies import Galaxies, Clusters
|
||||||
|
|
||||||
|
r_serv_tags = Flask_config.r_serv_tags
|
||||||
|
|
||||||
|
def get_taxonomie_from_tag(tag):
|
||||||
|
return tag.split(':')[0]
|
||||||
|
|
||||||
|
def get_galaxy_from_tag(tag):
|
||||||
|
galaxy = tag.split(':')[1]
|
||||||
|
galaxy = galaxy.split('=')[0]
|
||||||
|
return galaxy
|
||||||
|
|
||||||
|
def get_active_taxonomies():
|
||||||
|
return r_serv_tags.smembers('active_taxonomies')
|
||||||
|
|
||||||
|
def get_active_galaxies():
|
||||||
|
return r_serv_tags.smembers('active_galaxies')
|
||||||
|
|
||||||
|
def is_taxonomie_tag_enabled(taxonomie, tag):
|
||||||
|
if tag in r_serv_tags.smembers('active_tag_' + taxonomie):
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def is_galaxy_tag_enabled(taxonomie, galaxy):
|
||||||
|
if tag in r_serv_tags.smembers('active_tag_galaxies_' + galaxy):
|
||||||
|
return True
|
||||||
|
else:
|
||||||
|
return False
|
||||||
|
|
||||||
|
# Check if tags are enabled in AIL
|
||||||
|
def is_valid_tags_taxonomies_galaxy(list_tags, list_tags_galaxy):
|
||||||
|
if list_tags:
|
||||||
|
active_taxonomies = Tags.get_active_taxonomies()
|
||||||
|
|
||||||
|
for tag in list_tags:
|
||||||
|
taxonomie = get_taxonomie_from_tag(tag)
|
||||||
|
if taxonomie not in active_taxonomies:
|
||||||
|
return False
|
||||||
|
if not is_taxonomie_tag_enabled(taxonomie, tag):
|
||||||
|
return False
|
||||||
|
|
||||||
|
if list_tags_galaxy:
|
||||||
|
active_galaxies = Tags.get_active_galaxies()
|
||||||
|
|
||||||
|
for tag in list_tags_galaxy:
|
||||||
|
galaxy = get_galaxy_from_tag(tag)
|
||||||
|
if galaxy not in active_galaxies:
|
||||||
|
return False
|
||||||
|
if not is_galaxy_tag_enabled(galaxy, tag):
|
||||||
|
return False
|
||||||
|
return True
|
|
@ -92,7 +92,6 @@ def remove_submit_uuid(uuid):
|
||||||
r_serv_log_submit.expire(uuid + ':nb_sucess', expire_time)
|
r_serv_log_submit.expire(uuid + ':nb_sucess', expire_time)
|
||||||
r_serv_log_submit.expire(uuid + ':nb_end', expire_time)
|
r_serv_log_submit.expire(uuid + ':nb_end', expire_time)
|
||||||
r_serv_log_submit.expire(uuid + ':error', expire_time)
|
r_serv_log_submit.expire(uuid + ':error', expire_time)
|
||||||
r_serv_log_submit.srem(uuid + ':paste_submit_link', '')
|
|
||||||
r_serv_log_submit.expire(uuid + ':paste_submit_link', expire_time)
|
r_serv_log_submit.expire(uuid + ':paste_submit_link', expire_time)
|
||||||
|
|
||||||
# delete uuid
|
# delete uuid
|
||||||
|
@ -230,8 +229,6 @@ if __name__ == "__main__":
|
||||||
r_serv_log_submit.set(uuid + ':nb_total', -1)
|
r_serv_log_submit.set(uuid + ':nb_total', -1)
|
||||||
r_serv_log_submit.set(uuid + ':nb_end', 0)
|
r_serv_log_submit.set(uuid + ':nb_end', 0)
|
||||||
r_serv_log_submit.set(uuid + ':nb_sucess', 0)
|
r_serv_log_submit.set(uuid + ':nb_sucess', 0)
|
||||||
r_serv_log_submit.set(uuid + ':error', 'error:')
|
|
||||||
r_serv_log_submit.sadd(uuid + ':paste_submit_link', '')
|
|
||||||
|
|
||||||
|
|
||||||
r_serv_log_submit.set(uuid + ':processing', 1)
|
r_serv_log_submit.set(uuid + ':processing', 1)
|
||||||
|
|
|
@ -67,15 +67,15 @@ log_dir = os.path.join(os.environ['AIL_HOME'], 'logs')
|
||||||
if not os.path.isdir(log_dir):
|
if not os.path.isdir(log_dir):
|
||||||
os.makedirs(logs_dir)
|
os.makedirs(logs_dir)
|
||||||
|
|
||||||
log_filename = os.path.join(log_dir, 'flask_server.logs')
|
# log_filename = os.path.join(log_dir, 'flask_server.logs')
|
||||||
logger = logging.getLogger()
|
# logger = logging.getLogger()
|
||||||
formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
|
# formatter = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s')
|
||||||
handler_log = logging.handlers.TimedRotatingFileHandler(log_filename, when="midnight", interval=1)
|
# handler_log = logging.handlers.TimedRotatingFileHandler(log_filename, when="midnight", interval=1)
|
||||||
handler_log.suffix = '%Y-%m-%d.log'
|
# handler_log.suffix = '%Y-%m-%d.log'
|
||||||
handler_log.setFormatter(formatter)
|
# handler_log.setFormatter(formatter)
|
||||||
handler_log.setLevel(30)
|
# handler_log.setLevel(30)
|
||||||
logger.addHandler(handler_log)
|
# logger.addHandler(handler_log)
|
||||||
logger.setLevel(30)
|
# logger.setLevel(30)
|
||||||
|
|
||||||
# ========= =========#
|
# ========= =========#
|
||||||
|
|
||||||
|
@ -226,7 +226,7 @@ def login():
|
||||||
# login failed
|
# login failed
|
||||||
else:
|
else:
|
||||||
# set brute force protection
|
# set brute force protection
|
||||||
logger.warning("Login failed, ip={}, username={}".format(current_ip, username))
|
#logger.warning("Login failed, ip={}, username={}".format(current_ip, username))
|
||||||
r_cache.incr('failed_login_ip:{}'.format(current_ip))
|
r_cache.incr('failed_login_ip:{}'.format(current_ip))
|
||||||
r_cache.expire('failed_login_ip:{}'.format(current_ip), 300)
|
r_cache.expire('failed_login_ip:{}'.format(current_ip), 300)
|
||||||
r_cache.incr('failed_login_user_id:{}'.format(username))
|
r_cache.incr('failed_login_user_id:{}'.format(username))
|
||||||
|
|
|
@ -12,7 +12,6 @@ import sys
|
||||||
|
|
||||||
# FLASK #
|
# FLASK #
|
||||||
app = None
|
app = None
|
||||||
#secret_key = 'ail-super-secret_key01C'
|
|
||||||
|
|
||||||
# CONFIG #
|
# CONFIG #
|
||||||
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
|
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
|
||||||
|
@ -146,7 +145,7 @@ if HiveApi != False:
|
||||||
HiveApi = False
|
HiveApi = False
|
||||||
print('The Hive not connected')
|
print('The Hive not connected')
|
||||||
|
|
||||||
# VARIABLES #
|
#### VARIABLES ####
|
||||||
baseUrl = cfg.get("Flask", "baseurl")
|
baseUrl = cfg.get("Flask", "baseurl")
|
||||||
baseUrl = baseUrl.replace('/', '')
|
baseUrl = baseUrl.replace('/', '')
|
||||||
if baseUrl != '':
|
if baseUrl != '':
|
||||||
|
|
|
@ -23,6 +23,9 @@ import json
|
||||||
|
|
||||||
import Paste
|
import Paste
|
||||||
|
|
||||||
|
import Import_helper
|
||||||
|
import Tags
|
||||||
|
|
||||||
from pytaxonomies import Taxonomies
|
from pytaxonomies import Taxonomies
|
||||||
from pymispgalaxies import Galaxies, Clusters
|
from pymispgalaxies import Galaxies, Clusters
|
||||||
|
|
||||||
|
@ -108,44 +111,6 @@ def launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password, isfile =
|
||||||
# save UUID on disk
|
# save UUID on disk
|
||||||
r_serv_db.sadd('submitted:uuid', UUID)
|
r_serv_db.sadd('submitted:uuid', UUID)
|
||||||
|
|
||||||
|
|
||||||
def addTagsVerification(tags, tagsgalaxies):
|
|
||||||
|
|
||||||
list_tag = tags.split(',')
|
|
||||||
list_tag_galaxies = tagsgalaxies.split(',')
|
|
||||||
|
|
||||||
taxonomies = Taxonomies()
|
|
||||||
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
|
|
||||||
|
|
||||||
active_galaxies = r_serv_tags.smembers('active_galaxies')
|
|
||||||
|
|
||||||
if list_tag != ['']:
|
|
||||||
for tag in list_tag:
|
|
||||||
# verify input
|
|
||||||
tax = tag.split(':')[0]
|
|
||||||
if tax in active_taxonomies:
|
|
||||||
if tag in r_serv_tags.smembers('active_tag_' + tax):
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
return False
|
|
||||||
else:
|
|
||||||
return False
|
|
||||||
|
|
||||||
if list_tag_galaxies != ['']:
|
|
||||||
for tag in list_tag_galaxies:
|
|
||||||
# verify input
|
|
||||||
gal = tag.split(':')[1]
|
|
||||||
gal = gal.split('=')[0]
|
|
||||||
|
|
||||||
if gal in active_galaxies:
|
|
||||||
if tag in r_serv_tags.smembers('active_tag_galaxies_' + gal):
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
return False
|
|
||||||
else:
|
|
||||||
return False
|
|
||||||
return True
|
|
||||||
|
|
||||||
def date_to_str(date):
|
def date_to_str(date):
|
||||||
return "{0}-{1}-{2}".format(date.year, date.month, date.day)
|
return "{0}-{1}-{2}".format(date.year, date.month, date.day)
|
||||||
|
|
||||||
|
@ -279,11 +244,9 @@ def hive_create_case(hive_tlp, threat_level, hive_description, hive_case_title,
|
||||||
@login_required
|
@login_required
|
||||||
@login_analyst
|
@login_analyst
|
||||||
def PasteSubmit_page():
|
def PasteSubmit_page():
|
||||||
#active taxonomies
|
# Get all active tags/galaxy
|
||||||
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
|
active_taxonomies = Tags.get_active_taxonomies()
|
||||||
|
active_galaxies = Tags.get_active_galaxies()
|
||||||
#active galaxies
|
|
||||||
active_galaxies = r_serv_tags.smembers('active_galaxies')
|
|
||||||
|
|
||||||
return render_template("submit_items.html",
|
return render_template("submit_items.html",
|
||||||
active_taxonomies = active_taxonomies,
|
active_taxonomies = active_taxonomies,
|
||||||
|
@ -301,6 +264,9 @@ def submit():
|
||||||
ltagsgalaxies = request.form['tags_galaxies']
|
ltagsgalaxies = request.form['tags_galaxies']
|
||||||
paste_content = request.form['paste_content']
|
paste_content = request.form['paste_content']
|
||||||
|
|
||||||
|
print(ltags)
|
||||||
|
print(ltagsgalaxies)
|
||||||
|
|
||||||
is_file = False
|
is_file = False
|
||||||
if 'file' in request.files:
|
if 'file' in request.files:
|
||||||
file = request.files['file']
|
file = request.files['file']
|
||||||
|
@ -311,12 +277,16 @@ def submit():
|
||||||
submitted_tag = 'infoleak:submission="manual"'
|
submitted_tag = 'infoleak:submission="manual"'
|
||||||
|
|
||||||
#active taxonomies
|
#active taxonomies
|
||||||
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
|
active_taxonomies = Tags.get_active_taxonomies()
|
||||||
#active galaxies
|
#active galaxies
|
||||||
active_galaxies = r_serv_tags.smembers('active_galaxies')
|
active_galaxies = Tags.get_active_galaxies()
|
||||||
|
|
||||||
if ltags or ltagsgalaxies:
|
if ltags or ltagsgalaxies:
|
||||||
if not addTagsVerification(ltags, ltagsgalaxies):
|
|
||||||
|
list_tag = tags.split(',')
|
||||||
|
list_tag_galaxies = tagsgalaxies.split(',')
|
||||||
|
|
||||||
|
if not Tags.is_valid_tags_taxonomies_galaxy(ltags, ltagsgalaxies):
|
||||||
content = 'INVALID TAGS'
|
content = 'INVALID TAGS'
|
||||||
print(content)
|
print(content)
|
||||||
return content, 400
|
return content, 400
|
||||||
|
@ -358,7 +328,7 @@ def submit():
|
||||||
|
|
||||||
paste_content = full_path
|
paste_content = full_path
|
||||||
|
|
||||||
launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password ,True)
|
Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password ,True)
|
||||||
|
|
||||||
return render_template("submit_items.html",
|
return render_template("submit_items.html",
|
||||||
active_taxonomies = active_taxonomies,
|
active_taxonomies = active_taxonomies,
|
||||||
|
@ -381,7 +351,7 @@ def submit():
|
||||||
# clean file name
|
# clean file name
|
||||||
#id = clean_filename(paste_name)
|
#id = clean_filename(paste_name)
|
||||||
|
|
||||||
launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password)
|
Import_helper.create_import_queue(ltags, ltagsgalaxies, paste_content, UUID, password)
|
||||||
|
|
||||||
return render_template("submit_items.html",
|
return render_template("submit_items.html",
|
||||||
active_taxonomies = active_taxonomies,
|
active_taxonomies = active_taxonomies,
|
||||||
|
@ -433,10 +403,10 @@ def submit_status():
|
||||||
else:
|
else:
|
||||||
prog = 0
|
prog = 0
|
||||||
|
|
||||||
if error == 'error:':
|
if error:
|
||||||
isError = False
|
|
||||||
else:
|
|
||||||
isError = True
|
isError = True
|
||||||
|
else:
|
||||||
|
isError = False
|
||||||
|
|
||||||
if end == '0':
|
if end == '0':
|
||||||
end = False
|
end = False
|
||||||
|
|
|
@ -8,10 +8,13 @@
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
import sys
|
import sys
|
||||||
|
import uuid
|
||||||
import json
|
import json
|
||||||
import redis
|
import redis
|
||||||
import datetime
|
import datetime
|
||||||
|
|
||||||
|
import Import_helper
|
||||||
|
|
||||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
|
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
|
||||||
from flask_login import login_required
|
from flask_login import login_required
|
||||||
|
|
||||||
|
@ -20,6 +23,7 @@ from functools import wraps
|
||||||
# ============ VARIABLES ============
|
# ============ VARIABLES ============
|
||||||
import Flask_config
|
import Flask_config
|
||||||
|
|
||||||
|
|
||||||
app = Flask_config.app
|
app = Flask_config.app
|
||||||
cfg = Flask_config.cfg
|
cfg = Flask_config.cfg
|
||||||
baseUrl = Flask_config.baseUrl
|
baseUrl = Flask_config.baseUrl
|
||||||
|
@ -108,8 +112,20 @@ def authErrors(user_role):
|
||||||
else:
|
else:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
|
# ============ API CORE =============
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# ============ FUNCTIONS ============
|
# ============ FUNCTIONS ============
|
||||||
|
|
||||||
|
def is_valid_uuid_v4(header_uuid):
|
||||||
|
try:
|
||||||
|
header_uuid=header_uuid.replace('-', '')
|
||||||
|
uuid_test = uuid.UUID(hex=header_uuid, version=4)
|
||||||
|
return uuid_test.hex == header_uuid
|
||||||
|
except:
|
||||||
|
return False
|
||||||
|
|
||||||
def one():
|
def one():
|
||||||
return 1
|
return 1
|
||||||
|
|
||||||
|
@ -127,5 +143,57 @@ def items():
|
||||||
|
|
||||||
return Response(json.dumps({'test': 2}), mimetype='application/json')
|
return Response(json.dumps({'test': 2}), mimetype='application/json')
|
||||||
|
|
||||||
|
|
||||||
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
|
#
|
||||||
|
# POST JSON FORMAT
|
||||||
|
#
|
||||||
|
# {
|
||||||
|
# "type": "text", (default value)
|
||||||
|
# "tags": [], (default value)
|
||||||
|
# "default_ags": True, (default value)
|
||||||
|
# "galaxy" [], (default value)
|
||||||
|
# "text": "", mandatory if type = text
|
||||||
|
# }
|
||||||
|
#
|
||||||
|
# response: {"uuid": "uuid"}
|
||||||
|
#
|
||||||
|
# # # #
|
||||||
|
# GET
|
||||||
|
#
|
||||||
|
# {
|
||||||
|
# "uuid": "uuid", mandatory
|
||||||
|
# }
|
||||||
|
#
|
||||||
|
# response: {"uuid": "uuid"}
|
||||||
|
#
|
||||||
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
|
|
||||||
|
@restApi.route("api/import/item", methods=['POST'])
|
||||||
|
@token_required('admin')
|
||||||
|
def import_item():
|
||||||
|
data = request.get_json()
|
||||||
|
if not data:
|
||||||
|
return Response(json.dumps({'status': 'error', 'reason': 'Malformed JSON'}, indent=2, sort_keys=True), mimetype='application/json'), 400
|
||||||
|
|
||||||
|
# TODO: add submitted tag
|
||||||
|
|
||||||
|
UUID = 'uuuuuuu'
|
||||||
|
|
||||||
|
return Response(json.dumps({'uuid': UUID}, indent=2, sort_keys=True), mimetype='application/json')
|
||||||
|
|
||||||
|
@restApi.route("api/import/item/<UUID>", methods=['GET'])
|
||||||
|
@token_required('admin')
|
||||||
|
def import_item_uuid(UUID):
|
||||||
|
|
||||||
|
# Verify uuid
|
||||||
|
if not is_valid_uuid_v4(UUID):
|
||||||
|
Response(json.dumps({'status': 'error', 'reason': 'Invalid uuid'}), mimetype='application/json'), 400
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
return Response(json.dumps({'item_id': 4}), mimetype='application/json')
|
||||||
|
|
||||||
# ========= REGISTRATION =========
|
# ========= REGISTRATION =========
|
||||||
app.register_blueprint(restApi, url_prefix=baseUrl)
|
app.register_blueprint(restApi, url_prefix=baseUrl)
|
||||||
|
|
Loading…
Reference in a new issue