diff --git a/bin/Global.py b/bin/Global.py index 6115a53f..32a3656b 100755 --- a/bin/Global.py +++ b/bin/Global.py @@ -57,7 +57,6 @@ if __name__ == '__main__': while True: message = p.get_from_set() - #print(message) # Recovering the streamed message informations. if message is not None: splitted = message.split() diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 1eec715d..ec51e715 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -92,6 +92,11 @@ host = localhost port = 6380 db = 0 +[Redis_Log_submit] +host = localhost +port = 6380 +db = 1 + [Redis_Queues] host = localhost port = 6381 diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 975b7b2c..454427ea 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -120,3 +120,6 @@ publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags [Bitcoin] subscribe = Redis_Global publish = Redis_Duplicate,Redis_alertHandler,Redis_Tags + +[submit_paste] +publish = Redis_Mixer diff --git a/bin/submit_paste.py b/bin/submit_paste.py new file mode 100755 index 00000000..048a4c43 --- /dev/null +++ b/bin/submit_paste.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import configparser +import os +import sys +import gzip +import io +import redis +import base64 +import datetime + +from Helper import Process + +def add_tags(tags, tagsgalaxies, path): + list_tag = tags.split(',') + list_tag_galaxies = tagsgalaxies.split(',') + + if list_tag != ['']: + for tag in list_tag: + #add tag + r_serv_metadata.sadd('tag:'+path, tag) + r_serv_tags.sadd(tag, path) + #add new tag in list of all used tags + r_serv_tags.sadd('list_tags', tag) + + if list_tag_galaxies != ['']: + for tag in list_tag_galaxies: + #add tag + r_serv_metadata.sadd('tag:'+path, tag) + r_serv_tags.sadd(tag, path) + #add new tag in list of all used tags + r_serv_tags.sadd('list_tags', tag) + + +if __name__ == "__main__": + if len(sys.argv) != 6: + print('usage:', 'submit_paste.py', 'ltags', 'ltagsgalaxies', 'paste_content', 'paste_name', 'id') + exit(1) + + try: + ltags = sys.argv[1] + ltagsgalaxies = sys.argv[2] + paste_content = sys.argv[3] + paste_name = sys.argv[4] + id = sys.argv[5] + except: + print('unable to get elements') + exit(1) + + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + + cfg = configparser.ConfigParser() + cfg.read(configfile) + + r_serv_log_submit = redis.StrictRedis( + host=cfg.get("Redis_Log_submit", "host"), + port=cfg.getint("Redis_Log_submit", "port"), + db=cfg.getint("Redis_Log_submit", "db"), + decode_responses=True) + + r_serv_tags = redis.StrictRedis( + host=cfg.get("ARDB_Tags", "host"), + port=cfg.getint("ARDB_Tags", "port"), + db=cfg.getint("ARDB_Tags", "db"), + decode_responses=True) + + r_serv_metadata = redis.StrictRedis( + host=cfg.get("ARDB_Metadata", "host"), + port=cfg.getint("ARDB_Metadata", "port"), + db=cfg.getint("ARDB_Metadata", "db"), + decode_responses=True) + + # TODO put on config + expire_time = 10200 + + r_serv_log_submit.expire(id + ':end', expire_time) + r_serv_log_submit.expire(id + ':nb_total', expire_time) + r_serv_log_submit.expire(id + ':nb_end', expire_time) + r_serv_log_submit.expire(id + ':error', expire_time) + + config_section = 'submit_paste' + p = Process(config_section) + + now = datetime.datetime.now() + save_path = 'submitted/' + now.strftime("%Y") + '/' + now.strftime("%m") + '/' + now.strftime("%d") + '/' + id + '.gz' + + full_path = filename = os.path.join(os.environ['AIL_HOME'], + p.config.get("Directories", "pastes"), save_path) + + if os.path.isfile(full_path): + error = r_serv_log_submit.get(id + ':error') + r_serv_log_submit.set(id + ':error', error + '

File: ' + save_path + ' already exist in submitted pastes') + exit(1) + + + gzipencoded = gzip.compress(paste_content.encode()) + gzip64encoded = base64.standard_b64encode(gzipencoded).decode() + + # send paste to Global module + relay_message = "{0} {1}".format(save_path, gzip64encoded) + p.populate_set_out(relay_message, 'Mixer') + + # add tags + add_tags(ltags, ltagsgalaxies, full_path) + + r_serv_log_submit.incr(id + ':nb_end') + + + if r_serv_log_submit.get(id + ':nb_end') == r_serv_log_submit.get(id + ':nb_total'): + r_serv_log_submit.set(id + ':end', 1) + + exit(0) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index a03999ab..077c3ea3 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -28,6 +28,7 @@ cfg = Flask_config.cfg Flask_config.app = Flask(__name__, static_url_path='/static/') app = Flask_config.app +#app.secret_key = Flask_config.secret_key # ========= HEADER GENERATION ======== diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index 26edccfa..db74928f 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -10,6 +10,7 @@ import os # FLASK # app = None +#secret_key = 'ail-super-secret_key01C' # CONFIG # configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') @@ -35,6 +36,12 @@ r_serv_log = redis.StrictRedis( db=cfg.getint("Redis_Log", "db"), decode_responses=True) +r_serv_log_submit = redis.StrictRedis( + host=cfg.get("Redis_Log_submit", "host"), + port=cfg.getint("Redis_Log_submit", "port"), + db=cfg.getint("Redis_Log_submit", "db"), + decode_responses=True) + r_serv_charts = redis.StrictRedis( host=cfg.get("ARDB_Trending", "host"), port=cfg.getint("ARDB_Trending", "port"), diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py new file mode 100644 index 00000000..d86c168e --- /dev/null +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Flask functions and routes for the trending modules page +''' +import redis +from flask import Flask, render_template, jsonify, request, Blueprint, session + +'''import random''' + +import unicodedata +import string +import subprocess +import os +import sys +import datetime + +from pytaxonomies import Taxonomies +from pymispgalaxies import Galaxies, Clusters + +# ============ VARIABLES ============ +import Flask_config + +app = Flask_config.app +cfg = Flask_config.cfg +r_serv_tags = Flask_config.r_serv_tags +r_serv_log_submit = Flask_config.r_serv_log_submit + +PasteSubmit = Blueprint('PasteSubmit', __name__, template_folder='templates') + +valid_filename_chars = "-_ %s%s" % (string.ascii_letters, string.digits) + +# ============ FUNCTIONS ============ +def one(): + return 1 + +def clean_filename(filename, whitelist=valid_filename_chars, replace=' '): + # replace characters + for r in replace: + filename = filename.replace(r,'_') + + # keep only valid ascii chars + cleaned_filename = unicodedata.normalize('NFKD', filename).encode('ASCII', 'ignore').decode() + + # keep only whitelisted chars + return ''.join(c for c in cleaned_filename if c in whitelist) + +'''@app.before_request +def csrf_protect(): + if request.method == "POST": + token = session.pop('_csrf_token', None) + if not token or token != request.form.get('_csrf_token'): + abort(400) + +def generate_csrf_token(): + if '_csrf_token' not in session: + session['_csrf_token'] = some_random_string() + return session['_csrf_token'] + +app.jinja_env.globals['csrf_token'] = generate_csrf_token + +def some_random_string(): + N = 15 + return ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(N))''' + + +def addTagsVerification(tags, tagsgalaxies): + + list_tag = tags.split(',') + list_tag_galaxies = tagsgalaxies.split(',') + + taxonomies = Taxonomies() + active_taxonomies = r_serv_tags.smembers('active_taxonomies') + + active_galaxies = r_serv_tags.smembers('active_galaxies') + + if list_tag != ['']: + for tag in list_tag: + # verify input + tax = tag.split(':')[0] + if tax in active_taxonomies: + if tag in r_serv_tags.smembers('active_tag_' + tax): + pass + else: + return False + else: + return False + + if list_tag_galaxies != ['']: + for tag in list_tag_galaxies: + # verify input + gal = tag.split(':')[1] + gal = gal.split('=')[0] + + if gal in active_galaxies: + if tag in r_serv_tags.smembers('active_tag_galaxies_' + gal): + pass + else: + return False + else: + return False + return True +# ============= ROUTES ============== + +@PasteSubmit.route("/PasteSubmit/", methods=['GET']) +def PasteSubmit_page(): + #active taxonomies + active_taxonomies = r_serv_tags.smembers('active_taxonomies') + + #active galaxies + active_galaxies = r_serv_tags.smembers('active_galaxies') + + return render_template("PasteSubmit.html", + active_taxonomies = active_taxonomies, + active_galaxies = active_galaxies) + +@PasteSubmit.route("/PasteSubmit/submit", methods=['POST']) +def submit(): + + paste_name = request.form['paste_name'] + ltags = request.form['tags_taxonomies'] + ltagsgalaxies = request.form['tags_galaxies'] + paste_content = request.form['paste_content'] + + if paste_content != '': + if sys.getsizeof(paste_content) < 900000: + + if ltags or ltagsgalaxies: + if not addTagsVerification(ltags, ltagsgalaxies): + return 'INVALID TAGS' + + to_launch = os.environ['AIL_BIN'] + 'submit_paste.py' + # get id + id = str(r_serv_tags.get('submit_id')) + + if paste_name: + # clean file name + id = clean_filename(paste_name) + + # create logs + r_serv_log_submit.set(id + ':end', 0) + r_serv_log_submit.set(id + ':nb_total', 1) + r_serv_log_submit.set(id + ':nb_end', 0) + r_serv_log_submit.set(id + ':error', 'error:') + + #incr id + r_serv_tags.incr('submit_id') + + # add submitted tags + if(ltags != ''): + ltags = ltags + ',submitted' + else: + ltags ='submitted' + + # launch process + process = subprocess.Popen(["python", to_launch, ltags, ltagsgalaxies, paste_content, paste_name, id], + stdout=subprocess.PIPE) + + return render_template("submiting.html", + id = id) + + else: + return 'size error' + + return 'submit' + +@PasteSubmit.route("/PasteSubmit/submit_status", methods=['GET']) +def submit_status(): + id = request.args.get('id') + + if id: + end = r_serv_log_submit.get(id + ':end') + nb_total = r_serv_log_submit.get(id + ':nb_total') + nb_end = r_serv_log_submit.get(id + ':nb_end') + error = r_serv_log_submit.get(id + ':error') + if (end != None) and (nb_total != None) and (nb_end != None) and (error != None): + + in_progress = nb_end + ' / ' + nb_total + prog = int(int(nb_end) * 100 / int(nb_total)) + + if error == 'error:': + isError = False + else: + isError = True + + if end == '0': + end = False + else: + end = True + + return jsonify(end=end, + in_progress=in_progress, + prog=prog, + isError=isError, + error=error) + else: + return 'to do' + else: + return 'INVALID ID' + +# ========= REGISTRATION ========= +app.register_blueprint(PasteSubmit) diff --git a/var/www/modules/PasteSubmit/templates/PasteSubmit.html b/var/www/modules/PasteSubmit/templates/PasteSubmit.html new file mode 100644 index 00000000..338ef693 --- /dev/null +++ b/var/www/modules/PasteSubmit/templates/PasteSubmit.html @@ -0,0 +1,203 @@ + + + + + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + +
+
+ + +
+ + + + +
+
paste info +
+
+ +
+ +
+ +
+ + +
+ +
+ + +
+
+
+ +
+
Tags : +
    +
  • + +
    + +
    + +
    + + +
    + +
    • +
  • + +
    + +
    + +
    + + +
    + +
    • +
+
+
+ +
+ +
+ +
+ +
+ +
+ + +
+ +
+ + +
+ + + + + + + + + + + + + + diff --git a/var/www/modules/PasteSubmit/templates/header_PasteSubmit.html b/var/www/modules/PasteSubmit/templates/header_PasteSubmit.html new file mode 100644 index 00000000..525f51e3 --- /dev/null +++ b/var/www/modules/PasteSubmit/templates/header_PasteSubmit.html @@ -0,0 +1 @@ +
  • PasteSubmit
    • diff --git a/var/www/modules/PasteSubmit/templates/submiting.html b/var/www/modules/PasteSubmit/templates/submiting.html new file mode 100644 index 00000000..78430d7d --- /dev/null +++ b/var/www/modules/PasteSubmit/templates/submiting.html @@ -0,0 +1,318 @@ + + + + + + + + Analysis Information Leak framework Dashboard + + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + +
    +
    + + +
    + + + + +
    +
    paste info +
    +
    + +
    + +
    + +
    + + +
    + +
    + + +
    +
    +
    + +
    +
    Tags : +
      +
    • + +
      + +
      + +
      + + +
      + +
      • +
    • + +
      + +
      + +
      + + +
      + +
      • +
    +
    +
    + +
    + +
    + +
    + +
    + +
    +
    + +
    + + + +
    + +
    + + + + + + + + + + + + + + + + + + + + diff --git a/var/www/modules/Tags/Flask_Tags.py b/var/www/modules/Tags/Flask_Tags.py index a8dd4c8b..31877d70 100644 --- a/var/www/modules/Tags/Flask_Tags.py +++ b/var/www/modules/Tags/Flask_Tags.py @@ -64,6 +64,7 @@ def get_tags_with_synonyms(tag): else: return {'name':tag,'id':tag} + # ============= ROUTES ============== @Tags.route("/Tags/", methods=['GET']) diff --git a/var/www/modules/dashboard/templates/searchbox.html b/var/www/modules/dashboard/templates/searchbox.html index bfbf1413..b2513ee8 100644 --- a/var/www/modules/dashboard/templates/searchbox.html +++ b/var/www/modules/dashboard/templates/searchbox.html @@ -1,16 +1,16 @@
    -