diff --git a/bin/submit_paste.py b/bin/submit_paste.py index 7ee009f2..ab688740 100755 --- a/bin/submit_paste.py +++ b/bin/submit_paste.py @@ -79,6 +79,9 @@ class SubmitPaste(AbstractModule): password = self.r_serv_db.get(f'{uuid}:password') source = self.r_serv_db.get(f'{uuid}:source') + if source in ['crawled', 'tests']: + source = 'submitted' + self.redis_logger.debug(f'isfile UUID {isfile}') self.redis_logger.debug(f'source UUID {source}') self.redis_logger.debug(f'paste_content UUID {paste_content}') @@ -100,7 +103,7 @@ class SubmitPaste(AbstractModule): # textarea input paste self._manage_text(uuid, paste_content, ltags, ltagsgalaxies, source) - # new paste created from file, remove uuid ref + # new paste created from file, remove uuid ref self.remove_submit_uuid(uuid) @@ -108,12 +111,12 @@ class SubmitPaste(AbstractModule): """ Run Module endless process """ - + # Endless loop processing messages from the input queue while self.proceed: # Get one message (paste) from the QueueIn (copy of Redis_Global publish) nb_submit = self.r_serv_db.scard('submitted:uuid') - + if nb_submit > 0: try: uuid = self.r_serv_db.srandmember('submitted:uuid') @@ -122,7 +125,7 @@ class SubmitPaste(AbstractModule): self.compute(uuid) except Exception as err: self.redis_logger.error(f'Error in module {self.module_name}: {err}') - # Remove uuid ref + # Remove uuid ref self.remove_submit_uuid(uuid) else: # Wait before next process @@ -150,7 +153,7 @@ class SubmitPaste(AbstractModule): if os.path.exists(file_full_path): self.redis_logger.debug(f'file exists {file_full_path}') - + file_size = os.stat(file_full_path).st_size self.redis_logger.debug(f'file size {file_size}') # Verify file length @@ -168,7 +171,6 @@ class SubmitPaste(AbstractModule): content = f.read() self.r_serv_log_submit.set(uuid + ':nb_total', 1) self.create_paste(uuid, content.encode(), ltags, ltagsgalaxies, uuid, source) - self.remove_submit_uuid(uuid) except: self.abord_file_submission(uuid, "file error") @@ -261,15 +263,17 @@ class SubmitPaste(AbstractModule): # delete uuid self.r_serv_db.srem('submitted:uuid', uuid) self.redis_logger.debug(f'{uuid} all file submitted') + print(f'{uuid} all file submitted') def create_paste(self, uuid, paste_content, ltags, ltagsgalaxies, name, source=None): - + # # TODO: Use Item create + result = False now = datetime.datetime.now() source = source if source else 'submitted' - save_path = source + '/' + now.strftime("%Y") + '/' + now.strftime("%m") + '/' + now.strftime("%d") + '/' + name + '.gz' + save_path = source + '/' + now.strftime("%Y") + '/' + now.strftime("%m") + '/' + now.strftime("%d") + '/submitted_' + name + '.gz' full_path = filename = os.path.join(os.environ['AIL_HOME'], self.process.config.get("Directories", "pastes"), save_path) @@ -309,6 +313,7 @@ class SubmitPaste(AbstractModule): self.r_serv_log_submit.set(f'{uuid}:end', 1) self.redis_logger.debug(f' {rel_item_path} send to Global') + print(f' {rel_item_path} send to Global') self.r_serv_log_submit.sadd(f'{uuid}:paste_submit_link', rel_item_path) curr_date = datetime.date.today() @@ -328,13 +333,13 @@ class SubmitPaste(AbstractModule): gzip64encoded = base64.standard_b64encode(gzipencoded).decode() except: self.abord_file_submission(uuid, "file error") - + return gzip64encoded def addError(self, uuid, errorMessage): self.redis_logger.debug(errorMessage) - + print(errorMessage) error = self.r_serv_log_submit.get(f'{uuid}:error') if error != None: self.r_serv_log_submit.set(f'{uuid}:error', error + '

' + errorMessage) @@ -351,7 +356,7 @@ class SubmitPaste(AbstractModule): self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'submit_abord', 1) self.remove_submit_uuid(uuid) - + # # TODO: use Item function def get_item_date(self, item_filename): l_directory = item_filename.split('/') return f'{l_directory[-4]}{l_directory[-3]}{l_directory[-2]}' @@ -371,6 +376,6 @@ class SubmitPaste(AbstractModule): if __name__ == '__main__': - + module = SubmitPaste() module.run() diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index e7227888..04336d4e 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -45,7 +45,7 @@ redis_logger = publisher # Port of the redis instance used by pubsublogger redis_logger.port = 6380 # Channel name to publish logs -redis_logger.channel = 'front' +redis_logger.channel = 'Flask' sys.path.append('../../configs/keys') diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 85bd1cb2..6b4a002e 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -11,7 +11,6 @@ import os import sys import json import string -import subprocess import datetime import redis import unicodedata @@ -19,8 +18,10 @@ import uuid from io import BytesIO from Date import Date -from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, abort from functools import wraps + +# Flask +from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, abort from Role_Manager import login_admin, login_analyst from flask_login import login_required @@ -257,7 +258,7 @@ def PasteSubmit_page(): return render_template("submit_items.html", active_taxonomies = active_taxonomies, - active_galaxies = active_galaxies, + active_galaxies = active_galaxies, text_max_size = text_max_size, file_max_size = file_max_size, allowed_extensions = allowed_extensions) @@ -277,6 +278,13 @@ def submit(): paste_content = request.form['paste_content'] paste_source = request.form['paste_source'] + # limit source length + paste_source = paste_source.replace('/', '')[:80] + if paste_source in ['crawled', 'tests']: + content = f'Invalid source' + logger.info(paste_source) + return content, 400 + is_file = False if 'file' in request.files: file_import = request.files['file'] @@ -343,7 +351,7 @@ def submit(): #Flask verify the file size file_import.save(full_path) logger.debug('file saved') - + Import_helper.create_import_queue(ltags, ltagsgalaxies, full_path, UUID, password, True) return render_template("submit_items.html", diff --git a/var/www/modules/PasteSubmit/templates/submit_items.html b/var/www/modules/PasteSubmit/templates/submit_items.html index 1a8d4f75..8e19a042 100644 --- a/var/www/modules/PasteSubmit/templates/submit_items.html +++ b/var/www/modules/PasteSubmit/templates/submit_items.html @@ -100,7 +100,7 @@
{% if message %} -

{{ message }}

+

{{ message }}

{% endif %}
-
Submit Paste
+
Submit Item
@@ -150,8 +150,8 @@ Tags {% for taxo in active_taxonomies %} - + {% endfor %}
@@ -174,8 +174,8 @@ {% for galaxy in active_galaxies %} - + {% endfor %} @@ -320,22 +320,22 @@ }); {% for taxo in active_taxonomies %} - jQuery("#{{ taxo }}-id{{ loop.index0 }}").on("click", function (e) { - $.getJSON("{{ url_for('Tags.get_tags_taxonomie') }}?taxonomie={{ taxo }}", - function (data) { - ltags.setData(data) - }); - }); + jQuery("#{{ taxo }}-id{{ loop.index0 }}").on("click", function (e) { + $.getJSON("{{ url_for('Tags.get_tags_taxonomie') }}?taxonomie={{ taxo }}", + function (data) { + ltags.setData(data) + }); + }); {% endfor %} @@ -417,4 +417,4 @@ - \ No newline at end of file +