diff --git a/bin/submit_paste.py b/bin/submit_paste.py
index 7ee009f2..ab688740 100755
--- a/bin/submit_paste.py
+++ b/bin/submit_paste.py
@@ -79,6 +79,9 @@ class SubmitPaste(AbstractModule):
password = self.r_serv_db.get(f'{uuid}:password')
source = self.r_serv_db.get(f'{uuid}:source')
+ if source in ['crawled', 'tests']:
+ source = 'submitted'
+
self.redis_logger.debug(f'isfile UUID {isfile}')
self.redis_logger.debug(f'source UUID {source}')
self.redis_logger.debug(f'paste_content UUID {paste_content}')
@@ -100,7 +103,7 @@ class SubmitPaste(AbstractModule):
# textarea input paste
self._manage_text(uuid, paste_content, ltags, ltagsgalaxies, source)
- # new paste created from file, remove uuid ref
+ # new paste created from file, remove uuid ref
self.remove_submit_uuid(uuid)
@@ -108,12 +111,12 @@ class SubmitPaste(AbstractModule):
"""
Run Module endless process
"""
-
+
# Endless loop processing messages from the input queue
while self.proceed:
# Get one message (paste) from the QueueIn (copy of Redis_Global publish)
nb_submit = self.r_serv_db.scard('submitted:uuid')
-
+
if nb_submit > 0:
try:
uuid = self.r_serv_db.srandmember('submitted:uuid')
@@ -122,7 +125,7 @@ class SubmitPaste(AbstractModule):
self.compute(uuid)
except Exception as err:
self.redis_logger.error(f'Error in module {self.module_name}: {err}')
- # Remove uuid ref
+ # Remove uuid ref
self.remove_submit_uuid(uuid)
else:
# Wait before next process
@@ -150,7 +153,7 @@ class SubmitPaste(AbstractModule):
if os.path.exists(file_full_path):
self.redis_logger.debug(f'file exists {file_full_path}')
-
+
file_size = os.stat(file_full_path).st_size
self.redis_logger.debug(f'file size {file_size}')
# Verify file length
@@ -168,7 +171,6 @@ class SubmitPaste(AbstractModule):
content = f.read()
self.r_serv_log_submit.set(uuid + ':nb_total', 1)
self.create_paste(uuid, content.encode(), ltags, ltagsgalaxies, uuid, source)
- self.remove_submit_uuid(uuid)
except:
self.abord_file_submission(uuid, "file error")
@@ -261,15 +263,17 @@ class SubmitPaste(AbstractModule):
# delete uuid
self.r_serv_db.srem('submitted:uuid', uuid)
self.redis_logger.debug(f'{uuid} all file submitted')
+ print(f'{uuid} all file submitted')
def create_paste(self, uuid, paste_content, ltags, ltagsgalaxies, name, source=None):
-
+ # # TODO: Use Item create
+
result = False
now = datetime.datetime.now()
source = source if source else 'submitted'
- save_path = source + '/' + now.strftime("%Y") + '/' + now.strftime("%m") + '/' + now.strftime("%d") + '/' + name + '.gz'
+ save_path = source + '/' + now.strftime("%Y") + '/' + now.strftime("%m") + '/' + now.strftime("%d") + '/submitted_' + name + '.gz'
full_path = filename = os.path.join(os.environ['AIL_HOME'],
self.process.config.get("Directories", "pastes"), save_path)
@@ -309,6 +313,7 @@ class SubmitPaste(AbstractModule):
self.r_serv_log_submit.set(f'{uuid}:end', 1)
self.redis_logger.debug(f' {rel_item_path} send to Global')
+ print(f' {rel_item_path} send to Global')
self.r_serv_log_submit.sadd(f'{uuid}:paste_submit_link', rel_item_path)
curr_date = datetime.date.today()
@@ -328,13 +333,13 @@ class SubmitPaste(AbstractModule):
gzip64encoded = base64.standard_b64encode(gzipencoded).decode()
except:
self.abord_file_submission(uuid, "file error")
-
+
return gzip64encoded
def addError(self, uuid, errorMessage):
self.redis_logger.debug(errorMessage)
-
+ print(errorMessage)
error = self.r_serv_log_submit.get(f'{uuid}:error')
if error != None:
self.r_serv_log_submit.set(f'{uuid}:error', error + '
' + errorMessage)
@@ -351,7 +356,7 @@ class SubmitPaste(AbstractModule):
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'submit_abord', 1)
self.remove_submit_uuid(uuid)
-
+ # # TODO: use Item function
def get_item_date(self, item_filename):
l_directory = item_filename.split('/')
return f'{l_directory[-4]}{l_directory[-3]}{l_directory[-2]}'
@@ -371,6 +376,6 @@ class SubmitPaste(AbstractModule):
if __name__ == '__main__':
-
+
module = SubmitPaste()
module.run()
diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py
index e7227888..04336d4e 100644
--- a/var/www/modules/Flask_config.py
+++ b/var/www/modules/Flask_config.py
@@ -45,7 +45,7 @@ redis_logger = publisher
# Port of the redis instance used by pubsublogger
redis_logger.port = 6380
# Channel name to publish logs
-redis_logger.channel = 'front'
+redis_logger.channel = 'Flask'
sys.path.append('../../configs/keys')
diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py
index 85bd1cb2..6b4a002e 100644
--- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py
+++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py
@@ -11,7 +11,6 @@ import os
import sys
import json
import string
-import subprocess
import datetime
import redis
import unicodedata
@@ -19,8 +18,10 @@ import uuid
from io import BytesIO
from Date import Date
-from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, abort
from functools import wraps
+
+# Flask
+from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, abort
from Role_Manager import login_admin, login_analyst
from flask_login import login_required
@@ -257,7 +258,7 @@ def PasteSubmit_page():
return render_template("submit_items.html",
active_taxonomies = active_taxonomies,
- active_galaxies = active_galaxies,
+ active_galaxies = active_galaxies,
text_max_size = text_max_size,
file_max_size = file_max_size,
allowed_extensions = allowed_extensions)
@@ -277,6 +278,13 @@ def submit():
paste_content = request.form['paste_content']
paste_source = request.form['paste_source']
+ # limit source length
+ paste_source = paste_source.replace('/', '')[:80]
+ if paste_source in ['crawled', 'tests']:
+ content = f'Invalid source'
+ logger.info(paste_source)
+ return content, 400
+
is_file = False
if 'file' in request.files:
file_import = request.files['file']
@@ -343,7 +351,7 @@ def submit():
#Flask verify the file size
file_import.save(full_path)
logger.debug('file saved')
-
+
Import_helper.create_import_queue(ltags, ltagsgalaxies, full_path, UUID, password, True)
return render_template("submit_items.html",
diff --git a/var/www/modules/PasteSubmit/templates/submit_items.html b/var/www/modules/PasteSubmit/templates/submit_items.html
index 1a8d4f75..8e19a042 100644
--- a/var/www/modules/PasteSubmit/templates/submit_items.html
+++ b/var/www/modules/PasteSubmit/templates/submit_items.html
@@ -100,7 +100,7 @@
{{ message }}
+{{ message }}
{% endif %}