AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-10 00:28:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: [submit_paste] restrict source name

Browse source
This commit is contained in:
Terrtia 2021-05-28 17:37:46 +02:00
parent 75bc585242
commit a2ebd09c2a
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
4 changed files with 49 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
bin/submit_paste.py
View file

@ -79,6 +79,9 @@ class SubmitPaste(AbstractModule):
password = self.r_serv_db.get(f'{uuid}:password') password = self.r_serv_db.get(f'{uuid}:password')
source = self.r_serv_db.get(f'{uuid}:source') source = self.r_serv_db.get(f'{uuid}:source')
if source in ['crawled', 'tests']:
source = 'submitted'
self.redis_logger.debug(f'isfile UUID {isfile}') self.redis_logger.debug(f'isfile UUID {isfile}')
self.redis_logger.debug(f'source UUID {source}') self.redis_logger.debug(f'source UUID {source}')
self.redis_logger.debug(f'paste_content UUID {paste_content}') self.redis_logger.debug(f'paste_content UUID {paste_content}')
@ -100,7 +103,7 @@ class SubmitPaste(AbstractModule):
# textarea input paste # textarea input paste
self._manage_text(uuid, paste_content, ltags, ltagsgalaxies, source) self._manage_text(uuid, paste_content, ltags, ltagsgalaxies, source)
# new paste created from file, remove uuid ref # new paste created from file, remove uuid ref
self.remove_submit_uuid(uuid) self.remove_submit_uuid(uuid)
@ -108,12 +111,12 @@ class SubmitPaste(AbstractModule):
""" """
Run Module endless process Run Module endless process
""" """
# Endless loop processing messages from the input queue # Endless loop processing messages from the input queue
while self.proceed: while self.proceed:
# Get one message (paste) from the QueueIn (copy of Redis_Global publish) # Get one message (paste) from the QueueIn (copy of Redis_Global publish)
nb_submit = self.r_serv_db.scard('submitted:uuid') nb_submit = self.r_serv_db.scard('submitted:uuid')
if nb_submit > 0: if nb_submit > 0:
try: try:
uuid = self.r_serv_db.srandmember('submitted:uuid') uuid = self.r_serv_db.srandmember('submitted:uuid')
@ -122,7 +125,7 @@ class SubmitPaste(AbstractModule):
self.compute(uuid) self.compute(uuid)
except Exception as err: except Exception as err:
self.redis_logger.error(f'Error in module {self.module_name}: {err}') self.redis_logger.error(f'Error in module {self.module_name}: {err}')
# Remove uuid ref # Remove uuid ref
self.remove_submit_uuid(uuid) self.remove_submit_uuid(uuid)
else: else:
# Wait before next process # Wait before next process
@ -150,7 +153,7 @@ class SubmitPaste(AbstractModule):
if os.path.exists(file_full_path): if os.path.exists(file_full_path):
self.redis_logger.debug(f'file exists {file_full_path}') self.redis_logger.debug(f'file exists {file_full_path}')
file_size = os.stat(file_full_path).st_size file_size = os.stat(file_full_path).st_size
self.redis_logger.debug(f'file size {file_size}') self.redis_logger.debug(f'file size {file_size}')
# Verify file length # Verify file length
@ -168,7 +171,6 @@ class SubmitPaste(AbstractModule):
content = f.read() content = f.read()
self.r_serv_log_submit.set(uuid + ':nb_total', 1) self.r_serv_log_submit.set(uuid + ':nb_total', 1)
self.create_paste(uuid, content.encode(), ltags, ltagsgalaxies, uuid, source) self.create_paste(uuid, content.encode(), ltags, ltagsgalaxies, uuid, source)
self.remove_submit_uuid(uuid)
except: except:
self.abord_file_submission(uuid, "file error") self.abord_file_submission(uuid, "file error")
@ -261,15 +263,17 @@ class SubmitPaste(AbstractModule):
# delete uuid # delete uuid
self.r_serv_db.srem('submitted:uuid', uuid) self.r_serv_db.srem('submitted:uuid', uuid)
self.redis_logger.debug(f'{uuid} all file submitted') self.redis_logger.debug(f'{uuid} all file submitted')
print(f'{uuid} all file submitted')
def create_paste(self, uuid, paste_content, ltags, ltagsgalaxies, name, source=None): def create_paste(self, uuid, paste_content, ltags, ltagsgalaxies, name, source=None):
# # TODO: Use Item create
result = False result = False
now = datetime.datetime.now() now = datetime.datetime.now()
source = source if source else 'submitted' source = source if source else 'submitted'
save_path = source + '/' + now.strftime("%Y") + '/' + now.strftime("%m") + '/' + now.strftime("%d") + '/' + name + '.gz' save_path = source + '/' + now.strftime("%Y") + '/' + now.strftime("%m") + '/' + now.strftime("%d") + '/submitted_' + name + '.gz'
full_path = filename = os.path.join(os.environ['AIL_HOME'], full_path = filename = os.path.join(os.environ['AIL_HOME'],
self.process.config.get("Directories", "pastes"), save_path) self.process.config.get("Directories", "pastes"), save_path)
@ -309,6 +313,7 @@ class SubmitPaste(AbstractModule):
self.r_serv_log_submit.set(f'{uuid}:end', 1) self.r_serv_log_submit.set(f'{uuid}:end', 1)
self.redis_logger.debug(f' {rel_item_path} send to Global') self.redis_logger.debug(f' {rel_item_path} send to Global')
print(f' {rel_item_path} send to Global')
self.r_serv_log_submit.sadd(f'{uuid}:paste_submit_link', rel_item_path) self.r_serv_log_submit.sadd(f'{uuid}:paste_submit_link', rel_item_path)
curr_date = datetime.date.today() curr_date = datetime.date.today()
@ -328,13 +333,13 @@ class SubmitPaste(AbstractModule):
gzip64encoded = base64.standard_b64encode(gzipencoded).decode() gzip64encoded = base64.standard_b64encode(gzipencoded).decode()
except: except:
self.abord_file_submission(uuid, "file error") self.abord_file_submission(uuid, "file error")
return gzip64encoded return gzip64encoded
def addError(self, uuid, errorMessage): def addError(self, uuid, errorMessage):
self.redis_logger.debug(errorMessage) self.redis_logger.debug(errorMessage)
print(errorMessage)
error = self.r_serv_log_submit.get(f'{uuid}:error') error = self.r_serv_log_submit.get(f'{uuid}:error')
if error != None: if error != None:
self.r_serv_log_submit.set(f'{uuid}:error', error + '<br></br>' + errorMessage) self.r_serv_log_submit.set(f'{uuid}:error', error + '<br></br>' + errorMessage)
@ -351,7 +356,7 @@ class SubmitPaste(AbstractModule):
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'submit_abord', 1) self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'submit_abord', 1)
self.remove_submit_uuid(uuid) self.remove_submit_uuid(uuid)
# # TODO: use Item function
def get_item_date(self, item_filename): def get_item_date(self, item_filename):
l_directory = item_filename.split('/') l_directory = item_filename.split('/')
return f'{l_directory[-4]}{l_directory[-3]}{l_directory[-2]}' return f'{l_directory[-4]}{l_directory[-3]}{l_directory[-2]}'
@ -371,6 +376,6 @@ class SubmitPaste(AbstractModule):
if __name__ == '__main__': if __name__ == '__main__':
module = SubmitPaste() module = SubmitPaste()
module.run() module.run()

2
var/www/modules/Flask_config.py
View file

@ -45,7 +45,7 @@ redis_logger = publisher
# Port of the redis instance used by pubsublogger # Port of the redis instance used by pubsublogger
redis_logger.port = 6380 redis_logger.port = 6380
# Channel name to publish logs # Channel name to publish logs
redis_logger.channel = 'front' redis_logger.channel = 'Flask'
sys.path.append('../../configs/keys') sys.path.append('../../configs/keys')

16
var/www/modules/PasteSubmit/Flask_PasteSubmit.py
View file

@ -11,7 +11,6 @@ import os
import sys import sys
import json import json
import string import string
import subprocess
import datetime import datetime
import redis import redis
import unicodedata import unicodedata
@ -19,8 +18,10 @@ import uuid
from io import BytesIO from io import BytesIO
from Date import Date from Date import Date
from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, abort
from functools import wraps from functools import wraps
# Flask
from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, abort
from Role_Manager import login_admin, login_analyst from Role_Manager import login_admin, login_analyst
from flask_login import login_required from flask_login import login_required
@ -257,7 +258,7 @@ def PasteSubmit_page():
return render_template("submit_items.html", return render_template("submit_items.html",
active_taxonomies = active_taxonomies, active_taxonomies = active_taxonomies,
active_galaxies = active_galaxies, active_galaxies = active_galaxies,
text_max_size = text_max_size, text_max_size = text_max_size,
file_max_size = file_max_size, file_max_size = file_max_size,
allowed_extensions = allowed_extensions) allowed_extensions = allowed_extensions)
@ -277,6 +278,13 @@ def submit():
paste_content = request.form['paste_content'] paste_content = request.form['paste_content']
paste_source = request.form['paste_source'] paste_source = request.form['paste_source']
# limit source length
paste_source = paste_source.replace('/', '')[:80]
if paste_source in ['crawled', 'tests']:
content = f'Invalid source'
logger.info(paste_source)
return content, 400
is_file = False is_file = False
if 'file' in request.files: if 'file' in request.files:
file_import = request.files['file'] file_import = request.files['file']
@ -343,7 +351,7 @@ def submit():
#Flask verify the file size #Flask verify the file size
file_import.save(full_path) file_import.save(full_path)
logger.debug('file saved') logger.debug('file saved')
Import_helper.create_import_queue(ltags, ltagsgalaxies, full_path, UUID, password, True) Import_helper.create_import_queue(ltags, ltagsgalaxies, full_path, UUID, password, True)
return render_template("submit_items.html", return render_template("submit_items.html",

38
var/www/modules/PasteSubmit/templates/submit_items.html
View file

@ -100,7 +100,7 @@
<div class="col-12 col-lg-10" id="core_content"> <div class="col-12 col-lg-10" id="core_content">
{% if message %} {% if message %}
<p>{{ message }}</p> <p>{{ message }}</p>
{% endif %} {% endif %}
<form action="{{ url_for('PasteSubmit.submit') }}" id="pasteSubmitForm" method="post" <form action="{{ url_for('PasteSubmit.submit') }}" id="pasteSubmitForm" method="post"
@ -112,7 +112,7 @@
<div class="card mt-2 mb-4"> <div class="card mt-2 mb-4">
<div class="card-header py-3 d-flex flex-row align-items-center justify-content-between"> <div class="card-header py-3 d-flex flex-row align-items-center justify-content-between">
<h6 class="m-0 font-weight-bold text-primary">Submit Paste</h6> <h6 class="m-0 font-weight-bold text-primary">Submit Item</h6>
</div> </div>
<div class="card-body"> <div class="card-body">
@ -150,8 +150,8 @@
Tags <i class="fas fa-tags"></i></button> Tags <i class="fas fa-tags"></i></button>
<div class="dropdown-divider"></div> <div class="dropdown-divider"></div>
{% for taxo in active_taxonomies %} {% for taxo in active_taxonomies %}
<button class="dropdown-item" type="button" <button class="dropdown-item" type="button"
id="{{ taxo }}-id{{ loop.index0 }}">{{ taxo }}</button> id="{{ taxo }}-id{{ loop.index0 }}">{{ taxo }}</button>
{% endfor %} {% endfor %}
</div> </div>
</div> </div>
@ -174,8 +174,8 @@
<i class="fas fa-tags"></i></button> <i class="fas fa-tags"></i></button>
<div class="dropdown-divider"></div> <div class="dropdown-divider"></div>
{% for galaxy in active_galaxies %} {% for galaxy in active_galaxies %}
<button class="dropdown-item" type="button" <button class="dropdown-item" type="button"
id="{{ galaxy }}-idgalax{{ loop.index0 }}">{{ galaxy }}</button> id="{{ galaxy }}-idgalax{{ loop.index0 }}">{{ galaxy }}</button>
{% endfor %} {% endfor %}
</div> </div>
</div> </div>
@ -320,22 +320,22 @@
}); });
{% for taxo in active_taxonomies %} {% for taxo in active_taxonomies %}
jQuery("#{{ taxo }}-id{{ loop.index0 }}").on("click", function (e) { jQuery("#{{ taxo }}-id{{ loop.index0 }}").on("click", function (e) {
$.getJSON("{{ url_for('Tags.get_tags_taxonomie') }}?taxonomie={{ taxo }}", $.getJSON("{{ url_for('Tags.get_tags_taxonomie') }}?taxonomie={{ taxo }}",
function (data) { function (data) {
ltags.setData(data) ltags.setData(data)
}); });
}); });
{% endfor %} {% endfor %}
</script> </script>
<script> <script>
{% for galaxy in active_galaxies %} {% for galaxy in active_galaxies %}
jQuery("#{{ galaxy }}-idgalax{{ loop.index0 }}").on("click", function (e) { jQuery("#{{ galaxy }}-idgalax{{ loop.index0 }}").on("click", function (e) {
$.getJSON("{{ url_for('Tags.get_tags_galaxy') }}?galaxy={{ galaxy }}", $.getJSON("{{ url_for('Tags.get_tags_galaxy') }}?galaxy={{ galaxy }}",
function (data) { function (data) {
ltagsgalaxies.setData(data) ltagsgalaxies.setData(data)
}); });
}); });
{% endfor %} {% endfor %}
</script> </script>
@ -417,4 +417,4 @@
</script> </script>
</html> </html>