diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh
index ed05f676..704d64cc 100755
--- a/bin/LAUNCH.sh
+++ b/bin/LAUNCH.sh
@@ -163,6 +163,8 @@ function launching_scripts {
screen -S "Script_AIL" -X screen -t "Tags" bash -c './Tags.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
+ sleep 0.1
+ screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c './submit_paste.py; read x'
}
diff --git a/bin/submit_paste.py b/bin/submit_paste.py
index 048a4c43..12ae14af 100755
--- a/bin/submit_paste.py
+++ b/bin/submit_paste.py
@@ -9,8 +9,80 @@ import io
import redis
import base64
import datetime
+import time
+
+from sflock.main import unpack
+import sflock
from Helper import Process
+from pubsublogger import publisher
+
+def create_paste(uuid, paste_content, ltags, ltagsgalaxies, name):
+
+ now = datetime.datetime.now()
+ save_path = 'submitted/' + now.strftime("%Y") + '/' + now.strftime("%m") + '/' + now.strftime("%d") + '/' + name + '.gz'
+
+ full_path = filename = os.path.join(os.environ['AIL_HOME'],
+ p.config.get("Directories", "pastes"), save_path)
+
+ if os.path.isfile(full_path):
+ addError(uuid, 'File: ' + save_path + ' already exist in submitted pastes')
+ return 1
+
+ gzipencoded = gzip.compress(paste_content)
+ gzip64encoded = base64.standard_b64encode(gzipencoded).decode()
+
+ # send paste to Global module
+ relay_message = "{0} {1}".format(save_path, gzip64encoded)
+ p.populate_set_out(relay_message, 'Mixer')
+
+ # add tags
+ add_tags(ltags, ltagsgalaxies, full_path)
+
+ r_serv_log_submit.incr(uuid + ':nb_end')
+ r_serv_log_submit.incr(uuid + ':nb_sucess')
+
+ if r_serv_log_submit.get(uuid + ':nb_end') == r_serv_log_submit.get(uuid + ':nb_total'):
+ r_serv_log_submit.set(uuid + ':end', 1)
+
+ print(' {} send to Global'.format(save_path))
+ r_serv_log_submit.sadd(uuid + ':paste_submit_link', full_path)
+
+ return 0
+
+def addError(uuid, errorMessage):
+ print(errorMessage)
+ error = r_serv_log_submit.get(uuid + ':error')
+ if error != None:
+ r_serv_log_submit.set(uuid + ':error', error + '
' + errorMessage)
+ r_serv_log_submit.incr(uuid + ':nb_end')
+
+def abord_file_submission(uuid, errorMessage):
+ addError(uuid, errorMessage)
+ r_serv_log_submit.set(uuid + ':end', 1)
+ remove_submit_uuid(uuid)
+
+
+def remove_submit_uuid(uuid):
+ # save temp value on disk
+ r_serv_db.delete(uuid + ':ltags')
+ r_serv_db.delete(uuid + ':ltagsgalaxies')
+ r_serv_db.delete(uuid + ':paste_content')
+ r_serv_db.delete(uuid + ':isfile')
+ r_serv_db.delete(uuid + ':password')
+
+ r_serv_log_submit.expire(uuid + ':end', expire_time)
+ r_serv_log_submit.expire(uuid + ':processing', expire_time)
+ r_serv_log_submit.expire(uuid + ':nb_total', expire_time)
+ r_serv_log_submit.expire(uuid + ':nb_sucess', expire_time)
+ r_serv_log_submit.expire(uuid + ':nb_end', expire_time)
+ r_serv_log_submit.expire(uuid + ':error', expire_time)
+ r_serv_log_submit.srem(uuid + ':paste_submit_link', '')
+ r_serv_log_submit.expire(uuid + ':paste_submit_link', expire_time)
+
+ # delete uuid
+ r_serv_db.srem('submitted:uuid', uuid)
+ print('{} all file submitted'.format(uuid))
def add_tags(tags, tagsgalaxies, path):
list_tag = tags.split(',')
@@ -32,21 +104,22 @@ def add_tags(tags, tagsgalaxies, path):
#add new tag in list of all used tags
r_serv_tags.sadd('list_tags', tag)
+def verify_extention_filename(filename):
+ if not '.' in filename:
+ return True
+ else:
+ file_type = filename.rsplit('.', 1)[1]
+
+ #txt file
+ if file_type in ALLOWED_EXTENSIONS:
+ return True
+ else:
+ return False
if __name__ == "__main__":
- if len(sys.argv) != 6:
- print('usage:', 'submit_paste.py', 'ltags', 'ltagsgalaxies', 'paste_content', 'paste_name', 'id')
- exit(1)
- try:
- ltags = sys.argv[1]
- ltagsgalaxies = sys.argv[2]
- paste_content = sys.argv[3]
- paste_name = sys.argv[4]
- id = sys.argv[5]
- except:
- print('unable to get elements')
- exit(1)
+ publisher.port = 6380
+ publisher.channel = "Script"
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
if not os.path.exists(configfile):
@@ -57,6 +130,12 @@ if __name__ == "__main__":
cfg = configparser.ConfigParser()
cfg.read(configfile)
+ r_serv_db = redis.StrictRedis(
+ host=cfg.get("ARDB_DB", "host"),
+ port=cfg.getint("ARDB_DB", "port"),
+ db=cfg.getint("ARDB_DB", "db"),
+ decode_responses=True)
+
r_serv_log_submit = redis.StrictRedis(
host=cfg.get("Redis_Log_submit", "host"),
port=cfg.getint("Redis_Log_submit", "port"),
@@ -75,43 +154,117 @@ if __name__ == "__main__":
db=cfg.getint("ARDB_Metadata", "db"),
decode_responses=True)
- # TODO put on config
- expire_time = 10200
-
- r_serv_log_submit.expire(id + ':end', expire_time)
- r_serv_log_submit.expire(id + ':nb_total', expire_time)
- r_serv_log_submit.expire(id + ':nb_end', expire_time)
- r_serv_log_submit.expire(id + ':error', expire_time)
+ expire_time = 120
+ MAX_FILE_SIZE = 1000000000
+ ALLOWED_EXTENSIONS = 'txt'
config_section = 'submit_paste'
p = Process(config_section)
- now = datetime.datetime.now()
- save_path = 'submitted/' + now.strftime("%Y") + '/' + now.strftime("%m") + '/' + now.strftime("%d") + '/' + id + '.gz'
+ while True:
- full_path = filename = os.path.join(os.environ['AIL_HOME'],
- p.config.get("Directories", "pastes"), save_path)
+ # paste submitted
+ if r_serv_db.scard('submitted:uuid') > 0:
+ uuid = r_serv_db.srandmember('submitted:uuid')
- if os.path.isfile(full_path):
- error = r_serv_log_submit.get(id + ':error')
- r_serv_log_submit.set(id + ':error', error + '
File: ' + save_path + ' already exist in submitted pastes')
- exit(1)
+ # get temp value save on disk
+ ltags = r_serv_db.get(uuid + ':ltags')
+ ltagsgalaxies = r_serv_db.get(uuid + ':ltagsgalaxies')
+ paste_content = r_serv_db.get(uuid + ':paste_content')
+ isfile = r_serv_db.get(uuid + ':isfile')
+ password = r_serv_db.get(uuid + ':password')
+
+ # needed if redis is restarted
+ r_serv_log_submit.set(uuid + ':end', 0)
+ r_serv_log_submit.set(uuid + ':processing', 0)
+ r_serv_log_submit.set(uuid + ':nb_total', -1)
+ r_serv_log_submit.set(uuid + ':nb_end', 0)
+ r_serv_log_submit.set(uuid + ':nb_sucess', 0)
+ r_serv_log_submit.set(uuid + ':error', 'error:')
+ r_serv_log_submit.sadd(uuid + ':paste_submit_link', '')
- gzipencoded = gzip.compress(paste_content.encode())
- gzip64encoded = base64.standard_b64encode(gzipencoded).decode()
+ r_serv_log_submit.set(uuid + ':processing', 1)
- # send paste to Global module
- relay_message = "{0} {1}".format(save_path, gzip64encoded)
- p.populate_set_out(relay_message, 'Mixer')
+ if isfile == 'True':
+ file_full_path = paste_content
- # add tags
- add_tags(ltags, ltagsgalaxies, full_path)
+ if not os.path.exists(file_full_path):
+ abord_file_submission(uuid, "Server Error, the archive can't be found")
+ continue
- r_serv_log_submit.incr(id + ':nb_end')
+ #verify file lengh
+ if os.stat(file_full_path).st_size > MAX_FILE_SIZE:
+ abord_file_submission(uuid, 'File :{} too large'.format(file_full_path))
+
+ else:
+ filename = file_full_path.split('/')[-1]
+ if not '.' in filename:
+ # read file
+ with open(file_full_path,'r') as f:
+ content = f.read()
+ r_serv_log_submit.set(uuid + ':nb_total', 1)
+ create_paste(uuid, content.encode(), ltags, ltagsgalaxies, uuid)
+ remove_submit_uuid(uuid)
+
+ else:
+ file_type = filename.rsplit('.', 1)[1]
+
+ #txt file
+ if file_type == 'txt':
+ with open(file_full_path,'r') as f:
+ content = f.read()
+ r_serv_log_submit.set(uuid + ':nb_total', 1)
+ create_paste(uuid, content.encode(), ltags, ltagsgalaxies, uuid)
+ remove_submit_uuid(uuid)
+ #compressed file
+ else:
+ #decompress file
+ try:
+ if password == '':
+ files = unpack(file_full_path.encode())
+ #print(files.children)
+ else:
+ try:
+ files = unpack(file_full_path.encode(), password)
+ #print(files.children)
+ except sflock.exception.IncorrectUsageException:
+ abord_file_submission(uuid, "Wrong Password")
+ continue
+ except:
+ abord_file_submission(uuid, "file decompression error")
+ continue
+ print('unpacking {} file'.format(files.unpacker))
+ if(not files.children):
+ abord_file_submission(uuid, "Empty compressed file")
+ continue
+ # set number of files to submit
+ r_serv_log_submit.set(uuid + ':nb_total', len(files.children))
+ n = 1
+ for child in files.children:
+ if verify_extention_filename(child.filename.decode()):
+ create_paste(uuid, child.contents, ltags, ltagsgalaxies, uuid+'_'+ str(n) )
+ n = n + 1
+ else:
+ print('bad extention')
+ addError(uuid, 'Bad file extension: {}'.format(child.filename.decode()) )
+
+ except FileNotFoundError:
+ print('file not found')
+ addError(uuid, 'File not found: {}'.format(file_full_path), uuid )
+
+ remove_submit_uuid(uuid)
- if r_serv_log_submit.get(id + ':nb_end') == r_serv_log_submit.get(id + ':nb_total'):
- r_serv_log_submit.set(id + ':end', 1)
- exit(0)
+ # textarea input paste
+ else:
+ r_serv_log_submit.set(uuid + ':nb_total', 1)
+ create_paste(uuid, paste_content.encode(), ltags, ltagsgalaxies, uuid)
+ remove_submit_uuid(uuid)
+ time.sleep(0.5)
+
+ # wait for paste
+ else:
+ publisher.debug("Script submit_paste is Idling 10s")
+ time.sleep(3)
diff --git a/installing_deps.sh b/installing_deps.sh
index 6bb73f6b..6948c5d6 100755
--- a/installing_deps.sh
+++ b/installing_deps.sh
@@ -33,6 +33,9 @@ sudo pip install nose -y
sudo apt-get install libfuzzy-dev -y
sudo apt-get install build-essential libffi-dev automake autoconf libtool -y
+# sflock, gz requirement
+sudo apt-get install p7zip-full -y
+
# REDIS #
test ! -d redis/ && git clone https://github.com/antirez/redis.git
pushd redis/
diff --git a/pip3_packages_requirement.txt b/pip3_packages_requirement.txt
index 78b19d64..352c8679 100644
--- a/pip3_packages_requirement.txt
+++ b/pip3_packages_requirement.txt
@@ -56,6 +56,9 @@ pycountry
# To fetch Onion urls
PySocks
+# decompress files
+sflock
+
#ASN lookup requirements
#https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/adns-python/adns-python-1.2.1.tar.gz
https://github.com/trolldbois/python3-adns/archive/master.zip
diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py
index 60244814..85353a7d 100755
--- a/var/www/Flask_server.py
+++ b/var/www/Flask_server.py
@@ -7,7 +7,7 @@ import json
import datetime
import time
import calendar
-from flask import Flask, render_template, jsonify, request
+from flask import Flask, render_template, jsonify, request, Request
import flask
import importlib
import os
@@ -28,7 +28,7 @@ cfg = Flask_config.cfg
Flask_config.app = Flask(__name__, static_url_path='/static/')
app = Flask_config.app
-#app.secret_key = Flask_config.secret_key
+app.config['MAX_CONTENT_LENGTH'] = 900 * 1024 * 1024
# ========= HEADER GENERATION ========
diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py
index db74928f..80ec5967 100644
--- a/var/www/modules/Flask_config.py
+++ b/var/www/modules/Flask_config.py
@@ -84,6 +84,12 @@ r_serv_metadata = redis.StrictRedis(
db=cfg.getint("ARDB_Metadata", "db"),
decode_responses=True)
+r_serv_db = redis.StrictRedis(
+ host=cfg.get("ARDB_DB", "host"),
+ port=cfg.getint("ARDB_DB", "port"),
+ db=cfg.getint("ARDB_DB", "db"),
+ decode_responses=True)
+
# VARIABLES #
max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
@@ -91,3 +97,5 @@ max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number
DiffMaxLineLength = int(cfg.get("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
+
+UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')
diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py
index 1816d85c..0afa8e06 100644
--- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py
+++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py
@@ -5,9 +5,7 @@
Flask functions and routes for the trending modules page
'''
import redis
-from flask import Flask, render_template, jsonify, request, Blueprint
-
-'''import random'''
+from flask import Flask, render_template, jsonify, request, Blueprint, url_for
import unicodedata
import string
@@ -15,6 +13,7 @@ import subprocess
import os
import sys
import datetime
+import uuid
from pytaxonomies import Taxonomies
from pymispgalaxies import Galaxies, Clusters
@@ -25,21 +24,26 @@ import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
r_serv_tags = Flask_config.r_serv_tags
+r_serv_metadata = Flask_config.r_serv_metadata
+r_serv_db = Flask_config.r_serv_db
r_serv_log_submit = Flask_config.r_serv_log_submit
PasteSubmit = Blueprint('PasteSubmit', __name__, template_folder='templates')
valid_filename_chars = "-_ %s%s" % (string.ascii_letters, string.digits)
-ALLOWED_EXTENSIONS = set(['txt', 'zip', 'gzip'])
+ALLOWED_EXTENSIONS = set(['txt', 'zip', 'gz', 'tar.gz'])
+UPLOAD_FOLDER = Flask_config.UPLOAD_FOLDER
# ============ FUNCTIONS ============
def one():
return 1
def allowed_file(filename):
- return '.' in filename and \
- filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
+ if not '.' in filename:
+ return True
+ else:
+ return filename.rsplit('.', 1)[1].lower() in ALLOWED_EXTENSIONS
def clean_filename(filename, whitelist=valid_filename_chars, replace=' '):
# replace characters
@@ -52,23 +56,28 @@ def clean_filename(filename, whitelist=valid_filename_chars, replace=' '):
# keep only whitelisted chars
return ''.join(c for c in cleaned_filename if c in whitelist)
-'''@app.before_request
-def csrf_protect():
- if request.method == "POST":
- token = session.pop('_csrf_token', None)
- if not token or token != request.form.get('_csrf_token'):
- abort(400)
+def launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password, isfile = False):
-def generate_csrf_token():
- if '_csrf_token' not in session:
- session['_csrf_token'] = some_random_string()
- return session['_csrf_token']
+ print(UUID)
-app.jinja_env.globals['csrf_token'] = generate_csrf_token
+ # save temp value on disk
+ r_serv_db.set(UUID + ':ltags', ltags)
+ r_serv_db.set(UUID + ':ltagsgalaxies', ltagsgalaxies)
+ r_serv_db.set(UUID + ':paste_content', paste_content)
+ r_serv_db.set(UUID + ':password', password)
+ r_serv_db.set(UUID + ':isfile', isfile)
-def some_random_string():
- N = 15
- return ''.join(random.SystemRandom().choice(string.ascii_uppercase + string.digits) for _ in range(N))'''
+ r_serv_log_submit.set(UUID + ':end', 0)
+ r_serv_log_submit.set(UUID + ':processing', 0)
+ r_serv_log_submit.set(UUID + ':nb_total', -1)
+ r_serv_log_submit.set(UUID + ':nb_end', 0)
+ r_serv_log_submit.set(UUID + ':nb_sucess', 0)
+ r_serv_log_submit.set(UUID + ':error', 'error:')
+ r_serv_log_submit.sadd(UUID + ':paste_submit_link', '')
+
+
+ # save UUID on disk
+ r_serv_db.sadd('submitted:uuid', UUID)
def addTagsVerification(tags, tagsgalaxies):
@@ -124,7 +133,9 @@ def PasteSubmit_page():
@PasteSubmit.route("/PasteSubmit/submit", methods=['POST'])
def submit():
- paste_name = request.form['paste_name']
+ #paste_name = request.form['paste_name']
+
+ password = request.form['password']
ltags = request.form['tags_taxonomies']
ltagsgalaxies = request.form['tags_galaxies']
paste_content = request.form['paste_content']
@@ -133,68 +144,107 @@ def submit():
if not addTagsVerification(ltags, ltagsgalaxies):
return 'INVALID TAGS'
- if 'file' not in request.files:
+ # add submitted tags
+ if(ltags != ''):
+ ltags = ltags + ',submitted'
+ else:
+ ltags ='submitted'
+
+ print(request.files)
+
+ if 'file' in request.files:
+ print(request.files)
file = request.files['file']
+ if file:
- if file.filename == '':
+ if file and allowed_file(file.filename):
- if paste_content != '':
- if sys.getsizeof(paste_content) < 900000:
+ # get UUID
+ UUID = str(uuid.uuid4())
- to_launch = os.environ['AIL_BIN'] + 'submit_paste.py'
- # get id
- id = str(r_serv_tags.get('submit_id'))
-
- if paste_name:
- # clean file name
- id = clean_filename(paste_name)
-
- # create logs
- r_serv_log_submit.set(id + ':end', 0)
- r_serv_log_submit.set(id + ':nb_total', 1)
- r_serv_log_submit.set(id + ':nb_end', 0)
- r_serv_log_submit.set(id + ':error', 'error:')
-
- #incr id
- r_serv_tags.incr('submit_id')
-
- # add submitted tags
- if(ltags != ''):
- ltags = ltags + ',submitted'
- else:
- ltags ='submitted'
-
- # launch process
- process = subprocess.Popen(["python", to_launch, ltags, ltagsgalaxies, paste_content, paste_name, id],
- stdout=subprocess.PIPE)
-
- return render_template("submiting.html",
- id = id)
+ '''if paste_name:
+ # clean file name
+ UUID = clean_filename(paste_name)'''
+ if not '.' in file.filename:
+ full_path = os.path.join(UPLOAD_FOLDER, UUID)
else:
- return 'size error'
+ if file.filename[-6:] == 'tar.gz':
+ file_type = 'tar.gz'
+ else:
+ file_type = file.filename.rsplit('.', 1)[1]
+ name = UUID + '.' + file_type
+ full_path = os.path.join(UPLOAD_FOLDER, name)
- return 'submit'
+ #Flask verify the file size
+ file.save(full_path)
- if file and allowed_file(file.filename):
- print(file.read())
+ paste_content = full_path
- return 'error'
+ launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password ,True)
+
+ return render_template("submiting.html",
+ UUID = UUID)
+
+ else:
+ print('wrong file type')
+
+
+ if paste_content != '':
+ if sys.getsizeof(paste_content) < 900000:
+
+ # get id
+ UUID = str(uuid.uuid4())
+ print(UUID)
+
+ #if paste_name:
+ # clean file name
+ #id = clean_filename(paste_name)
+
+ launch_submit(ltags, ltagsgalaxies, paste_content, UUID, password)
+
+ return render_template("submiting.html",
+ UUID = UUID)
+
+ else:
+ return 'size error'
+
+ return 'submit'
+
+
+ return PasteSubmit_page()
@PasteSubmit.route("/PasteSubmit/submit_status", methods=['GET'])
def submit_status():
- id = request.args.get('id')
+ UUID = request.args.get('UUID')
- if id:
- end = r_serv_log_submit.get(id + ':end')
- nb_total = r_serv_log_submit.get(id + ':nb_total')
- nb_end = r_serv_log_submit.get(id + ':nb_end')
- error = r_serv_log_submit.get(id + ':error')
- if (end != None) and (nb_total != None) and (nb_end != None) and (error != None):
+ if UUID:
+ end = r_serv_log_submit.get(UUID + ':end')
+ nb_total = r_serv_log_submit.get(UUID + ':nb_total')
+ nb_end = r_serv_log_submit.get(UUID + ':nb_end')
+ error = r_serv_log_submit.get(UUID + ':error')
+ processing = r_serv_log_submit.get(UUID + ':processing')
+ nb_sucess = r_serv_log_submit.get(UUID + ':nb_sucess')
+ paste_submit_link = list(r_serv_log_submit.smembers(UUID + ':paste_submit_link'))
- in_progress = nb_end + ' / ' + nb_total
- prog = int(int(nb_end) * 100 / int(nb_total))
+ if (end != None) and (nb_total != None) and (nb_end != None) and (error != None) and (processing != None) and (paste_submit_link != None):
+
+ link = ''
+ if paste_submit_link:
+ for paste in paste_submit_link:
+ url = url_for('showsavedpastes.showsavedpaste') + '?paste=' + paste
+ link += '' + paste +''
+
+ if nb_total == '-1':
+ in_progress = nb_sucess + ' / '
+ else:
+ in_progress = nb_sucess + ' / ' + nb_total
+
+ if int(nb_total) != 0:
+ prog = int(int(nb_end) * 100 / int(nb_total))
+ else:
+ prog = 0
if error == 'error:':
isError = False
@@ -206,15 +256,29 @@ def submit_status():
else:
end = True
+ if processing == '0':
+ processing = False
+ else:
+ processing = True
+
return jsonify(end=end,
in_progress=in_progress,
prog=prog,
+ link=link,
+ processing=processing,
isError=isError,
error=error)
else:
+ # FIXME TODO
+ print(end)
+ print(nb_total)
+ print(nb_end)
+ print(error)
+ print(processing)
+ print(nb_sucess)
return 'to do'
else:
- return 'INVALID ID'
+ return 'INVALID UUID'
# ========= REGISTRATION =========
app.register_blueprint(PasteSubmit)
diff --git a/var/www/modules/PasteSubmit/templates/PasteSubmit.html b/var/www/modules/PasteSubmit/templates/PasteSubmit.html
index b2d01a6e..3a463694 100644
--- a/var/www/modules/PasteSubmit/templates/PasteSubmit.html
+++ b/var/www/modules/PasteSubmit/templates/PasteSubmit.html
@@ -36,22 +36,18 @@