From 85fb63141f27720b09cffc0dbd9c716cb0cedad2 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 22 Aug 2017 17:16:06 +0200 Subject: [PATCH 1/5] Added AIL reset script --- reset_AIL.sh | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) create mode 100755 reset_AIL.sh diff --git a/reset_AIL.sh b/reset_AIL.sh new file mode 100755 index 00000000..e87dfb67 --- /dev/null +++ b/reset_AIL.sh @@ -0,0 +1,57 @@ +#!/bin/bash + +RED="\\033[1;31m" +DEFAULT="\\033[0;39m" +GREEN="\\033[1;32m" + +[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; + +# Make sure the reseting is intentional +num=$(( ( RANDOM % 100 ) + 1 )) + +echo -e $RED"To reset the platform, enter the following number: "$DEFAULT $num +read userInput + +if [ $userInput -eq $num ] +then + echo "Reseting AIL..." +else + echo "Wrong number" + exit 1; +fi + + +# Kill all screens +screen -ls | grep Detached | cut -d. -f1 | awk '{print $1}' | xargs kill + +set -e + +# Access dirs and delete +cd $AIL_HOME + +pushd dumps/ +rm * +echo 'cleaned dumps' +popd + +pushd indexdir/ +rm -r * +echo 'cleaned indexdir' +popd + +pushd LEVEL_DB_DATA/ +rm -r * +echo 'cleaned LEVEL_DB_DATA' +popd + +pushd logs/ +rm * +echo 'cleaned logs' +popd + +pushd PASTES/ +rm -r * +echo 'cleaned PASTES' +popd + +echo -e $GREEN"* AIL has been reset *"$DEFAULT From 429cc96d2ad22214f3589056f130c51717636be9 Mon Sep 17 00:00:00 2001 From: = Date: Tue, 22 Aug 2017 15:17:49 +0200 Subject: [PATCH 2/5] Improvement of import_dir.py. Added feed name parameter, send the correct path (without potential ../) and gzipEncode it if needed. --- bin/import_dir.py | 73 ++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 69 insertions(+), 4 deletions(-) diff --git a/bin/import_dir.py b/bin/import_dir.py index 7977b303..1300cb43 100755 --- a/bin/import_dir.py +++ b/bin/import_dir.py @@ -3,15 +3,57 @@ import zmq import base64 +import StringIO +import gzip import argparse import os -import time +import time, datetime +import mimetypes + +''' +' +' Import content/pastes into redis. +' If content is not compressed yet, compress it. +' +' /!\ WARNING /!\ + Content to be imported must be placed in a directory tree of the form + root/ + | + +-- Year/ + | + +-- Month/ + | + +-- Day/ + | + +-- Content + e.g.: + ~/to_import/2017/08/22/paste1.gz +' +''' + +import StringIO +import gzip +def is_hierachy_valid(path): + var = path.split('/') + try: + newDate = datetime.datetime(int(var[-4]), int(var[-3]), int(var[-2])) + correctDate = True + except ValueError: + correctDate = False + except IndexError: + correctDate = False + except: + correctDate = False + return correctDate + if __name__ == "__main__": parser = argparse.ArgumentParser(description='Take files from a directory and push them into a 0MQ feed.') parser.add_argument('-d', '--directory', type=str, required=True, help='Root directory to import') parser.add_argument('-p', '--port', type=int, default=5556, help='Zero MQ port') parser.add_argument('-c', '--channel', type=str, default='102', help='Zero MQ channel') + parser.add_argument('-n', '--name', type=str, default='import_dir', help='Name of the feeder') + parser.add_argument('--hierarchy', type=int, default=1, help='Number of parent directory forming the name') args = parser.parse_args() @@ -22,7 +64,30 @@ if __name__ == "__main__": for dirname, dirnames, filenames in os.walk(args.directory): for filename in filenames: - messagedata = open(os.path.join(dirname, filename)).read() - print(os.path.join(dirname, filename)) - socket.send('{} {} {}'.format(args.channel, os.path.join(dirname, filename), base64.b64encode(messagedata))) + complete_path = os.path.join(dirname, filename) + + #take wanted path of the file + wanted_path = os.path.realpath(complete_path) + wanted_path = wanted_path.split('/') + wanted_path = '/'.join(wanted_path[-(4+args.hierarchy):]) + + messagedata = open(complete_path).read() + + #if paste do not have a 'date hierarchy' ignore it + if not is_hierachy_valid(complete_path): + print('/!\ hierarchy not valid, should have the format yyyy/mm/dd/paste.gz /!\ ') + print(complete_path) + break + + #verify that the data is gzipEncoded. if not compress it + if 'text' in str(mimetypes.guess_type(complete_path)[0]): + out = StringIO.StringIO() + with gzip.GzipFile(fileobj=out, mode="w") as f: + f.write(messagedata) + messagedata = out.getvalue() + wanted_path += '.gz' + + print(args.name+'>'+wanted_path) + path_to_send = args.name + '>' + wanted_path + socket.send('{} {} {}'.format(args.channel, path_to_send, base64.b64encode(messagedata))) time.sleep(.2) From 74ff401e1d5cd6ccf6304111f53ab82320b863a3 Mon Sep 17 00:00:00 2001 From: = Date: Thu, 24 Aug 2017 16:41:30 +0200 Subject: [PATCH 3/5] Fixed bug: No CR between indexes in all_index.txt file --- bin/Indexer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/Indexer.py b/bin/Indexer.py index 29990bfd..96c071f5 100755 --- a/bin/Indexer.py +++ b/bin/Indexer.py @@ -119,7 +119,7 @@ if __name__ == "__main__": indexname = str(timestamp) #update all_index with open(indexRegister_path, "a") as f: - f.write(str(timestamp)) + f.write('/n'+str(timestamp)+) #create new dir os.mkdir(indexpath) ix = create_in(indexpath, schema) From b2d4dd4a1f573e64041e471e88485cfbaee6257e Mon Sep 17 00:00:00 2001 From: = Date: Thu, 24 Aug 2017 16:43:05 +0200 Subject: [PATCH 4/5] Typo --- bin/Indexer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/Indexer.py b/bin/Indexer.py index 96c071f5..cb0a6b9d 100755 --- a/bin/Indexer.py +++ b/bin/Indexer.py @@ -119,7 +119,7 @@ if __name__ == "__main__": indexname = str(timestamp) #update all_index with open(indexRegister_path, "a") as f: - f.write('/n'+str(timestamp)+) + f.write('/n'+str(timestamp)) #create new dir os.mkdir(indexpath) ix = create_in(indexpath, schema) From a8dc9231c6e53d83b7d404f19f674e2d44a6ee2d Mon Sep 17 00:00:00 2001 From: = Date: Thu, 24 Aug 2017 16:43:42 +0200 Subject: [PATCH 5/5] Typo2.. --- bin/Indexer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/Indexer.py b/bin/Indexer.py index cb0a6b9d..be4c899c 100755 --- a/bin/Indexer.py +++ b/bin/Indexer.py @@ -119,7 +119,7 @@ if __name__ == "__main__": indexname = str(timestamp) #update all_index with open(indexRegister_path, "a") as f: - f.write('/n'+str(timestamp)) + f.write('\n'+str(timestamp)) #create new dir os.mkdir(indexpath) ix = create_in(indexpath, schema)