From 01ff202ae81eebe2be33dc2ba09bc921902f30cd Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 12 Sep 2019 13:25:29 +0200 Subject: [PATCH] chg: [update] add v2.2 update + fix default update + fix Empty Item.get_tags() + add new LAUNCHER options --- bin/LAUNCH.sh | 172 ++++++++++++++++++-------------- bin/packages/Item.py | 3 + bin/packages/Tag.py | 2 +- bin/packages/config.cfg.sample | 5 + update/default_update/Update.sh | 8 +- update/v2.2/Update.py | 122 ++++++++++++++++++++++ var/www/modules/Flask_config.py | 6 +- 7 files changed, 236 insertions(+), 82 deletions(-) create mode 100755 update/v2.2/Update.py diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 4d6619c8..d87ef21e 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -76,12 +76,15 @@ function helptext { Usage: ----- LAUNCH.sh - [-l | --launchAuto] - [-k | --killAll] - [-u | --update] - [-c | --configUpdate] - [-t | --thirdpartyUpdate] - [-h | --help] + [-l | --launchAuto] LAUNCH DB + Scripts + [-k | --killAll] Kill DB + Scripts + [-ks | --killscript] Scripts + [-u | --update] Update AIL + [-c | --crawler] LAUNCH Crawlers + [-f | --launchFeeder] LAUNCH Pystemon feeder + [-t | --thirdpartyUpdate] Update Web + [-m | --menu] Display Advanced Menu + [-h | --help] Help " } @@ -398,6 +401,18 @@ function launch_feeder { fi } +function killscript { + if [[ $islogged || $isqueued || $isscripted || $isflasked || $isfeeded || $iscrawler ]]; then + echo -e $GREEN"Killing Script"$DEFAULT + kill $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler + sleep 0.2 + echo -e $ROSE`screen -ls`$DEFAULT + echo -e $GREEN"\t* $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler killed."$DEFAULT + else + echo -e $RED"\t* No script to kill"$DEFAULT + fi +} + function killall { if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted || $isflasked || $isfeeded || $iscrawler ]]; then if [[ $isredis ]]; then @@ -457,76 +472,82 @@ function launch_all { launch_flask; } -#If no params, display the menu +function menu_display { + + options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Flask" "Killall" "Shutdown" "Update" "Update-config" "Update-thirdparty") + + menu() { + echo "What do you want to Launch?:" + for i in ${!options[@]}; do + printf "%3d%s) %s\n" $((i+1)) "${choices[i]:- }" "${options[i]}" + done + [[ "$msg" ]] && echo "$msg"; : + } + + prompt="Check an option (again to uncheck, ENTER when done): " + + while menu && read -rp "$prompt" numinput && [[ "$numinput" ]]; do + for num in $numinput; do + [[ "$num" != *[![:digit:]]* ]] && (( num > 0 && num <= ${#options[@]} )) || { + msg="Invalid option: $num"; break + } + ((num--)); msg="${options[num]} was ${choices[num]:+un}checked" + [[ "${choices[num]}" ]] && choices[num]="" || choices[num]="+" + done + done + + for i in ${!options[@]}; do + if [[ "${choices[i]}" ]]; then + case ${options[i]} in + Redis) + launch_redis; + ;; + Ardb) + launch_ardb; + ;; + Logs) + launch_logs; + ;; + Queues) + launch_queues; + ;; + Scripts) + launch_scripts; + ;; + Flask) + launch_flask; + ;; + Crawler) + launching_crawler; + ;; + Killall) + killall; + ;; + Shutdown) + shutdown; + ;; + Update) + update; + ;; + Update-config) + checking_configuration; + ;; + Update-thirdparty) + update_thirdparty; + ;; + esac + fi + done + + exit + +} + + +#If no params, display the help [[ $@ ]] || { helptext; - - options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Flask" "Killall" "Shutdown" "Update" "Update-config" "Update-thirdparty") - - menu() { - echo "What do you want to Launch?:" - for i in ${!options[@]}; do - printf "%3d%s) %s\n" $((i+1)) "${choices[i]:- }" "${options[i]}" - done - [[ "$msg" ]] && echo "$msg"; : - } - - prompt="Check an option (again to uncheck, ENTER when done): " - while menu && read -rp "$prompt" numinput && [[ "$numinput" ]]; do - for num in $numinput; do - [[ "$num" != *[![:digit:]]* ]] && (( num > 0 && num <= ${#options[@]} )) || { - msg="Invalid option: $num"; break - } - ((num--)); msg="${options[num]} was ${choices[num]:+un}checked" - [[ "${choices[num]}" ]] && choices[num]="" || choices[num]="+" - done - done - - for i in ${!options[@]}; do - if [[ "${choices[i]}" ]]; then - case ${options[i]} in - Redis) - launch_redis; - ;; - Ardb) - launch_ardb; - ;; - Logs) - launch_logs; - ;; - Queues) - launch_queues; - ;; - Scripts) - launch_scripts; - ;; - Flask) - launch_flask; - ;; - Crawler) - launching_crawler; - ;; - Killall) - killall; - ;; - Shutdown) - shutdown; - ;; - Update) - update; - ;; - Update-config) - checking_configuration; - ;; - Update-thirdparty) - update_thirdparty; - ;; - esac - fi - done - - exit } #echo "$@" @@ -547,6 +568,10 @@ while [ "$1" != "" ]; do ;; -k | --killAll ) killall; ;; + -ks | --killscript ) killscript; + ;; + -m | --menu ) menu_display; + ;; -u | --update ) update; ;; -t | --thirdpartyUpdate ) update_thirdparty; @@ -559,7 +584,6 @@ while [ "$1" != "" ]; do exit ;; -kh | --khelp ) helptext; - ;; * ) helptext exit 1 diff --git a/bin/packages/Item.py b/bin/packages/Item.py index 2a464a5b..264ef300 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -22,6 +22,9 @@ def exist_item(item_id): else: return False +def get_item_id(full_path): + return full_path.replace(PASTES_FOLDER, '', 1) + def get_item_date(item_id): l_directory = item_id.split('/') return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2]) diff --git a/bin/packages/Tag.py b/bin/packages/Tag.py index dd1e858c..70d7e72e 100755 --- a/bin/packages/Tag.py +++ b/bin/packages/Tag.py @@ -82,7 +82,7 @@ def get_item_tags(item_id): if tags: return list(tags) else: - return '[]' + return [] # TEMPLATE + API QUERY def add_items_tag(tags=[], galaxy_tags=[], item_id=None): diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 0d32293b..c59e5c6b 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -180,6 +180,11 @@ host = localhost port = 6382 db = 3 +[ARDB_Tracker] +host = localhost +port = 6382 +db = 3 + [ARDB_Hashs] host = localhost db = 1 diff --git a/update/default_update/Update.sh b/update/default_update/Update.sh index 5809fb38..189ae846 100755 --- a/update/default_update/Update.sh +++ b/update/default_update/Update.sh @@ -20,8 +20,8 @@ export PATH=$AIL_FLASK:$PATH GREEN="\\033[1;32m" DEFAULT="\\033[0;39m" -echo -e $GREEN"Shutting down AIL ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -k +echo -e $GREEN"Shutting down AIL Script ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -ks wait echo "" @@ -37,8 +37,8 @@ echo "" echo "" echo "" -echo -e $GREEN"Shutting down ARDB ..."$DEFAULT -bash ${AIL_BIN}/LAUNCH.sh -k +echo -e $GREEN"Killing Script ..."$DEFAULT +bash ${AIL_BIN}/LAUNCH.sh -ks wait echo "" diff --git a/update/v2.2/Update.py b/update/v2.2/Update.py new file mode 100755 index 00000000..6a762f6c --- /dev/null +++ b/update/v2.2/Update.py @@ -0,0 +1,122 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import re +import sys +import time +import redis +import datetime +import configparser + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) +import Item +import Term + + +def rreplace(s, old, new, occurrence): + li = s.rsplit(old, occurrence) + return new.join(li) + + +if __name__ == '__main__': + + start_deb = time.time() + + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg.sample') + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + cfg = configparser.ConfigParser() + cfg.read(configfile) + + r_serv_term_stats = redis.StrictRedis( + host=cfg.get("ARDB_Trending", "host"), + port=cfg.getint("ARDB_Trending", "port"), + db=cfg.getint("ARDB_Trending", "db"), + decode_responses=True) + + r_serv_termfreq = redis.StrictRedis( + host=cfg.get("ARDB_TermFreq", "host"), + port=cfg.getint("ARDB_TermFreq", "port"), + db=cfg.getint("ARDB_TermFreq", "db"), + decode_responses=True) + + r_serv_term_stats.flushdb() + + #convert all regex: + all_regex = r_serv_termfreq.smembers('TrackedRegexSet') + for regex in all_regex: + tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)) ) + mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)) ) + + new_term = regex[1:-1] + res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test') + if res[1] == 200: + term_uuid = res[0]['uuid'] + list_items = r_serv_termfreq.smembers('regex_{}'.format(regex)) + for paste_item in list_items: + item_id = Item.get_item_id(paste_item) + item_date = Item.get_item_date(item_id) + Term.add_tracked_item(term_uuid, item_id, item_date) + + # Invalid Tracker => remove it + else: + print('Invalid Regex Removed: {}'.format(regex)) + print(res[0]) + # allow reprocess + r_serv_termfreq.srem('TrackedRegexSet', regex) + + all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet') + for token in all_tokens: + tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)) ) + mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)) ) + + res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test') + if res[1] == 200: + term_uuid = res[0]['uuid'] + list_items = r_serv_termfreq.smembers('tracked_{}'.format(token)) + for paste_item in list_items: + item_id = Item.get_item_id(paste_item) + item_date = Item.get_item_date(item_id) + Term.add_tracked_item(term_uuid, item_id, item_date) + # Invalid Tracker => remove it + else: + print('Invalid Token Removed: {}'.format(token)) + print(res[0]) + # allow reprocess + r_serv_termfreq.srem('TrackedSetTermSet', token) + + all_set = r_serv_termfreq.smembers('TrackedSetSet') + for curr_set in all_set: + tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)) ) + mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)) ) + + to_remove = ',{}'.format(curr_set.split(',')[-1]) + new_set = rreplace(curr_set, to_remove, '', 1) + new_set = new_set[2:] + new_set = new_set.replace(',', '') + + res = Term.parse_json_term_to_add({"term": new_set, "type": 'set', "nb_words": 1, "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test') + if res[1] == 200: + term_uuid = res[0]['uuid'] + list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set)) + for paste_item in list_items: + item_id = Item.get_item_id(paste_item) + item_date = Item.get_item_date(item_id) + Term.add_tracked_item(term_uuid, item_id, item_date) + # Invalid Tracker => remove it + else: + print('Invalid Set Removed: {}'.format(curr_set)) + print(res[0]) + # allow reprocess + r_serv_termfreq.srem('TrackedSetSet', curr_set) + + r_serv_termfreq.flushdb() + + #Set current ail version + r_serv.set('ail:version', 'v2.2') + + #Set current ail version + r_serv.hset('ail:update_date', 'v2.2', datetime.datetime.now().strftime("%Y%m%d")) diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index 0e3852e7..49f2919c 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -61,9 +61,9 @@ r_serv_sentiment = redis.StrictRedis( decode_responses=True) r_serv_term = redis.StrictRedis( - host=cfg.get("ARDB_TermFreq", "host"), - port=cfg.getint("ARDB_TermFreq", "port"), - db=cfg.getint("ARDB_TermFreq", "db"), + host=cfg.get("ARDB_Tracker", "host"), + port=cfg.getint("ARDB_Tracker", "port"), + db=cfg.getint("ARDB_Tracker", "db"), decode_responses=True) r_serv_cred = redis.StrictRedis(