mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
chg: [update] add v2.2 update + fix default update + fix Empty Item.get_tags() + add new LAUNCHER options
This commit is contained in:
Terrtia
parent
934d631f7a
commit
01ff202ae8
7 changed files with 236 additions and 82 deletions
172
bin/LAUNCH.sh
172
bin/LAUNCH.sh
|
|
@ -76,12 +76,15 @@ function helptext {
|
|||
Usage:
|
||||
-----
|
||||
LAUNCH.sh
|
||||
[-l | --launchAuto]
|
||||
[-k | --killAll]
|
||||
[-u | --update]
|
||||
[-c | --configUpdate]
|
||||
[-t | --thirdpartyUpdate]
|
||||
[-h | --help]
|
||||
[-l | --launchAuto] LAUNCH DB + Scripts
|
||||
[-k | --killAll] Kill DB + Scripts
|
||||
[-ks | --killscript] Scripts
|
||||
[-u | --update] Update AIL
|
||||
[-c | --crawler] LAUNCH Crawlers
|
||||
[-f | --launchFeeder] LAUNCH Pystemon feeder
|
||||
[-t | --thirdpartyUpdate] Update Web
|
||||
[-m | --menu] Display Advanced Menu
|
||||
[-h | --help] Help
|
||||
"
|
||||
}
|
||||
|
||||
|
|
@ -398,6 +401,18 @@ function launch_feeder {
|
|||
fi
|
||||
}
|
||||
|
||||
function killscript {
|
||||
if [[ $islogged || $isqueued || $isscripted || $isflasked || $isfeeded || $iscrawler ]]; then
|
||||
echo -e $GREEN"Killing Script"$DEFAULT
|
||||
kill $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler
|
||||
sleep 0.2
|
||||
echo -e $ROSE`screen -ls`$DEFAULT
|
||||
echo -e $GREEN"\t* $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler killed."$DEFAULT
|
||||
else
|
||||
echo -e $RED"\t* No script to kill"$DEFAULT
|
||||
fi
|
||||
}
|
||||
|
||||
function killall {
|
||||
if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted || $isflasked || $isfeeded || $iscrawler ]]; then
|
||||
if [[ $isredis ]]; then
|
||||
|
|
@ -457,76 +472,82 @@ function launch_all {
|
|||
launch_flask;
|
||||
}
|
||||
|
||||
#If no params, display the menu
|
||||
function menu_display {
|
||||
|
||||
options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Flask" "Killall" "Shutdown" "Update" "Update-config" "Update-thirdparty")
|
||||
|
||||
menu() {
|
||||
echo "What do you want to Launch?:"
|
||||
for i in ${!options[@]}; do
|
||||
printf "%3d%s) %s\n" $((i+1)) "${choices[i]:- }" "${options[i]}"
|
||||
done
|
||||
[[ "$msg" ]] && echo "$msg"; :
|
||||
}
|
||||
|
||||
prompt="Check an option (again to uncheck, ENTER when done): "
|
||||
|
||||
while menu && read -rp "$prompt" numinput && [[ "$numinput" ]]; do
|
||||
for num in $numinput; do
|
||||
[[ "$num" != *[![:digit:]]* ]] && (( num > 0 && num <= ${#options[@]} )) || {
|
||||
msg="Invalid option: $num"; break
|
||||
}
|
||||
((num--)); msg="${options[num]} was ${choices[num]:+un}checked"
|
||||
[[ "${choices[num]}" ]] && choices[num]="" || choices[num]="+"
|
||||
done
|
||||
done
|
||||
|
||||
for i in ${!options[@]}; do
|
||||
if [[ "${choices[i]}" ]]; then
|
||||
case ${options[i]} in
|
||||
Redis)
|
||||
launch_redis;
|
||||
;;
|
||||
Ardb)
|
||||
launch_ardb;
|
||||
;;
|
||||
Logs)
|
||||
launch_logs;
|
||||
;;
|
||||
Queues)
|
||||
launch_queues;
|
||||
;;
|
||||
Scripts)
|
||||
launch_scripts;
|
||||
;;
|
||||
Flask)
|
||||
launch_flask;
|
||||
;;
|
||||
Crawler)
|
||||
launching_crawler;
|
||||
;;
|
||||
Killall)
|
||||
killall;
|
||||
;;
|
||||
Shutdown)
|
||||
shutdown;
|
||||
;;
|
||||
Update)
|
||||
update;
|
||||
;;
|
||||
Update-config)
|
||||
checking_configuration;
|
||||
;;
|
||||
Update-thirdparty)
|
||||
update_thirdparty;
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
done
|
||||
|
||||
exit
|
||||
|
||||
}
|
||||
|
||||
|
||||
#If no params, display the help
|
||||
[[ $@ ]] || {
|
||||
|
||||
helptext;
|
||||
|
||||
options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Flask" "Killall" "Shutdown" "Update" "Update-config" "Update-thirdparty")
|
||||
|
||||
menu() {
|
||||
echo "What do you want to Launch?:"
|
||||
for i in ${!options[@]}; do
|
||||
printf "%3d%s) %s\n" $((i+1)) "${choices[i]:- }" "${options[i]}"
|
||||
done
|
||||
[[ "$msg" ]] && echo "$msg"; :
|
||||
}
|
||||
|
||||
prompt="Check an option (again to uncheck, ENTER when done): "
|
||||
while menu && read -rp "$prompt" numinput && [[ "$numinput" ]]; do
|
||||
for num in $numinput; do
|
||||
[[ "$num" != *[![:digit:]]* ]] && (( num > 0 && num <= ${#options[@]} )) || {
|
||||
msg="Invalid option: $num"; break
|
||||
}
|
||||
((num--)); msg="${options[num]} was ${choices[num]:+un}checked"
|
||||
[[ "${choices[num]}" ]] && choices[num]="" || choices[num]="+"
|
||||
done
|
||||
done
|
||||
|
||||
for i in ${!options[@]}; do
|
||||
if [[ "${choices[i]}" ]]; then
|
||||
case ${options[i]} in
|
||||
Redis)
|
||||
launch_redis;
|
||||
;;
|
||||
Ardb)
|
||||
launch_ardb;
|
||||
;;
|
||||
Logs)
|
||||
launch_logs;
|
||||
;;
|
||||
Queues)
|
||||
launch_queues;
|
||||
;;
|
||||
Scripts)
|
||||
launch_scripts;
|
||||
;;
|
||||
Flask)
|
||||
launch_flask;
|
||||
;;
|
||||
Crawler)
|
||||
launching_crawler;
|
||||
;;
|
||||
Killall)
|
||||
killall;
|
||||
;;
|
||||
Shutdown)
|
||||
shutdown;
|
||||
;;
|
||||
Update)
|
||||
update;
|
||||
;;
|
||||
Update-config)
|
||||
checking_configuration;
|
||||
;;
|
||||
Update-thirdparty)
|
||||
update_thirdparty;
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
done
|
||||
|
||||
exit
|
||||
}
|
||||
|
||||
#echo "$@"
|
||||
|
|
@ -547,6 +568,10 @@ while [ "$1" != "" ]; do
|
|||
;;
|
||||
-k | --killAll ) killall;
|
||||
;;
|
||||
-ks | --killscript ) killscript;
|
||||
;;
|
||||
-m | --menu ) menu_display;
|
||||
;;
|
||||
-u | --update ) update;
|
||||
;;
|
||||
-t | --thirdpartyUpdate ) update_thirdparty;
|
||||
|
|
@ -559,7 +584,6 @@ while [ "$1" != "" ]; do
|
|||
exit
|
||||
;;
|
||||
-kh | --khelp ) helptext;
|
||||
|
||||
;;
|
||||
* ) helptext
|
||||
exit 1
|
||||
|
|
|
|||
|
|
@ -22,6 +22,9 @@ def exist_item(item_id):
|
|||
else:
|
||||
return False
|
||||
|
||||
def get_item_id(full_path):
|
||||
return full_path.replace(PASTES_FOLDER, '', 1)
|
||||
|
||||
def get_item_date(item_id):
|
||||
l_directory = item_id.split('/')
|
||||
return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
|
||||
|
|
|
|||
|
|
@ -82,7 +82,7 @@ def get_item_tags(item_id):
|
|||
if tags:
|
||||
return list(tags)
|
||||
else:
|
||||
return '[]'
|
||||
return []
|
||||
|
||||
# TEMPLATE + API QUERY
|
||||
def add_items_tag(tags=[], galaxy_tags=[], item_id=None):
|
||||
|
|
|
|||
|
|
@ -180,6 +180,11 @@ host = localhost
|
|||
port = 6382
|
||||
db = 3
|
||||
|
||||
[ARDB_Tracker]
|
||||
host = localhost
|
||||
port = 6382
|
||||
db = 3
|
||||
|
||||
[ARDB_Hashs]
|
||||
host = localhost
|
||||
db = 1
|
||||
|
|
|
|||
|
|
@ -20,8 +20,8 @@ export PATH=$AIL_FLASK:$PATH
|
|||
GREEN="\\033[1;32m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
|
||||
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -k
|
||||
echo -e $GREEN"Shutting down AIL Script ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
echo ""
|
||||
|
|
@ -37,8 +37,8 @@ echo ""
|
|||
echo ""
|
||||
|
||||
echo ""
|
||||
echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -k
|
||||
echo -e $GREEN"Killing Script ..."$DEFAULT
|
||||
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||
wait
|
||||
|
||||
echo ""
|
||||
|
|
|
|||
122
update/v2.2/Update.py
Executable file
122
update/v2.2/Update.py
Executable file
|
|
@ -0,0 +1,122 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
import configparser
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||
import Item
|
||||
import Term
|
||||
|
||||
|
||||
def rreplace(s, old, new, occurrence):
|
||||
li = s.rsplit(old, occurrence)
|
||||
return new.join(li)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg.sample')
|
||||
if not os.path.exists(configfile):
|
||||
raise Exception('Unable to find the configuration file. \
|
||||
Did you set environment variables? \
|
||||
Or activate the virtualenv.')
|
||||
cfg = configparser.ConfigParser()
|
||||
cfg.read(configfile)
|
||||
|
||||
r_serv_term_stats = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_Trending", "host"),
|
||||
port=cfg.getint("ARDB_Trending", "port"),
|
||||
db=cfg.getint("ARDB_Trending", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_termfreq = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_TermFreq", "host"),
|
||||
port=cfg.getint("ARDB_TermFreq", "port"),
|
||||
db=cfg.getint("ARDB_TermFreq", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_term_stats.flushdb()
|
||||
|
||||
#convert all regex:
|
||||
all_regex = r_serv_termfreq.smembers('TrackedRegexSet')
|
||||
for regex in all_regex:
|
||||
tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)) )
|
||||
mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)) )
|
||||
|
||||
new_term = regex[1:-1]
|
||||
res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
|
||||
if res[1] == 200:
|
||||
term_uuid = res[0]['uuid']
|
||||
list_items = r_serv_termfreq.smembers('regex_{}'.format(regex))
|
||||
for paste_item in list_items:
|
||||
item_id = Item.get_item_id(paste_item)
|
||||
item_date = Item.get_item_date(item_id)
|
||||
Term.add_tracked_item(term_uuid, item_id, item_date)
|
||||
|
||||
# Invalid Tracker => remove it
|
||||
else:
|
||||
print('Invalid Regex Removed: {}'.format(regex))
|
||||
print(res[0])
|
||||
# allow reprocess
|
||||
r_serv_termfreq.srem('TrackedRegexSet', regex)
|
||||
|
||||
all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet')
|
||||
for token in all_tokens:
|
||||
tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)) )
|
||||
mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)) )
|
||||
|
||||
res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
|
||||
if res[1] == 200:
|
||||
term_uuid = res[0]['uuid']
|
||||
list_items = r_serv_termfreq.smembers('tracked_{}'.format(token))
|
||||
for paste_item in list_items:
|
||||
item_id = Item.get_item_id(paste_item)
|
||||
item_date = Item.get_item_date(item_id)
|
||||
Term.add_tracked_item(term_uuid, item_id, item_date)
|
||||
# Invalid Tracker => remove it
|
||||
else:
|
||||
print('Invalid Token Removed: {}'.format(token))
|
||||
print(res[0])
|
||||
# allow reprocess
|
||||
r_serv_termfreq.srem('TrackedSetTermSet', token)
|
||||
|
||||
all_set = r_serv_termfreq.smembers('TrackedSetSet')
|
||||
for curr_set in all_set:
|
||||
tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)) )
|
||||
mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)) )
|
||||
|
||||
to_remove = ',{}'.format(curr_set.split(',')[-1])
|
||||
new_set = rreplace(curr_set, to_remove, '', 1)
|
||||
new_set = new_set[2:]
|
||||
new_set = new_set.replace(',', '')
|
||||
|
||||
res = Term.parse_json_term_to_add({"term": new_set, "type": 'set', "nb_words": 1, "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
|
||||
if res[1] == 200:
|
||||
term_uuid = res[0]['uuid']
|
||||
list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set))
|
||||
for paste_item in list_items:
|
||||
item_id = Item.get_item_id(paste_item)
|
||||
item_date = Item.get_item_date(item_id)
|
||||
Term.add_tracked_item(term_uuid, item_id, item_date)
|
||||
# Invalid Tracker => remove it
|
||||
else:
|
||||
print('Invalid Set Removed: {}'.format(curr_set))
|
||||
print(res[0])
|
||||
# allow reprocess
|
||||
r_serv_termfreq.srem('TrackedSetSet', curr_set)
|
||||
|
||||
r_serv_termfreq.flushdb()
|
||||
|
||||
#Set current ail version
|
||||
r_serv.set('ail:version', 'v2.2')
|
||||
|
||||
#Set current ail version
|
||||
r_serv.hset('ail:update_date', 'v2.2', datetime.datetime.now().strftime("%Y%m%d"))
|
||||
|
|
@ -61,9 +61,9 @@ r_serv_sentiment = redis.StrictRedis(
|
|||
decode_responses=True)
|
||||
|
||||
r_serv_term = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_TermFreq", "host"),
|
||||
port=cfg.getint("ARDB_TermFreq", "port"),
|
||||
db=cfg.getint("ARDB_TermFreq", "db"),
|
||||
host=cfg.get("ARDB_Tracker", "host"),
|
||||
port=cfg.getint("ARDB_Tracker", "port"),
|
||||
db=cfg.getint("ARDB_Tracker", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_cred = redis.StrictRedis(
|
||||
|
|
|
|||
Loading…
Reference in a new issue