chg: [update] add v2.2 update + fix default update + fix Empty Item.get_tags() + add new LAUNCHER options

bin/LAUNCH.sh

@@ -76,12 +76,15 @@ function helptext {
     Usage:
     -----
     LAUNCH.sh
-      [-l | --launchAuto]
+      [-l  | --launchAuto]         LAUNCH DB + Scripts
-      [-k | --killAll]
+      [-k  | --killAll]            Kill DB + Scripts
-      [-u | --update]
+      [-ks | --killscript]         Scripts
-      [-c | --configUpdate]
+      [-u  | --update]             Update AIL
-      [-t | --thirdpartyUpdate]
+      [-c  | --crawler]            LAUNCH Crawlers
-      [-h | --help]
+      [-f  | --launchFeeder]       LAUNCH Pystemon feeder
+      [-t  | --thirdpartyUpdate]   Update Web
+      [-m  | --menu]               Display Advanced Menu
+      [-h  | --help]               Help
     "
 }
 
@@ -398,6 +401,18 @@ function launch_feeder {
     fi
 }
 
+function killscript {
+    if [[ $islogged || $isqueued || $isscripted || $isflasked || $isfeeded || $iscrawler ]]; then
+        echo -e $GREEN"Killing Script"$DEFAULT
+        kill $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler
+        sleep 0.2
+        echo -e $ROSE`screen -ls`$DEFAULT
+        echo -e $GREEN"\t* $islogged $isqueued $isscripted $isflasked $isfeeded $iscrawler killed."$DEFAULT
+    else
+        echo -e $RED"\t* No script to kill"$DEFAULT
+    fi
+}
+
 function killall {
     if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted || $isflasked || $isfeeded || $iscrawler ]]; then
         if [[ $isredis ]]; then
@@ -457,10 +472,7 @@ function launch_all {
     launch_flask;
 }
 
-#If no params, display the menu
+function menu_display {
-[[ $@ ]] || {
-
-    helptext;
 
   options=("Redis" "Ardb" "Logs" "Queues" "Scripts" "Flask" "Killall" "Shutdown" "Update" "Update-config" "Update-thirdparty")
 
@@ -473,6 +485,7 @@ function launch_all {
   }
 
   prompt="Check an option (again to uncheck, ENTER when done): "
+
   while menu && read -rp "$prompt" numinput && [[ "$numinput" ]]; do
       for num in $numinput; do
           [[ "$num" != *[![:digit:]]* ]] && (( num > 0 && num <= ${#options[@]} )) || {
@@ -527,6 +540,14 @@ function launch_all {
   done
 
   exit
+
+}
+
+
+#If no params, display the help
+[[ $@ ]] || {
+
+    helptext;
 }
 
 #echo "$@"
@@ -547,6 +568,10 @@ while [ "$1" != "" ]; do
                                       ;;
         -k | --killAll )              killall;
                                       ;;
+        -ks | --killscript )          killscript;
+                                      ;;
+        -m | --menu )                 menu_display;
+                                      ;;
         -u | --update )               update;
                                       ;;
         -t | --thirdpartyUpdate )     update_thirdparty;
@@ -559,7 +584,6 @@ while [ "$1" != "" ]; do
                                       exit
                                       ;;
         -kh | --khelp )               helptext;
-
                                       ;;
         * )                           helptext
                                       exit 1

bin/packages/Item.py

@@ -22,6 +22,9 @@ def exist_item(item_id):
     else:
         return False
 
+def get_item_id(full_path):
+    return full_path.replace(PASTES_FOLDER, '', 1)
+
 def get_item_date(item_id):
     l_directory = item_id.split('/')
     return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])

bin/packages/Tag.py

@@ -82,7 +82,7 @@ def get_item_tags(item_id):
     if tags:
         return list(tags)
     else:
-        return '[]'
+        return []
 
 # TEMPLATE + API QUERY
 def add_items_tag(tags=[], galaxy_tags=[], item_id=None):

bin/packages/config.cfg.sample

@@ -180,6 +180,11 @@ host = localhost
 port = 6382
 db = 3
 
+[ARDB_Tracker]
+host = localhost
+port = 6382
+db = 3
+
 [ARDB_Hashs]
 host = localhost
 db = 1

update/default_update/Update.sh

@@ -20,8 +20,8 @@ export PATH=$AIL_FLASK:$PATH
 GREEN="\\033[1;32m"
 DEFAULT="\\033[0;39m"
 
-echo -e $GREEN"Shutting down AIL ..."$DEFAULT
+echo -e $GREEN"Shutting down AIL Script ..."$DEFAULT
-bash ${AIL_BIN}/LAUNCH.sh -k
+bash ${AIL_BIN}/LAUNCH.sh -ks
 wait
 
 echo ""
@@ -37,8 +37,8 @@ echo ""
 echo ""
 
 echo ""
-echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
+echo -e $GREEN"Killing Script ..."$DEFAULT
-bash ${AIL_BIN}/LAUNCH.sh -k
+bash ${AIL_BIN}/LAUNCH.sh -ks
 wait
 
 echo ""

update/v2.2/Update.py

@@ -0,0 +1,122 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import re
+import sys
+import time
+import redis
+import datetime
+import configparser
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
+import Item
+import Term
+
+
+def rreplace(s, old, new, occurrence):
+    li = s.rsplit(old, occurrence)
+    return new.join(li)
+
+
+if __name__ == '__main__':
+
+    start_deb = time.time()
+
+    configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg.sample')
+    if not os.path.exists(configfile):
+        raise Exception('Unable to find the configuration file. \
+                        Did you set environment variables? \
+                        Or activate the virtualenv.')
+    cfg = configparser.ConfigParser()
+    cfg.read(configfile)
+
+    r_serv_term_stats = redis.StrictRedis(
+        host=cfg.get("ARDB_Trending", "host"),
+        port=cfg.getint("ARDB_Trending", "port"),
+        db=cfg.getint("ARDB_Trending", "db"),
+        decode_responses=True)
+
+    r_serv_termfreq = redis.StrictRedis(
+        host=cfg.get("ARDB_TermFreq", "host"),
+        port=cfg.getint("ARDB_TermFreq", "port"),
+        db=cfg.getint("ARDB_TermFreq", "db"),
+        decode_responses=True)
+
+    r_serv_term_stats.flushdb()
+
+    #convert all regex:
+    all_regex = r_serv_termfreq.smembers('TrackedRegexSet')
+    for regex in all_regex:
+        tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)) )
+        mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)) )
+
+        new_term = regex[1:-1]
+        res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
+        if res[1] == 200:
+            term_uuid = res[0]['uuid']
+            list_items = r_serv_termfreq.smembers('regex_{}'.format(regex))
+            for paste_item in list_items:
+                item_id = Item.get_item_id(paste_item)
+                item_date = Item.get_item_date(item_id)
+                Term.add_tracked_item(term_uuid, item_id, item_date)
+
+        # Invalid Tracker => remove it
+        else:
+            print('Invalid Regex Removed: {}'.format(regex))
+        print(res[0])
+        # allow reprocess
+        r_serv_termfreq.srem('TrackedRegexSet', regex)
+
+    all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet')
+    for token in all_tokens:
+        tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)) )
+        mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)) )
+
+        res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
+        if res[1] == 200:
+            term_uuid = res[0]['uuid']
+            list_items = r_serv_termfreq.smembers('tracked_{}'.format(token))
+            for paste_item in list_items:
+                item_id = Item.get_item_id(paste_item)
+                item_date = Item.get_item_date(item_id)
+                Term.add_tracked_item(term_uuid, item_id, item_date)
+        # Invalid Tracker => remove it
+        else:
+            print('Invalid Token Removed: {}'.format(token))
+        print(res[0])
+        # allow reprocess
+        r_serv_termfreq.srem('TrackedSetTermSet', token)
+
+    all_set = r_serv_termfreq.smembers('TrackedSetSet')
+    for curr_set in all_set:
+        tags = list( r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)) )
+        mails = list( r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)) )
+
+        to_remove = ',{}'.format(curr_set.split(',')[-1])
+        new_set = rreplace(curr_set, to_remove, '', 1)
+        new_set = new_set[2:]
+        new_set = new_set.replace(',', '')
+
+        res = Term.parse_json_term_to_add({"term": new_set, "type": 'set', "nb_words": 1, "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
+        if res[1] == 200:
+            term_uuid = res[0]['uuid']
+            list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set))
+            for paste_item in list_items:
+                item_id = Item.get_item_id(paste_item)
+                item_date = Item.get_item_date(item_id)
+                Term.add_tracked_item(term_uuid, item_id, item_date)
+        # Invalid Tracker => remove it
+        else:
+            print('Invalid Set Removed: {}'.format(curr_set))
+        print(res[0])
+        # allow reprocess
+        r_serv_termfreq.srem('TrackedSetSet', curr_set)
+
+    r_serv_termfreq.flushdb()
+
+    #Set current ail version
+    r_serv.set('ail:version', 'v2.2')
+
+    #Set current ail version
+    r_serv.hset('ail:update_date', 'v2.2', datetime.datetime.now().strftime("%Y%m%d"))

var/www/modules/Flask_config.py

@@ -61,9 +61,9 @@ r_serv_sentiment = redis.StrictRedis(
         decode_responses=True)
 
 r_serv_term = redis.StrictRedis(
-        host=cfg.get("ARDB_TermFreq", "host"),
+        host=cfg.get("ARDB_Tracker", "host"),
-        port=cfg.getint("ARDB_TermFreq", "port"),
+        port=cfg.getint("ARDB_Tracker", "port"),
-        db=cfg.getint("ARDB_TermFreq", "db"),
+        db=cfg.getint("ARDB_Tracker", "db"),
         decode_responses=True)
 
 r_serv_cred = redis.StrictRedis(