chg: [Update v2.7] add update v2.7 scripts

bin/packages/Date.py

@@ -79,6 +79,9 @@ class Date(object):
         comp_day = str(computed_date.day).zfill(2)
         return comp_year + comp_month + comp_day
 
+def get_today_date_str():
+    return datetime.date.today().strftime("%Y%m%d")
+
 def date_add_day(date, num_day=1):
     new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day)
     new_date = str(new_date).replace('-', '')

bin/update-background.py

@@ -86,11 +86,31 @@ if __name__ == "__main__":
         update_file = os.path.join(os.environ['AIL_HOME'], 'update', new_version, 'Update_screenshots.py')
         process = subprocess.run(['python' ,update_file])
 
-        update_progress = r_serv_db.get('ail:current_background_script_stat')
+        update_progress = r_serv.get('ail:current_background_script_stat')
         if update_progress:
             if int(update_progress) == 100:
                 r_serv.delete('ail:update_in_progress')
                 r_serv.delete('ail:current_background_script')
                 r_serv.delete('ail:current_background_script_stat')
                 r_serv.delete('ail:current_background_update')
-                r_serv_db.srem('ail:to_update', new_version)
+                r_serv.srem('ail:to_update', new_version)
+
+    elif r_serv.sismember('ail:to_update', 'v2.7'):
+        new_version = 'v2.7'
+        r_serv.delete('ail:update_error')
+        r_serv.delete('ail:current_background_script_stat')
+        r_serv.set('ail:update_in_progress', new_version)
+        r_serv.set('ail:current_background_update', new_version)
+        r_serv.set('ail:current_background_script', 'domain tags update')
+
+        update_file = os.path.join(os.environ['AIL_HOME'], 'update', new_version, 'Update_domain_tags.py')
+        process = subprocess.run(['python' ,update_file])
+
+        update_progress = r_serv.get('ail:current_background_script_stat')
+        if update_progress:
+            if int(update_progress) == 100:
+                r_serv.delete('ail:update_in_progress')
+                r_serv.delete('ail:current_background_script')
+                r_serv.delete('ail:current_background_script_stat')
+                r_serv.delete('ail:current_background_update')
+                r_serv.srem('ail:to_update', new_version)

update/v1.5/Update-ARDB_Tags.py

@@ -65,6 +65,9 @@ if __name__ == '__main__':
                 tag_metadata[tag]['last_seen'] = int(tag_metadata[tag]['last_seen'])
             nb_tags_to_update += r_serv_tag.scard(tag)
 
+        if nb_tags_to_update == 0:
+            nb_tags_to_update = 1
+
         for tag in tags_list:
 
             all_item =  r_serv_tag.smembers(tag)

update/v2.7/Update.py

@@ -0,0 +1,43 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import re
+import sys
+import time
+import redis
+import datetime
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+new_version = 'v2.7'
+
+if __name__ == '__main__':
+
+    start_deb = time.time()
+
+    config_loader = ConfigLoader.ConfigLoader()
+    r_serv = config_loader.get_redis_conn("ARDB_DB")
+    r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
+    r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
+    config_loader = None
+
+    #Set current update_in_progress
+    r_serv.set('ail:update_in_progress', new_version)
+    r_serv.set('ail:current_background_update', new_version)
+
+    r_serv.sadd('ail:to_update', new_version)
+
+    #### Update tags ####
+    r_serv_tags.sunionstore('list_tags:item', 'list_tags', [])
+    r_serv_onion.sunionstore('domain_update_v2.7', 'full_onion_up', [])
+    r_serv_onion.delete('incorrect_domain')
+    r_serv.set('ail:update_v2.7:deletetagrange', 1)
+    ####     ####
+
+    #Set current ail version
+    r_serv.set('ail:version', new_version)
+
+    #Set current ail version
+    r_serv.hset('ail:update_date', new_version, datetime.datetime.now().strftime("%Y%m%d"))

update/v2.7/Update.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
+
+export PATH=$AIL_HOME:$PATH
+export PATH=$AIL_REDIS:$PATH
+export PATH=$AIL_ARDB:$PATH
+export PATH=$AIL_BIN:$PATH
+export PATH=$AIL_FLASK:$PATH
+
+GREEN="\\033[1;32m"
+DEFAULT="\\033[0;39m"
+
+echo -e $GREEN"Shutting down AIL ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+bash ${AIL_BIN}/LAUNCH.sh -lav &
+wait
+echo ""
+
+echo ""
+echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
+echo ""
+python ${AIL_HOME}/update/v2.7/Update.py
+wait
+echo ""
+echo ""
+
+echo ""
+echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+exit 0

update/v2.7/Update_domain_tags.py

@@ -0,0 +1,119 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import sys
+import time
+import redis
+import datetime
+
+from hashlib import sha256
+
+from pyfaup.faup import Faup
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
+import Date
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+import Tag
+
+def sanitize_domain(domain):
+    faup.decode(domain)
+    domain_sanitized = faup.get()
+    domain_sanitized = domain_sanitized['domain']
+    try:
+        domain_sanitized = domain_sanitized.decode()
+    except:
+        pass
+    return domain_sanitized.lower()
+
+def delete_domain_tag_daterange():
+    all_domains_tags = Tag.get_all_obj_tags('domain')
+    nb_updated = 0
+    nb_to_update = len(all_domains_tags)
+    if nb_to_update == 0:
+        nb_to_update = 1
+    refresh_time = time.time()
+    l_dates = Date.substract_date('20191008', Date.get_today_date_str())
+    for tag in all_domains_tags:
+        for date_day in l_dates:
+            r_serv_tags.delete('domain:{}:{}'.format(tag, date_day))
+        nb_updated += 1
+        refresh_time = update_progress(refresh_time, nb_updated, nb_to_update)
+    r_serv_db.delete('ail:update_v2.7:deletetagrange')
+
+def update_domain_tags(domain):
+    domain_sanitized = sanitize_domain(domain)
+    if domain != domain_sanitized:
+        r_serv_onion.sadd('incorrect_domain', domain)
+        domain = domain_sanitized
+
+    domain_tags = Tag.get_obj_tag(domain)
+    for tag in domain_tags:
+        # delete incorrect tags
+        if tag == 'infoleak:submission="crawler"' and tag == 'infoleak:submission="manual"':
+            r_serv_metadata.srem('tag:{}'.format(domain), tag)
+        else:
+            Tag.add_global_tag(tag, object_type='domain')
+            r_serv_tags.sadd('{}:{}'.format('domain', tag), domain)
+
+def update_progress(refresh_time, nb_updated, nb_elem_to_update):
+    if time.time() - refresh_time > 10:
+        progress = int((nb_updated * 100) / nb_elem_to_update)
+        print('{}/{}    updated    {}%'.format(nb_updated, nb_elem_to_update, progress))
+        r_serv_db.set('ail:current_background_script_stat', progress)
+        refresh_time = time.time()
+
+    return refresh_time
+
+def update_db():
+    nb_updated = 0
+    nb_to_update = r_serv_onion.scard('domain_update_v2.7')
+    refresh_time = time.time()
+    r_serv_db.set('ail:current_background_script_stat', 0)
+    r_serv_db.set('ail:current_background_script', 'domain tags update')
+    domain = r_serv_onion.spop('domain_update_v2.7')
+    while domain is not None:
+        update_domain_tags(domain)
+        nb_updated += 1
+        refresh_time = update_progress(refresh_time, nb_updated, nb_to_update)
+        domain = r_serv_onion.spop('domain_update_v2.7')
+    if r_serv_db.exists('ail:update_v2.7:deletetagrange'):
+        r_serv_db.set('ail:current_background_script_stat', 0)
+        r_serv_db.set('ail:current_background_script', 'tags: remove deprecated keys')
+        delete_domain_tag_daterange()
+
+if __name__ == '__main__':
+
+    start_deb = time.time()
+    faup = Faup()
+
+    config_loader = ConfigLoader.ConfigLoader()
+
+    r_serv_db = config_loader.get_redis_conn("ARDB_DB")
+    r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
+    r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
+    config_loader = None
+
+    update_version = 'v2.7'
+
+    r_serv_db.set('ail:update_in_progress', update_version)
+    r_serv_db.set('ail:current_background_update', update_version)
+
+    r_serv_db.set('ail:current_background_script_stat', 0)
+    r_serv_db.set('ail:current_background_script', 'tags update')
+
+    update_db()
+
+    r_serv_db.set('ail:current_background_script_stat', 100)
+
+
+    end = time.time()
+    print('ALL domains tags updated in {} s'.format(end - start_deb))
+
+    r_serv_db.delete('ail:update_in_progress')
+    r_serv_db.delete('ail:current_background_script')
+    r_serv_db.delete('ail:current_background_script_stat')
+    r_serv_db.delete('ail:current_background_update')
+    r_serv_db.srem('ail:to_update', update_version)

var/www/modules/Flask_config.py

@@ -90,6 +90,8 @@ dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_me
                            'v2.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
                                                               'update_warning_message_notice_me': 'missing from the UI.'},
                            'v2.6':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
+                                                           'update_warning_message_notice_me': 'missing from the UI.'},
+                           'v2.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be',
                                                             'update_warning_message_notice_me': 'missing from the UI.'}
                         }