chg: [tracker + update] add update v3.7 + add map item_id:tracker_uuid (data retention) + fix tracker first_seen/last_seen

bin/lib/Tracker.py

@@ -67,6 +67,24 @@ def verify_mail_list(mail_list):
 ##-- UTILS --##
 ###############
 
+def get_all_tracker_type():
+    return ['word', 'set', 'regex', 'yara']
+
+def get_all_tracker_uuid():
+    return r_serv_tracker.smembers(f'trackers:all')
+
+def get_all_tracker_by_type(tracker_type):
+    r_serv_tracker.smembers(f'trackers:all:{tracker_type}')
+
+# def get_all_tracker():
+#     l_keys_name = []
+#     for tracker_type in get_all_tracker_type():
+#         l_keys_name.append(f'all:tracker:{tracker_type}')
+#     return r_serv_tracker.sunion(l_keys_name[0], *l_keys_name[1:])
+
+def get_all_tracker_by_type(tracker_type):
+    return r_serv_tracker.smembers(f'all:tracker:{tracker_type}')
+
 def get_tracker_by_uuid(tracker_uuid):
     return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'tracked')
 
@@ -94,17 +112,20 @@ def get_tracker_uuid_sources(tracker_uuid):
 def get_tracker_description(tracker_uuid):
     return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'description')
 
+def get_tracker_date(tracker_uuid):
+    return r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'date')
+
 def get_tracker_first_seen(tracker_uuid):
-    res = r_serv_tracker.zrange('tracker:stat:{}'.format(tracker_uuid), 0, 0)
+    res = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'first_seen')
     if res:
-        return res[0]
+        return res
     else:
         return None
 
 def get_tracker_last_seen(tracker_uuid):
-    res = r_serv_tracker.zrevrange('tracker:stat:{}'.format(tracker_uuid), 0, 0)
+    res = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'last_seen')
     if res:
-        return res[0]
+        return res
     else:
         return None
 
@@ -112,7 +133,7 @@ def get_tracker_metedata(tracker_uuid, user_id=False, description=False, level=F
     dict_uuid = {}
     dict_uuid['tracker'] = get_tracker_by_uuid(tracker_uuid)
     dict_uuid['type'] = get_tracker_type(tracker_uuid)
-    dict_uuid['date'] = r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'date')
+    dict_uuid['date'] = get_tracker_date(tracker_uuid)
     dict_uuid['description'] = get_tracker_description(tracker_uuid)
     dict_uuid['first_seen'] = get_tracker_first_seen(tracker_uuid)
     dict_uuid['last_seen'] = get_tracker_last_seen(tracker_uuid)
@@ -131,15 +152,12 @@ def get_tracker_metedata(tracker_uuid, user_id=False, description=False, level=F
     dict_uuid['uuid'] = tracker_uuid
     return dict_uuid
 
-################################################################################
-################################################################################
-################################################################################
-# # TODO: FIXME
+# tracker sparkline
 def get_tracker_sparkline(tracker_uuid, num_day=6):
     date_range_sparkline = Date.get_date_range(num_day)
     sparklines_value = []
     for date_day in date_range_sparkline:
-        nb_seen_this_day = r_serv_tracker.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day))
+        nb_seen_this_day = r_serv_tracker.zscore(f'tracker:stat:{tracker_uuid}', int(date_day))
         if nb_seen_this_day is None:
             nb_seen_this_day = 0
         sparklines_value.append(int(nb_seen_this_day))
@@ -159,11 +177,40 @@ def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to):
 def add_tracked_item(tracker_uuid, item_id):
     item_date = item_basic.get_item_date(item_id)
     # track item
-    # r_serv_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid)
+    r_serv_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid)
     res = r_serv_tracker.sadd(f'tracker:item:{tracker_uuid}:{item_date}', item_id)
     # track nb item by date
     if res == 1:
-        r_serv_tracker.zincrby('tracker:stat:{}'.format(tracker_uuid), int(item_date), 1)
+        nb_items = r_serv_tracker.zincrby('tracker:stat:{}'.format(tracker_uuid), int(item_date), 1)
+        if nb_items == 1:
+            update_tracker_daterange(tracker_uuid, item_date)
+
+def set_tracker_first_seen(tracker_uuid, date):
+    r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'first_seen', int(date))
+
+def set_tracker_last_seen(tracker_uuid, date):
+    r_serv_tracker.hset(f'tracker:{tracker_uuid}', 'last_seen', int(date))
+
+# # TODO: ADD CACHE ???
+def update_tracker_daterange(tracker_uuid, date, op='add'):
+    date = int(date)
+    first_seen = get_tracker_first_seen(tracker_uuid)
+
+    if op == 'add':
+        if not first_seen:
+            set_tracker_first_seen(tracker_uuid, date)
+            set_tracker_last_seen(tracker_uuid, date)
+        else:
+            first_seen = int(first_seen)
+            last_seen = int(get_tracker_last_seen(tracker_uuid))
+            if date < first_seen:
+                set_tracker_first_seen(tracker_uuid, date)
+            if date > last_seen:
+                set_tracker_last_seen(tracker_uuid, date)
+
+    if op == 'del':
+        pass
+
 
 def remove_tracked_item(item_id):
     item_date = item_basic.get_item_date(item_id)
@@ -237,13 +284,40 @@ def api_is_allowed_to_edit_tracker(tracker_uuid, user_id):
 
 #### FIX DB ####
 def fix_tracker_stats_per_day(tracker_uuid):
-    date_from = get_tracker_first_seen(tracker_uuid)
-    date_to = get_tracker_last_seen(tracker_uuid)
+    date_from = get_tracker_date(tracker_uuid)
+    date_to = Date.get_today_date_str()
     # delete stats
     r_serv_tracker.delete(f'tracker:stat:{tracker_uuid}')
     # create new stats
     for date_day in Date.substract_date(date_from, date_to):
-        pass
+        date_day = int(date_day)
+
+        nb_items = r_serv_tracker.scard(f'tracker:item:{tracker_uuid}:{date_day}')
+        if nb_items:
+            r_serv_tracker.zincrby('tracker:stat:{}'.format(tracker_uuid), int(date_day), nb_items)
+
+        # update first_seen/last_seen
+        update_tracker_daterange(tracker_uuid, date_day)
+
+def fix_tracker_item_link(tracker_uuid):
+    date_from = get_tracker_first_seen(tracker_uuid)
+    date_to = get_tracker_last_seen(tracker_uuid)
+
+    for date_day in Date.substract_date(date_from, date_to):
+        l_items = r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}')
+        for item_id in l_items:
+            r_serv_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid)
+
+def fix_all_tracker_uuid_list():
+    r_serv_tracker.delete(f'trackers:all')
+    r_serv_tracker.delete(f'trackers:all:{tracker_type}')
+    for tracker_type in get_all_tracker_type():
+        l_tracker = get_all_tracker_by_type(tracker_type)
+        for tracker in l_tracker:
+            l_tracker_uuid = get_tracker_uuid_list(tracker, tracker_type)
+            for tracker_uuid in l_tracker_uuid:
+                r_serv_tracker.sadd(f'trackers:all', tracker_uuid)
+                r_serv_tracker.sadd(f'trackers:all:{tracker_type}', tracker_uuid)
 
 ##-- FIX DB --##
 
@@ -359,6 +433,9 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti
     # create tracker - uuid map
     r_serv_tracker.sadd('all:tracker_uuid:{}:{}'.format(tracker_type, tracker), tracker_uuid)
 
+    r_serv_tracker.sadd(f'trackers:all', tracker_uuid)
+    r_serv_tracker.sadd(f'trackers:all:{tracker_type}', tracker_uuid)
+
     # add display level set
     if level == 0: # user only
         r_serv_tracker.sadd('user:tracker:{}'.format(user_id), tracker_uuid)
@@ -1121,7 +1198,21 @@ def api_delete_retro_hunt_task(task_uuid):
     else:
         return (delete_retro_hunt_task(task_uuid), 200)
 
-#if __name__ == '__main__':
+# if __name__ == '__main__':
+    # fix_all_tracker_uuid_list()
+    # res = get_all_tracker_uuid()
+    # print(len(res))
+
+    # import Term
+    # Term.delete_term('5262ab6c-8784-4a55-b0ff-a471018414b4')
+
+    #fix_tracker_stats_per_day('5262ab6c-8784-4a55-b0ff-a471018414b4')
+
+    # tracker_uuid = '5262ab6c-8784-4a55-b0ff-a471018414b4'
+    # fix_tracker_item_link(tracker_uuid)
+    # res = get_item_all_trackers_uuid('archive/')
+    # print(res)
+
     #res = is_valid_yara_rule('rule dummy {  }')
 
     # res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None, sources=['crawled', 'pastebin.com', 'rt/pastebin.com'])

bin/packages/Term.py

@@ -290,11 +290,16 @@ def parse_tracked_term_to_delete(dict_input, user_id):
     delete_term(term_uuid)
     return ({"uuid": term_uuid}, 200)
 
+# # TODO: MOVE IN TRACKER
 def delete_term(term_uuid):
     term = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked')
     term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type')
     level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
     r_serv_term.srem('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid)
+
+    r_serv_term.srem(f'trackers:all', term_uuid)
+    r_serv_term.srem(f'trackers:all:{term_type}', term_uuid)
+
     # Term not tracked by other users
     if not r_serv_term.exists('all:tracker_uuid:{}:{}'.format(term_type, term)):
         r_serv_term.srem('all:tracker:{}'.format(term_type), term)
@@ -323,10 +328,15 @@ def delete_term(term_uuid):
     r_serv_term.delete('tracker:sources:{}'.format(term_uuid))
 
     # remove item set
-    all_item_date = r_serv_term.zrange('tracker:stat:{}'.format(term_uuid), 0, -1)
-    for date in all_item_date:
-        r_serv_term.delete('tracker:item:{}:{}'.format(term_uuid, date))
-    r_serv_term.delete('tracker:stat:{}'.format(term_uuid))
+    #########################3
+    all_item_date = r_serv_term.zrange(f'tracker:stat:{term_uuid}', 0, -1, withscores=True)
+    if all_item_date:
+        all_item_date = dict(all_item_date)
+        for date in all_item_date:
+            for item_id in r_serv_term.smembers(f'tracker:item:{term_uuid}:{date}'):
+                r_serv_term.srem(f'obj:trackers:item:{item_id}', term_uuid)
+            r_serv_term.delete(f'tracker:item:{term_uuid}:{date}')
+        r_serv_term.delete('tracker:stat:{}'.format(term_uuid))
 
     if term_type == 'yara':
         # delete custom rule

bin/update-background.py

@@ -65,3 +65,4 @@ if __name__ == "__main__":
     launch_background_upgrade('v2.6', ['Update_screenshots.py'])
     launch_background_upgrade('v2.7', ['Update_domain_tags.py'])
     launch_background_upgrade('v3.4', ['Update_domain.py'])
+    launch_background_upgrade('v3.7', ['Update_trackers.py'])

update/v3.7/Update.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import re
+import sys
+import time
+import redis
+import datetime
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+import Tracker
+
+sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
+from ail_updater import AIL_Updater
+
+class Updater(AIL_Updater):
+    """default Updater."""
+
+    def __init__(self, version):
+        super(Updater, self).__init__(version)
+
+    def update(self):
+        """
+        Update Domain Languages
+        """
+        print('Fixing Tracker_uuid list ...')
+        Tracker.fix_all_tracker_uuid_list()
+        nb = 0
+        for tracker_uuid in get_all_tracker_uuid:
+            self.r_serv.sadd('trackers_update_v3.7', tracker_uuid)
+            nb += 1
+
+        self.r_serv.set('update:nb_elem_to_convert', nb)
+        self.r_serv.set('update:nb_elem_converted',0)
+
+        # Add background update
+        self.r_serv.sadd('ail:to_update', self.version)
+
+if __name__ == '__main__':
+
+    updater = Updater('v3.7')
+    updater.run_update()

update/v3.7/Update.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+
+[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
+[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
+
+export PATH=$AIL_HOME:$PATH
+export PATH=$AIL_REDIS:$PATH
+export PATH=$AIL_ARDB:$PATH
+export PATH=$AIL_BIN:$PATH
+export PATH=$AIL_FLASK:$PATH
+
+GREEN="\\033[1;32m"
+DEFAULT="\\033[0;39m"
+
+echo -e $GREEN"Shutting down AIL ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+# SUBMODULES #
+git submodule update
+
+
+echo -e $GREEN"Updating thirdparty ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ut
+wait
+
+echo ""
+echo -e $GREEN"Updating AIL VERSION ..."$DEFAULT
+echo ""
+python ${AIL_HOME}/update/v3.7/Update.py
+wait
+echo ""
+echo ""
+
+
+echo ""
+echo -e $GREEN"Shutting down ARDB ..."$DEFAULT
+bash ${AIL_BIN}/LAUNCH.sh -ks
+wait
+
+exit 0

update/v3.7/Update_trackers.py

@@ -0,0 +1,70 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import os
+import re
+import sys
+import time
+import redis
+import datetime
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+import Domain
+
+def update_update_stats():
+    nb_updated = int(r_serv_db.get('update:nb_elem_converted'))
+    progress = int((nb_updated * 100) / nb_elem_to_update)
+    print('{}/{}    updated    {}%'.format(nb_updated, nb_elem_to_update, progress))
+    r_serv_db.set('ail:current_background_script_stat', progress)
+
+def update_domain_language(domain_obj, item_id):
+    domain_name = domain_obj.get_domain_name()
+    Domain.add_domain_languages_by_item_id(domain_name, item_id)
+
+if __name__ == '__main__':
+
+    start_deb = time.time()
+
+    config_loader = ConfigLoader.ConfigLoader()
+    r_serv_db = config_loader.get_redis_conn("ARDB_DB")
+    r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
+    config_loader = None
+
+    r_serv_db.set('ail:current_background_script', 'trackers update')
+
+    nb_elem_to_update = r_serv_db.get('update:nb_elem_to_convert')
+    if not nb_elem_to_update:
+        nb_elem_to_update = 1
+    else:
+        nb_elem_to_update = int(nb_elem_to_update)
+
+
+
+    while True:
+        tracker_uuid = r_serv_onion.spop('trackers_update_v3.7')
+        if tracker_uuid is not None:
+            date_from =
+            date_to =
+            
+            # FIX STATS
+
+            print(tracker_uuid)
+            # get all dates
+
+            # get items id
+
+            # convert
+            domain = Domain.Domain(domain)
+            for domain_history in domain.get_domain_history():
+                domain_item = domain.get_domain_items_crawled(epoch=domain_history[1]) # item_tag
+                if "items" in domain_item:
+                    for item_dict in domain_item['items']:
+                        update_domain_language(domain, item_dict['id'])
+
+            r_serv_db.incr('update:nb_elem_converted')
+            update_update_stats()
+
+        else:
+            r_serv_db.set('ail:current_background_script_stat', 100)
+            sys.exit(0)

var/www/modules/Flask_config.py

@@ -109,7 +109,9 @@ dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_me
                            'v2.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be',
                                                             'update_warning_message_notice_me': 'missing from the UI.'},
                            'v3.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Languages can be',
-                                                            'update_warning_message_notice_me': 'missing from the UI.'}
+                                                            'update_warning_message_notice_me': 'missing from the UI.'},
+                           'v3.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Tracker first_seen/last_seen can be',
+                                                             'update_warning_message_notice_me': 'missing from the UI.'}
                         }
 
 UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')

var/www/modules/hunter/templates/showTracker.html

@@ -209,13 +209,15 @@
 								<div class="col-md-6">
 									<div class="input-group" id="date-range-from">
 										<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
-										<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ tracker_metadata['date_from'] }}" name="date_from" autocomplete="off">
+										<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from" autocomplete="off"
+										{%if tracker_metadata['date_from']%}value="{{ tracker_metadata['date_from'] }}"{%else%}value="{{tracker_metadata['first_seen']}}"{%endif%}>
 									</div>
 								</div>
 								<div class="col-md-6">
 									<div class="input-group" id="date-range-to">
 										<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
-										<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ tracker_metadata['date_to'] }}" name="date_to" autocomplete="off">
+										<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to" autocomplete="off"
+										{%if tracker_metadata['date_to']%}value="{{ tracker_metadata['date_to'] }}"{%else%}value="{{tracker_metadata['last_seen']}}"{%endif%}>
 									</div>
 								</div>
 							</div>
@@ -270,7 +272,7 @@
 <script>
 $(document).ready(function(){
 		$('#div_edit_mails').hide();
-		$('#div_edit_tags').hide();edit_description
+		$('#div_edit_tags').hide();
 		$('#div_edit_description').hide();
 		$("#page-Decoded").addClass("active");