chg: [dashboard] add echart feeder graph + cleanup olg graph libs + dashboard, show today nb objects

bin/lib/ail_stats.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import datetime
+import os
+import sys
+import time
+
+# from datetime import datetime
+from logging import lastResort
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.ConfigLoader import ConfigLoader
+from lib.objects import ail_objects
+
+
+# Config
+config_loader = ConfigLoader()
+r_stats = config_loader.get_db_conn("Kvrocks_Stats")
+# r_cache = config_loader.get_redis_conn("Redis_Cache")
+config_loader = None
+
+
+def get_feeders():
+    return r_stats.smembers(f'feeders:name')
+
+def get_current_feeder_timestamp(timestamp):
+    return int(timestamp - (timestamp % 30))
+
+def get_next_feeder_timestamp(timestamp):
+    return int(timestamp + 30 - (timestamp % 30))
+
+def get_feeders_by_time(timestamp):  # TODO
+    feeders = {}
+    for row in r_stats.zrange(f'feeders:{timestamp}', 0, -1, withscores=True):
+        feeders[row[0]] = int(row[1])
+    return feeders
+
+def get_feeders_dashboard_full():
+    timestamp = get_current_feeder_timestamp(int(time.time()))
+    print(timestamp)
+    # timestamp = 1731491970
+    f_dashboard = {}
+
+    feeders = get_feeders()
+    d_time = []
+    for i in range(timestamp - 30*20, timestamp +30, 30):
+        t_feeders = get_feeders_by_time(i)
+        for feeder in feeders:
+            if feeder not in f_dashboard:
+                f_dashboard[feeder] = []
+            if feeder in t_feeders:
+                f_dashboard[feeder].append(t_feeders[feeder])
+            else:
+                f_dashboard[feeder].append(0)
+        d_time.append(datetime.datetime.utcfromtimestamp(i).strftime('%H:%M:%S'))
+    return {'data': f_dashboard, 'dates': d_time}
+
+def get_feeders_dashboard():
+    timestamp = get_current_feeder_timestamp(int(time.time()))
+    print(timestamp)
+
+    f_dashboard = {}
+    t_feeders = get_feeders_by_time(timestamp)
+    for feeder in get_feeders():
+        if feeder in t_feeders:
+            f_dashboard[feeder] = t_feeders[feeder]
+        else:
+            f_dashboard[feeder] = 0
+
+    date = datetime.datetime.utcfromtimestamp(timestamp).strftime('%H:%M:%S')
+    return {'data': f_dashboard, 'date': date}
+
+
+def add_feeders(timestamp, feeders):
+    if feeders:
+        r = r_stats.zadd(f'feeders:{timestamp}', feeders)
+        print(r)
+        for feeder in feeders:
+            r_stats.sadd(f'feeders:name', feeder)
+        # cleanup keys
+        r_stats.sadd(f'feeders:timestamps', timestamp)
+
+def get_nb_objs_today():
+    date = datetime.date.today().strftime("%Y%m%d")
+    nb_objs = ail_objects.get_nb_objects_by_date(date)
+    return nb_objs
+
+def get_nb_objs_dashboard():
+    date = datetime.date.today().strftime("%Y%m%d")
+    return ail_objects.get_nb_objects_dashboard(date)
+
+
+

bin/lib/objects/BarCodes.py

@@ -150,6 +150,19 @@ class Barcodes(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('barcode', Barcode)
 
+    def get_name(self):
+        return 'Barcodes'
+
+    def get_icon(self):
+        return {'fa': 'fas', 'icon': 'barcode'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_barcode.objects_barcodes')
+        else:
+            url = f'{baseurl}/objects/barcodes'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search  # TODO
 

bin/lib/objects/CookiesNames.py

@@ -109,6 +109,19 @@ class CookiesNames(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('cookie-name', CookieName)
 
+    def get_name(self):
+        return 'Cookie-Names'
+
+    def get_icon(self):
+        return {'fa': 'fas', 'icon': 'cookie-bite'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_cookie_name.objects_cookies_names')
+        else:
+            url = f'{baseurl}/objects/cookie-name'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search  # TODO
 

bin/lib/objects/Cves.py

@@ -16,7 +16,7 @@ sys.path.append(os.environ['AIL_BIN'])
 # Import Project packages
 ##################################
 from lib.ConfigLoader import ConfigLoader
-from lib.objects.abstract_daterange_object import AbstractDaterangeObject
+from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects
 from packages import Date
 
 config_loader = ConfigLoader()
@@ -97,6 +97,29 @@ class Cve(AbstractDaterangeObject):
         except requests.exceptions.ReadTimeout:
             return {'error': f'Timeout Error'}
 
+class Cves(AbstractDaterangeObjects):
+    """
+        Barcodes Objects
+    """
+    def __init__(self):
+        super().__init__('cve', Cve)
+
+    def get_name(self):
+        return 'Cves'
+
+    def get_icon(self):
+        return {'fa': 'fas', 'icon': 'bug'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_cve.objects_cves')
+        else:
+            url = f'{baseurl}/objects/cves'
+        return url
+
+    def sanitize_id_to_search(self, name_to_search):
+        return name_to_search  # TODO
+
 
 # TODO  ADD SEARCH FUNCTION
 

bin/lib/objects/DomHashs.py

@@ -114,6 +114,19 @@ class DomHashs(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('dom-hash', DomHash)
 
+    def get_name(self):
+        return 'DomHashs'
+
+    def get_icon(self):
+        return {'fa': 'fas', 'icon': 'align-left'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_dom_hash.objects_dom_hashs')
+        else:
+            url = f'{baseurl}/objects/dom-hashs'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search
 

bin/lib/objects/Etags.py

@@ -109,6 +109,19 @@ class Etags(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('etag', Etag)
 
+    def get_name(self):
+        return 'Etags'
+
+    def get_icon(self):
+        return {'fa': 'fas', 'icon': 'tag'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_etag.objects_etags')
+        else:
+            url = f'{baseurl}/objects/etags'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search  # TODO
 

bin/lib/objects/Favicons.py

@@ -135,6 +135,19 @@ class Favicons(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('favicon', Favicon)
 
+    def get_name(self):
+        return 'Favicons'
+
+    def get_icon(self):
+        return {'fa': 'fas', 'icon': 'star-half'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_favicon.objects_favicons')
+        else:
+            url = f'{baseurl}/objects/favicons'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search  # TODO
 

bin/lib/objects/FilesNames.py

@@ -84,6 +84,20 @@ class FilesNames(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('file-name', FileName)
 
+    def get_name(self):
+        return 'File-Names'
+
+    def get_icon(self):
+        return {'fa': 'far', 'icon': 'file'}
+
+    def get_link(self, flask_context=False):
+        pass
+        # if flask_context:
+        #     url = url_for('objects_favicon.objects_favicons')
+        # else:
+        #     url = f'{baseurl}/objects/favicons'
+        # return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search
 

bin/lib/objects/HHHashs.py

@@ -126,6 +126,19 @@ class HHHashs(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('hhhash', HHHash)
 
+    def get_name(self):
+        return 'HHHashs'
+
+    def get_icon(self):
+        return {'fas': 'far', 'icon': 'align-left'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_hhhash.objects_hhhashs')
+        else:
+            url = f'{baseurl}/objects/hhhashs'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search  # TODO
 

bin/lib/objects/Images.py

@@ -152,6 +152,19 @@ class Images(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('image', Image)
 
+    def get_name(self):
+        return 'Images'
+
+    def get_icon(self):
+        return {'fas': 'fas', 'icon': 'image'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_image.objects_images')
+        else:
+            url = f'{baseurl}/objects/images'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search  # TODO
 

bin/lib/objects/Ocrs.py

@@ -323,6 +323,19 @@ class Ocrs(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('ocr', Ocr)
 
+    def get_name(self):
+        return 'Ocrs'
+
+    def get_icon(self):
+        return {'fas': 'far', 'icon': 'expand'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_ocr.objects_ocrs')
+        else:
+            url = f'{baseurl}/objects/ocrs'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search  # TODO
 

bin/lib/objects/QrCodes.py

@@ -150,6 +150,19 @@ class Qrcodes(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('qrcode', Qrcode)
 
+    def get_name(self):
+        return 'Qrcodes'
+
+    def get_icon(self):
+        return {'fas': 'far', 'icon': 'qrcode'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_qrcode.objects_qrcodes')
+        else:
+            url = f'{baseurl}/objects/qrcodes'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search  # TODO
 

bin/lib/objects/Titles.py

@@ -104,6 +104,19 @@ class Titles(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('title', Title)
 
+    def get_name(self):
+        return 'Titles'
+
+    def get_icon(self):
+        return {'fas': 'far', 'icon': 'heading'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_title.objects_titles')
+        else:
+            url = f'{baseurl}/objects/titles'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search
 

bin/lib/objects/abstract_daterange_object.py

@@ -193,6 +193,18 @@ class AbstractDaterangeObjects(ABC):
         self.type = obj_type
         self.obj_class = obj_class
 
+    @abstractmethod
+    def get_name(self):
+        pass
+
+    @abstractmethod
+    def get_icon(self):
+        pass
+
+    @abstractmethod
+    def get_link(self, flask_context=False):
+        pass
+
     ################################################
     ################################################
 

bin/lib/objects/ail_objects.py

@@ -25,7 +25,7 @@ from lib.objects import ChatSubChannels
 from lib.objects import ChatThreads
 from lib.objects import CryptoCurrencies
 from lib.objects import CookiesNames
-from lib.objects.Cves import Cve
+from lib.objects import Cves
 from lib.objects.Decodeds import Decoded, get_all_decodeds_objects, get_nb_decodeds_objects
 from lib.objects.Domains import Domain
 from lib.objects import Etags
@@ -44,9 +44,36 @@ from lib.objects import Titles
 from lib.objects import UsersAccount
 from lib.objects import Usernames
 
-config_loader = ConfigLoader()
+# config_loader = ConfigLoader()
-
+#
-config_loader = None
+# config_loader = None
+# TODO INIT objs classes ????
+OBJECTS_CLASS = {
+    'barcode': {'obj': BarCodes.Barcode, 'objs': BarCodes.Barcodes},
+    'chat': {'obj': Chats.Chat, 'objs': None}, ## SUBTYPE #########################################
+    'chat-subchannel': {'obj': ChatSubChannels.ChatSubChannel, 'objs': None}, ######   ######
+    'chat-thread': {'obj': ChatThreads.ChatThread, 'objs': None},    ######   ######
+    'cookie-name': {'obj': CookiesNames.CookieName, 'objs': CookiesNames.CookiesNames},
+    'cve': {'obj': Cves.Cve, 'objs': Cves.Cves},
+    'cryptocurrency': {'obj': CryptoCurrencies.CryptoCurrency, 'objs': None}, ## SUBTYPE #########################################
+    'decoded': {'obj': Decoded, 'objs': None}, ###############################################################################################
+    'domain': {'obj': Domain, 'objs': None}, ####################################################################################################
+    'dom-hash': {'obj': DomHashs.DomHash, 'objs': DomHashs.DomHashs},
+    'etag': {'obj': Etags.Etag, 'objs': Etags.Etags},
+    'favicon': {'obj': Favicons.Favicon, 'objs': Favicons.Favicons},
+    'file-name': {'obj': FilesNames.FileName, 'objs': FilesNames.FilesNames},
+    'hhhash': {'obj': HHHashs.HHHash, 'objs': HHHashs.HHHashs},
+    'item': {'obj': Item, 'objs': None}, ######
+    'image': {'obj': Images.Image, 'objs': Images.Images},
+    'message': {'obj': Messages.Message, 'objs': None}, ######
+    'ocr': {'obj': Ocrs.Ocr, 'objs': Ocrs.Ocrs},
+    'pgp': {'obj': Pgps.Pgp, 'objs': None}, ## SUBTYPE ###########################################################################
+    'qrcode': {'obj': QrCodes.Qrcode, 'objs': QrCodes.Qrcodes},
+    'screenshot': {'obj': Screenshots.Screenshot, 'objs': None}, ######
+    'title': {'obj': Titles.Title, 'objs': Titles.Titles},
+    'user-account': {'obj': UsersAccount.UserAccount, 'objs': None}, ## SUBTYPE ###########################################################################
+    'username': {'obj': Usernames.Username, 'objs': None}, ## SUBTYPE ###########################################################################
+}
 
 
 def is_valid_object_type(obj_type):
@@ -70,67 +97,29 @@ def sanitize_objs_types(objs, default=False):
             l_types = get_all_objects()
     return l_types
 
+
 #### OBJECT ####
 
+def get_obj_class(obj_type):
+    if obj_type in OBJECTS_CLASS:
+        return OBJECTS_CLASS[obj_type]['obj']
+
+def get_objs_class(obj_type):
+    if obj_type in OBJECTS_CLASS:
+        return OBJECTS_CLASS[obj_type]['objs']
+
 def get_object(obj_type, subtype, obj_id):
     if subtype == 'None':
         subtype = None
     obj_id = str(obj_id)
+    obj_class = OBJECTS_CLASS[obj_type]['obj']
+    if not obj_class:
+        raise AILObjectUnknown(f'Unknown AIL object: {obj_type} {subtype} {obj_id}')
     if not subtype:
-        if obj_type == 'item':
+        return obj_class(obj_id)
-            return Item(obj_id)
-        elif obj_type == 'domain':
-            return Domain(obj_id)
-        elif obj_type == 'decoded':
-            return Decoded(obj_id)
-        elif obj_type == 'cookie-name':
-            return CookiesNames.CookieName(obj_id)
-        elif obj_type == 'cve':
-            return Cve(obj_id)
-        elif obj_type == 'etag':
-            return Etags.Etag(obj_id)
-        elif obj_type == 'favicon':
-            return Favicons.Favicon(obj_id)
-        elif obj_type == 'file-name':
-            return FilesNames.FileName(obj_id)
-        elif obj_type == 'dom-hash':
-            return DomHashs.DomHash(obj_id)
-        elif obj_type == 'hhhash':
-            return HHHashs.HHHash(obj_id)
-        elif obj_type == 'image':
-            return Images.Image(obj_id)
-        elif obj_type == 'message':
-            return Messages.Message(obj_id)
-        elif obj_type == 'ocr':
-            return Ocrs.Ocr(obj_id)
-        elif obj_type == 'barcode':
-            return BarCodes.Barcode(obj_id)
-        elif obj_type == 'qrcode':
-            return QrCodes.Qrcode(obj_id)
-        elif obj_type == 'screenshot':
-            return Screenshots.Screenshot(obj_id)
-        elif obj_type == 'title':
-            return Titles.Title(obj_id)
-        else:
-            raise AILObjectUnknown(f'Unknown AIL object: {obj_type} {subtype} {obj_id}')
     # SUBTYPES
     else:
-        if obj_type == 'chat':
+        obj_class(obj_id, subtype)
-            return Chats.Chat(obj_id, subtype)
-        elif obj_type == 'chat-subchannel':
-            return ChatSubChannels.ChatSubChannel(obj_id, subtype)
-        elif obj_type == 'chat-thread':
-            return ChatThreads.ChatThread(obj_id, subtype)
-        elif obj_type == 'cryptocurrency':
-            return CryptoCurrencies.CryptoCurrency(obj_id, subtype)
-        elif obj_type == 'pgp':
-            return Pgps.Pgp(obj_id, subtype)
-        elif obj_type == 'user-account':
-            return UsersAccount.UserAccount(obj_id, subtype)
-        elif obj_type == 'username':
-            return Usernames.Username(obj_id, subtype)
-        else:
-            raise AILObjectUnknown(f'Unknown AIL object: {obj_type} {subtype} {obj_id}')
 
 def exists_obj(obj_type, subtype, obj_id):
     obj = get_object(obj_type, subtype, obj_id)
@@ -172,6 +161,32 @@ def api_get_object_global_id(global_id):
 
 #### --API-- ####
 
+
+#### OBJECTS ####
+
+def get_nb_objects_by_date(date):
+    objs = {}
+    for obj_type in get_all_objects():
+        objs_class = get_objs_class(obj_type)
+        if objs_class:
+            objs_class = objs_class()
+            objs[obj_type] = objs_class.get_nb_by_date(date)
+    return objs
+
+def get_nb_objects_dashboard(date, flask_context=True):
+    objs = {}
+    for obj_type in get_all_objects():
+        objs_class = get_objs_class(obj_type)
+        if objs_class:
+            objs_class = objs_class()
+            objs[obj_type] = {}
+            objs[obj_type]['nb'] = objs_class.get_nb_by_date(date)
+            objs[obj_type]['name'] = objs_class.get_name()
+            objs[obj_type]['icon'] = objs_class.get_icon()
+            objs[obj_type]['link'] = objs_class.get_link(flask_context=flask_context)
+    return objs
+
+
 #########################################################################################
 #########################################################################################
 #########################################################################################
@@ -241,6 +256,9 @@ def add_obj_tags(obj_type, subtype, id, tags):
 
 # -TAGS- #
 
+#### OBJ META ####
+
+
 def get_object_meta(obj_type, subtype, id, options=set(), flask_context=False):
     obj = get_object(obj_type, subtype, id)
     meta = obj.get_meta(options=options)

bin/modules/CodeReader.py

@@ -176,4 +176,3 @@ class CodeReader(AbstractModule):
 if __name__ == '__main__':
     module = CodeReader()
     module.run()
-    

bin/modules/Mixer.py

@@ -31,16 +31,17 @@ Note that the hash of the content is defined as the sha1(gzip64encoded).
 """
 import os
 import sys
-
-import hashlib
 import time
 
+# import hashlib
+
 sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
 from modules.abstract_module import AbstractModule
 from lib.ConfigLoader import ConfigLoader
+from lib import ail_stats
 
 
 class Mixer(AbstractModule):
@@ -51,12 +52,14 @@ class Mixer(AbstractModule):
 
         config_loader = ConfigLoader()
         self.r_cache = config_loader.get_redis_conn("Redis_Mixer_Cache")
-        # self.r_cache_s = config_loader.get_redis_conn("Redis_Log_submit")
 
-        self.pending_seconds = 5
+        self.pending_seconds = 1
 
         self.refresh_time = 30
-        self.last_refresh = time.time()
+        timestamp = int(time.time())
+        self.last_refresh = int(timestamp - (timestamp % 30))
+        if timestamp > self.last_refresh:
+            self.last_refresh += 30
 
         self.operation_mode = config_loader.get_config_int("Module_Mixer", "operation_mode")
         print(f'Operation mode {self.operation_mode}')
@@ -64,71 +67,25 @@ class Mixer(AbstractModule):
         self.ttl_key = config_loader.get_config_int("Module_Mixer", "ttl_duplicate")
         self.default_feeder_name = config_loader.get_config_str("Module_Mixer", "default_unnamed_feed_name")
 
-        self.nb_processed_items = 0
         self.feeders_processed = {}
-        self.feeders_duplicate = {}
 
         self.logger.info(f"Module: {self.module_name} Launched")
 
-    # TODO Save stats in cache
-    # def get_feeders(self):
-    #     return self.r_cache_s.smembers("mixer_cache:feeders")
-    #
-    # def get_feeder_nb_last_processed(self, feeder):
-    #     nb = self.r_cache_s.hget("mixer_cache:feeders:last_processed", feeder)
-    #     if nb:
-    #         return int(nb)
-    #     else:
-    #         return 0
-    #
-    # def get_cache_feeders_nb_last_processed(self):
-    #     feeders = {}
-    #     for feeder in self.get_feeders():
-    #         feeders[feeder] = self.get_feeder_nb_last_processed(feeder)
-    #     return feeders
-
-    def clear_feeders_stat(self):
-        pass
-        # self.r_cache_s.delete("mixer_cache:feeders:last_processed")
-
     def increase_stat_processed(self, feeder):
-        self.nb_processed_items += 1
         try:
             self.feeders_processed[feeder] += 1
         except KeyError:
             self.feeders_processed[feeder] = 1
 
-    def increase_stat_duplicate(self, feeder):
-        self.nb_processed_items += 1
-        try:
-            self.feeders_duplicate[feeder] += 1
-        except KeyError:
-            self.feeders_duplicate[feeder] = 1
-
-    # TODO Save stats in cache
     def refresh_stats(self):
-        if int(time.time() - self.last_refresh) > self.refresh_time:
+        timestamp = int(time.time())
-            # update internal feeder
+        if timestamp >= self.last_refresh:
-            to_print = f'Mixer; ; ; ;mixer_all All_feeders Processed {self.nb_processed_items} item(s) in {self.refresh_time}sec'
+            timestamp = timestamp - timestamp % self.refresh_time
-            print(to_print)
+            print('update', timestamp)
-            self.redis_logger.info(to_print)
+            print(self.feeders_processed)
-            self.nb_processed_items = 0
+            ail_stats.add_feeders(timestamp, self.feeders_processed)
-
+            self.feeders_processed = {}
-            for feeder in self.feeders_processed:
+            self.last_refresh = self.last_refresh + 30
-                to_print = f'Mixer; ; ; ;mixer_{feeder} {feeder} Processed {self.feeders_processed[feeder]} item(s) in {self.refresh_time}sec'
-                print(to_print)
-                self.redis_logger.info(to_print)
-                self.feeders_processed[feeder] = 0
-
-            for feeder in self.feeders_duplicate:
-                to_print = f'Mixer; ; ; ;mixer_{feeder} {feeder} Duplicated {self.feeders_duplicate[feeder]} item(s) in {self.refresh_time}sec'
-                print(to_print)
-                self.redis_logger.info(to_print)
-                self.feeders_duplicate[feeder] = 0
-
-            self.last_refresh = time.time()
-            self.clear_feeders_stat()
-            time.sleep(0.5)
 
     def computeNone(self):
         self.refresh_stats()
@@ -163,22 +120,19 @@ class Mixer(AbstractModule):
                 self.queue.rename_message_obj(self.obj.id, obj_id)
 
 
-        relay_message = gzip64encoded
+        # # TODO only work for item object
-        # print(relay_message)
+        # # Avoid any duplicate coming from any sources
-
+        # if self.operation_mode == 1:
-        # TODO only work for item object
+        #     digest = hashlib.sha1(gzip64encoded.encode('utf8')).hexdigest()
-        # Avoid any duplicate coming from any sources
+        #     if self.r_cache.exists(digest):  # Content already exists
-        if self.operation_mode == 1:
+        #         # STATS
-            digest = hashlib.sha1(gzip64encoded.encode('utf8')).hexdigest()
+        #         self.increase_stat_duplicate(feeder_name)
-            if self.r_cache.exists(digest):  # Content already exists
+        #     else:  # New content
-                # STATS
+        #         self.r_cache.sadd(digest, feeder_name)
-                self.increase_stat_duplicate(feeder_name)
+        #         self.r_cache.expire(digest, self.ttl_key)
-            else:  # New content
+        #
-                self.r_cache.sadd(digest, feeder_name)
+        #         self.increase_stat_processed(feeder_name)
-                self.r_cache.expire(digest, self.ttl_key)
+        #         self.add_message_to_queue(message=relay_message)
-
-                self.increase_stat_processed(feeder_name)
-                self.add_message_to_queue(message=relay_message)
 
         # Need To Be Fixed, Currently doesn't check the source (-> same as operation 1)
         # # Keep duplicate coming from different sources
@@ -213,12 +167,10 @@ class Mixer(AbstractModule):
         #             self.increase_stat_duplicate(feeder_name)
 
         # No Filtering
-        else:
+        # else:
-            self.increase_stat_processed(feeder_name)
+
-            if self.obj.type == 'item':
+        self.increase_stat_processed(feeder_name)
-                self.add_message_to_queue(obj=self.obj, message=gzip64encoded)
+        self.add_message_to_queue(obj=self.obj, message=gzip64encoded)
-            else:
-                self.add_message_to_queue(obj=self.obj, message=gzip64encoded)
 
 
 if __name__ == "__main__":

requirements.txt

@@ -77,6 +77,7 @@ phonenumbers>8.12.1
 # Web
 flask>=2.3.3
 flask-login
+flask-sock
 bcrypt>3.1.6
 pyotp
 segno

var/www/Flask_server.py

@@ -6,13 +6,13 @@ import sys
 import ssl
 import json
 import time
-import uuid
 import random
 import logging
 import logging.config
 
 from flask import Flask, render_template, jsonify, request, Request, Response, session, redirect, url_for
 from flask_login import LoginManager, current_user, login_user, logout_user, login_required
+from flask_sock import Sock
 
 import importlib
 from os.path import join
@@ -28,6 +28,7 @@ from lib.ail_users import AILUser, get_session_user
 from lib import Tag
 from lib import ail_core
 from lib import ail_logger
+from lib import ail_stats
 
 from packages.git_status import clear_git_meta_cache
 
@@ -47,6 +48,7 @@ from blueprints.hunters import hunters
 from blueprints.old_endpoints import old_endpoints
 from blueprints.ail_2_ail_sync import ail_2_ail_sync
 from blueprints.settings_b import settings_b
+from blueprints.objects_objs import objects_objs
 from blueprints.objects_cve import objects_cve
 from blueprints.objects_decoded import objects_decoded
 from blueprints.objects_subtypes import objects_subtypes
@@ -134,6 +136,7 @@ app.register_blueprint(old_endpoints, url_prefix=baseUrl)
 app.register_blueprint(ail_2_ail_sync, url_prefix=baseUrl)
 app.register_blueprint(settings_b, url_prefix=baseUrl)
 app.register_blueprint(objects_cve, url_prefix=baseUrl)
+app.register_blueprint(objects_objs, url_prefix=baseUrl)
 app.register_blueprint(objects_decoded, url_prefix=baseUrl)
 app.register_blueprint(objects_subtypes, url_prefix=baseUrl)
 app.register_blueprint(objects_title, url_prefix=baseUrl)
@@ -163,7 +166,7 @@ login_manager.init_app(app)
 # ========= LOGIN MANAGER ========
 
 @login_manager.user_loader
-def load_user(session_id):  # TODO USE Alternative ID
+def load_user(session_id):
     # print(session)
     user_id = get_session_user(session_id)
     if user_id:
@@ -186,9 +189,7 @@ try:
 except IOError:
     pass
 
-# Dynamically import routes and functions from modules
+# Dynamically import routes and functions from modules # # # # TODO REMOVE ME ################################################
-# Also, prepare header.html
-to_add_to_header_dico = {}
 for root, dirs, files in os.walk(os.path.join(Flask_dir, 'modules')):
     sys.path.append(join(root))
 
@@ -204,36 +205,13 @@ for root, dirs, files in os.walk(os.path.join(Flask_dir, 'modules')):
                 continue
             name = name.strip('.py')
             importlib.import_module(name)
-        elif name == 'header_{}.html'.format(module_name):
-            with open(join(root, name), 'r') as f:
-                to_add_to_header_dico[module_name] = f.read()
 
-# create header.html
-with open(os.path.join(Flask_dir, 'templates', 'header_base.html'), 'r') as f:
-    complete_header = f.read()
-modified_header = complete_header
-
-# Add the header in the supplied order
-for module_name, txt in list(to_add_to_header_dico.items()):
-    to_replace = '<!--{}-->'.format(module_name)
-    if to_replace in complete_header:
-        modified_header = modified_header.replace(to_replace, txt)
-        del to_add_to_header_dico[module_name]
-
-# Add the header for no-supplied order
-to_add_to_header = []
-for module_name, txt in to_add_to_header_dico.items():
-    to_add_to_header.append(txt)
-
-modified_header = modified_header.replace('<!--insert here-->', '\n'.join(to_add_to_header))
-
-# Write the header.html file
-with open(os.path.join(Flask_dir, 'templates', 'header.html'), 'w') as f:
-    f.write(modified_header)
 
 # ========= JINJA2 FUNCTIONS ========
 def list_len(s):
     return len(s)
+
+
 app.jinja_env.filters['list_len'] = list_len
 
 
@@ -316,6 +294,33 @@ def page_not_found(e):
     return render_template('error/404.html'), 404
 
 
+# ========== WEBSOCKET ============
+
+app.config['SOCK_SERVER_OPTIONS'] = {'ping_interval': 25}
+sock = Sock(app)
+
+@login_required
+@sock.route('/ws/dashboard')
+def ws_dashboard(ws):
+    # TODO wait %30
+    next_feeders = ail_stats.get_next_feeder_timestamp(int(time.time())) + 1
+    try:
+        while True:
+            # TODO CHECK IF NEEDED
+            # if ws.closed:
+            #     print('WebSocket connection closed')
+            #     break
+            if int(time.time()) >= next_feeders:
+                feeders = ail_stats.get_feeders_dashboard()
+                # feeders['data']['telegram'] = 600
+                # feeders['data']['test'] = 1300
+                ws.send(json.dumps({'feeders': feeders}))
+                next_feeders = next_feeders + 30
+            time.sleep(1)
+    except Exception as e:  # ConnectionClosed ?
+        print("WEBSOCKET", e)
+
+
 # ========== INITIAL taxonomies ============
 default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"]
 # enable default taxonomies

var/www/blueprints/objects_cve.py

@@ -30,7 +30,7 @@ bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
 
 
 # ============ FUNCTIONS ============
-@objects_cve.route("/objects/cve", methods=['GET'])
+@objects_cve.route("/objects/cves", methods=['GET'])
 @login_required
 @login_read_only
 def objects_cves():

var/www/blueprints/objects_objs.py

@@ -0,0 +1,48 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+'''
+    Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
+'''
+
+import os
+import sys
+import json
+
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file, stream_with_context
+from flask_login import login_required
+
+# Import Role_Manager
+from Role_Manager import login_admin, login_read_only
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.objects import ail_objects
+from lib import ail_stats
+
+
+
+# ============ BLUEPRINT ============
+objects_objs = Blueprint('objects_objs', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects'))
+
+
+# ============ VARIABLES ============
+bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
+
+
+# ============ FUNCTIONS ============
+@objects_objs.route("/objects", methods=['GET'])
+@login_required
+@login_read_only
+def objects():
+    nb_objects = ail_stats.get_nb_objs_dashboard()
+    print(nb_objects)
+    feeders_dashboard = ail_stats.get_feeders_dashboard_full()
+    return render_template("objs_dashboard.html", feeders_dashboard=feeders_dashboard, nb_objects=nb_objects)
+
+
+
+# ============= ROUTES ==============
+

var/www/blueprints/objects_title.py

@@ -34,7 +34,7 @@ def create_json_response(data, status_code):
     return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
 
 # ============= ROUTES ==============
-@objects_title.route("/objects/title", methods=['GET'])
+@objects_title.route("/objects/titles", methods=['GET'])
 @login_required
 @login_read_only
 def objects_titles():

var/www/templates/objects/block_obj_button.html

@@ -0,0 +1,9 @@
+<a class="icon-button btn-outline-dark px-2" href="{{ url }}">
+    <div class="icon-wrapper text-center">
+        <i class="fas fa-{{ icon }} fa-4x"></i>
+        <span class="badge badge-pill badge-danger notification-badge">
+            {{ nb }}
+        </span>
+    </div>
+    <div class="h4"><b>{{ name }}</b></div>
+</a>

var/www/templates/objects/objs_dashboard.html

@@ -0,0 +1,484 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <title>Objects - AIL</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
+
+  <!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+
+  <!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/echarts.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+
+
+{#     <script src="https://cdn.jsdelivr.net/npm/echarts/dist/echarts.min.js"></script>#}
+
+{#    <script defer src="{{ url_for('static', filename='js/regular.js')}}"></script>#}
+{#    <script defer src="{{ url_for('static', filename='js/solid.js')}}"></script>#}
+{#    <script defer src="{{ url_for('static', filename='js/fontawesome.js')}}"></script>#}
+
+
+
+    <style>
+        .icon-button {
+      position: relative;
+      display: inline-block;
+      border: none;
+      background: none;
+      {#padding: 0;#}
+      margin: 0;
+      cursor: pointer;
+      outline: inherit;
+	    transition: color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;
+}
+    }
+
+    .icon-wrapper {
+      position: relative;
+      display: inline-block;
+    }
+
+    .notification-badge {
+      position: absolute;
+      top: -10px;
+      right: -10px;
+      font-size: 14px;
+      background: forestgreen;
+    }
+  </style>
+
+</head>
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+
+
+		<div class="row">
+
+            <div class="col-12 col-xl-6">
+                <div id="feeders_dashboard" style="width: 100%; height:600px;"></div>
+            </div>
+
+            <div class="col-12 col-xl-6 mt-4">
+
+                <div class="row">
+
+                    {% for obj_type in nb_objects %}
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            {% with name=nb_objects[obj_type]['name'], icon=nb_objects[obj_type]['icon']['icon'], nb=nb_objects[obj_type]['nb'], url=nb_objects[obj_type]['link'] %}
+                                {% include 'objects/block_obj_button.html' %}
+                            {% endwith %}
+                        </div>
+                    {% endfor %}
+
+
+                    <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                        <a class="icon-button btn-outline-dark px-2" href="{{url_for('chats_explorer.chats_explorer_protocols')}}">
+                            <div class="icon-wrapper text-center">
+                                <i class="fas fa-comment fa-4x"></i>
+                                <span class="badge badge-pill badge-danger notification-badge">
+                                    384556
+                                    <span class="sr-only">Chats by days</span>
+                                </span>
+                            </div>
+                            <div class="h4"><b>Chats</b></div>
+                        </a>
+                    </div>
+                </div>
+
+            </div>
+
+			<div class="col-12" id="core_content">
+
+                <div class="container mt-5">
+                    <div class="row">
+
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            <a class="icon-button btn-outline-dark px-2" href="{{url_for('chats_explorer.chats_explorer_protocols')}}">
+                                <div class="icon-wrapper text-center">
+                                    <i class="fas fa-comment fa-6x"></i>
+                                    <span class="badge badge-pill badge-danger notification-badge">
+                                        384556
+                                        <span class="sr-only">Chats by days</span>
+                                    </span>
+                                </div>
+                                <div class="h4"><b>Chats</b></div>
+                            </a>
+                        </div>
+
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            <a class="btn" style="position:relative;display: inline-block;">
+                                <div class="icon-wrapper">
+                                    <i class="fas fa-bug fa-6x"></i>
+                                    <span class="badge badge-pill badge-danger notification-badge">
+                                        5896
+                                    </span>
+                                </div>
+                                <div class="h4"><b>CVEs</b></div>
+                            </a>
+                        </div>
+
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            <div class="icon-wrapper">
+                                <i class="fas fa-bug fa-6x"></i>
+                                <span class="badge badge-pill badge-danger notification-badge">
+                                    5896
+                                </span>
+                            </div>
+                            <div class="h4"><b>CVEs</b></div>
+                        </div>
+
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            <div class="icon-wrapper">
+                                <i class="fas fa-bug fa-6x"></i>
+                                <span class="badge badge-pill badge-danger notification-badge">
+                                    5896
+                                </span>
+                            </div>
+                            <div class="h4"><b>CVEs</b></div>
+                        </div>
+
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            <div class="icon-wrapper">
+                                <i class="fas fa-bug fa-6x"></i>
+                                <span class="badge badge-pill badge-danger notification-badge">
+                                    5896
+                                </span>
+                            </div>
+                            <div class="h4"><b>CVEs</b></div>
+                        </div>
+
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            <div class="icon-wrapper">
+                                <i class="fas fa-bug fa-6x"></i>
+                                <span class="badge badge-pill badge-danger notification-badge">
+                                    5896
+                                </span>
+                            </div>
+                            <div class="h4"><b>CVEs</b></div>
+                        </div>
+
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            <div class="icon-wrapper">
+                                <i class="fas fa-bug fa-6x"></i>
+                                <span class="badge badge-pill badge-danger notification-badge">
+                                    5896
+                                </span>
+                            </div>
+                            <div class="h4"><b>CVEs</b></div>
+                        </div>
+
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            <div class="icon-wrapper">
+                                <i class="fas fa-bug fa-6x"></i>
+                                <span class="badge badge-pill badge-danger notification-badge">
+                                    5896
+                                </span>
+                            </div>
+                            <div class="h4"><b>CVEs</b></div>
+                        </div>
+
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            <div class="icon-wrapper">
+                                <i class="fas fa-bug fa-6x"></i>
+                                <span class="badge badge-pill badge-danger notification-badge">
+                                    5896
+                                </span>
+                            </div>
+                            <div class="h4"><b>CVEs</b></div>
+                        </div>
+
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            <div class="icon-wrapper">
+                                <i class="fas fa-bug fa-6x"></i>
+                                <span class="badge badge-pill badge-danger notification-badge">
+                                    5896
+                                </span>
+                            </div>
+                            <div class="h4"><b>CVEs</b></div>
+                        </div>
+
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            <div class="icon-wrapper">
+                                <i class="fas fa-bug fa-6x"></i>
+                                <span class="badge badge-pill badge-danger notification-badge">
+                                    5896
+                                </span>
+                            </div>
+                            <div class="h4"><b>CVEs</b></div>
+                        </div>
+
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            <div class="icon-wrapper">
+                                <i class="fas fa-bug fa-6x"></i>
+                                <span class="badge badge-pill badge-danger notification-badge">
+                                    5896
+                                </span>
+                            </div>
+                            <div class="h4"><b>CVEs</b></div>
+                        </div>
+
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            <div class="icon-wrapper">
+                                <i class="fas fa-bug fa-6x"></i>
+                                <span class="badge badge-pill badge-danger notification-badge">
+                                    5896
+                                </span>
+                            </div>
+                            <div class="h4"><b>CVEs</b></div>
+                        </div>
+
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            <div class="icon-wrapper">
+                                <i class="fas fa-bug fa-6x"></i>
+                                <span class="badge badge-pill badge-danger notification-badge">
+                                    5896
+                                </span>
+                            </div>
+                            <div class="h4"><b>CVEs</b></div>
+                        </div>
+
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            <div class="icon-wrapper">
+                                <i class="fas fa-bug fa-6x"></i>
+                                <span class="badge badge-pill badge-danger notification-badge">
+                                    5896
+                                </span>
+                            </div>
+                            <div class="h4"><b>CVEs</b></div>
+                        </div>
+
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            <div class="icon-wrapper">
+                                <i class="fas fa-bug fa-6x"></i>
+                                <span class="badge badge-pill badge-danger notification-badge">
+                                    5896
+                                </span>
+                            </div>
+                            <div class="h4"><b>CVEs</b></div>
+                        </div>
+
+                        <div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
+                            <div class="icon-wrapper">
+                                <i class="fas fa-bug fa-6x"></i>
+                                <span class="badge badge-pill badge-danger notification-badge">
+                                    5896
+                                </span>
+                            </div>
+                            <div class="h4"><b>CVEs</b></div>
+                        </div>
+
+                    </div>
+
+
+
+                </div>
+
+
+
+                <span class="fa-6x">
+                    <span class="fa-layers fa-fw">
+                        <i class="fa-solid  fa-comment"></i>
+                        <span class="fa-layers-counter" style="background:Tomato">1,419</span>
+                    </span>
+                    <span class="fa-layers fa-fw">
+                        <i class="fa-solid  fa-bug"></i>
+                        <span class="fa-layers-counter" style="background:Tomato">1,419</span>
+                    </span>
+                    <span class="fa-layers fa-fw">
+                        <i class="fa-solid  fa-cookie-bite"></i>
+                        <span class="fa-layers-counter" style="background:Tomato">1,419</span>
+                    </span>
+                    <span class="fa-layers fa-fw">
+                        <i class="fa-solid  fa-tag"></i>
+                        <span class="fa-layers-counter" style="background:Tomato">1,419</span>
+                    </span>
+                    <span class="fa-layers fa-fw">
+                        <i class="fa-solid  fa-align-left"></i>
+                        <span class="fa-layers-counter" style="background:Tomato">1,419</span>
+                    </span>
+                    <span class="fa-layers fa-fw">
+                        <i class="fa-solid  fa-skull-crossbones"></i>
+                        <span class="fa-layers-counter" style="background:Tomato">1,419</span>
+                    </span>
+                    <span class="fa-layers fa-fw">
+                        <i class="fa-solid  fa-star-half"></i>
+                        <span class="fa-layers-counter" style="background:Tomato">1,419</span>
+                    </span>
+                    <span class="fa-layers fa-fw">
+                        <i class="fa-solid  fa-image"></i>
+                        <span class="fa-layers-counter" style="background:Tomato">1,419</span>
+                    </span>
+                    <span class="fa-layers fa-fw">
+                        <i class="fa-solid  fa-expand"></i>
+                        <span class="fa-layers-counter" style="background:Tomato">1,419</span>
+                    </span>
+                    <span class="fa-layers fa-fw">
+                        <i class="fa-solid  fa-barcode"></i>
+                        <span class="fa-layers-counter" style="background:Tomato">1,419</span>
+                    </span>
+                    <span class="fa-layers fa-fw">
+                        <i class="fa-solid  fa-qrcode"></i>
+                        <span class="fa-layers-counter" style="background:Tomato">1,419</span>
+                    </span>
+                    <span class="fa-layers fa-fw">
+                        <i class="fa-solid  fa-heading"></i>
+                        <span class="fa-layers-counter" style="background:Tomato">1,419</span>
+                    </span>
+                    <span class="fa-layers fa-fw">
+                        <i class="fa-solid  fa-lock-open"></i>
+                        <span class="fa-layers-counter" style="background:Tomato">1,419</span>
+                    </span>
+                    <span class="fa-layers fa-fw">
+                        <i class="fa-solid  fa-key"></i>
+                        <span class="fa-layers-counter" style="background:Tomato">1,419</span>
+                    </span>
+                    <span class="fa-layers fa-fw">
+                        <i class="fa-solid  fa-coins"></i>
+                        <span class="fa-layers-counter" style="background:Tomato">1,419</span>
+                    </span>
+                    <span class="fa-layers fa-fw">
+                        <i class="fa-solid  fa-user"></i>
+                        <span class="fa-layers-counter" style="background:Tomato">1,419</span>
+                    </span>
+                    <span class="fa-layers fa-fw">
+                        <i class="fa-solid  fa-user-circle"></i>
+                        <span class="fa-layers-counter" style="background:Tomato">1,419</span>
+                    </span>
+                </span>
+
+
+			</div>
+		</div>
+	</div>
+
+<script>
+    var chart = {};
+    $(document).ready(function(){
+        $("#page-Decoded").addClass("active");
+        $("#nav_dashboard_{{obj_type}}").addClass("active");
+    });
+</script>
+
+<script>
+    // Init Dashboard
+    var feederChart = echarts.init(document.getElementById('feeders_dashboard'));
+    window.addEventListener('resize', function() {
+        feederChart.resize();
+      });
+    var optionFeeder;
+    var maxDataLength = 21;
+
+    var feeders_names = [{% for feeder in feeders_dashboard['data'] %}'{{ feeder }}', {% endfor %}]
+    var feeders_xaxis = [{% for date in feeders_dashboard['dates'] %}'{{ date }}', {% endfor %}]
+    var feeders_data = {
+        {% for feeder in feeders_dashboard['data'] %}
+            {{ feeder }}: {{ feeders_dashboard['data'][feeder] }},
+        {% endfor %}
+    }
+
+
+
+optionFeeder = {
+  title: {
+    text: 'Feeders'
+  },
+  tooltip: {
+    trigger: 'axis',
+    axisPointer: {
+      type: 'cross',
+      label: {
+        backgroundColor: '#6a7985'
+      }
+    }
+  },
+  legend: {
+    data: feeders_names
+  },
+  grid: {
+    left: '3%',
+    right: '4%',
+    bottom: '3%',
+    containLabel: true
+  },
+  xAxis: [
+    {
+      type: 'category',
+      boundaryGap: false,
+      data: feeders_xaxis
+    }
+  ],
+  yAxis: [
+    {
+      type: 'value'
+    }
+  ],
+  series: [
+      {% for feeder in feeders_dashboard['data'] %}
+          {
+              name: '{{ feeder }}',
+              type: 'line',
+              stack: 'Total',
+              areaStyle: {},
+              emphasis: {
+                  focus: 'series'
+              },
+              data: feeders_data['{{ feeder }}']
+          },
+      {% endfor %}
+  ]
+};
+
+optionFeeder && feederChart.setOption(optionFeeder);
+
+{# TODO UPDATE NEW FEEDER NAME#}
+function updateFeederChart(data) {
+    let new_date = data['feeders']['date']
+    feeders_xaxis.push(new_date)
+    feeders_xaxis.shift()
+
+    for (const f_name in data['feeders']['data']) {
+        if (f_name in feeders_data) {
+            feeders_data[f_name].push(data['feeders']['data'][f_name])
+            feeders_data[f_name].shift()
+        } else {
+            let fdata = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, data['feeders']['data'][f_name]]
+            feeders_names.push(f_name)
+            feeders_data[f_name] = fdata
+            // add new feeder
+            optionFeeder.legend.data.push(f_name)
+            optionFeeder.series.push({name: f_name, type: 'line', stack: 'Total', areaStyle: {}, emphasis: {focus: 'series'}, data: feeders_data[f_name]})
+
+        }
+    }
+    feederChart.setOption(optionFeeder)
+}
+
+// WebSocket
+var socket = new WebSocket("{{ url_for('ws_dashboard') }}");
+socket.wsocket = function(event) {
+    console.log("WebSocket connection opened:", event);
+};
+socket.onmessage = function(event) {
+    let data = JSON.parse(event.data);
+    updateFeederChart(data);
+};
+socket.onerror = function(error) {
+    console.error('WebSocket error:', error);
+};
+socket.onclose = function(event) {
+    console.log('WebSocket connection closed:', event);
+    // Optionally implement reconnection logic here
+};
+
+
+</script>
+
+</body>
+</html>

var/www/update_thirdparty.sh

@@ -5,28 +5,43 @@
 # submodules
 git submodule update
 
-wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
-
-BOOTSTRAP_VERSION='4.2.1'
-FONT_AWESOME_VERSION='6.6.0'
-
-D3_JS_VERSION='5.16.0'
-wget https://d3js.org/d3.v7.min.js -O ./static/js/d3.v7.min.js
-
 rm -rf temp
 mkdir temp
 
-wget https://github.com/twbs/bootstrap/releases/download/v${BOOTSTRAP_VERSION}/bootstrap-${BOOTSTRAP_VERSION}-dist.zip -O temp/bootstrap${BOOTSTRAP_VERSION}.zip
+#### D3JS ####
-wget https://github.com/FortAwesome/Font-Awesome/archive/${FONT_AWESOME_VERSION}.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip
+#### TODO UPDATE ALL D3 JS to V7
+wget https://d3js.org/d3.v7.min.js -O ./static/js/d3.v7.min.js
+
+# OLD
+D3_JS_VERSION='5.16.0'
 wget https://github.com/d3/d3/releases/download/v${D3_JS_VERSION}/d3.zip -O  temp/d3_${D3_JS_VERSION}.zip
+unzip -qq temp/d3_${D3_JS_VERSION}.zip -d temp/
+
+mv temp/d3.min.js ./static/js/
+#### ####
+
+#### FONT_AWESOME ####
+FONT_AWESOME_VERSION='6.6.0'
+wget https://github.com/FortAwesome/Font-Awesome/archive/${FONT_AWESOME_VERSION}.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip
+unzip temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/
+
+rm -rf ./static/webfonts/
+mv temp/Font-Awesome-${FONT_AWESOME_VERSION}/css/all.min.css ./static/css/font-awesome.min.css
+mv temp/Font-Awesome-${FONT_AWESOME_VERSION}/webfonts ./static/webfonts
+
+rm -rf ./static/fonts/ ./static/font-awesome/
+mv temp/font-awesome/ ./static
+#### ####
+
+BOOTSTRAP_VERSION='4.2.1'
+
+wget https://github.com/twbs/bootstrap/releases/download/v${BOOTSTRAP_VERSION}/bootstrap-${BOOTSTRAP_VERSION}-dist.zip -O temp/bootstrap${BOOTSTRAP_VERSION}.zip
 
 # dateRangePicker
 wget https://github.com/moment/moment/archive/2.24.0.zip -O temp/moment.zip
 wget https://github.com/longbill/jquery-date-range-picker/archive/v0.20.0.zip -O temp/daterangepicker.zip
 
 unzip -qq temp/bootstrap${BOOTSTRAP_VERSION}.zip -d temp/
-unzip temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/
-unzip -qq temp/d3_${D3_JS_VERSION}.zip -d temp/
 
 unzip -qq temp/moment.zip -d temp/
 unzip -qq temp/daterangepicker.zip -d temp/
@@ -36,19 +51,10 @@ mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/js/bootstrap.min.js.map ./static/js/
 mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css ./static/css/bootstrap4.min.css
 mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css.map ./static/css/bootstrap4.min.css.map
 
-rm -rf ./static/webfonts/
-mv temp/Font-Awesome-${FONT_AWESOME_VERSION}/css/all.min.css ./static/css/font-awesome.min.css
-mv temp/Font-Awesome-${FONT_AWESOME_VERSION}/webfonts ./static/webfonts
-
-rm -rf ./static/js/plugins
-
-rm -rf ./static/fonts/ ./static/font-awesome/
-mv temp/font-awesome/ ./static/
 
 rm -rf ./static/css/plugins/
 mv temp/jquery-date-range-picker-0.20.0/dist/daterangepicker.min.css ./static/css/
 
-mv temp/d3.min.js ./static/js/
 mv temp/moment-2.24.0/min/moment.min.js ./static/js/
 mv temp/jquery-date-range-picker-0.20.0/dist/jquery.daterangepicker.min.js ./static/js/
 
@@ -57,8 +63,6 @@ wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js
 
 #Ressources for dataTable
 wget https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js -O ./static/js/jquery.dataTables.min.js
-wget https://cdn.datatables.net/plug-ins/1.10.20/integration/bootstrap/3/dataTables.bootstrap.css -O ./static/css/dataTables.bootstrap.css
-wget https://cdn.datatables.net/plug-ins/1.10.20/integration/bootstrap/3/dataTables.bootstrap.js -O ./static/js/dataTables.bootstrap.js
 
 wget https://cdn.datatables.net/1.10.20/css/dataTables.bootstrap4.min.css -O ./static/css/dataTables.bootstrap.min.css
 wget https://cdn.datatables.net/1.10.20/js/dataTables.bootstrap4.min.js -O ./static/js/dataTables.bootstrap.min.js
@@ -71,20 +75,10 @@ mv temp/floating-ui-${POPPER_VERSION}/dist/umd/popper.min.js ./static/js/
 mv temp/floating-ui-${POPPER_VERSION}/dist/umd/popper.min.js.map ./static/js/
 
 #Ressource for graph
-# DASHBOARD   # TODO REFACTOR DASHBOARD GRAPHS
+# DASHBOARD   # TODO Extract from github
-wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.js -O ./static/js/jquery.flot.js
+wget https://cdn.jsdelivr.net/npm/echarts@5.5.1/dist/echarts.min.js - O ./static/js/echarts.min.js
-wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.pie.js -O ./static/js/jquery.flot.pie.js
-wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.time.js -O ./static/js/jquery.flot.time.js
-wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.stack.js -O ./static/js/jquery.flot.stack.js
 
-#Ressources for sparkline and canvasJS and slider
-#wget http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js
-#wget https://canvasjs.com/assets/script/canvasjs.min.js -O ./static/js/jquery.canvasjs.min.js
 
-wget https://jqueryui.com/resources/download/jquery-ui-1.12.1.zip -O temp/jquery-ui.zip
-unzip -qq temp/jquery-ui.zip -d temp/
-mv temp/jquery-ui-1.12.1/jquery-ui.min.js ./static/js/jquery-ui.min.js
-mv temp/jquery-ui-1.12.1/jquery-ui.min.css ./static/css/jquery-ui.min.css
 
 # INSTALL YARA
 YARA_VERSION="4.3.0"