chg: [dashboard] add echart feeder graph + cleanup olg graph libs + dashboard, show today nb objects

bin/lib/ail_stats.py

@@ -0,0 +1,97 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+import datetime
+import os
+import sys
+import time
+
+# from datetime import datetime
+from logging import lastResort
+
+sys.path.append(os.environ['AIL_BIN'])
+##################################
+# Import Project packages
+##################################
+from lib.ConfigLoader import ConfigLoader
+from lib.objects import ail_objects
+
+
+# Config
+config_loader = ConfigLoader()
+r_stats = config_loader.get_db_conn("Kvrocks_Stats")
+# r_cache = config_loader.get_redis_conn("Redis_Cache")
+config_loader = None
+
+
+def get_feeders():
+    return r_stats.smembers(f'feeders:name')
+
+def get_current_feeder_timestamp(timestamp):
+    return int(timestamp - (timestamp % 30))
+
+def get_next_feeder_timestamp(timestamp):
+    return int(timestamp + 30 - (timestamp % 30))
+
+def get_feeders_by_time(timestamp):  # TODO
+    feeders = {}
+    for row in r_stats.zrange(f'feeders:{timestamp}', 0, -1, withscores=True):
+        feeders[row[0]] = int(row[1])
+    return feeders
+
+def get_feeders_dashboard_full():
+    timestamp = get_current_feeder_timestamp(int(time.time()))
+    print(timestamp)
+    # timestamp = 1731491970
+    f_dashboard = {}
+
+    feeders = get_feeders()
+    d_time = []
+    for i in range(timestamp - 30*20, timestamp +30, 30):
+        t_feeders = get_feeders_by_time(i)
+        for feeder in feeders:
+            if feeder not in f_dashboard:
+                f_dashboard[feeder] = []
+            if feeder in t_feeders:
+                f_dashboard[feeder].append(t_feeders[feeder])
+            else:
+                f_dashboard[feeder].append(0)
+        d_time.append(datetime.datetime.utcfromtimestamp(i).strftime('%H:%M:%S'))
+    return {'data': f_dashboard, 'dates': d_time}
+
+def get_feeders_dashboard():
+    timestamp = get_current_feeder_timestamp(int(time.time()))
+    print(timestamp)
+
+    f_dashboard = {}
+    t_feeders = get_feeders_by_time(timestamp)
+    for feeder in get_feeders():
+        if feeder in t_feeders:
+            f_dashboard[feeder] = t_feeders[feeder]
+        else:
+            f_dashboard[feeder] = 0
+
+    date = datetime.datetime.utcfromtimestamp(timestamp).strftime('%H:%M:%S')
+    return {'data': f_dashboard, 'date': date}
+
+
+def add_feeders(timestamp, feeders):
+    if feeders:
+        r = r_stats.zadd(f'feeders:{timestamp}', feeders)
+        print(r)
+        for feeder in feeders:
+            r_stats.sadd(f'feeders:name', feeder)
+        # cleanup keys
+        r_stats.sadd(f'feeders:timestamps', timestamp)
+
+def get_nb_objs_today():
+    date = datetime.date.today().strftime("%Y%m%d")
+    nb_objs = ail_objects.get_nb_objects_by_date(date)
+    return nb_objs
+
+def get_nb_objs_dashboard():
+    date = datetime.date.today().strftime("%Y%m%d")
+    return ail_objects.get_nb_objects_dashboard(date)
+
+
+

bin/lib/objects/BarCodes.py

@@ -150,6 +150,19 @@ class Barcodes(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('barcode', Barcode)
 
+    def get_name(self):
+        return 'Barcodes'
+
+    def get_icon(self):
+        return {'fa': 'fas', 'icon': 'barcode'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_barcode.objects_barcodes')
+        else:
+            url = f'{baseurl}/objects/barcodes'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search  # TODO
 

bin/lib/objects/CookiesNames.py

@@ -109,6 +109,19 @@ class CookiesNames(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('cookie-name', CookieName)
 
+    def get_name(self):
+        return 'Cookie-Names'
+
+    def get_icon(self):
+        return {'fa': 'fas', 'icon': 'cookie-bite'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_cookie_name.objects_cookies_names')
+        else:
+            url = f'{baseurl}/objects/cookie-name'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search  # TODO
 

bin/lib/objects/Cves.py

@@ -16,7 +16,7 @@ sys.path.append(os.environ['AIL_BIN'])
 # Import Project packages
 ##################################
 from lib.ConfigLoader import ConfigLoader
-from lib.objects.abstract_daterange_object import AbstractDaterangeObject
+from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects
 from packages import Date
 
 config_loader = ConfigLoader()
@@ -97,6 +97,29 @@ class Cve(AbstractDaterangeObject):
         except requests.exceptions.ReadTimeout:
             return {'error': f'Timeout Error'}
 
+class Cves(AbstractDaterangeObjects):
+    """
+        Barcodes Objects
+    """
+    def __init__(self):
+        super().__init__('cve', Cve)
+
+    def get_name(self):
+        return 'Cves'
+
+    def get_icon(self):
+        return {'fa': 'fas', 'icon': 'bug'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_cve.objects_cves')
+        else:
+            url = f'{baseurl}/objects/cves'
+        return url
+
+    def sanitize_id_to_search(self, name_to_search):
+        return name_to_search  # TODO
+
 
 # TODO  ADD SEARCH FUNCTION
 

bin/lib/objects/DomHashs.py

@@ -114,6 +114,19 @@ class DomHashs(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('dom-hash', DomHash)
 
+    def get_name(self):
+        return 'DomHashs'
+
+    def get_icon(self):
+        return {'fa': 'fas', 'icon': 'align-left'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_dom_hash.objects_dom_hashs')
+        else:
+            url = f'{baseurl}/objects/dom-hashs'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search
 

bin/lib/objects/Etags.py

@@ -109,6 +109,19 @@ class Etags(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('etag', Etag)
 
+    def get_name(self):
+        return 'Etags'
+
+    def get_icon(self):
+        return {'fa': 'fas', 'icon': 'tag'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_etag.objects_etags')
+        else:
+            url = f'{baseurl}/objects/etags'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search  # TODO
 

bin/lib/objects/Favicons.py

@@ -135,6 +135,19 @@ class Favicons(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('favicon', Favicon)
 
+    def get_name(self):
+        return 'Favicons'
+
+    def get_icon(self):
+        return {'fa': 'fas', 'icon': 'star-half'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_favicon.objects_favicons')
+        else:
+            url = f'{baseurl}/objects/favicons'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search  # TODO
 

bin/lib/objects/FilesNames.py

@@ -84,6 +84,20 @@ class FilesNames(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('file-name', FileName)
 
+    def get_name(self):
+        return 'File-Names'
+
+    def get_icon(self):
+        return {'fa': 'far', 'icon': 'file'}
+
+    def get_link(self, flask_context=False):
+        pass
+        # if flask_context:
+        #     url = url_for('objects_favicon.objects_favicons')
+        # else:
+        #     url = f'{baseurl}/objects/favicons'
+        # return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search
 

bin/lib/objects/HHHashs.py

@@ -126,6 +126,19 @@ class HHHashs(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('hhhash', HHHash)
 
+    def get_name(self):
+        return 'HHHashs'
+
+    def get_icon(self):
+        return {'fas': 'far', 'icon': 'align-left'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_hhhash.objects_hhhashs')
+        else:
+            url = f'{baseurl}/objects/hhhashs'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search  # TODO
 

bin/lib/objects/Images.py

@@ -152,6 +152,19 @@ class Images(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('image', Image)
 
+    def get_name(self):
+        return 'Images'
+
+    def get_icon(self):
+        return {'fas': 'fas', 'icon': 'image'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_image.objects_images')
+        else:
+            url = f'{baseurl}/objects/images'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search  # TODO
 

bin/lib/objects/Ocrs.py

@@ -323,6 +323,19 @@ class Ocrs(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('ocr', Ocr)
 
+    def get_name(self):
+        return 'Ocrs'
+
+    def get_icon(self):
+        return {'fas': 'far', 'icon': 'expand'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_ocr.objects_ocrs')
+        else:
+            url = f'{baseurl}/objects/ocrs'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search  # TODO
 

bin/lib/objects/QrCodes.py

@@ -150,6 +150,19 @@ class Qrcodes(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('qrcode', Qrcode)
 
+    def get_name(self):
+        return 'Qrcodes'
+
+    def get_icon(self):
+        return {'fas': 'far', 'icon': 'qrcode'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_qrcode.objects_qrcodes')
+        else:
+            url = f'{baseurl}/objects/qrcodes'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search  # TODO
 

bin/lib/objects/Titles.py

@@ -104,6 +104,19 @@ class Titles(AbstractDaterangeObjects):
     def __init__(self):
         super().__init__('title', Title)
 
+    def get_name(self):
+        return 'Titles'
+
+    def get_icon(self):
+        return {'fas': 'far', 'icon': 'heading'}
+
+    def get_link(self, flask_context=False):
+        if flask_context:
+            url = url_for('objects_title.objects_titles')
+        else:
+            url = f'{baseurl}/objects/titles'
+        return url
+
     def sanitize_id_to_search(self, name_to_search):
         return name_to_search
 

bin/lib/objects/abstract_daterange_object.py

@@ -193,6 +193,18 @@ class AbstractDaterangeObjects(ABC):
         self.type = obj_type
         self.obj_class = obj_class
 
+    @abstractmethod
+    def get_name(self):
+        pass
+
+    @abstractmethod
+    def get_icon(self):
+        pass
+
+    @abstractmethod
+    def get_link(self, flask_context=False):
+        pass
+
     ################################################
     ################################################
 

bin/lib/objects/ail_objects.py

@@ -25,7 +25,7 @@ from lib.objects import ChatSubChannels
 from lib.objects import ChatThreads
 from lib.objects import CryptoCurrencies
 from lib.objects import CookiesNames
-from lib.objects.Cves import Cve
+from lib.objects import Cves
 from lib.objects.Decodeds import Decoded, get_all_decodeds_objects, get_nb_decodeds_objects
 from lib.objects.Domains import Domain
 from lib.objects import Etags
@@ -44,9 +44,36 @@ from lib.objects import Titles
 from lib.objects import UsersAccount
 from lib.objects import Usernames
 
-config_loader = ConfigLoader()
-
-config_loader = None
+# config_loader = ConfigLoader()
+#
+# config_loader = None
+# TODO INIT objs classes ????
+OBJECTS_CLASS = {
+    'barcode': {'obj': BarCodes.Barcode, 'objs': BarCodes.Barcodes},
+    'chat': {'obj': Chats.Chat, 'objs': None}, ## SUBTYPE #########################################
+    'chat-subchannel': {'obj': ChatSubChannels.ChatSubChannel, 'objs': None}, ######   ######
+    'chat-thread': {'obj': ChatThreads.ChatThread, 'objs': None},    ######   ######
+    'cookie-name': {'obj': CookiesNames.CookieName, 'objs': CookiesNames.CookiesNames},
+    'cve': {'obj': Cves.Cve, 'objs': Cves.Cves},
+    'cryptocurrency': {'obj': CryptoCurrencies.CryptoCurrency, 'objs': None}, ## SUBTYPE #########################################
+    'decoded': {'obj': Decoded, 'objs': None}, ###############################################################################################
+    'domain': {'obj': Domain, 'objs': None}, ####################################################################################################
+    'dom-hash': {'obj': DomHashs.DomHash, 'objs': DomHashs.DomHashs},
+    'etag': {'obj': Etags.Etag, 'objs': Etags.Etags},
+    'favicon': {'obj': Favicons.Favicon, 'objs': Favicons.Favicons},
+    'file-name': {'obj': FilesNames.FileName, 'objs': FilesNames.FilesNames},
+    'hhhash': {'obj': HHHashs.HHHash, 'objs': HHHashs.HHHashs},
+    'item': {'obj': Item, 'objs': None}, ######
+    'image': {'obj': Images.Image, 'objs': Images.Images},
+    'message': {'obj': Messages.Message, 'objs': None}, ######
+    'ocr': {'obj': Ocrs.Ocr, 'objs': Ocrs.Ocrs},
+    'pgp': {'obj': Pgps.Pgp, 'objs': None}, ## SUBTYPE ###########################################################################
+    'qrcode': {'obj': QrCodes.Qrcode, 'objs': QrCodes.Qrcodes},
+    'screenshot': {'obj': Screenshots.Screenshot, 'objs': None}, ######
+    'title': {'obj': Titles.Title, 'objs': Titles.Titles},
+    'user-account': {'obj': UsersAccount.UserAccount, 'objs': None}, ## SUBTYPE ###########################################################################
+    'username': {'obj': Usernames.Username, 'objs': None}, ## SUBTYPE ###########################################################################
+}
 
 
 def is_valid_object_type(obj_type):
@@ -70,67 +97,29 @@ def sanitize_objs_types(objs, default=False):
             l_types = get_all_objects()
     return l_types
 
+
 #### OBJECT ####
 
+def get_obj_class(obj_type):
+    if obj_type in OBJECTS_CLASS:
+        return OBJECTS_CLASS[obj_type]['obj']
+
+def get_objs_class(obj_type):
+    if obj_type in OBJECTS_CLASS:
+        return OBJECTS_CLASS[obj_type]['objs']
+
 def get_object(obj_type, subtype, obj_id):
     if subtype == 'None':
         subtype = None
     obj_id = str(obj_id)
+    obj_class = OBJECTS_CLASS[obj_type]['obj']
+    if not obj_class:
+        raise AILObjectUnknown(f'Unknown AIL object: {obj_type} {subtype} {obj_id}')
     if not subtype:
-        if obj_type == 'item':
-            return Item(obj_id)
-        elif obj_type == 'domain':
-            return Domain(obj_id)
-        elif obj_type == 'decoded':
-            return Decoded(obj_id)
-        elif obj_type == 'cookie-name':
-            return CookiesNames.CookieName(obj_id)
-        elif obj_type == 'cve':
-            return Cve(obj_id)
-        elif obj_type == 'etag':
-            return Etags.Etag(obj_id)
-        elif obj_type == 'favicon':
-            return Favicons.Favicon(obj_id)
-        elif obj_type == 'file-name':
-            return FilesNames.FileName(obj_id)
-        elif obj_type == 'dom-hash':
-            return DomHashs.DomHash(obj_id)
-        elif obj_type == 'hhhash':
-            return HHHashs.HHHash(obj_id)
-        elif obj_type == 'image':
-            return Images.Image(obj_id)
-        elif obj_type == 'message':
-            return Messages.Message(obj_id)
-        elif obj_type == 'ocr':
-            return Ocrs.Ocr(obj_id)
-        elif obj_type == 'barcode':
-            return BarCodes.Barcode(obj_id)
-        elif obj_type == 'qrcode':
-            return QrCodes.Qrcode(obj_id)
-        elif obj_type == 'screenshot':
-            return Screenshots.Screenshot(obj_id)
-        elif obj_type == 'title':
-            return Titles.Title(obj_id)
-        else:
-            raise AILObjectUnknown(f'Unknown AIL object: {obj_type} {subtype} {obj_id}')
+        return obj_class(obj_id)
     # SUBTYPES
     else:
-        if obj_type == 'chat':
-            return Chats.Chat(obj_id, subtype)
-        elif obj_type == 'chat-subchannel':
-            return ChatSubChannels.ChatSubChannel(obj_id, subtype)
-        elif obj_type == 'chat-thread':
-            return ChatThreads.ChatThread(obj_id, subtype)
-        elif obj_type == 'cryptocurrency':
-            return CryptoCurrencies.CryptoCurrency(obj_id, subtype)
-        elif obj_type == 'pgp':
-            return Pgps.Pgp(obj_id, subtype)
-        elif obj_type == 'user-account':
-            return UsersAccount.UserAccount(obj_id, subtype)
-        elif obj_type == 'username':
-            return Usernames.Username(obj_id, subtype)
-        else:
-            raise AILObjectUnknown(f'Unknown AIL object: {obj_type} {subtype} {obj_id}')
+        obj_class(obj_id, subtype)
 
 def exists_obj(obj_type, subtype, obj_id):
     obj = get_object(obj_type, subtype, obj_id)
@@ -172,6 +161,32 @@ def api_get_object_global_id(global_id):
 
 #### --API-- ####
 
+
+#### OBJECTS ####
+
+def get_nb_objects_by_date(date):
+    objs = {}
+    for obj_type in get_all_objects():
+        objs_class = get_objs_class(obj_type)
+        if objs_class:
+            objs_class = objs_class()
+            objs[obj_type] = objs_class.get_nb_by_date(date)
+    return objs
+
+def get_nb_objects_dashboard(date, flask_context=True):
+    objs = {}
+    for obj_type in get_all_objects():
+        objs_class = get_objs_class(obj_type)
+        if objs_class:
+            objs_class = objs_class()
+            objs[obj_type] = {}
+            objs[obj_type]['nb'] = objs_class.get_nb_by_date(date)
+            objs[obj_type]['name'] = objs_class.get_name()
+            objs[obj_type]['icon'] = objs_class.get_icon()
+            objs[obj_type]['link'] = objs_class.get_link(flask_context=flask_context)
+    return objs
+
+
 #########################################################################################
 #########################################################################################
 #########################################################################################
@@ -241,6 +256,9 @@ def add_obj_tags(obj_type, subtype, id, tags):
 
 # -TAGS- #
 
+#### OBJ META ####
+
+
 def get_object_meta(obj_type, subtype, id, options=set(), flask_context=False):
     obj = get_object(obj_type, subtype, id)
     meta = obj.get_meta(options=options)

bin/modules/CodeReader.py

@@ -176,4 +176,3 @@ class CodeReader(AbstractModule):
 if __name__ == '__main__':
     module = CodeReader()
     module.run()
-    

bin/modules/Mixer.py

@@ -31,16 +31,17 @@ Note that the hash of the content is defined as the sha1(gzip64encoded).
 """
 import os
 import sys
-
-import hashlib
 import time
 
+# import hashlib
+
 sys.path.append(os.environ['AIL_BIN'])
 ##################################
 # Import Project packages
 ##################################
 from modules.abstract_module import AbstractModule
 from lib.ConfigLoader import ConfigLoader
+from lib import ail_stats
 
 
 class Mixer(AbstractModule):
@@ -51,12 +52,14 @@ class Mixer(AbstractModule):
 
         config_loader = ConfigLoader()
         self.r_cache = config_loader.get_redis_conn("Redis_Mixer_Cache")
-        # self.r_cache_s = config_loader.get_redis_conn("Redis_Log_submit")
 
-        self.pending_seconds = 5
+        self.pending_seconds = 1
 
         self.refresh_time = 30
-        self.last_refresh = time.time()
+        timestamp = int(time.time())
+        self.last_refresh = int(timestamp - (timestamp % 30))
+        if timestamp > self.last_refresh:
+            self.last_refresh += 30
 
         self.operation_mode = config_loader.get_config_int("Module_Mixer", "operation_mode")
         print(f'Operation mode {self.operation_mode}')
@@ -64,71 +67,25 @@ class Mixer(AbstractModule):
         self.ttl_key = config_loader.get_config_int("Module_Mixer", "ttl_duplicate")
         self.default_feeder_name = config_loader.get_config_str("Module_Mixer", "default_unnamed_feed_name")
 
-        self.nb_processed_items = 0
         self.feeders_processed = {}
-        self.feeders_duplicate = {}
 
         self.logger.info(f"Module: {self.module_name} Launched")
 
-    # TODO Save stats in cache
-    # def get_feeders(self):
-    #     return self.r_cache_s.smembers("mixer_cache:feeders")
-    #
-    # def get_feeder_nb_last_processed(self, feeder):
-    #     nb = self.r_cache_s.hget("mixer_cache:feeders:last_processed", feeder)
-    #     if nb:
-    #         return int(nb)
-    #     else:
-    #         return 0
-    #
-    # def get_cache_feeders_nb_last_processed(self):
-    #     feeders = {}
-    #     for feeder in self.get_feeders():
-    #         feeders[feeder] = self.get_feeder_nb_last_processed(feeder)
-    #     return feeders
-
-    def clear_feeders_stat(self):
-        pass
-        # self.r_cache_s.delete("mixer_cache:feeders:last_processed")
-
     def increase_stat_processed(self, feeder):
-        self.nb_processed_items += 1
         try:
             self.feeders_processed[feeder] += 1
         except KeyError:
             self.feeders_processed[feeder] = 1
 
-    def increase_stat_duplicate(self, feeder):
-        self.nb_processed_items += 1
-        try:
-            self.feeders_duplicate[feeder] += 1
-        except KeyError:
-            self.feeders_duplicate[feeder] = 1
-
-    # TODO Save stats in cache
     def refresh_stats(self):
-        if int(time.time() - self.last_refresh) > self.refresh_time:
-            # update internal feeder
-            to_print = f'Mixer; ; ; ;mixer_all All_feeders Processed {self.nb_processed_items} item(s) in {self.refresh_time}sec'
-            print(to_print)
-            self.redis_logger.info(to_print)
-            self.nb_processed_items = 0
-
-            for feeder in self.feeders_processed:
-                to_print = f'Mixer; ; ; ;mixer_{feeder} {feeder} Processed {self.feeders_processed[feeder]} item(s) in {self.refresh_time}sec'
-                print(to_print)
-                self.redis_logger.info(to_print)
-                self.feeders_processed[feeder] = 0
-
-            for feeder in self.feeders_duplicate:
-                to_print = f'Mixer; ; ; ;mixer_{feeder} {feeder} Duplicated {self.feeders_duplicate[feeder]} item(s) in {self.refresh_time}sec'
-                print(to_print)
-                self.redis_logger.info(to_print)
-                self.feeders_duplicate[feeder] = 0
-
-            self.last_refresh = time.time()
-            self.clear_feeders_stat()
-            time.sleep(0.5)
+        timestamp = int(time.time())
+        if timestamp >= self.last_refresh:
+            timestamp = timestamp - timestamp % self.refresh_time
+            print('update', timestamp)
+            print(self.feeders_processed)
+            ail_stats.add_feeders(timestamp, self.feeders_processed)
+            self.feeders_processed = {}
+            self.last_refresh = self.last_refresh + 30
 
     def computeNone(self):
         self.refresh_stats()
@@ -163,22 +120,19 @@ class Mixer(AbstractModule):
                 self.queue.rename_message_obj(self.obj.id, obj_id)
 
 
-        relay_message = gzip64encoded
-        # print(relay_message)
-
-        # TODO only work for item object
-        # Avoid any duplicate coming from any sources
-        if self.operation_mode == 1:
-            digest = hashlib.sha1(gzip64encoded.encode('utf8')).hexdigest()
-            if self.r_cache.exists(digest):  # Content already exists
-                # STATS
-                self.increase_stat_duplicate(feeder_name)
-            else:  # New content
-                self.r_cache.sadd(digest, feeder_name)
-                self.r_cache.expire(digest, self.ttl_key)
-
-                self.increase_stat_processed(feeder_name)
-                self.add_message_to_queue(message=relay_message)
+        # # TODO only work for item object
+        # # Avoid any duplicate coming from any sources
+        # if self.operation_mode == 1:
+        #     digest = hashlib.sha1(gzip64encoded.encode('utf8')).hexdigest()
+        #     if self.r_cache.exists(digest):  # Content already exists
+        #         # STATS
+        #         self.increase_stat_duplicate(feeder_name)
+        #     else:  # New content
+        #         self.r_cache.sadd(digest, feeder_name)
+        #         self.r_cache.expire(digest, self.ttl_key)
+        #
+        #         self.increase_stat_processed(feeder_name)
+        #         self.add_message_to_queue(message=relay_message)
 
         # Need To Be Fixed, Currently doesn't check the source (-> same as operation 1)
         # # Keep duplicate coming from different sources
@@ -213,12 +167,10 @@ class Mixer(AbstractModule):
         #             self.increase_stat_duplicate(feeder_name)
 
         # No Filtering
-        else:
-            self.increase_stat_processed(feeder_name)
-            if self.obj.type == 'item':
-                self.add_message_to_queue(obj=self.obj, message=gzip64encoded)
-            else:
-                self.add_message_to_queue(obj=self.obj, message=gzip64encoded)
+        # else:
+
+        self.increase_stat_processed(feeder_name)
+        self.add_message_to_queue(obj=self.obj, message=gzip64encoded)
 
 
 if __name__ == "__main__":