chg: [cleanup] remove ARDB + fix hive case

bin/DB_KVROCKS_MIGRATION.py

@@ -252,6 +252,8 @@ def trackers_migration():
         for id in old_Tracker.get_retro_hunt_items_by_daterange(task_uuid, meta['date_from'], meta['date_to']):
             Tracker.save_retro_hunt_match(task_uuid, id)
 
+    Tracker._fix_db_custom_tags()
+
 
 def investigations_migration():
     print('INVESTIGATION MIGRATION...')

bin/Helper.py

@@ -140,12 +140,6 @@ class Process(object):
                 db=self.config.get('RedisPubSub', 'db'),
                 decode_responses=True)
 
-            self.serv_statistics = redis.StrictRedis(
-                host=self.config.get('ARDB_Statistics', 'host'),
-                port=self.config.get('ARDB_Statistics', 'port'),
-                db=self.config.get('ARDB_Statistics', 'db'),
-                decode_responses=True)
-
             self.moduleNum = os.getpid()
 
     def populate_set_in(self):
@@ -181,9 +175,9 @@ class Process(object):
             try:
                 if '.gz' in message:
                     path = message.split(".")[-2].split("/")[-1]
-                    #find start of path with AIL_HOME
+                    # find start of path with AIL_HOME
                     index_s = message.find(os.environ['AIL_HOME'])
-                    #Stop when .gz
+                    # Stop when .gz
                     index_e = message.find(".gz")+3
                     if(index_s == -1):
                         complete_path = message[0:index_e]
@@ -200,7 +194,6 @@ class Process(object):
                 self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
 
                 curr_date = datetime.date.today()
-                self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'paste_by_modules_in:'+self.subscriber_name, 1)
                 return message
 
             except:
@@ -238,6 +231,3 @@ class Process(object):
                 continue
             self.pubsub.publish(message)
 
-    def incr_module_timeout_statistic(self):
-        curr_date = datetime.date.today()
-        self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'paste_by_modules_timeout:'+self.subscriber_name, 1)

bin/LAUNCH.sh

@@ -237,6 +237,8 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Telegram" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Telegram.py; read x"
     sleep 0.1
+    screen -S "Script_AIL" -X screen -t "Tools" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Tools.py; read x"
+    sleep 0.1
 
     screen -S "Script_AIL" -X screen -t "Hosts" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Hosts.py; read x"
     sleep 0.1
@@ -283,8 +285,6 @@ function launching_scripts {
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x"
     sleep 0.1
-    screen -S "Script_AIL" -X screen -t "Tools" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tools.py; read x"
-    sleep 0.1
     screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
     sleep 0.1
     screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./IPAddress.py; read x"

bin/Release.py

@@ -1,69 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-import time
-from lib.objects.Items import Item
-from pubsublogger import publisher
-from Helper import Process
-import re
-
-import signal
-
-class TimeoutException(Exception):
-    pass
-
-def timeout_handler(signum, frame):
-    raise TimeoutException
-
-signal.signal(signal.SIGALRM, timeout_handler)
-
-'''
-This module takes its input from the global module.
-It applies some regex and publish matched content
-'''
-
-if __name__ == "__main__":
-    publisher.port = 6380
-    publisher.channel = "Script"
-    config_section = "Release"
-    p = Process(config_section)
-    max_execution_time = p.config.getint("Curve", "max_execution_time")
-    publisher.info("Release scripts to find release names")
-
-    movie = "[a-zA-Z0-9.]+\.[0-9]{4}.[a-zA-Z0-9.]+\-[a-zA-Z]+"
-    tv = "[a-zA-Z0-9.]+\.S[0-9]{2}E[0-9]{2}.[a-zA-Z0-9.]+\.[a-zA-Z0-9.]+\-[a-zA-Z0-9]+"
-    xxx = "[a-zA-Z0-9._]+.XXX.[a-zA-Z0-9.]+\-[a-zA-Z0-9]+"
-
-    regexs = [movie, tv, xxx]
-
-    regex = '|'.join(regexs)
-    while True:
-        signal.alarm(max_execution_time)
-        filepath = p.get_from_set()
-        if filepath is None:
-            publisher.debug("Script Release is Idling 10s")
-            print('Sleeping')
-            time.sleep(10)
-            continue
-
-        item = Item(filepath)
-        content = item.get_content()
-
-        #signal.alarm(max_execution_time)
-        try:
-            releases = set(re.findall(regex, content))
-            if len(releases) == 0:
-                continue
-
-            to_print = f'Release;{item.get_source()};{item.get_date()};{item.get_basename()};{len(releases)} releases;{item.get_id()}'
-            print(to_print)
-            if len(releases) > 30:
-                publisher.warning(to_print)
-            else:
-                publisher.info(to_print)
-
-        except TimeoutException:
-            p.incr_module_timeout_statistic()
-            print(f"{item.get_id()} processing timeout")
-            continue
-        else:
-            signal.alarm(0)

bin/SourceCode.py

@@ -1,55 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-import time
-from lib.objects.Items import Item
-from pubsublogger import publisher
-from Helper import Process
-import re
-
-if __name__ == "__main__":
-    publisher.port = 6380
-    publisher.channel = "Script"
-    config_section = "SourceCode"
-    p = Process(config_section)
-    publisher.info("Finding Source Code")
-
-    critical = 0  # AS TO BE IMPORTANT, MIGHT BE REMOVED
-
-    # RELEVANT LANGUAGES
-    shell = r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+\:\~\$"
-    c = r"\#include\ \<[a-z\/]+.h\>"
-    php = r"\<\?php"
-    python = r"import\ [\w]+"
-    bash = r"#!\/[\w]*\/bash"
-    javascript = r"function\(\)"
-    ruby = r"require \ [\w]+"
-    adr = r"0x[a-f0-9]{2}"
-
-    # asm = r"\"((?s).{1}x[0-9a-f]{2}){3,}" ISSUES WITH FINDALL, pattern like \x54\xaf\x23\..
-
-    languages = [shell, c, php, bash, python, javascript, bash, ruby, adr]
-    regex = '|'.join(languages)
-    print(regex)
-
-    while True:
-        message = p.get_from_set()
-        if message is None:
-            publisher.debug("Script Source Code is Idling 10s")
-            print('Sleeping')
-            time.sleep(10)
-            continue
-
-        filepath, count = message.split()
-
-        item = Item(filepath)
-        content = item.get_content()
-        match_set = set(re.findall(regex, content))
-        if len(match_set) == 0:
-            continue
-
-        to_print = f'SourceCode;{item.get_source()};{item.get_date()};{item.get_basename()};{item.get_id()}'
-
-        if len(match_set) > critical:
-            publisher.warning(to_print)
-        else:
-            publisher.info(to_print)

bin/export/Export.py

@@ -72,6 +72,25 @@ def is_hive_connected():
 
 HIVE_CLIENT = get_hive_client()
 
+def sanitize_threat_level_hive(threat_level):
+    try:
+        int(threat_level)
+        if 1 <= threat_level <= 3:
+            return threat_level
+        else:
+            return 2
+    except:
+        return 2
+
+def sanitize_tlp_hive(tlp):
+    try:
+        int(tlp)
+        if 0 <= tlp <= 3:
+            return tlp
+        else:
+            return 2
+    except:
+        return 2
 
 def create_thehive_alert(item_id, tag_trigger):
     item = Item(item_id)

bin/helper/dump_crawler_history.py

@@ -1,76 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-
-import os
-import sys
-import gzip
-import datetime
-import redis
-import json
-import time
-
-import shutil
-
-sys.path.append(os.environ['AIL_BIN'])
-sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
-from HiddenServices import HiddenServices
-from Helper import Process
-
-def substract_date(date_from, date_to):
-    date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
-    date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
-    delta = date_to - date_from # timedelta
-    l_date = []
-    for i in range(delta.days + 1):
-        date = date_from + datetime.timedelta(i)
-        l_date.append( date.strftime('%Y%m%d') )
-    return l_date
-
-config_section = 'Keys'
-p = Process(config_section)
-
-r_serv_onion = redis.StrictRedis(
-    host=p.config.get("ARDB_Onion", "host"),
-    port=p.config.getint("ARDB_Onion", "port"),
-    db=p.config.getint("ARDB_Onion", "db"),
-    decode_responses=True)
-
-date_from = '20190614'
-date_to = '20190615'
-service_type = 'onion'
-date_range = substract_date(date_from, date_to)
-
-dir_path = os.path.join(os.environ['AIL_HOME'], 'temp')
-
-domain_skipped = []
-
-for date in date_range:
-    domains_up = list(r_serv_onion.smembers('{}_up:{}'.format(service_type, date)))
-    if domains_up:
-        save_path = os.path.join(dir_path, date[0:4], date[4:6], date[6:8])
-        try:
-            os.makedirs(save_path)
-        except FileExistsError:
-            pass
-    for domain in domains_up:
-        print(domain)
-        h = HiddenServices(domain, 'onion')
-        item_core = h.get_domain_crawled_core_item()
-        if 'root_item' in item_core:
-            l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item'])
-            try:
-                res = h.create_domain_basic_archive(l_pastes)
-                filename = os.path.join(save_path, '{}'.format(domain))
-                with open(filename, 'wb') as f:
-                    shutil.copyfileobj(res, f)
-                    print('done')
-            except Exception as e:
-                print('skipped')
-                domain_skipped.append(domain)
-                pass
-
-print()
-print()
-print('DOMAINS SKIPPED: ')
-for domain in domain_skipped:
-    print(domain)

bin/helper/reprocess_bitcoin.py

@@ -1,50 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-
-import os
-import sys
-import gzip
-import base64
-import uuid
-import datetime
-import base64
-import redis
-import json
-import time
-
-sys.path.append(os.environ['AIL_BIN'])
-from Helper import Process
-
-def substract_date(date_from, date_to):
-    date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
-    date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
-    delta = date_to - date_from # timedelta
-    l_date = []
-    for i in range(delta.days + 1):
-        date = date_from + datetime.timedelta(i)
-        l_date.append( date.strftime('%Y%m%d') )
-    return l_date
-
-config_section = 'Global'
-p = Process(config_section)
-
-r_tags = redis.StrictRedis(
-    host=p.config.get("ARDB_Tags", "host"),
-    port=p.config.getint("ARDB_Tags", "port"),
-    db=p.config.getint("ARDB_Tags", "db"),
-    decode_responses=True)
-
-tag = 'infoleak:automatic-detection="bitcoin-address"'
-
-# get tag first/last seen
-first_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'first_seen')
-last_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
-
-l_dates = substract_date(first_seen, last_seen)
-
-# get all tagged items
-for date in l_dates:
-    daily_tagged_items = r_tags.smembers('{}:{}'.format(tag, date))
-
-    for item in daily_tagged_items:
-        p.populate_set_out(item)

bin/helper/reprocess_pgp_message.py

@@ -1,50 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-
-import os
-import sys
-import gzip
-import base64
-import uuid
-import datetime
-import base64
-import redis
-import json
-import time
-
-sys.path.append(os.environ['AIL_BIN'])
-from Helper import Process
-
-def substract_date(date_from, date_to):
-    date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
-    date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
-    delta = date_to - date_from # timedelta
-    l_date = []
-    for i in range(delta.days + 1):
-        date = date_from + datetime.timedelta(i)
-        l_date.append( date.strftime('%Y%m%d') )
-    return l_date
-
-config_section = 'Keys'
-p = Process(config_section)
-
-r_tags = redis.StrictRedis(
-    host=p.config.get("ARDB_Tags", "host"),
-    port=p.config.getint("ARDB_Tags", "port"),
-    db=p.config.getint("ARDB_Tags", "db"),
-    decode_responses=True)
-
-tag = 'infoleak:automatic-detection="pgp-message"'
-
-# get tag first/last seen
-first_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'first_seen')
-last_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
-
-l_dates = substract_date(first_seen, last_seen)
-
-# get all tagged items
-for date in l_dates:
-    daily_tagged_items = r_tags.smembers('{}:{}'.format(tag, date))
-
-    for item in daily_tagged_items:
-        p.populate_set_out(item, 'PgpDump')

bin/import/importer.py

@@ -10,7 +10,6 @@ Recieve Json Items (example: Twitter feeder)
 import os
 import importlib
 import json
-import redis
 import sys
 import time
 

bin/lib/Config_DB.py

@@ -15,7 +15,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
 from lib import ConfigLoader
 
 config_loader = ConfigLoader.ConfigLoader()
-r_serv_db = config_loader.get_redis_conn("ARDB_DB")
+r_serv_db = config_loader.get_redis_conn("_DB")
 config_loader = None
 
 #### TO PUT IN CONFIG

bin/lib/MispModules.py

@@ -3,6 +3,7 @@
 import os
 import json
 import requests
+import sys
 import configparser
 
 misp_module_url = 'http://localhost:6666'
@@ -16,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
 from lib import ConfigLoader
 
 config_loader = ConfigLoader.ConfigLoader()
-r_serv = config_loader.get_redis_conn("ARDB_DB")
+r_serv = config_loader.get_redis_conn("_DB")
 config_loader = None
 
 def init_config(config_path=default_config_path):
@@ -114,6 +115,7 @@ def parse_module_enrichment_response(misp_module_response):
         #print(response_types)
     return response_values
 
+
 if __name__ == "__main__":
 
     load_modules_list()

bin/lib/Statistics.py

@@ -12,8 +12,7 @@ sys.path.append(os.environ['AIL_BIN'])
 from lib import ConfigLoader
 
 config_loader = ConfigLoader.ConfigLoader()
-r_statistics = config_loader.get_redis_conn("ARDB_Statistics")
+r_statistics = config_loader.get_db_conn("Kvrocks_Stats")
-# r_serv_trend = ConfigLoader().get_redis_conn("ARDB_Trending")
 config_loader = None
 
 PIE_CHART_MAX_CARDINALITY = 8
@@ -113,11 +112,11 @@ def update_module_stats(module_name, num, keyword, date):
 
     if r_statistics.zcard(f'top_{module_name}_set_{date}') < PIE_CHART_MAX_CARDINALITY:
         r_statistics.zadd(f'top_{module_name}_set_{date}', {keyword: float(keyword_total_sum)})
-    else: # zset at full capacity
+    else:  # zset at full capacity
         member_set = r_statistics.zrangebyscore(f'top_{module_name}_set_{date}', '-inf', '+inf', withscores=True, start=0, num=1)
         # Member set is a list of (value, score) pairs
         if int(member_set[0][1]) < keyword_total_sum:
-            #remove min from set and add the new one
+            # remove min from set and add the new one
             r_statistics.zrem(f'top_{module_name}_set_{date}', member_set[0][0])
             r_statistics.zadd(f'top_{module_name}_set_{date}', {keyword: float(keyword_total_sum)})
 

bin/lib/Tag.py

@@ -620,7 +620,7 @@ def update_tag_metadata(tag, date, delete=False): # # TODO: delete Tags
 # old
 # r_tags.smembers(f'{tag}:{date}')
 # r_tags.smembers(f'{obj_type}:{tag}')
-def get_tag_objects(obj_type, subtype='', date=''):
+def get_tag_objects(tag, obj_type, subtype='', date=''):
     if obj_type == 'item':
         return r_tags.smembers(f'{obj_type}:{subtype}:{tag}:{date}')
     else:
@@ -1079,6 +1079,10 @@ def get_modal_add_tags(object_id, object_type='item', object_subtype=''):
     return {"active_taxonomies": get_active_taxonomies(), "active_galaxies": get_active_galaxies(),
             "object_id": object_id, "object_type": object_type, "object_subtype": object_subtype}
 
+#####################################################################################
+#####################################################################################
+#####################################################################################
+
 ######## NEW VERSION ########
 def create_custom_tag(tag):
     r_tags.sadd('tags:custom', tag)
@@ -1137,28 +1141,45 @@ def get_enabled_tags_with_synonyms_ui():
 
 ## Objects tags ##
 
-###################################################################################
-###################################################################################
-###################################################################################
-###################################################################################
 ###################################################################################
 ###################################################################################
 ###################################################################################
 ###################################################################################
 ###################################################################################
 
-def add_global_tag(tag, object_type=None):
+# TYPE -> taxonomy/galaxy/custom
-    '''
-    Create a set of all tags used in AIL (all + by object)
 
-    :param tag: tag
+class Tag:
-    :type tag: str
+
-    :param object_type: object type
+    def __int__(self, t_type, t_id, obj='item'):
-    :type object_type: str
+        self.type = t_type
-    '''
+        self.id = t_id
-    r_tags.sadd('list_tags', tag)
+        self.obj = obj
-    if object_type:
+
-        r_tags.sadd('list_tags:{}'.format(object_type), tag)
+    def get_first_seen(self):
+        pass
+
+    def get_last_seen(self):
+        pass
+
+    def get_color(self):
+        pass
+
+    def is_enabled(self):
+        pass
+
+    def get_meta(self):
+        meta = {'first_seen': self.get_first_seen(),
+                'last_seen': self.get_last_seen(),
+                'obj': self.obj,
+                'tag': self.id,
+                'type': self.type}
+
+
+###################################################################################
+###################################################################################
+###################################################################################
+###################################################################################
 
 def add_obj_tags(object_id, object_subtype, object_type, tags=[], galaxy_tags=[]):
     for tag in tags:
@@ -1201,23 +1222,6 @@ def api_add_obj_tags(tags=[], galaxy_tags=[], object_id=None, object_type="item"
     res_dict['type'] = object_type
     return res_dict, 200
 
-# def add_tag(object_type, tag, object_id, obj_date=None):
-#     # new tag
-#     if not is_obj_tagged(object_id, tag):
-#         # # TODO: # FIXME: sanitize object_type
-#         if obj_date:
-#             try:
-#                 obj_date = int(obj_date)
-#             except:
-#                 obj_date = None
-#         if not obj_date:
-#             obj_date = get_obj_date(object_type, object_id)
-#         add_global_tag(tag, object_type=object_type)
-#         add_obj_tag(object_type, object_id, tag, obj_date=obj_date)
-#         update_tag_metadata(tag, obj_date, object_type=object_type)
-#
-#     # create tags stats  # # TODO:  put me in cache
-#     r_tags.hincrby('daily_tags:{}'.format(datetime.date.today().strftime("%Y%m%d")), tag, 1)
 
 # def delete_obj_tag(object_type, object_id, tag, obj_date):
 #     if object_type=="item": # # TODO: # FIXME: # REVIEW: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

bin/lib/Tracker.py

@@ -551,9 +551,11 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti
 
     # create tracker tags list
     for tag in tags:
-        r_serv_tracker.sadd(f'tracker:tags:{tracker_uuid}', escape(tag))
+        tag = escape(tag)
+        r_serv_tracker.sadd(f'tracker:tags:{tracker_uuid}', tag)
+        Tag.create_custom_tag(tag)
 
-    # create tracker tags mail notification list
+    # create tracker mail notification list
     for mail in mails:
         r_serv_tracker.sadd(f'tracker:mail:{tracker_uuid}', escape(mail))
 
@@ -1030,7 +1032,9 @@ def create_retro_hunt_task(name, rule, date_from, date_to, creator, sources=[],
     for source in sources:
         r_serv_tracker.sadd(f'tracker:retro_hunt:task:sources:{task_uuid}', escape(source))
     for tag in tags:
-        r_serv_tracker.sadd(f'tracker:retro_hunt:task:tags:{task_uuid}', escape(tag))
+        tag = escape(tag)
+        r_serv_tracker.sadd(f'tracker:retro_hunt:task:tags:{task_uuid}', tag)
+        Tag.create_custom_tag(tag)
     for mail in mails:
         r_serv_tracker.sadd(f'tracker:retro_hunt:task:mails:{task_uuid}', escape(mail))
 
@@ -1364,14 +1368,13 @@ def get_trackers_tags():
 def _fix_db_custom_tags():
     for tag in get_trackers_tags():
         if not Tag.is_taxonomie_tag(tag) and not Tag.is_galaxy_tag(tag):
-            print(tag)
             Tag.create_custom_tag(tag)
 
 #### -- ####
 
-#if __name__ == '__main__':
+if __name__ == '__main__':
 
-    #_fix_db_custom_tags()
+    _fix_db_custom_tags()
     # fix_all_tracker_uuid_list()
     # res = get_all_tracker_uuid()
     # print(len(res))

bin/lib/Users.py

@@ -17,7 +17,6 @@ from lib.ConfigLoader import ConfigLoader
 
 # Config
 config_loader = ConfigLoader()
-# r_serv_db = config_loader.get_redis_conn("ARDB_DB")
 r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
 config_loader = None
 

bin/lib/ail_updates.py

@@ -10,42 +10,42 @@ sys.path.append(os.environ['AIL_BIN'])
 # Import Project packages
 ##################################
 from lib.ConfigLoader import ConfigLoader
-from lib import Users
 
 config_loader = ConfigLoader()
-#r_serv_db = config_loader.get_redis_conn("Kvrocks_DB")
+r_db = config_loader.get_redis_conn("Kvrocks_DB")
-r_serv = config_loader.get_redis_conn("ARDB_DB")
 config_loader = None
 
 BACKGROUND_UPDATES = {
-    'v1.5':{
+    'v1.5': {
-            'nb_updates': 5,
+        'nb_updates': 5,
-            'message': 'Tags and Screenshots'
+        'message': 'Tags and Screenshots'
-           },
+    },
-    'v2.4':{
+    'v2.4': {
-            'nb_updates': 1,
+        'nb_updates': 1,
-            'message': ' Domains Tags and Correlations'
+        'message': ' Domains Tags and Correlations'
-           },
+    },
-    'v2.6':{
+    'v2.6': {
-            'nb_updates': 1,
+        'nb_updates': 1,
-            'message': 'Domains Tags and Correlations'
+        'message': 'Domains Tags and Correlations'
-           },
+    },
-    'v2.7':{
+    'v2.7': {
-            'nb_updates': 1,
+        'nb_updates': 1,
-            'message': 'Domains Tags'
+        'message': 'Domains Tags'
-           },
+    },
-    'v3.4':{
+    'v3.4': {
-            'nb_updates': 1,
+        'nb_updates': 1,
-            'message': 'Domains Languages'
+        'message': 'Domains Languages'
-           },
+    },
-    'v3.7':{
+    'v3.7': {
-            'nb_updates': 1,
+        'nb_updates': 1,
-            'message': 'Trackers first_seen/last_seen'
+        'message': 'Trackers first_seen/last_seen'
-           }
+    }
 }
 
+
 def get_ail_version():
-    return r_serv.get('ail:version')
+    return r_db.get('ail:version')
+
 
 def get_ail_float_version():
     version = get_ail_version()
@@ -55,20 +55,23 @@ def get_ail_float_version():
         version = 0
     return version
 
+
 def get_ail_all_updates(date_separator='-'):
-    dict_update = r_serv.hgetall('ail:update_date')
+    dict_update = r_db.hgetall('ail:update_date')
     if date_separator:
         for version in dict_update:
             u_date = dict_update[version]
             dict_update[version] = f'{u_date[0:4]}{date_separator}{u_date[4:6]}{date_separator}{u_date[6:8]}'
     return dict_update
 
+
 def add_ail_update(version):
-    #Add new AIL version
+    # Add new AIL version
-    r_serv.hset('ail:update_date', version, datetime.datetime.now().strftime("%Y%m%d"))
+    r_db.hset('ail:update_date', version, datetime.datetime.now().strftime("%Y%m%d"))
-    #Set current ail version
+    # Set current ail version
     if float(version[1:]) > get_ail_float_version():
-        r_serv.set('ail:version', version)
+        r_db.set('ail:version', version)
+
 
 def check_version(version):
     if version[0] != 'v':
@@ -83,75 +86,94 @@ def check_version(version):
         return False
     return True
 
+
 #### UPDATE BACKGROUND ####
 
 def exits_background_update_to_launch():
-    return r_serv.scard('ail:update:to_update') != 0
+    return r_db.scard('ail:update:to_update') != 0
+
 
 def is_version_in_background_update(version):
-    return r_serv.sismember('ail:update:to_update', version)
+    return r_db.sismember('ail:update:to_update', version)
+
 
 def get_all_background_updates_to_launch():
-    return r_serv.smembers('ail:update:to_update')
+    return r_db.smembers('ail:update:to_update')
+
 
 def get_current_background_update():
-    return r_serv.get('ail:update:update_in_progress')
+    return r_db.get('ail:update:update_in_progress')
+
 
 def get_current_background_update_script():
-    return r_serv.get('ail:update:current_background_script')
+    return r_db.get('ail:update:current_background_script')
+
 
 def get_current_background_update_script_path(version, script_name):
     return os.path.join(os.environ['AIL_HOME'], 'update', version, script_name)
 
+
 def get_current_background_nb_update_completed():
-    return r_serv.scard('ail:update:update_in_progress:completed')
+    return r_db.scard('ail:update:update_in_progress:completed')
+
 
 def get_current_background_update_progress():
-    progress = r_serv.get('ail:update:current_background_script_stat')
+    progress = r_db.get('ail:update:current_background_script_stat')
     if not progress:
         progress = 0
     return int(progress)
 
+
 def get_background_update_error():
-    return r_serv.get('ail:update:error')
+    return r_db.get('ail:update:error')
+
 
 def add_background_updates_to_launch(version):
-    return r_serv.sadd('ail:update:to_update', version)
+    return r_db.sadd('ail:update:to_update', version)
+
 
 def start_background_update(version):
-    r_serv.delete('ail:update:error')
+    r_db.delete('ail:update:error')
-    r_serv.set('ail:update:update_in_progress', version)
+    r_db.set('ail:update:update_in_progress', version)
+
 
 def set_current_background_update_script(script_name):
-    r_serv.set('ail:update:current_background_script', script_name)
+    r_db.set('ail:update:current_background_script', script_name)
-    r_serv.set('ail:update:current_background_script_stat', 0)
+    r_db.set('ail:update:current_background_script_stat', 0)
+
 
 def set_current_background_update_progress(progress):
-    r_serv.set('ail:update:current_background_script_stat', progress)
+    r_db.set('ail:update:current_background_script_stat', progress)
+
 
 def set_background_update_error(error):
-    r_serv.set('ail:update:error', error)
+    r_db.set('ail:update:error', error)
+
 
 def end_background_update_script():
-    r_serv.sadd('ail:update:update_in_progress:completed')
+    r_db.sadd('ail:update:update_in_progress:completed')
+
 
 def end_background_update(version):
-    r_serv.delete('ail:update:update_in_progress')
+    r_db.delete('ail:update:update_in_progress')
-    r_serv.delete('ail:update:current_background_script')
+    r_db.delete('ail:update:current_background_script')
-    r_serv.delete('ail:update:current_background_script_stat')
+    r_db.delete('ail:update:current_background_script_stat')
-    r_serv.delete('ail:update:update_in_progress:completed')
+    r_db.delete('ail:update:update_in_progress:completed')
-    r_serv.srem('ail:update:to_update', version)
+    r_db.srem('ail:update:to_update', version)
+
 
 def clear_background_update():
-    r_serv.delete('ail:update:error')
+    r_db.delete('ail:update:error')
-    r_serv.delete('ail:update:update_in_progress')
+    r_db.delete('ail:update:update_in_progress')
-    r_serv.delete('ail:update:current_background_script')
+    r_db.delete('ail:update:current_background_script')
-    r_serv.delete('ail:update:current_background_script_stat')
+    r_db.delete('ail:update:current_background_script_stat')
-    r_serv.delete('ail:update:update_in_progress:completed')
+    r_db.delete('ail:update:update_in_progress:completed')
+
 
 def get_update_background_message(version):
     return BACKGROUND_UPDATES[version]['message']
 
+
 # TODO: Detect error in subprocess
 def get_update_background_metadata():
     dict_update = {}
@@ -162,7 +184,7 @@ def get_update_background_metadata():
         dict_update['script_progress'] = get_current_background_update_progress()
         dict_update['nb_update'] = BACKGROUND_UPDATES[dict_update['version']]['nb_updates']
         dict_update['nb_completed'] = get_current_background_nb_update_completed()
-        dict_update['progress'] = int(dict_update['nb_completed']*100/dict_update['nb_update'])
+        dict_update['progress'] = int(dict_update['nb_completed'] * 100 / dict_update['nb_update'])
         dict_update['error'] = get_background_update_error()
     return dict_update
 
@@ -170,14 +192,6 @@ def get_update_background_metadata():
 ##-- UPDATE BACKGROUND --##
 
 
-
-
-
-
-
-
-##########################
-
 if __name__ == '__main__':
     res = check_version('v3.1..1')
     print(res)

bin/lib/objects/Decodeds.py

@@ -36,7 +36,6 @@ except:
 config_loader = ConfigLoader()
 r_objects = config_loader.get_db_conn("Kvrocks_Objects")
 
-r_metadata = config_loader.get_redis_conn("ARDB_Metadata")
 HASH_DIR = config_loader.get_config_str('Directories', 'hash')
 baseurl = config_loader.get_config_str("Notifications", "ail_domain")
 config_loader = None

bin/lib/objects/abstract_subtype_object.py

@@ -8,7 +8,6 @@ Base Class for AIL Objects
 ##################################
 import os
 import sys
-from abc import abstractmethod
 
 # from flask import url_for
 
@@ -25,7 +24,6 @@ from packages import Date
 
 # LOAD CONFIG
 config_loader = ConfigLoader()
-r_metadata = config_loader.get_redis_conn("ARDB_Metadata")
 r_object = config_loader.get_db_conn("Kvrocks_Objects")
 config_loader = None
 
@@ -51,13 +49,6 @@ class AbstractSubtypeObject(AbstractObject):
     def exists(self):
         return r_object.exists(f'meta:{self.type}:{self.subtype}:{self.id}')
 
-    # def exists(self):
-    #     res = r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id)
-    #     if res is not None:
-    #         return True
-    #     else:
-    #         return False
-
     def get_first_seen(self, r_int=False):
         first_seen = r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'first_seen')
         if r_int:
@@ -79,11 +70,11 @@ class AbstractSubtypeObject(AbstractObject):
             return last_seen
 
     def get_nb_seen(self):
-        return int(r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id))
+        return int(r_object.zscore(f'{self.type}_all:{self.subtype}', self.id))
 
     # # TODO: CHECK RESULT
     def get_nb_seen_by_date(self, date_day):
-        nb = r_metadata.hget(f'{self.type}:{self.subtype}:{date_day}', self.id)
+        nb = r_object.hget(f'{self.type}:{self.subtype}:{date_day}', self.id)
         if nb is None:
             return 0
         else:
@@ -134,9 +125,9 @@ class AbstractSubtypeObject(AbstractObject):
         self.update_daterange(date)
         update_obj_date(date, self.type, self.subtype)
         # daily
-        r_metadata.hincrby(f'{self.type}:{self.subtype}:{date}', self.id, 1)
+        r_object.hincrby(f'{self.type}:{self.subtype}:{date}', self.id, 1)
         # all subtypes
-        r_metadata.zincrby(f'{self.type}_all:{self.subtype}', 1, self.id)
+        r_object.zincrby(f'{self.type}_all:{self.subtype}', 1, self.id)
 
         #######################################################################
         #######################################################################
@@ -158,14 +149,5 @@ class AbstractSubtypeObject(AbstractObject):
     def _delete(self):
         pass
 
-
-    ####################################
-    #
-    # _get_items
-    # get_metadata
-    #
-    #
-
-
 def get_all_id(obj_type, subtype):
-    return r_metadata.zrange(f'{obj_type}_all:{subtype}', 0, -1)
+    return r_object.zrange(f'{obj_type}_all:{subtype}', 0, -1)

bin/modules/Credential.py

@@ -69,8 +69,7 @@ class Credential(AbstractModule):
 
         # Database
         config_loader = ConfigLoader.ConfigLoader()
-        # self.server_cred = config_loader.get_redis_conn("ARDB_TermCred")
+        # self.server_cred = config_loader.get_redis_conn("_TermCred")
-        self.server_statistics = config_loader.get_redis_conn("ARDB_Statistics")
 
         # Config values
         self.minimumLengthThreshold = config_loader.get_config_int("Credential", "minimumLengthThreshold")

bin/modules/Global.py

@@ -43,6 +43,7 @@ from modules.abstract_module import AbstractModule
 from lib.ConfigLoader import ConfigLoader
 from lib.data_retention_engine import update_obj_date
 from lib import item_basic
+# from lib import Statistics
 
 class Global(AbstractModule):
     """
@@ -52,8 +53,6 @@ class Global(AbstractModule):
     def __init__(self):
         super(Global, self).__init__()
 
-        self.r_stats = ConfigLoader().get_redis_conn("ARDB_Statistics")
-
         self.processed_item = 0
         self.time_last_stats = time.time()
 
@@ -197,12 +196,13 @@ class Global(AbstractModule):
             self.redis_logger.warning(f'Global; Incomplete file: {filename}')
             print(f'Global; Incomplete file: {filename}')
             # save daily stats
-            self.r_stats.zincrby('module:Global:incomplete_file', 1, datetime.datetime.now().strftime('%Y%m%d'))
+            # self.r_stats.zincrby('module:Global:incomplete_file', 1, datetime.datetime.now().strftime('%Y%m%d'))
+            # Statistics.
         except OSError:
             self.redis_logger.warning(f'Global; Not a gzipped file: {filename}')
             print(f'Global; Not a gzipped file: {filename}')
             # save daily stats
-            self.r_stats.zincrby('module:Global:invalid_file', 1, datetime.datetime.now().strftime('%Y%m%d'))
+            # self.r_stats.zincrby('module:Global:invalid_file', 1, datetime.datetime.now().strftime('%Y%m%d'))
 
         return curr_file_content
 

bin/modules/LibInjection.py

@@ -13,7 +13,6 @@ It tries to identify SQL Injections with libinjection.
 
 import os
 import sys
-import urllib.request
 import pylibinjection
 
 from datetime import datetime
@@ -28,6 +27,7 @@ sys.path.append(os.environ['AIL_BIN'])
 from modules.abstract_module import AbstractModule
 from lib.ConfigLoader import ConfigLoader
 from lib.objects.Items import Item
+from lib import Statistics
 
 class LibInjection(AbstractModule):
     """docstring for LibInjection module."""
@@ -37,9 +37,6 @@ class LibInjection(AbstractModule):
 
         self.faup = Faup()
 
-        config_loader = ConfigLoader()
-        self.server_statistics = config_loader.get_redis_conn("ARDB_Statistics")
-
         self.redis_logger.info(f"Module: {self.module_name} Launched")
 
     def compute(self, message):
@@ -94,7 +91,7 @@ class LibInjection(AbstractModule):
                 tld = url_parsed['tld']
             if tld is not None:
                 date = datetime.now().strftime("%Y%m")
-                self.server_statistics.hincrby(f'SQLInjection_by_tld:{date}', tld, 1)
+                Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
 
 
 if __name__ == "__main__":

bin/modules/Onion.py

@@ -34,7 +34,6 @@ class Onion(AbstractModule):
 
         config_loader = ConfigLoader()
         self.r_cache = config_loader.get_redis_conn("Redis_Cache")
-        self.r_onion = config_loader.get_redis_conn("ARDB_Onion")
 
         self.pending_seconds = config_loader.get_config_int("Onion", "max_execution_time")
         # regex timeout
@@ -91,7 +90,7 @@ class Onion(AbstractModule):
 
         if onion_urls:
             if crawlers.is_crawler_activated():
-                for domain in domains:# TODO LOAD DEFAULT SCREENSHOT + HAR
+                for domain in domains:  # TODO LOAD DEFAULT SCREENSHOT + HAR
                     task_uuid = crawlers.add_crawler_task(domain, parent=item.get_id())
                     if task_uuid:
                         print(f'{domain} added to crawler queue: {task_uuid}')
@@ -109,6 +108,3 @@ if __name__ == "__main__":
     module = Onion()
     # module.compute('submitted/2022/10/10/submitted_705d1d92-7e9a-4a44-8c21-ccd167bfb7db.gz 9')
     module.run()
-
-
-# 5ajw6aqf3ep7sijnscdzw77t7xq4xjpsy335yb2wiwgouo7yfxtjlmid.onion to debian.org

bin/modules/SQLInjectionDetection.py

@@ -27,6 +27,7 @@ sys.path.append(os.environ['AIL_BIN'])
 from modules.abstract_module import AbstractModule
 from lib.ConfigLoader import ConfigLoader
 from lib.objects.Items import Item
+from lib import Statistics
 
 class SQLInjectionDetection(AbstractModule):
     """docstring for SQLInjectionDetection module."""
@@ -40,9 +41,6 @@ class SQLInjectionDetection(AbstractModule):
 
         self.faup = Faup()
 
-        config_loader = ConfigLoader()
-        self.server_statistics = config_loader.get_redis_conn("ARDB_Statistics")
-
         self.redis_logger.info(f"Module: {self.module_name} Launched")
 
     def compute(self, message):
@@ -75,7 +73,7 @@ class SQLInjectionDetection(AbstractModule):
                 except:
                     pass
                 date = datetime.now().strftime("%Y%m")
-                self.server_statistics.hincrby(f'SQLInjection_by_tld:{date}', tld, 1)
+                Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
 
     # Try to detect if the url passed might be an sql injection by applying the regex
     # defined above on it.

bin/modules/SentimentAnalysis.py

@@ -62,7 +62,7 @@ class SentimentAnalysis(AbstractModule):
         self.sentiment_lexicon_file = ConfigLoader.ConfigLoader().get_config_str("Directories", "sentiment_lexicon_file")
 
         # REDIS_LEVEL_DB #
-        self.db = ConfigLoader.ConfigLoader().get_redis_conn("ARDB_Sentiment")
+        self.db = ConfigLoader.ConfigLoader().get_redis_conn("_Sentiment")
 
         self.time1 = time.time()
 

bin/packages/config.cfg.docker-compose-sample

@@ -1,253 +0,0 @@
-[Directories]
-bloomfilters = Blooms
-dicofilters = Dicos
-pastes = PASTES
-hash = HASHS
-crawled = crawled
-crawled_screenshot = CRAWLED_SCREENSHOT
-
-wordtrending_csv = var/www/static/csv/wordstrendingdata
-wordsfile = files/wordfile
-
-protocolstrending_csv = var/www/static/csv/protocolstrendingdata
-protocolsfile = files/protocolsfile
-
-tldstrending_csv = var/www/static/csv/tldstrendingdata
-tldsfile = faup/src/data/mozilla.tlds
-
-domainstrending_csv = var/www/static/csv/domainstrendingdata
-
-pystemonpath = /opt/pystemon/
-
-sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt
-
-##### Notifications ######
-[Notifications]
-ail_domain = http://localhost:7000
-sender = sender@example.com
-sender_host = smtp.example.com
-sender_port = 1337
-sender_pw = None
-
-# optional for using with authenticated SMTP over SSL
-# sender_pw = securepassword
-
-##### Flask #####
-[Flask]
-#Proxying requests to the app
-baseUrl = /
-#Number of logs to display in the dashboard
-max_dashboard_logs = 15
-#Maximum number of character to display in the toolip
-max_preview_char = 250
-#Maximum number of character to display in the modal
-max_preview_modal = 800
-#Default number of header to display in trending graphs
-default_display = 10
-#Number of minutes displayed for the number of processed pastes.
-minute_processed_paste = 10
-#Maximum line length authorized to make a diff between duplicates
-DiffMaxLineLength = 10000
-
-#### Modules ####
-[BankAccount]
-max_execution_time = 60
-
-[Categ]
-#Minimum number of match between the paste and the category file
-matchingThreshold=1
-
-[Credential]
-#Minimum length that a credential must have to be considered as such
-minimumLengthThreshold=3
-#Will be pushed as alert if the number of credentials is greater to that number
-criticalNumberToAlert=8
-#Will be considered as false positive if less that X matches from the top password list
-minTopPassList=5
-
-[Curve]
-max_execution_time = 90
-
-[Onion]
-max_execution_time = 180
-
-[Base64]
-path = Base64/
-max_execution_time = 60
-
-[Binary]
-path = Base64/
-max_execution_time = 60
-
-[Hex]
-path = Base64/
-max_execution_time = 60
-
-[Modules_Duplicates]
-#Number of month to look back
-maximum_month_range = 3
-#The value where two pastes are considerate duplicate for ssdeep.
-threshold_duplicate_ssdeep = 50
-#The value where two pastes are considerate duplicate for tlsh.
-threshold_duplicate_tlsh = 52
-#Minimum size of the paste considered
-min_paste_size = 0.3
-
-[Module_ModuleInformation]
-#Threshold to deduce if a module is stuck or not, in seconds.
-threshold_stucked_module=600
-
-[Module_Mixer]
-#Define the configuration of the mixer, possible value: 1, 2 or 3
-operation_mode = 3
-#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
-ttl_duplicate = 86400
-default_unnamed_feed_name = unnamed_feeder
-
-[RegexForTermsFrequency]
-max_execution_time = 60
-
-##### Redis #####
-[Redis_Cache]
-host = localhost
-port = 6379
-db = 0
-
-[Redis_Log]
-host = localhost
-port = 6380
-db = 0
-
-[Redis_Log_submit]
-host = localhost
-port = 6380
-db = 1
-
-[Redis_Queues]
-host = localhost
-port = 6381
-db = 0
-
-[Redis_Data_Merging]
-host = localhost
-port = 6379
-db = 1
-
-[Redis_Paste_Name]
-host = localhost
-port = 6379
-db = 2
-
-[Redis_Mixer_Cache]
-host = localhost
-port = 6381
-db = 1
-
-##### ARDB #####
-[ARDB_Curve]
-host = localhost
-port = 6382
-db = 1
-
-[ARDB_Sentiment]
-host = localhost
-port = 6382
-db = 4
-
-[ARDB_TermFreq]
-host = localhost
-port = 6382
-db = 2
-
-[ARDB_TermCred]
-host = localhost
-port = 6382
-db = 5
-
-[ARDB_DB]
-host = localhost
-port = 6382
-db = 0
-
-[ARDB_Trending]
-host = localhost
-port = 6382
-db = 3
-
-[ARDB_Hashs]
-host = localhost
-db = 1
-
-[ARDB_Tags]
-host = localhost
-port = 6382
-db = 6
-
-[ARDB_Metadata]
-host = localhost
-port = 6382
-db = 7
-
-[ARDB_Statistics]
-host = localhost
-port = 6382
-db = 8
-
-[ARDB_Onion]
-host = localhost
-port = 6382
-db = 9
-
-[Url]
-cc_critical = DE
-
-[DomClassifier]
-cc = DE
-cc_tld = r'\.de$'
-dns = 8.8.8.8
-
-[Mail]
-dns = 8.8.8.8
-
-[Web]
-dns = 149.13.33.69
-
-# Indexer configuration
-[Indexer]
-type = whoosh
-path = indexdir
-register = indexdir/all_index.txt
-#size in Mb
-index_max_size = 2000
-
-[ailleakObject]
-maxDuplicateToPushToMISP=10
-
-###############################################################################
-
-# For multiple feed, add them with "," without space
-# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
-[ZMQ_Global]
-#address = tcp://crf.circl.lu:5556
-address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
-channel = 102
-bind = tcp://127.0.0.1:5556
-
-[ZMQ_Url]
-address = tcp://127.0.0.1:5004
-channel = urls
-
-[ZMQ_FetchedOnion]
-address = tcp://127.0.0.1:5005
-channel = FetchedOnion
-
-[RedisPubSub]
-host = localhost
-port = 6381
-db = 0
-
-[Crawler]
-activate_crawler = False
-crawler_depth_limit = 1
-splash_url_onion = http://172.17.0.1
-splash_onion_port = 8050

bin/packages/modules.cfg

@@ -12,10 +12,6 @@ publish = Redis_Mixer,Redis_Tags
 subscribe = Redis_Mixer
 publish = Redis_Global,Redis_ModuleStats
 
-#[PreProcessFeed]
-#subscribe = Redis_preProcess1
-#publish = Redis_Mixer
-
 [Duplicates]
 subscribe = Redis_Duplicate
 
@@ -62,7 +58,7 @@ subscribe = Redis_Global
 
 [Categ]
 subscribe = Redis_Global
-publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Urls,Redis_Credential,Redis_SourceCode,Redis_Cve,Redis_ApiKey,Redis_SyncModule
+publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Urls,Redis_Credential,Redis_Cve,Redis_ApiKey,Redis_SyncModule
 
 [CreditCards]
 subscribe = Redis_CreditCards
@@ -78,13 +74,9 @@ publish = Redis_ModuleStats,Redis_Tags
 
 [Onion]
 subscribe = Redis_Onion
-publish = Redis_ValidOnion,Redis_Tags,Redis_Crawler
+publish = Redis_Tags,Redis_Crawler
 #publish = Redis_ValidOnion,ZMQ_FetchedOnion,Redis_Tags,Redis_Crawler
 
-# TODO remove me
-[DumpValidOnion]
-subscribe = Redis_ValidOnion
-
 [Urls]
 subscribe = Redis_Urls
 publish = Redis_Url
@@ -112,15 +104,8 @@ subscribe = Redis_SyncModule
 [MISP_The_hive_feeder]
 subscribe = Redis_Tags_feed
 
-#[send_to_queue]
+#[SentimentAnalysis]
-#subscribe = Redis_Cve
+#subscribe = Redis_Global
-#publish = Redis_Tags
-
-[SentimentAnalysis]
-subscribe = Redis_Global
-
-[Release]
-subscribe = Redis_Global
 
 [Credential]
 subscribe = Redis_Credential
@@ -130,9 +115,10 @@ publish = Redis_Duplicate,Redis_ModuleStats,Redis_Tags
 subscribe = Redis_Cve
 publish = Redis_Tags
 
-[Phone]
+# Disabled 
-subscribe = Redis_Global
+#[Phone]
-publish = Redis_Tags
+#subscribe = Redis_Global
+#publish = Redis_Tags
 
 [Keys]
 subscribe = Redis_Global
@@ -159,7 +145,6 @@ subscribe = Redis
 publish = Redis_Mixer
 
 [Crawler]
-subscribe = Redis_Crawler
 publish = Redis_Mixer,Redis_Tags
 
 [IP]
@@ -169,3 +154,11 @@ publish = Redis_Tags
 [Zerobins]
 subscribe = Redis_Url
 
+#[PreProcessFeed]
+#subscribe = Redis_preProcess1
+#publish = Redis_Mixer
+
+# [My_Module]
+# subscribe = Redis_Global
+# publish = Redis_Tags
+

bin/template.py

@@ -36,11 +36,11 @@ class Template(AbstractModule):
         # Send module state to logs
         self.redis_logger.info(f'Module {self.module_name} initialized')
 
-    def computeNone(self):
+    # def computeNone(self):
-        """
+    #     """
-        Do something when there is no message in the queue
+    #     Do something when there is no message in the queue
-        """
+    #     """
-        self.redis_logger.debug("No message in queue")
+    #     self.redis_logger.debug("No message in queue")
 
     def compute(self, message):
         """

var/www/blueprints/import_export.py

@@ -81,7 +81,7 @@ def import_object_file():
     return render_template("import_object.html", all_imported_obj=all_imported_obj, error=error)
 
 
-@import_export.route("/objects/misp/export", methods=['GET'])
+@import_export.route("/misp/objects/export", methods=['GET'])
 @login_required
 @login_analyst
 def objects_misp_export():
@@ -91,7 +91,7 @@ def objects_misp_export():
     return render_template("export_object.html", object_types=object_types, to_export=to_export)
 
 
-@import_export.route("/objects/misp/export/post", methods=['POST'])
+@import_export.route("/misp/objects/export/post", methods=['POST'])
 @login_required
 @login_analyst
 def objects_misp_export_post():
@@ -159,7 +159,7 @@ def objects_misp_export_post():
                                misp_url=event['url'])
 
 
-@import_export.route("/objects/misp/export/add", methods=['GET'])
+@import_export.route("/misp/objects/export/add", methods=['GET'])
 @login_required
 @login_analyst
 def add_object_id_to_export():
@@ -181,7 +181,7 @@ def add_object_id_to_export():
     return redirect(url_for('import_export.objects_misp_export'))
 
 
-@import_export.route("/objects/misp/export/delete", methods=['GET'])
+@import_export.route("/misp/objects/export/delete", methods=['GET'])
 @login_required
 @login_analyst
 def delete_object_id_to_export():
@@ -194,7 +194,7 @@ def delete_object_id_to_export():
     return jsonify(success=True)
 
 
-@import_export.route("/import_export/investigation", methods=['GET'])
+@import_export.route("/investigation/misp/export", methods=['GET'])
 @login_required
 @login_analyst
 def export_investigation():
@@ -206,3 +206,24 @@ def export_investigation():
         return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True),
                         mimetype='application/json'), 400
     return redirect(url_for('investigations_b.show_investigation', uuid=investigation_uuid))
+
+
+@import_export.route("/thehive/objects/case/export", methods=['POST'])
+@login_required
+@login_analyst
+def create_thehive_case():
+    description = request.form['hive_description']
+    title = request.form['hive_case_title']
+    threat_level = Export.sanitize_threat_level_hive(request.form['threat_level_hive'])
+    tlp = Export.sanitize_tlp_hive(request.form['hive_tlp'])
+    item_id = request.form['obj_id']
+
+    item = Item(item_id)
+    if not item.exists():
+        abort(404)
+
+    case_id = Export.create_thehive_case(item_id, title=title, tlp=tlp, threat_level=threat_level, description=description)
+    if case_id:
+        return redirect(Export.get_case_url(case_id))
+    else:
+        return 'error'

var/www/modules/Flask_config.py

@@ -30,15 +30,9 @@ r_cache = config_loader.get_redis_conn("Redis_Cache")
 r_serv_log = config_loader.get_redis_conn("Redis_Log")
 r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit")
 
-
+r_serv_charts = config_loader.get_redis_conn("ARDB_Trending")  # -> TODO MIGRATE Stats Graphs
-
+r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")  # -> TODO MIGRATE /correlation/ subtypes objects
-r_serv_charts = config_loader.get_redis_conn("ARDB_Trending")
+r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")  # -> TODO MIGRATE AUTO CRAWLER
-r_serv_sentiment = config_loader.get_redis_conn("ARDB_Sentiment")
-r_serv_term = config_loader.get_redis_conn("ARDB_Tracker")
-r_serv_cred = config_loader.get_redis_conn("ARDB_TermCred")
-r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
-r_serv_statistics = config_loader.get_redis_conn("ARDB_Statistics")
-r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
 
 # # # # # # #
 r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
@@ -51,72 +45,45 @@ redis_logger.port = 6380
 # Channel name to publish logs
 redis_logger.channel = 'Flask'
 
-
 sys.path.append('../../configs/keys')
-# MISP #
-try:
-    from pymisp import PyMISP
-    from mispKEYS import misp_url, misp_key, misp_verifycert
-    pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
-    misp_event_url = misp_url + '/events/view/'
-    print('Misp connected')
-except:
-    print('Misp not connected')
-    pymisp = False
-    misp_event_url = '#'
-# The Hive #
-try:
-    from thehive4py.api import TheHiveApi
-    import thehive4py.exceptions
-    from theHiveKEYS import the_hive_url, the_hive_key, the_hive_verifycert
-    if the_hive_url == '':
-        HiveApi = False
-        hive_case_url = '#'
-        print('The HIVE not connected')
-    else:
-        HiveApi = TheHiveApi(the_hive_url, the_hive_key, cert=the_hive_verifycert)
-        hive_case_url = the_hive_url+'/index.html#/case/id_here/details'
-except:
-    print('The HIVE not connected')
-    HiveApi = False
-    hive_case_url = '#'
-
-if HiveApi != False:
-    try:
-        HiveApi.get_alert(0)
-        print('The Hive connected')
-    except thehive4py.exceptions.AlertException:
-        HiveApi = False
-        print('The Hive not connected')
 
 #### VARIABLES ####
 baseUrl = config_loader.get_config_str("Flask", "baseurl")
 baseUrl = baseUrl.replace('/', '')
 if baseUrl != '':
-    baseUrl = '/'+baseUrl
+    baseUrl = '/' + baseUrl
 
-max_preview_char = int(config_loader.get_config_str("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
+max_preview_char = int(
-max_preview_modal = int(config_loader.get_config_str("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
+    config_loader.get_config_str("Flask", "max_preview_char"))  # Maximum number of character to display in the tooltip
+max_preview_modal = int(
+    config_loader.get_config_str("Flask", "max_preview_modal"))  # Maximum number of character to display in the modal
 
 max_tags_result = 50
 
-DiffMaxLineLength =  int(config_loader.get_config_str("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value
+DiffMaxLineLength = int(config_loader.get_config_str("Flask",
+                                                     "DiffMaxLineLength"))  # Use to display the estimated percentage instead of a raw value
 
 bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
 
-dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_message': 'An Update is running on the background. Some informations like Tags, screenshot can be',
+dict_update_description = {'v1.5': {'nb_background_update': 5,
-                                   'update_warning_message_notice_me': 'missing from the UI.'},
+                                    'update_warning_message': 'An Update is running on the background. Some informations like Tags, screenshot can be',
-                           'v2.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
+                                    'update_warning_message_notice_me': 'missing from the UI.'},
-                                                              'update_warning_message_notice_me': 'missing from the UI.'},
+                           'v2.4': {'nb_background_update': 1,
-                           'v2.6':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
+                                    'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
-                                                           'update_warning_message_notice_me': 'missing from the UI.'},
+                                    'update_warning_message_notice_me': 'missing from the UI.'},
-                           'v2.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be',
+                           'v2.6': {'nb_background_update': 1,
-                                                            'update_warning_message_notice_me': 'missing from the UI.'},
+                                    'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
-                           'v3.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Languages can be',
+                                    'update_warning_message_notice_me': 'missing from the UI.'},
-                                                            'update_warning_message_notice_me': 'missing from the UI.'},
+                           'v2.7': {'nb_background_update': 1,
-                           'v3.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Tracker first_seen/last_seen can be',
+                                    'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be',
-                                                             'update_warning_message_notice_me': 'missing from the UI.'}
+                                    'update_warning_message_notice_me': 'missing from the UI.'},
-                        }
+                           'v3.4': {'nb_background_update': 1,
+                                    'update_warning_message': 'An Update is running on the background. Some informations like Domain Languages can be',
+                                    'update_warning_message_notice_me': 'missing from the UI.'},
+                           'v3.7': {'nb_background_update': 1,
+                                    'update_warning_message': 'An Update is running on the background. Some informations like Tracker first_seen/last_seen can be',
+                                    'update_warning_message_notice_me': 'missing from the UI.'}
+                           }
 
 UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')
 
@@ -137,10 +104,10 @@ SUBMIT_PASTE_TEXT_MAX_SIZE = int(config_loader.get_config_str("SubmitPaste", "TE
 SUBMIT_PASTE_FILE_MAX_SIZE = int(config_loader.get_config_str("SubmitPaste", "FILE_MAX_SIZE"))
 SUBMIT_PASTE_FILE_ALLOWED_EXTENSIONS = [item.strip() for item in config_loader.get_config_str("SubmitPaste", "FILE_ALLOWED_EXTENSIONS").split(',')]
 
-
 # VT
 try:
     from virusTotalKEYS import vt_key
+
     if vt_key != '':
         vt_auth = vt_key
         vt_enabled = True

var/www/modules/PasteSubmit/Flask_PasteSubmit.py

@@ -11,10 +11,8 @@ import re
 import os
 import sys
 import string
-import datetime
 import unicodedata
 import uuid
-from io import BytesIO
 
 from functools import wraps
 
@@ -42,9 +40,6 @@ import Flask_config
 app = Flask_config.app
 baseUrl = Flask_config.baseUrl
 
-
-
-r_serv_metadata = Flask_config.r_serv_metadata # TODO REMOVE ME
 r_serv_db = Flask_config.r_serv_db # TODO REMOVE ME
 r_serv_log_submit = Flask_config.r_serv_log_submit # TODO REMOVE ME
 
@@ -317,35 +312,16 @@ def submit_status():
     else:
         return 'INVALID UUID'
 
-# TODO MOVE ME IN import_export blueprint
-@PasteSubmit.route("/PasteSubmit/create_hive_case", methods=['POST'])
-@login_required
-@login_analyst
-def create_hive_case():
 
-    tlp = request.form['hive_tlp']
+######################################################################################################
-    if tlp:
+######################################################################################################
-        tlp = int(tlp)
+######################################################################################################
-    else:
+######################################################################################################
-        tlp = 2
-    threat_level = request.form['threat_level_hive']
-    if threat_level:
-        threat_level = int(threat_level)
-    else:
-        threat_level = 2
-    description = request.form['hive_description']
-    title = request.form['hive_case_title']
-    item_id = request.form['obj_id']
 
-    if (0 <= tlp <= 3) and (1 <= threat_level <= 4):
+# TODO MIGRATE TAGS PUSH
 
-        case_id = Export.create_thehive_case(item_id, title=title, tlp=tlp, threat_level=threat_level, description=description)
+# TEMPORARY: 2 SET OF CUSTOM + infoleak tags ?????????
-        if case_id:
-            return redirect(get_case_url(case_id))
-        else:
-            return 'error'
 
-    return 'error'
 
 @PasteSubmit.route("/PasteSubmit/edit_tag_export")
 @login_required
@@ -410,8 +386,8 @@ def edit_tag_export():
                             status_hive=status_hive,
                             nb_tags_whitelist_misp=nb_tags_whitelist_misp,
                             nb_tags_whitelist_hive=nb_tags_whitelist_hive,
-                            flag_misp=flag_misp,
+                            flag_misp=True,
-                            flag_hive=flag_hive)
+                            flag_hive=True)
 
 @PasteSubmit.route("/PasteSubmit/tag_export_edited", methods=['POST'])
 @login_required

var/www/modules/hiddenServices/Flask_hiddenServices.py

@@ -27,7 +27,6 @@ app = Flask_config.app
 baseUrl = Flask_config.baseUrl
 r_cache = Flask_config.r_cache
 r_serv_onion = Flask_config.r_serv_onion
-r_serv_metadata = Flask_config.r_serv_metadata
 bootstrap_label = Flask_config.bootstrap_label
 
 hiddenServices = Blueprint('hiddenServices', __name__, template_folder='templates')

var/www/modules/restApi/Flask_restApi.py

@@ -41,8 +41,6 @@ app = Flask_config.app
 baseUrl = Flask_config.baseUrl
 r_cache = Flask_config.r_cache
 r_serv_db = Flask_config.r_serv_db
-r_serv_onion = Flask_config.r_serv_onion
-r_serv_metadata = Flask_config.r_serv_metadata
 
 
 restApi = Blueprint('restApi', __name__, template_folder='templates')
@@ -258,7 +256,8 @@ def add_item_tags():
     tags = data.get('tags', [])
     galaxy = data.get('galaxy', [])
 
-    res = Tag.api_add_obj_tags(tags=tags, galaxy_tags=galaxy, object_id=object_id, object_type="item")
+    # res = Tag.api_add_obj_tags(tags=tags, galaxy_tags=galaxy, object_id=object_id, object_type="item")
+    res = {'error': 'disabled endpoint'}, 500
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

var/www/modules/sentiment/Flask_sentiment.py

@@ -1,160 +1,160 @@
 #!/usr/bin/env python3
 # -*-coding:UTF-8 -*
 
-'''
+# '''
-    Flask functions and routes for the trending modules page
+#     Flask functions and routes for the trending modules page
-'''
+# '''
-import os
+# import os
-import sys
+# import sys
-import datetime
+# import datetime
-import calendar
+# import calendar
-import flask
+# import flask
-from flask import Flask, render_template, jsonify, request, Blueprint
+# from flask import Flask, render_template, jsonify, request, Blueprint
-
+#
-from Role_Manager import login_admin, login_analyst, login_read_only
+# from Role_Manager import login_admin, login_analyst, login_read_only
-from flask_login import login_required
+# from flask_login import login_required
-
+#
-sys.path.append(os.environ['AIL_BIN'])
+# sys.path.append(os.environ['AIL_BIN'])
-##################################
+# ##################################
-# Import Project packages
+# # Import Project packages
-##################################
+# ##################################
-from packages.Date import Date
+# from packages.Date import Date
-
+#
-# ============ VARIABLES ============
+# # ============ VARIABLES ============
-import Flask_config
+# import Flask_config
-
+#
-app = Flask_config.app
+# app = Flask_config.app
-baseUrl = Flask_config.baseUrl
+# baseUrl = Flask_config.baseUrl
-r_serv_charts = Flask_config.r_serv_charts
+# r_serv_charts = Flask_config.r_serv_charts
-r_serv_sentiment = Flask_config.r_serv_sentiment
+# r_serv_sentiment = Flask_config.r_serv_sentiment
-
+#
-sentiments = Blueprint('sentiments', __name__, template_folder='templates')
+# sentiments = Blueprint('sentiments', __name__, template_folder='templates')
-
+#
-# ============ FUNCTIONS ============
+# # ============ FUNCTIONS ============
-
+#
-def get_date_range(num_day):
+# def get_date_range(num_day):
-    curr_date = datetime.date.today()
+#     curr_date = datetime.date.today()
-    date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
+#     date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
-    date_list = []
+#     date_list = []
-
+#
-    for i in range(0, num_day+1):
+#     for i in range(0, num_day+1):
-        date_list.append(date.substract_day(i))
+#         date_list.append(date.substract_day(i))
-    return date_list
+#     return date_list
-
+#
-
+#
-# ============ ROUTES ============
+# # ============ ROUTES ============
-
+#
-@sentiments.route("/sentiment_analysis_trending/")
+# @sentiments.route("/sentiment_analysis_trending/")
-@login_required
+# @login_required
-@login_read_only
+# @login_read_only
-def sentiment_analysis_trending():
+# def sentiment_analysis_trending():
-    return render_template("sentiment_analysis_trending.html")
+#     return render_template("sentiment_analysis_trending.html")
-
+#
-
+#
-@sentiments.route("/sentiment_analysis_getplotdata/", methods=['GET'])
+# @sentiments.route("/sentiment_analysis_getplotdata/", methods=['GET'])
-@login_required
+# @login_required
-@login_read_only
+# @login_read_only
-def sentiment_analysis_getplotdata():
+# def sentiment_analysis_getplotdata():
-    # Get the top providers based on number of pastes
+#     # Get the top providers based on number of pastes
-    oneHour = 60*60
+#     oneHour = 60*60
-    sevenDays = oneHour*24*7
+#     sevenDays = oneHour*24*7
-    dateStart = datetime.datetime.now()
+#     dateStart = datetime.datetime.now()
-    dateStart = dateStart.replace(minute=0, second=0, microsecond=0)
+#     dateStart = dateStart.replace(minute=0, second=0, microsecond=0)
-    dateStart_timestamp = calendar.timegm(dateStart.timetuple())
+#     dateStart_timestamp = calendar.timegm(dateStart.timetuple())
-
+#
-    getAllProviders = request.args.get('getProviders')
+#     getAllProviders = request.args.get('getProviders')
-    provider = request.args.get('provider')
+#     provider = request.args.get('provider')
-    allProvider = request.args.get('all')
+#     allProvider = request.args.get('all')
-    if getAllProviders == 'True':
+#     if getAllProviders == 'True':
-        if allProvider == "True":
+#         if allProvider == "True":
-            range_providers = r_serv_charts.smembers('all_provider_set')
+#             range_providers = r_serv_charts.smembers('all_provider_set')
-
+#
-            return jsonify(list(range_providers))
+#             return jsonify(list(range_providers))
-        else:
+#         else:
-            range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8)
+#             range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8)
-            # if empty, get yesterday top providers
+#             # if empty, get yesterday top providers
-            range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers
+#             range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers
-
+#
-
+#
-            # if still empty, takes from all providers
+#             # if still empty, takes from all providers
-            if range_providers == []:
+#             if range_providers == []:
-                print('today provider empty')
+#                 print('today provider empty')
-                range_providers = r_serv_charts.smembers('all_provider_set')
+#                 range_providers = r_serv_charts.smembers('all_provider_set')
-
+#
-            return jsonify(list(range_providers))
+#             return jsonify(list(range_providers))
-
+#
-    elif provider is not None:
+#     elif provider is not None:
-        to_return = {}
+#         to_return = {}
-
+#
-        cur_provider_name = provider + '_'
+#         cur_provider_name = provider + '_'
-        list_date = {}
+#         list_date = {}
-        for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour):
+#         for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour):
-            cur_set_name = cur_provider_name + str(cur_timestamp)
+#             cur_set_name = cur_provider_name + str(cur_timestamp)
-
+#
-            list_value = []
+#             list_value = []
-            for cur_id in r_serv_sentiment.smembers(cur_set_name):
+#             for cur_id in r_serv_sentiment.smembers(cur_set_name):
-                cur_value = (r_serv_sentiment.get(cur_id))
+#                 cur_value = (r_serv_sentiment.get(cur_id))
-                list_value.append(cur_value)
+#                 list_value.append(cur_value)
-            list_date[cur_timestamp] = list_value
+#             list_date[cur_timestamp] = list_value
-        to_return[provider] = list_date
+#         to_return[provider] = list_date
-
+#
-        return jsonify(to_return)
+#         return jsonify(to_return)
-    return "Bad request"
+#     return "Bad request"
-
+#
-
+#
-
+#
-@sentiments.route("/sentiment_analysis_plot_tool/")
+# @sentiments.route("/sentiment_analysis_plot_tool/")
-@login_required
+# @login_required
-@login_read_only
+# @login_read_only
-def sentiment_analysis_plot_tool():
+# def sentiment_analysis_plot_tool():
-    return render_template("sentiment_analysis_plot_tool.html")
+#     return render_template("sentiment_analysis_plot_tool.html")
-
+#
-
+#
-
+#
-@sentiments.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET'])
+# @sentiments.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET'])
-@login_required
+# @login_required
-@login_read_only
+# @login_read_only
-def sentiment_analysis_plot_tool_getdata():
+# def sentiment_analysis_plot_tool_getdata():
-    getProviders = request.args.get('getProviders')
+#     getProviders = request.args.get('getProviders')
-
+#
-    if getProviders == 'True':
+#     if getProviders == 'True':
-        providers = []
+#         providers = []
-        for cur_provider in r_serv_charts.smembers('all_provider_set'):
+#         for cur_provider in r_serv_charts.smembers('all_provider_set'):
-            providers.append(cur_provider)
+#             providers.append(cur_provider)
-        return jsonify(providers)
+#         return jsonify(providers)
-
+#
-    else:
+#     else:
-        query = request.args.get('query')
+#         query = request.args.get('query')
-        query = query.split(',')
+#         query = query.split(',')
-        Qdate = request.args.get('Qdate')
+#         Qdate = request.args.get('Qdate')
-
+#
-        date1 = (Qdate.split('-')[0]).split('/')
+#         date1 = (Qdate.split('-')[0]).split('/')
-        date1 = datetime.date(int(date1[2]), int(date1[0]), int(date1[1]))
+#         date1 = datetime.date(int(date1[2]), int(date1[0]), int(date1[1]))
-
+#
-        date2 = (Qdate.split('-')[1]).split('/')
+#         date2 = (Qdate.split('-')[1]).split('/')
-        date2 = datetime.date(int(date2[2]), int(date2[0]), int(date2[1]))
+#         date2 = datetime.date(int(date2[2]), int(date2[0]), int(date2[1]))
-
+#
-        timestamp1 = calendar.timegm(date1.timetuple())
+#         timestamp1 = calendar.timegm(date1.timetuple())
-        timestamp2 = calendar.timegm(date2.timetuple())
+#         timestamp2 = calendar.timegm(date2.timetuple())
-
+#
-        oneHour = 60*60
+#         oneHour = 60*60
-        oneDay = oneHour*24
+#         oneDay = oneHour*24
-
+#
-        to_return = {}
+#         to_return = {}
-        for cur_provider in query:
+#         for cur_provider in query:
-            list_date = {}
+#             list_date = {}
-            cur_provider_name = cur_provider + '_'
+#             cur_provider_name = cur_provider + '_'
-            for cur_timestamp in range(int(timestamp1), int(timestamp2)+oneDay, oneHour):
+#             for cur_timestamp in range(int(timestamp1), int(timestamp2)+oneDay, oneHour):
-                cur_set_name = cur_provider_name + str(cur_timestamp)
+#                 cur_set_name = cur_provider_name + str(cur_timestamp)
-
+#
-                list_value = []
+#                 list_value = []
-                for cur_id in r_serv_sentiment.smembers(cur_set_name):
+#                 for cur_id in r_serv_sentiment.smembers(cur_set_name):
-                    cur_value = (r_serv_sentiment.get(cur_id))
+#                     cur_value = (r_serv_sentiment.get(cur_id))
-                    list_value.append(cur_value)
+#                     list_value.append(cur_value)
-                list_date[cur_timestamp] = list_value
+#                 list_date[cur_timestamp] = list_value
-            to_return[cur_provider] = list_date
+#             to_return[cur_provider] = list_date
-
+#
-        return jsonify(to_return)
+#         return jsonify(to_return)
-
+#
-# ========= REGISTRATION =========
+# # ========= REGISTRATION =========
-app.register_blueprint(sentiments, url_prefix=baseUrl)
+# app.register_blueprint(sentiments, url_prefix=baseUrl)

var/www/modules/sentiment/templates/header_sentiment.html

@@ -1,7 +1,7 @@
-<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-heart"></i> Sentiment Analysis
+{#<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-heart"></i> Sentiment Analysis#}
-    <span class="caret"></span></a>
+{#    <span class="caret"></span></a>#}
-    <ul class="dropdown-menu">
+{#    <ul class="dropdown-menu">#}
-      <li><a href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-bar-chart-o"> </i> Sentiment trending</a></li>
+{#      <li><a href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-bar-chart-o"> </i> Sentiment trending</a></li>#}
-      <li><a href="{{ url_for('sentiments.sentiment_analysis_plot_tool') }}"><i class="fa fa-wrench"> </i> Sentiment plot Tool</a></li>
+{#      <li><a href="{{ url_for('sentiments.sentiment_analysis_plot_tool') }}"><i class="fa fa-wrench"> </i> Sentiment plot Tool</a></li>#}
-    </ul>
+{#    </ul>#}
-</li>
+{#</li>#}

var/www/modules/terms/Flask_terms.py

@@ -1,356 +0,0 @@
-#!/usr/bin/env python3
-# -*-coding:UTF-8 -*
-
-'''
-    Flask functions and routes for the trending modules page
-
-    note: The matching of credential against supplied credential is done using Levenshtein distance
-'''
-import os
-import sys
-import flask
-from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, Response
-
-from Role_Manager import login_admin, login_analyst, login_user_no_api, login_read_only
-from flask_login import login_required, current_user
-import Levenshtein
-
-sys.path.append(os.environ['AIL_BIN'])
-##################################
-# Import Project packages
-##################################
-from lib.objects.Items import Item
-
-# ============ VARIABLES ============
-import Flask_config
-
-app = Flask_config.app
-baseUrl = Flask_config.baseUrl
-r_serv_term = Flask_config.r_serv_term
-r_serv_cred = Flask_config.r_serv_cred
-r_serv_db = Flask_config.r_serv_db
-bootstrap_label = Flask_config.bootstrap_label
-
-terms = Blueprint('terms', __name__, template_folder='templates')
-
-'''TERM'''
-DEFAULT_MATCH_PERCENT = 50
-
-#tracked
-TrackedTermsSet_Name = "TrackedSetTermSet"
-TrackedTermsDate_Name = "TrackedTermDate"
-#black
-BlackListTermsDate_Name = "BlackListTermDate"
-BlackListTermsSet_Name = "BlackListSetTermSet"
-#regex
-TrackedRegexSet_Name = "TrackedRegexSet"
-TrackedRegexDate_Name = "TrackedRegexDate"
-#set
-TrackedSetSet_Name = "TrackedSetSet"
-TrackedSetDate_Name = "TrackedSetDate"
-
-# notifications enabled/disabled
-# same value as in `bin/NotificationHelper.py`
-TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
-
-# associated notification email addresses for a specific term`
-# same value as in `bin/NotificationHelper.py`
-# Keys will be e.g. TrackedNotificationEmails_<TERMNAME>
-TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
-TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
-
-'''CRED'''
-REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
-REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername'
-REDIS_KEY_NUM_PATH = 'uniqNumForUsername'
-REDIS_KEY_ALL_CRED_SET = 'AllCredentials'
-REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
-REDIS_KEY_ALL_PATH_SET = 'AllPath'
-REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev'
-REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
-
-
-
-# ============ FUNCTIONS ============
-
-def Term_getValueOverRange(word, startDate, num_day, per_paste=""):
-    passed_days = 0
-    oneDay = 60*60*24
-    to_return = []
-    curr_to_return = 0
-    for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay):
-        value = r_serv_term.hget(per_paste+str(timestamp), word)
-        curr_to_return += int(value) if value is not None else 0
-        for i in num_day:
-            if passed_days == i-1:
-                to_return.append(curr_to_return)
-        passed_days += 1
-    return to_return
-
-#Mix suplied username, if extensive is set, slice username(s) with different windows
-def mixUserName(supplied, extensive=False):
-    #e.g.: John Smith
-    terms = supplied.split()[:2]
-    usernames = []
-    if len(terms) == 1:
-        terms.append(' ')
-
-    #john, smith, John, Smith, JOHN, SMITH
-    usernames += [terms[0].lower()]
-    usernames += [terms[1].lower()]
-    usernames += [terms[0][0].upper() + terms[0][1:].lower()]
-    usernames += [terms[1][0].upper() + terms[1][1:].lower()]
-    usernames += [terms[0].upper()]
-    usernames += [terms[1].upper()]
-
-    #johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN
-    usernames += [(terms[0].lower() + terms[1].lower()).strip()]
-    usernames += [(terms[1].lower() + terms[0].lower()).strip()]
-    usernames += [(terms[0].upper() + terms[1].lower()).strip()]
-    usernames += [(terms[0].lower() + terms[1].upper()).strip()]
-    usernames += [(terms[1].upper() + terms[0].lower()).strip()]
-    usernames += [(terms[1].lower() + terms[0].upper()).strip()]
-    #Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith
-    usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
-    usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
-    usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
-    usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
-    usernames += [(terms[0].lower() + terms[1][0].upper()).strip()]
-    usernames += [(terms[0].upper() + terms[1][0].lower()).strip()]
-    usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
-    usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
-    usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
-
-    if not extensive:
-        return usernames
-
-    #Slice the supplied username(s)
-    mixedSupplied = supplied.replace(' ','')
-    minWindow = 3 if len(mixedSupplied)/2 < 4 else len(mixedSupplied)/2
-    for winSize in range(3,len(mixedSupplied)):
-        for startIndex in range(0, len(mixedSupplied)-winSize):
-            usernames += [mixedSupplied[startIndex:startIndex+winSize]]
-
-    filtered_usernames = []
-    for usr in usernames:
-        if len(usr) > 2:
-            filtered_usernames.append(usr)
-    return filtered_usernames
-
-def save_tag_to_auto_push(list_tag):
-    for tag in set(list_tag):
-        #limit tag length
-        if len(tag) > 49:
-            tag = tag[0:48]
-        r_serv_db.sadd('list_export_tags', tag)
-
-# ============ ROUTES ============
-
- # TODO: remove + clean
-
-# @terms.route("/terms_plot_tool/")
-# @login_required
-# @login_read_only
-# def terms_plot_tool():
-#     term =  request.args.get('term')
-#     if term is not None:
-#         return render_template("terms_plot_tool.html", term=term)
-#     else:
-#         return render_template("terms_plot_tool.html", term="")
-#
-#
-# @terms.route("/terms_plot_tool_data/")
-# @login_required
-# @login_read_only
-# def terms_plot_tool_data():
-#     oneDay = 60*60*24
-#     range_start =  datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
-#     range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0)
-#     range_start = calendar.timegm(range_start.timetuple())
-#     range_end =  datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0;
-#     range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0)
-#     range_end = calendar.timegm(range_end.timetuple())
-#     term =  request.args.get('term')
-#
-#     per_paste = request.args.get('per_paste')
-#     if per_paste == "1" or per_paste is None:
-#         per_paste = "per_paste_"
-#     else:
-#         per_paste = ""
-#
-#     if term is None:
-#         return "None"
-#
-#     else:
-#         value_range = []
-#         for timestamp in range(range_start, range_end+oneDay, oneDay):
-#             value = r_serv_term.hget(per_paste+str(timestamp), term)
-#             curr_value_range = int(value) if value is not None else 0
-#             value_range.append([timestamp, curr_value_range])
-#         value_range.insert(0,term)
-#         return jsonify(value_range)
-#
-
-# @terms.route("/terms_plot_top/"
-# @login_required
-# @login_read_only
-# def terms_plot_top():
-#     per_paste = request.args.get('per_paste')
-#     per_paste = per_paste if per_paste is not None else 1
-#     return render_template("terms_plot_top.html", per_paste=per_paste)
-# 
-#
-# @terms.route("/terms_plot_top_data/")
-# @login_required
-# @login_read_only
-# def terms_plot_top_data():
-#     oneDay = 60*60*24
-#     today = datetime.datetime.now()
-#     today = today.replace(hour=0, minute=0, second=0, microsecond=0)
-#     today_timestamp = calendar.timegm(today.timetuple())
-#
-#     per_paste = request.args.get('per_paste')
-#     if per_paste == "1" or per_paste is None:
-#         per_paste = "per_paste_"
-#     else:
-#         per_paste = ""
-#
-#     set_day = per_paste + "TopTermFreq_set_day_" + str(today_timestamp)
-#     set_week = per_paste + "TopTermFreq_set_week";
-#     set_month = per_paste + "TopTermFreq_set_month";
-#
-#     the_set = per_paste + request.args.get('set')
-#     num_day = int(request.args.get('num_day'))
-#
-#     if the_set is None:
-#         return "None"
-#     else:
-#         to_return = []
-#         if "TopTermFreq_set_day" in the_set:
-#             the_set += "_" + str(today_timestamp)
-#
-#         for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20):
-#             position = {}
-#             position['day'] = r_serv_term.zrevrank(set_day, term)
-#             position['day'] = position['day']+1 if position['day'] is not None else "<20"
-#             position['week'] = r_serv_term.zrevrank(set_week, term)
-#             position['week'] = position['week']+1 if position['week'] is not None else "<20"
-#             position['month'] = r_serv_term.zrevrank(set_month, term)
-#             position['month'] = position['month']+1 if position['month'] is not None else "<20"
-#             value_range = []
-#             for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay):
-#                 value = r_serv_term.hget(per_paste+str(timestamp), term)
-#                 curr_value_range = int(value) if value is not None else 0
-#                 value_range.append([timestamp, curr_value_range])
-#
-#             to_return.append([term, value_range, tot_value, position])
-#
-#         return jsonify(to_return)
-
-
-@terms.route("/credentials_tracker/")
-@login_required
-@login_read_only
-def credentials_tracker():
-    return render_template("credentials_tracker.html")
-
-@terms.route("/credentials_management_query_paste/", methods=['GET', 'POST'])
-@login_required
-@login_user_no_api
-def credentials_management_query_paste():
-    cred = request.args.get('cred')
-    allPath = request.json['allPath']
-
-    paste_info = []
-    for pathNum in allPath:
-        path = r_serv_cred.hget(REDIS_KEY_ALL_PATH_SET_REV, pathNum)
-        item = Item(path)
-        p_date = item.get_date(separator=True)
-        p_source = item.get_source()
-        p_content = item.get_content()
-        p_encoding = item.get_mimetype()
-        p_size = item.get_size()
-        p_mime = p_encoding
-        lineinfo = item.get_meta_lines(content=p_content)
-        p_lineinfo = lineinfo['nb'], lineinfo['max_length']
-        if p_content != 0:
-            p_content = p_content[0:400]
-        paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content})
-
-    return jsonify(paste_info)
-
-@terms.route("/credentials_management_action/", methods=['GET'])
-@login_required
-@login_user_no_api
-def cred_management_action():
-
-    supplied =  request.args.get('term')
-    action = request.args.get('action')
-    section = request.args.get('section')
-    extensive = request.args.get('extensive')
-    extensive = True if extensive == "true" else False
-
-    if extensive:
-        #collectDico
-        AllUsernameInRedis = r_serv_cred.hgetall(REDIS_KEY_ALL_CRED_SET).keys()
-    uniq_num_set = set()
-    if action == "seek":
-        possibilities = mixUserName(supplied, extensive)
-        for poss in possibilities:
-            num = r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET, poss)
-            if num is not None:
-                uniq_num_set.add(num)
-            for num in r_serv_cred.smembers(poss):
-                uniq_num_set.add(num)
-        #Extensive /!\
-        if extensive:
-            iter_num = 0
-            tot_iter = len(AllUsernameInRedis)*len(possibilities)
-            for tempUsername in AllUsernameInRedis:
-                for poss in possibilities:
-                    #FIXME print progress
-                    if(iter_num % int(tot_iter/20) == 0):
-                        #print("searching: {}% done".format(int(iter_num/tot_iter*100)), sep=' ', end='\r', flush=True)
-                        print("searching: {}% done".format(float(iter_num)/float(tot_iter)*100))
-                    iter_num += 1
-
-                    if poss in tempUsername:
-                        num = (r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET, tempUsername))
-                        if num is not None:
-                            uniq_num_set.add(num)
-                        for num in r_serv_cred.smembers(tempUsername):
-                            uniq_num_set.add(num)
-
-    data = {'usr': [], 'path': [], 'numPaste': [], 'simil': []}
-    for Unum in uniq_num_set:
-        levenRatio = 2.0
-        username = (r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum))
-
-        # Calculate Levenshtein distance, ignore negative ratio
-        supp_splitted = supplied.split()
-        supp_mixed = supplied.replace(' ','')
-        supp_splitted.append(supp_mixed)
-        for indiv_supplied in supp_splitted:
-            levenRatio = float(Levenshtein.ratio(indiv_supplied, username))
-            levenRatioStr = "{:.1%}".format(levenRatio)
-
-        data['usr'].append(username)
-
-
-        allPathNum = list(r_serv_cred.smembers(REDIS_KEY_MAP_CRED_TO_PATH+'_'+Unum))
-
-        data['path'].append(allPathNum)
-        data['numPaste'].append(len(allPathNum))
-        data['simil'].append(levenRatioStr)
-
-    to_return = {}
-    to_return["section"] = section
-    to_return["action"] = action
-    to_return["term"] = supplied
-    to_return["data"] = data
-
-    return jsonify(to_return)
-
-
-# ========= REGISTRATION =========
-app.register_blueprint(terms, url_prefix=baseUrl)

var/www/modules/terms/templates/credentials_tracker.html

@@ -1,253 +0,0 @@
-<!DOCTYPE html>
-<html>
-
-    <head>
-        <meta charset="utf-8">
-        <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-        <title>Credentials Tracker - AIL</title>
-        <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
-
-        <!-- Core CSS -->
-        <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
-        <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
-        <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
-        <link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
-        <!-- JS -->
-        <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
-        <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
-        <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
-        <script src="{{ url_for('static', filename='js/jquery.flot.pie.js') }}"></script>
-        <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
-        <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
-        <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
-        <script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
-        <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
-        <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
-        <script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
-
-        <style>
-          .btn-link {
-              color: #000000
-          }
-        </style>
-
-    </head>
-    <body>
-        <!-- Modal -->
-        <div id="mymodal" class="modal fade" role="dialog">
-          <div class="modal-dialog modal-lg">
-
-            <!-- Modal content-->
-            <div id="mymodalcontent" class="modal-content">
-              <div id="mymodalbody" class="modal-body" max-width="8500px">
-                <p>Loading paste information...</p>
-                <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
-              </div>
-              <div class="modal-footer">
-                <a id="button_show_plot" target="_blank" href=""><button type="button" class="btn btn-info">Plot term</button></a>
-                <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
-              </div>
-            </div>
-          </div>
-        </div>
-        {% include 'navbar.html' %}
-
-        <div id="page-wrapper">
-            <div class="row">
-              <div class="col-lg-12">
-              <h1 class="page-header" data-page="page-termsfrequency" >Credential seeker</h1>
-              </div>
-				  <!-- /.col-lg-12 -->
-    <!-- Panel OPTIONS -->
-    <div class="row">
-        <div class="col-lg-12">
-            <div class="row">
-                <div class="col-lg-12">
-                    <div id="panel-today" class="panel panel-success">
-                        <div class="panel-heading">
-                            <strong>Credential seeker</strong>
-                        </div>
-                        <div class="panel-body">
-
-                            <label><input type="checkbox" id="extensive" style="margin-bottom: 10px;"><strong>Extensive search (takes time)</strong></input></label>
-                            <div class="form-group input-group" style="margin-bottom: 30px;">
-                                <span class="input-group-addon"><span class="glyphicon glyphicon-screenshot"></span></span>
-                                <input id="seekInput" class="form-control" placeholder="Credential to seek." type="text" style="max-width: 400px;">
-                                <button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="seek" data-action="seek"> Seek</button>
-                                <span id="nodata" class="alert alert-info" style="margin-left: 10px; display: none; padding: 6px;">No data</span>
-                            </div>
-
-                            <table class="table table-striped table-bordered table-hover" id="myTable">
-                                <thead>
-                                     <tr>
-                                        <th style="max-width: 800px;">Username</th>
-                                        <th>Similarity</th>
-                                        <th># concerned paste(s)</th>
-                                        <th>Action</th>
-                                     </tr>
-                                </thead>
-                                <tbody>
-                                </tbody>
-                            </table>
-                        <!-- /.panel-body -->
-                    </div>
-                </div>
-                <!-- /.panel -->
-            </div>
-            <!-- /.panel -->
-        </div>
-    </div>
-
-</div>
-<!-- /.row -->
-</div>
-            <!-- /#page-wrapper -->
-
-            <script>
-                $(document).ready(function(){
-                    activePage = $('h1.page-header').attr('data-page');
-                    $("#"+activePage).addClass("active");
-
-                    $('[data-toggle="tooltip"]').tooltip();
-                    table_track = $('#myTable').DataTable({ "order": [[ 1, "dec" ]] });
-
-                    table_track.on( 'draw.dt', function () {
-                        perform_binding();
-                    });
-
-                    $("#seekInput").on("keyup", function(event){
-                        if(event.keyCode == 13){
-                            $("#followTermBtn").click();
-                            $("#seekInput").val("");
-                        }
-                    });
-
-                    perform_binding();
-
-                    $("#mymodal").on('hidden.bs.modal', function () {
-                        $("#mymodalbody").html("<p>Loading paste information...</p>");
-                        var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
-                        $("#mymodalbody").append(loading_gif); // Show the loading GIF
-                    });
-                });
-            </script>
-
-            <script>
-                function perform_binding() {
-                    $(".btn-interaction").off("click.interaction");
-                    $(".btn-interaction").on("click.interaction", perform_operation);
-                }
-
-                function perform_modal_binding() {
-                    // On click, get html content from url and update the corresponding modal
-                    $("[data-toggle='modal']").on("click.openmodal", function (event) {
-                        event.preventDefault();
-                        var the_modal=$(this);
-                        var url = "{{ url_for('terms.credentials_management_query_paste')  }}?cred=" + encodeURIComponent($(this).attr('data-term'));
-                        $.ajax({
-                            type: 'POST',
-                            url: url,
-                            dataType: "json",
-                            data: JSON.stringify({ 'allPath': JSON.parse($(this).attr('data-path')) }),
-                            contentType : "application/json"
-                        }).done(function (data) {
-                            if (data.length != 0) {
-                                var html_to_add = "";
-                                html_to_add += "<table id=\"modal-table\" class=\"table table-striped\">";
-                                html_to_add += "<thead>";
-                                  html_to_add += "<tr>";
-                                    html_to_add += "<th>Source</th>";
-                                    html_to_add += "<th>Date</th>";
-                                    html_to_add += "<th>Encoding</th>";
-                                    html_to_add += "<th>Size (Kb)</th>";
-                                    html_to_add += "<th># lines</th>";
-                                    html_to_add += "<th>Max length</th>";
-                                    html_to_add += "<th>Preview</th>";
-                                  html_to_add += "</tr>";
-                                html_to_add += "</thead>";
-                                html_to_add += "<tbody>";
-                                for (i=0; i<data.length; i++) {
-                                    curr_data = data[i];
-                                        html_to_add += "<tr>";
-                                            html_to_add += "<td>"+curr_data.source+"</td>";
-                                            html_to_add += "<td>"+curr_data.date+"</td>";
-                                            html_to_add += "<td>"+curr_data.encoding+"</td>";
-                                            html_to_add += "<td>"+curr_data.size+"</td>";
-                                            html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
-                                            html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";
-                                            html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content.replace(/\"/g, "\'")+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('objects_item.showItem') }}?id="+curr_data.path+"&num=0\">  <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
-
-                                        html_to_add += "</tr>";
-                                }
-                                html_to_add += "</tbody>";
-                                html_to_add += "</table>";
-                                $("#mymodalbody").html(html_to_add);
-                                $("[data-toggle=popover]").popover();
-	                        $('#modal-table').DataTable();
-                            } else {
-                                $("#mymodalbody").html("No paste containing this term has been received yet.");
-                            }
-                        }
-                    )});
-                }
-
-                function perform_operation(){
-                    var curr_section = $(this).attr('data-section');
-                    var curr_action = $(this).attr('data-action');
-                    if (curr_action == "add") {
-                        var curr_term = $('#'+curr_section+'Input').val();
-                    } else if (curr_action == "seek") {
-                        var curr_term = $('#'+curr_section+'Input').val();
-                    } else {
-                        var curr_term = $(this).attr('data-content');
-                    }
-                    var data_to_send = { section: curr_section, action:curr_action, term: curr_term, extensive: $("#extensive").is(":checked")};
-
-                    if (curr_term != "") {
-                        //console.log(data_to_send);
-                        $.get("{{ url_for('terms.cred_management_action') }}", data_to_send, function(data, status){
-                            if(status == "success") {
-                                var json = data;
-
-                                if(json.action == "add") {
-                                    //not used for the moment
-
-                                } else if (json.action == "seek") {
-                                    table_track.clear().draw();
-                                    var rep = json.data;
-                                    //var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Track this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"add\">";
-                                    var action_button = "";
-                                    if (rep.usr.length == 0) {
-                                        $( "#nodata" ).removeClass('alert-info').addClass('alert-danger');
-                                        $( "#nodata" ).text("No data");
-                                        $( "#nodata" ).fadeIn( "fast", function() {
-                                            setTimeout(function() {
-                                                $( "#nodata" ).fadeOut("fast");
-                                            }, 2000);
-                                        });
-                                    } else {
-                                        for(i=0; i < rep.usr.length; i++) {
-                                            $( "#nodata" ).removeClass('alert-danger').addClass('alert-info');
-                                            $( "#nodata" ).text(curr_term);
-                                            $( "#nodata" ).fadeIn( "fast");
-                                            toAdd = "</button><span data-toggle=\"modal\" data-target=\"#mymodal\" data-term=\""+rep.usr[i]+"\" data-path=\"["+rep.path[i]+"]\" ><button class=\"btn-link\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Show concerned paste(s)\"><span class=\"glyphicon glyphicon-info-sign\"></span></button></span>";
-                                            table_track.row.add( [
-                                                    rep.usr[i],
-                                                    rep.simil[i],
-                                                    rep.numPaste[i],
-                                                    toAdd+action_button ] ).draw( false );
-                                        }
-                                        perform_binding();
-                                        perform_modal_binding();
-                                    }
-                                }
-                            }
-                        });
-                    }
-                }
-            </script>
-
-        </body>
-
-</html>

var/www/modules/terms/templates/header_terms.html

@@ -1,6 +0,0 @@
-<li id='page-termsfrequency'><a class="dropdown-toggle" data-toggle="dropdown" href="{{ url_for('terms.credentials_tracker') }}"><i class="fa fa-eye"></i> Terms frequency
-    <span class="caret"></span></a>
-    <ul class="dropdown-menu">
-      <li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials seeker</a></li>
-    </ul>
-</li>

var/www/modules/terms/templates/terms_management.html

@@ -1,591 +0,0 @@
-<!DOCTYPE html>
-<html>
-
-<head>
-  <meta charset="utf-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-  <title>Terms Management</title>
-  <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
-
-  <!-- Core CSS -->
-  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
-  <link href="{{ url_for('static', filename='css/switch_checkbox.css') }}" rel="stylesheet" type="text/css" />
-  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
-  <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
-
-  <style>
-      .sparkLineStats ul {
-          padding-left:0;
-          list-style:none
-      }
-
-      .btn-link {
-          color: #000000
-      }
-
-      .popover-content {
-         white-space:pre-wrap;
-         word-wrap:break-word;
-      }
-      .mouse_pointer{
-				cursor: pointer;
-			}
-      .lb-md {
-        font-size: 16px;
-      }
-  </style>
-</head>
-<body>
-
-    <!-- Modal -->
-    <div id="mymodal" class="modal fade" role="dialog">
-      <div class="modal-dialog modal-lg">
-
-        <!-- Modal content-->
-        <div id="mymodalcontent" class="modal-content">
-          <div id="mymodalbody" class="modal-body" max-width="8500px">
-            <p>Loading paste information...</p>
-            <img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
-          </div>
-          <div class="modal-footer">
-            <a id="button_show_plot" target="_blank" href=""><button type="button" class="btn btn-info">Plot term</button></a>
-            <button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
-          </div>
-        </div>
-      </div>
-    </div>
-
-   {% include 'navbar.html' %}
-
-<div id="page-wrapper">
-  <div class="row">
-      <div class="col-lg-12">
-          <h1 class="page-header" data-page="page-termsfrequency" >Terms frequency: Management interface</h1>
-      </div>
-      <!-- /.col-lg-12 -->
-  </div>
-    <!-- /.row -->
-    <div class="row">
-
-    <!-- Panel OPTIONS -->
-    <div class="row">
-        <div class="col-lg-12">
-            <div class="row">
-              {% set uniq_id = namespace(modal_id=0)%}
-                <div class="col-lg-12">
-                   <label class="switch">
-                       <input id="per_paste" class="switch-input" value="per_paste" type="checkbox" onclick="reload_per_paste()">
-                       <span class="switch-label" data-on="On" data-off="Off"></span>
-                       <span class="switch-handle"></span>
-                   </label>
-                   <strong style="top: 3px; position: relative;">1 term per paste</strong>
-
-                    <div id="panel-today" class="panel panel-success">
-                        <div class="panel-heading">
-                            <strong>Manage tracked terms</strong>
-                        </div>
-                        <div class="panel-body">
-
-                            <div style="margin-bottom: 10px;">
-                                <table>
-                                    <tr><td><b>Regex</b>: surround the term by '<b>/</b>'. </td> <td><b style="margin-left: 20px;">/([a-z])\w+([a-z])\n/</b></td></tr>
-                                    <tr><td><b>Set of terms</b>: surround the list by '<b>\</b>'. </td> <td><b style="margin-left: 20px;">\[term1, term2, ...]\</b></td></tr>
-                                    <tr><td> - To set a custom matching <b>threshold</b> (defaut=50), append it at the end as a inner list '<b>[thresh]</b>'. </td> <td><b style="margin-left: 20px;">\[term1, term2, ..., [75]]\</b></td></tr>
-                                </table>
-                            </div>
-                            <div class="form-group input-group" style="margin-bottom: 30px;">
-                                <span class="input-group-addon"><span class="fa fa-eye"></span></span>
-                                <input id="followTermInput" class="form-control" placeholder="Term to track." type="text" style="max-width: 400px;">
-                                <input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (optional, space separated)" type="text" style="max-width: 400px;">
-                                <input id="followTermTag" class="form-control" placeholder="Tags (optional, space separated)" type="text" style="max-width: 400px;">
-                                <button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button>
-                            </div>
-
-                            <table class="table table-striped table-bordered table-hover" id="myTable">
-                                <thead>
-                                     <tr>
-                                        <th style="max-width: 800px;">Term</th>
-                                        <th>Added date</th>
-                                        <th>Day occurence</th>
-                                        <th>Week occurence</th>
-                                        <th>Month occurence</th>
-                                        <th># tracked paste</th>
-                                        <th>Action</th>
-                                        <th>Notification E-Mails</th>
-                                     </tr>
-                                </thead>
-                                <tbody>
-                                <!-- SET -->
-                                {% for set in trackSet_list %}
-                                  <tr style="background-color: #cdffca;">
-                                      <td>
-                                          <span class="term_name">{{ set }}</span>
-                                          <div>
-                                            {% for tag in notificationTagsTermMapping[set] %}
-                                            <a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
-                                              <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
-                                            </a>
-                                            {%  endfor %}
-                                            {% if notificationTagsTermMapping[set] %}
-                                              <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
-
-                                              <div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
-                                      					<div class="modal-dialog">
-
-                                      						<!-- Modal content-->
-                                      						<div id="mymodalcontent" class="modal-content">
-                                      							<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
-                                      								<h2 class="text-center">Remove Custom Tag</h2>
-
-                                      							</div>
-
-                                      							<div class="modal-body">
-                                                      <form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
-                                                        {% for tag in notificationTagsTermMapping[set] %}
-                                                          <div class="form-check">
-                                                             <input type="hidden" class="form-control" name="term" value="{{ set }}">
-                                                            <input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
-                                                            <label class="form-check-label">
-                                                              <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
-                                                            </label>
-                                                            <br>
-                                                          </div>
-                                                        {%  endfor %}
-                                                      </form>
-                                      							</div>
-
-                                      							<div class="modal-footer">
-                                      								<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
-                                      										<span class="glyphicon glyphicon-trash"></span>
-                                      										<span class="label-icon">Remove Tags</span>
-                                      								</button>
-                                      								<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
-
-                                      							</div>
-                                      						</div>
-                                      					</div>
-                                      				</div>
-                                              {% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
-                                            {%  endif %}
-                                          </div>
-                                      </td>
-                                      <td>{{ trackSet_list_values[loop.index0][3] }}</td>
-                                      <td>{{ trackSet_list_values[loop.index0][0] }}</td>
-                                      <td>{{ trackSet_list_values[loop.index0][1] }}</td>
-                                      <td>{{ trackSet_list_values[loop.index0][2] }}</td>
-                                      <td>{{ trackSet_list_num_of_paste[loop.index0] }}</td>
-                                      <td><p style="margin: 0px; white-space: nowrap;">
-                                          <span data-toggle="modal" data-target="#mymodal" data-term="{{ set }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
-                                          <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
-                                          &nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ set }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[set] %} checked {% endif %}>
-                                      </p></td>
-                                      <td>
-                                        {% for email in notificationEMailTermMapping[set] %}
-                                          <a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{set}}">
-                                            <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
-                                                <span class="glyphicon glyphicon-trash" style="color:Red;" ></span>
-                                            </div>
-                                          </a>
-                                          {{ email }}
-                                          <br>
-                                        {%  endfor %}
-                                      </td>
-                                  </tr>
-                                {% endfor %}
-                                <!-- REGEX -->
-                                {% for regex in trackReg_list %}
-                                  <tr style="background-color: #fffdca;">
-                                      <td>
-                                          <span class="term_name">{{ regex }}</span>
-                                          <div>
-                                            {% for tag in notificationTagsTermMapping[regex] %}
-                                            <a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
-                                              <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
-                                            </a>
-                                            {%  endfor %}
-                                            {% if notificationTagsTermMapping[regex] %}
-                                              <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
-
-                                              <div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
-                                      					<div class="modal-dialog">
-
-                                      						<!-- Modal content-->
-                                      						<div id="mymodalcontent" class="modal-content">
-                                      							<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
-                                      								<h2 class="text-center">Remove Custom Tag</h2>
-
-                                      							</div>
-
-                                      							<div class="modal-body">
-                                                      <form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
-                                                        {% for tag in notificationTagsTermMapping[regex] %}
-                                                          <div class="form-check">
-                                                             <input type="hidden" class="form-control" name="term" value="{{ regex }}">
-                                                            <input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
-                                                            <label class="form-check-label">
-                                                              <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
-                                                            </label>
-                                                            <br>
-                                                          </div>
-                                                        {%  endfor %}
-                                                      </form>
-                                      							</div>
-
-                                      							<div class="modal-footer">
-                                      								<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
-                                      										<span class="glyphicon glyphicon-trash"></span>
-                                      										<span class="label-icon">Remove Tags</span>
-                                      								</button>
-                                      								<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
-
-                                      							</div>
-                                      						</div>
-                                      					</div>
-                                      				</div>
-                                              {% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
-                                            {%  endif %}
-                                          </div>
-                                      </td>
-                                      <td>{{ trackReg_list_values[loop.index0][3] }}</td>
-                                      <td>{{ trackReg_list_values[loop.index0][0] }}</td>
-                                      <td>{{ trackReg_list_values[loop.index0][1] }}</td>
-                                      <td>{{ trackReg_list_values[loop.index0][2] }}</td>
-                                      <td>{{ trackReg_list_num_of_paste[loop.index0] }}</td>
-                                      <td><p style="margin: 0px; white-space: nowrap;">
-                                          <span data-toggle="modal" data-target="#mymodal" data-term="{{ regex }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
-                                          <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ regex }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
-                                          &nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ regex }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[regex] %} checked {% endif %}>
-                                      </p></td>
-                                      <td>
-                                        {% for email in notificationEMailTermMapping[regex] %}
-                                          <a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{regex}}">
-                                            <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
-                                              <span class="glyphicon glyphicon-trash" style="color:Red;"></span>
-                                            </div>
-                                          </a>
-                                          {{ email }}
-                                          <br>
-                                        {%  endfor %}
-                                      </td>
-                                  </tr>
-                                {% endfor %}
-                                <!-- Normal term -->
-                                {% for term in track_list %}
-                                  <tr>
-                                      <td>
-                                          <span class="term_name">{{ term }}</span>
-                                          <div>
-                                            {% for tag in notificationTagsTermMapping[term] %}
-                                            <a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
-                                              <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
-                                            </a>
-                                            {%  endfor %}
-                                            {% if notificationTagsTermMapping[term] %}
-                                              <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
-
-                                              <div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
-                                      					<div class="modal-dialog">
-
-                                      						<!-- Modal content-->
-                                      						<div id="mymodalcontent" class="modal-content">
-                                      							<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
-                                      								<h2 class="text-center">Remove Custom Tag</h2>
-
-                                      							</div>
-
-                                      							<div class="modal-body">
-                                                      <form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
-                                                        {% for tag in notificationTagsTermMapping[term] %}
-                                                          <div class="form-check">
-                                                             <input type="hidden" class="form-control" name="term" value="{{ term }}">
-                                                            <input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
-                                                            <label class="form-check-label">
-                                                              <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
-                                                            </label>
-                                                            <br>
-                                                          </div>
-                                                        {%  endfor %}
-                                                      </form>
-                                      							</div>
-
-                                      							<div class="modal-footer">
-                                      								<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
-                                      										<span class="glyphicon glyphicon-trash"></span>
-                                      										<span class="label-icon">Remove Tags</span>
-                                      								</button>
-                                      								<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
-
-                                      							</div>
-                                      						</div>
-                                      					</div>
-                                      				</div>
-                                              {% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
-                                            {%  endif %}
-                                          </div>
-                                      </td>
-                                      <td>{{ track_list_values[loop.index0][3] }}</td>
-                                      <td>{{ track_list_values[loop.index0][0] }}</td>
-                                      <td>{{ track_list_values[loop.index0][1] }}</td>
-                                      <td>{{ track_list_values[loop.index0][2] }}</td>
-                                      <td>{{ track_list_num_of_paste[loop.index0] }}</td>
-                                      <td><p style="margin: 0px; white-space: nowrap;">
-                                          <span data-toggle="modal" data-target="#mymodal" data-term="{{ term }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
-                                          <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ term }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
-                                          &nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ term }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[term] %} checked {% endif %}>
-                                      </p></td>
-                                      <td>
-                                        {% for email in notificationEMailTermMapping[term] %}
-                                          <a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{term}}">
-                                            <div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
-                                                <span class="glyphicon glyphicon-trash" style="color:Red;"></span>
-                                            </div>
-                                          </a>
-                                          {{ email }}
-                                          <br>
-                                        {%  endfor %}
-                                      </td>
-                                  </tr>
-                                {% endfor %}
-                                </tbody>
-                            </table>
-                        <!-- /.panel-body -->
-                    </div>
-                </div>
-                <!-- /.panel -->
-            </div>
-            <!-- /.panel -->
-        </div>
-    </div>
-
-     <!-- Panel OPTIONS -->
-        <div class="col-lg-12">
-            <div class="row">
-                <div class="col-lg-12">
-                    <div id="panel-today" class="panel panel-danger">
-                        <div class="panel-heading">
-                            <strong>Manage blacklisted terms</strong>
-                        </div>
-                        <div class="panel-body">
-
-                            <div class="form-group input-group" style="margin-bottom: 30px;">
-                                <span class="input-group-addon"><span class="fa fa-eye-slash "></span></span>
-                                <input id="blacklistTermInput" class="form-control" placeholder="Term to track" type="text" style="max-width: 400px;">
-                                <button id="blacklistTermBtn" class="btn btn-danger btn-interaction" style="margin-left: 10px;" data-section="blacklistTerm" data-action="add"> Black list a term</button>
-                            </div>
-
-                            <table class="table table-striped table-bordered table-hover" id="myTable2">
-                                <thead>
-                                     <tr>
-                                        <th style="max-width: 800px;">Term</th>
-                                        <th>Added date</th>
-                                        <th>Action</th>
-                                     </tr>
-                                </thead>
-                                <tbody>
-                                {% for term, date in black_list %}
-                                  <tr>
-                                      <td>{{ term }}</td>
-                                      <td>{{ date }}</td>
-                                      <td><p style="margin: 0px;">
-                                          <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="right" title="Remove this term" data-content="{{ term }}" data-section="blacklistTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
-                                      </p></td>
-                                  </tr>
-                                {% endfor %}
-                                </tbody>
-                            </table>
-                        <!-- /.panel-body -->
-                    </div>
-                </div>
-                <!-- /.panel -->
-            </div>
-            <!-- /.panel -->
-        </div>
-    </div>
-
-    <!-- /.row -->
-    </div>
-<!-- /#page-wrapper -->
-</div>
-
-
-<!-- import graph function -->
-<script>
-    function reload_per_paste() {
-        var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
-        window.location.href = {{ url_for('terms.terms_management') }}+"?per_paste="+checked;
-    }
-
-
-    var table_track;
-    var table_black;
-
-function bindEventsForCurrentPage() {
-    // On click, get html content from url and update the corresponding modal
-    $("[data-toggle='modal']").unbind().on("click.openmodal", function (event) {
-        //console.log(data);
-        event.preventDefault();
-        var the_modal=$(this);
-        var url = "{{ url_for('terms.terms_management_query_paste')  }}?term=" + encodeURIComponent($(this).attr('data-term'));
-        $.getJSON(url, function (data) {
-            if (data.length != 0) {
-                var html_to_add = "";
-                html_to_add += "<table id=\"modal-table\" class=\"table table-striped\">";
-                html_to_add += "<thead>";
-                html_to_add += "<tr>";
-                html_to_add += "<th>Source</th>";
-                html_to_add += "<th>Date</th>";
-                html_to_add += "<th>Mime</th>";
-                html_to_add += "<th>Size (Kb)</th>";
-                html_to_add += "<th># lines</th>";
-                html_to_add += "<th>Max length</th>";
-                html_to_add += "<th>Preview</th>";
-                html_to_add += "</tr>";
-                html_to_add += "</thead>";
-                html_to_add += "<tbody>";
-                for (i=0; i<data.length; i++) {
-                    curr_data = data[i];
-                    html_to_add += "<tr>";
-                    html_to_add += "<td>"+curr_data.source+"</td>";
-                    html_to_add += "<td>"+curr_data.date+"</td>";
-                    html_to_add += "<td>"+curr_data.mime+"</td>";
-                    html_to_add += "<td>"+curr_data.size+"</td>";
-                    html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
-                    html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";
-                    html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content.replace(/\"/g, "\'")+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('objects_item.showItem') }}?id="+curr_data.path+"&num=0\">  <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
-
-                    html_to_add += "</tr>";
-                }
-                html_to_add += "</tbody>";
-                html_to_add += "</table>";
-                $("#mymodalbody").html(html_to_add);
-                $("[data-toggle=popover]").popover();
-                $("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
-                $('#modal-table').DataTable();
-            } else {
-                $("#mymodalbody").html("No paste containing this term has been received yet.");
-                $("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
-            }
-        });
-
-    });
-
-}
-
-
-    $(document).ready(function(){
-        bindEventsForCurrentPage();
-        activePage = $('h1.page-header').attr('data-page');
-        $("#"+activePage).addClass("active");
-        if({{ per_paste }} ==  1) {
-            $("#per_paste").attr('checked', true)
-        }
-
-        $('[data-toggle="tooltip"]').tooltip();
-	table_track = $('#myTable').DataTable();
-	table_black = $('#myTable2').DataTable();
-
-        table_track.on( 'draw.dt', function () {
-            perform_binding();
-        });
-        table_black.on( 'draw.dt', function () {
-            perform_binding();
-        });
-
-
-        $("#followTermInput").keyup(function(event){
-            if(event.keyCode == 13){
-                $("#followTermBtn").click();
-                $("#followTermInput").val("");
-            }
-        });
-
-        $("#blacklistTermInput").keyup(function(event){
-            if(event.keyCode == 13){
-                $("#blacklistTermBtn").click();
-                $("#blacklistTermInput").val("");
-            }
-        });
-
-        perform_binding();
-
-    $("#mymodal").on('hidden.bs.modal', function () {
-        $("#mymodalbody").html("<p>Loading paste information...</p>");
-        var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
-        $("#mymodalbody").append(loading_gif); // Show the loading GIF
-    });
-
-
-   });
-</script>
-
-<script>
-
-
-
-function perform_binding() {
-    $(".btn-interaction").unbind("click.interaction");
-    $(".btn-interaction").bind("click.interaction", perform_operation);
-}
-
-function perform_operation(){
-    var curr_section = $(this).attr('data-section');
-    var curr_action = $(this).attr('data-action');
-    var row_tr = $(this).closest("tr");
-    if (curr_action == "add") {
-        var curr_term = $('#'+curr_section+'Input').val();
-        var email_addresses = $('#followTermEMailNotificationReceiversInput').val();
-        var tags = $('#followTermTag').val();
-    } else {
-        var curr_term = $(this).attr('data-content');
-        var email_addresses = "";
-    }
-    var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses, tags: tags};
-
-    if (curr_term != "") {
-        //console.log(data_to_send);
-        $.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){
-            if(status == "success") {
-                var json = data;
-
-                if(json.section == "followTerm") {
-                    if(json.action == "add") {
-                        // query data
-                        $.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
-                            reload_per_paste();
-                        });
-                    } else if (json.action == "delete") {
-                      row_tr.remove()
-                    }
-                } else if(json.section == "blacklistTerm"){
-                    if(json.action == "add") {
-                        $.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
-                            console.log(data2);
-                            var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"blacklistTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
-                            table_black.row.add( [ json.term, data2[3], action_button ] ).draw( false );
-                            perform_binding();
-                        });
-                    } else if (json.action == "delete") {
-                       // Find indexes of row which have the term in the first column
-                       var index = table_black.rows().eq( 0 ).filter( function (rowIdx) {
-                           return table_black.cell( rowIdx, 0 ).data() === json.term;
-                       } );
-                       table_black.rows(index).remove().draw( false );
-                    }
-                }
-
-
-
-            }
-        });
-    }
-}
-</script>

var/www/modules/terms/templates/terms_plot_tool.html

@@ -1,275 +0,0 @@
-<!DOCTYPE html>
-<html>
-
-<head>
-  <meta charset="utf-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-  <title>Terms Plot Tool - AIL</title>
-  <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
-
-  <!-- Core CSS -->
-  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
-  <link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
-  <link href="{{ url_for('static', filename='css/switch_checkbox.css') }}" rel="stylesheet" type="text/css" />
-  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
-  <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery-ui.min.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
-
-  <style>
-      .sparkLineStats ul {
-          padding-left:0;
-          list-style:none
-      }
-  </style>
-</head>
-<body>
-
-   {% include 'navbar.html' %}
-
-<div id="page-wrapper">
-  <div class="row">
-      <div class="col-lg-12">
-          <h1 class="page-header" data-page="page-termsfrequency" >Terms plot tool</h1>
-      </div>
-      <!-- /.col-lg-12 -->
-  </div>
-    <!-- /.row -->
-    <div class="row">
-
-    <!-- Panel OPTIONS -->
-    <div class="row">
-        <div class="col-lg-12">
-            <div class="row">
-                <div class="col-lg-12">
-                    <div id="panel-today" class="panel panel-default">
-                        <div class="panel-heading">
-                            <strong>Select options</strong>
-                        </div>
-                        <div class="panel-body">
-                            <div aria-disabled="false" class="slider sliderRange sliderBlue ui-slider ui-slider-horizontal ui-widget ui-widget-content ui-corner-all" style="margin-bottom: 5px;"></div>
-                            <strong>Date:</strong> <input type="text" id="amount" readonly style="border:0; color:#f6931f; font-weight:bold;"> <button id="plot-month" onclick="replot();" class="btn btn-info" style="float: right;">Replot</button>
-
-                            <div class="form-group input-group" style="margin-top: 30px;">
-                                <span class="input-group-addon"><span class="glyphicon glyphicon-stats"></span></span>
-                                <input id="TermInput" class="form-control" placeholder="Term to plot" type="text" style="max-width: 400px;" data-init-plot="{{ term }}">
-                                <button id="plot-btn" class="btn btn-info" style="margin-left: 10px;"><span class="fa fa-caret-down"> Plot a term</button>
-                                <button id="plot-btn-add" class="btn btn-success" style="margin-left: 6px;"><span class="fa fa-plus"></span> Add the term to the chart</button>
-                            </div>
-
-                            <div class='pull_right' style="float: right;">
-                                <label class="switch">
-                                    <input id="per_paste" class="switch-input" value="per_paste" type="checkbox" checked>
-                                    <span class="switch-label" data-on="On" data-off="Off"></span>
-                                    <span class="switch-handle"></span>
-                                </label>
-                                <strong style="top: 3px; position: relative;">1 term per paste</strong>
-                            </div>
-
-                        </div>
-                        <!-- /.panel-body -->
-                    </div>
-                </div>
-                <!-- /.panel -->
-            </div>
-            <!-- /.panel -->
-        </div>
-    </div>
-
-    <!-- Panel PLOT -->
-    <div class="row">
-        <div class="col-lg-12">
-            <div class="row">
-                <div class="col-lg-12">
-                    <div id="panel-today" class="panel panel-default">
-                        <div class="panel-heading">
-                            <strong>Graph</strong>
-                        </div>
-                        <div class="panel-body">
-                            <div id="graph" style="height: 300px;"></div>
-                        </div>
-                        <!-- /.panel-body -->
-                    </div>
-                </div>
-                <!-- /.panel -->
-            </div>
-            <!-- /.panel -->
-        </div>
-    </div>
-
-
-    <!-- /.row -->
-    </div>
-<!-- /#page-wrapper -->
-</div>
-
-
-<div style="position: absolute; border: 1px solid rgb(255, 221, 221); padding: 2px; background-color: #333; color:white; opacity: 0.8; top: 423px; left: 616px; display: none;" id="tooltip"></div>
-
-
-<!-- import graph function -->
-<script>
-    $(document).ready(function(){
-        activePage = $('h1.page-header').attr('data-page');
-        $("#"+activePage).addClass("active");
-
-
-        /* Create the slider and button*/
-        $('#plot-btn-add').hide();
-        var today = Date.now();
-        var old_day = today - (12*31*24*60*60)*1000;
-        $( ".sliderRange" ).slider({
-            range: true,
-            min: old_day,
-            max: today,
-            values: [ today - (7*24*60*60)*1000, today ],
-            step: 24*60*60*1000,
-            slide: function( event, ui ) {
-                $( "#amount" ).val( new Date(ui.values[ 0 ]).toLocaleDateString() + " - " + new Date(ui.values[ 1 ]).toLocaleDateString() );
-            }
-        });
-
-        $( "#amount" ).val( new Date($( ".sliderRange" ).slider( "values", 0 )).toLocaleDateString() +
-          " - " + new Date($( ".sliderRange" ).slider( "values", 1 )).toLocaleDateString() );
-
-        $('#plot-btn').click(plotData);
-        $('#plot-btn-add').click(addData);
-
-        $("#TermInput").val($("#TermInput").attr("data-init-plot"));
-        if($("#TermInput").attr("data-init-plot") != "") {
-            $("#plot-btn").click();
-        }
-
-        $("#TermInput").keyup(function(event){
-            if(event.keyCode == 13){
-                $("#plot-btn").click();
-                $("#TermInput").val("");
-            }
-        });
-
-   });
-</script>
-
-<script>
-var plot;
-var graph_data = [];
-var plotted_terms = [];
-var graph_options = {
-                        series: {
-                            lines: {
-                                show: true,
-                                lineWidth: 2
-                            },
-                            bars: {show: false, barWidth: 60*60*1000},
-                            shadowSize: 0
-                        },
-                        grid: {
-                            hoverable: true,
-                            clickable: true,
-                            tickColor: "#f9f9f9",
-                            borderWidth: 0
-                        },
-                        xaxis: {
-                              mode: "time",
-                              timeformat: "%m/%d",
-                              minTickSize: [1, "day"]
-                        },
-                        yaxis: {
-                            autoscaleMargin: 0.1,
-                        },
-                    }
-
-
-function plotData() {
-    $('#plot-btn-add').show("fast");
-    var curthis = $(this);
-    var term = $('#TermInput').val();
-    plotted_terms = [term]
-    var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
-    var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
-    var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
-
-    $.getJSON("{{ url_for('terms.terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
-        graph_data = [];
-        var to_plot = [];
-        var curr_data = [];
-        for(i=1; i<data.length; i++) {
-            curr_data.push([data[i][0]*1000, data[i][1]]);
-        }
-        to_plot.push({ data: curr_data, label: term});
-        graph_data.push({ data: curr_data, label: term});
-        plot = $.plot($("#graph"), to_plot, graph_options);
-
-        $("#graph").bind("plothover", function (event, pos, item) {
-            if (item) {
-                var date = new Date(item.datapoint[0]);
-                var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
-                var y = item.datapoint[1];
-
-                $("#tooltip").html(item.series.label + " for "+x + " = " + y)
-                    .css({top: item.pageY-15, left: item.pageX+5})
-                    .fadeIn(200);
-            } else {
-                $("#tooltip").hide();
-            }
-        });
-
-        $("#TermInput").val("");
-    })
-}
-
-
-function addData() {
-    var curthis = $(this);
-    var term = $('#TermInput').val();
-    plotted_terms.push(term)
-    var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
-    var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
-    var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
-
-    $.getJSON("{{ url_for('terms.terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
-        var curr_data = [];
-        for(i=1; i<data.length; i++) {
-                curr_data.push([data[i][0]*1000, data[i][1]]);
-        }
-        graph_data.push({ data: curr_data, label: term});
-        plot = $.plot($("#graph"), graph_data, graph_options);
-        $("#TermInput").val("");
-    })
-}
-
-
-function replot() {
-    graph_data = [];
-    promises = [];
-    for(i=0; i<plotted_terms.length; i++) {
-        var term = plotted_terms[i];
-        var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
-        var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
-        var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
-
-        promises.push($.getJSON("{{ url_for('terms.terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
-            var curr_data = [];
-            for(i=1; i<data.length; i++) {
-                    curr_data.push([data[i][0]*1000, data[i][1]]);
-            }
-            graph_data.push({ data: curr_data, label: data[0]});
-            $("#TermInput").val("");
-        }))
-    }
-
-    $.when.apply($, promises).done( function () {
-        plot = $.plot($("#graph"), graph_data, graph_options);
-    });
-
-}
-
-</script>

var/www/modules/terms/templates/terms_plot_top.html

@@ -1,576 +0,0 @@
-<!DOCTYPE html>
-<html>
-
-<head>
-  <meta charset="utf-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
-  <title>Terms Plot Top - AIL</title>
-  <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
-
-  <!-- Core CSS -->
-  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
-  <link href="{{ url_for('static', filename='css/switch_checkbox.css') }}" rel="stylesheet" type="text/css" />
-  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
-  <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
-  <script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
-
-  <style>
-      .sparkLineStats ul {
-          padding-left:0;
-          list-style:none
-      }
-      .table > tbody > tr > td {
-          padding: 5px;
-      }
-  </style>
-</head>
-<body>
-
-   {% include 'navbar.html' %}
-
-<div id="page-wrapper">
-  <div class="row">
-      <div class="col-lg-12">
-          <h1 class="page-header" data-page="page-termsfrequency" >Terms frequency: Top set information</h1>
-      </div>
-      <!-- /.col-lg-12 -->
-  </div>
-    <!-- /.row -->
-    <div class="row">
-
-    <!-- Panel OPTIONS -->
-    <div class="row">
-        <div class="col-lg-12">
-            <div class="row">
-                <div class="col-lg-12">
-                    <div class='pull_right'>
-                        <label class="switch">
-                            <input id="per_paste" class="switch-input" value="per_paste" type="checkbox" onclick="reload_per_paste()">
-                            <span class="switch-label" data-on="On" data-off="Off"></span>
-                            <span class="switch-handle"></span>
-                        </label>
-                        <strong style="top: 3px; position: relative;">1 term per paste</strong>
-                    <div>
-                    <div id="panel-today" class="panel panel-info">
-                        <div class="panel-heading">
-                            <strong>Today</strong>
-                        </div>
-                        <div class="panel-body">
-                            <div id="panel-today" class="panel panel-default">
-                                <div class="panel-heading">
-                                    <strong>Today top word</strong>
-                                </div>
-                                <div class="panel-body">
-                                    <div class="col-lg-6">
-                                    <table class="table table-striped">
-                                        <thead>
-                                            <tr>
-                                                <th>Term</th>
-                                                <th>Value</th>
-                                                <th>Action</th>
-                                                <th>Show</th>
-                                                <th data-toggle="tooltip" data-placement="left" title="Position of the term in the week and month set">Position</th>
-                                            </tr>
-                                        </thead>
-                                        <tbody id="table-today">
-                                        </tbody>
-                                    </table>
-                                    </div>
-                                    <div class="col-lg-6">
-                                    <table class="table table-striped">
-                                        <thead>
-                                            <tr>
-                                                <th>Term</th>
-                                                <th>Value</th>
-                                                <th>Action</th>
-                                                <th>Show</th>
-                                                <th data-toggle="tooltip" data-placement="left" title="Position of the term in the week and month set">Position</th>
-                                            </tr>
-                                        </thead>
-                                        <tbody id="table-today2">
-                                        </tbody>
-                                    </table>
-                                    </div>
-                                </div>
-                                <!-- /.panel-body -->
-                            </div>
-                            <div id="panel-today" class="panel panel-default">
-                                <div class="panel-heading">
-                                    <strong>Graph</strong>
-                                </div>
-                                <div class="panel-body">
-                                    <div id="graph-today" style="height: 400px;"></div>
-                                </div>
-                                <!-- /.panel-body -->
-                            </div>
-
-                        </div>
-                        <!-- /.panel-body -->
-                    </div>
-                </div>
-                <!-- /.panel -->
-            </div>
-            <!-- /.panel -->
-        </div>
-    </div>
-
-    <!-- Panel OPTIONS -->
-    <div class="row">
-        <div class="col-lg-12">
-            <div class="row">
-                <div class="col-lg-12">
-                    <div id="panel-today" class="panel panel-info">
-                        <div class="panel-heading">
-                            <strong>Week</strong>
-                        </div>
-                        <div class="panel-body">
-                            <div id="panel-week" class="panel panel-default">
-                                <div class="panel-heading">
-                                    <strong>Week top word</strong>
-                                </div>
-                                <div class="panel-body">
-                                    <div class="col-lg-6">
-                                     <table class="table table-striped">
-                                        <thead>
-                                            <tr>
-                                                <th>Term</th>
-                                                <th>Value</th>
-                                                <th>Action</th>
-                                                <th>Show</th>
-                                                <th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and month set">Position</th>
-                                            </tr>
-                                        </thead>
-                                        <tbody id="table-week">
-                                        </tbody>
-                                    </table>
-                                    </div>
-                                    <div class="col-lg-6">
-                                    <table class="table table-striped">
-                                        <thead>
-                                            <tr>
-                                                <th>Term</th>
-                                                <th>Value</th>
-                                                <th>Action</th>
-                                                <th>Show</th>
-                                                <th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and month set">Position</th>
-                                            </tr>
-                                        </thead>
-                                        <tbody id="table-week2">
-                                        </tbody>
-                                    </table>
-                                    </div>
-                                </div>
-                                <!-- /.panel-body -->
-                            </div>
-                            <div id="panel-week" class="panel panel-default">
-                                <div class="panel-heading">
-                                    <strong>Graph</strong>
-                                </div>
-                                <div class="panel-body">
-                                    <div id="graph-week" style="height: 400px;"></div>
-                                </div>
-                                <!-- /.panel-body -->
-                            </div>
-
-                        </div>
-                        <!-- /.panel-body -->
-                    </div>
-                </div>
-                <!-- /.panel -->
-            </div>
-            <!-- /.panel -->
-        </div>
-    </div>
-
-    <!-- Panel OPTIONS -->
-    <div class="row">
-        <div class="col-lg-12">
-            <div class="row">
-                <div class="col-lg-12">
-                    <div id="panel-today" class="panel panel-info">
-                        <div class="panel-heading">
-                            <strong>Month</strong>
-                        </div>
-                        <div class="panel-body">
-                            <div id="panel-month" class="panel panel-default">
-                                <div class="panel-heading">
-                                    <strong>Month top word</strong>
-                                </div>
-                                <div class="panel-body">
-                                    <div class="col-lg-6">
-                                     <table class="table table-striped">
-                                        <thead>
-                                            <tr>
-                                                <th>Term</th>
-                                                <th>Value</th>
-                                                <th>Action</th>
-                                                <th>Show</th>
-                                                <th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and week set">Position</th>
-                                            </tr>
-                                        </thead>
-                                        <tbody id="table-month">
-                                        </tbody>
-                                    </table>
-                                    </div>
-                                    <div class="col-lg-6">
-                                    <table class="table table-striped">
-                                        <thead>
-                                            <tr>
-                                                <th>Term</th>
-                                                <th>Value</th>
-                                                <th>Action</th>
-                                                <th>Show</th>
-                                                <th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and week set">Position</th>
-                                            </tr>
-                                        </thead>
-                                        <tbody id="table-month2">
-                                        </tbody>
-                                    </table>
-                                    </div>
-
-                                </div>
-                                <!-- /.panel-body -->
-                            </div>
-                            <div id="panel-month" class="panel panel-default">
-                                <div class="panel-heading">
-                                    <strong>Graph</strong>
-                                </div>
-                                <div class="panel-body">
-                                    <div id="graph-month" style="height: 400px;"></div>
-                                </div>
-                                <!-- /.panel-body -->
-                            </div>
-
-                        </div>
-                        <!-- /.panel-body -->
-                    </div>
-                </div>
-                <!-- /.panel -->
-            </div>
-            <!-- /.panel -->
-        </div>
-    </div>
-
-
-
-    <!-- /.row -->
-    </div>
-<!-- /#page-wrapper -->
-</div>
-
-<div style="position: absolute; border: 1px solid rgb(255, 221, 221); padding: 2px; background-color: #333; color:white; opacity: 0.8; top: 423px; left: 616px; display: none;" id="tooltip"></div>
-
-<!-- import graph function -->
-<script>
-    $(document).ready(function(){
-        activePage = $('h1.page-header').attr('data-page');
-        $("#"+activePage).addClass("active");
-        if({{ per_paste }} ==  1) {
-            $("#per_paste").attr('checked', true)
-        }
-   });
-
-    function reload_per_paste() {
-        var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
-        window.location.href = {{ url_for('terms.terms_plot_top') }}+"?per_paste="+checked;
-    }
-</script>
-
-
-<script>
-var graph_options = {
-                        series: {
-                            lines: {
-                                show: true,
-                                lineWidth: 2
-                            },
-                            bars: {show: false, barWidth: 60*60*1000},
-                            shadowSize: 0
-                        },
-                        grid: {
-                            hoverable: true,
-                            clickable: true,
-                            tickColor: "#f9f9f9",
-                            borderWidth: 0
-                        },
-                        legend: { show: true,
-                                  noColumns: 1,
-                                  position: "nw",
-                                  labelFormatter: function(label, series) {
-                                      return "<a href=\"#\" onClick=\"hide_or_show2("+series.idx+", "+series.graphNum+"); return false;\" >" + label + "</a>";
-                                  }
-                        },
-                        xaxis: {
-                              mode: "time",
-                              timeformat: "%m/%d",
-                              minTickSize: [1, "day"]
-                        },
-                        yaxis: {
-                            //transform: function (v) { return v < 1 ? v : Math.log(v); }
-                            autoscaleMargin: 0.1,
-                        },
-                        tooltip: true,
-                        tooltipOpts: {
-                            content: " %s (%x.1 is %y.4) ",
-                            shifts: {
-                                x: -60,
-                                y: 25
-                            }
-                        }
-                    }
-
-
-set_today = "TopTermFreq_set_day";
-set_week = "TopTermFreq_set_week";
-set_month = "TopTermFreq_set_month";
-default_num_curves = 8;
-per_paste = {{ per_paste }}
-
-var plot_today;
-var plot_week;
-var plot_month;
-
-var promises = []; // Used to know when everything has been received
-
-promises.push($.getJSON("{{ url_for('terms.terms_plot_top_data') }}", { set: set_today, num_day: 5, per_paste: per_paste }, function(data, status){
-    data.sort(function(a, b){return b[2]-a[2];});
-    // Sort data
-    var table_today = $("#table-today")
-    var table_today2 = $("#table-today2")
-    var to_plot = [];
-    var unchecked_label = [];
-    for(i=0; i<data.length; i++) {
-        var highlight = data[i][3].week == "<20"? " style=\"background-color: lightgreen;\" " : "";
-        var curr_data = [];
-        for(j=0; j<data[i][1].length; j++) {
-            curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
-        }
-        if (i>=default_num_curves) {
-            unchecked_label.push(data[i][0]);
-        }
-        to_plot.push({ data: curr_data, label: data[i][0], idx: i});
-        if ( i < (data.length/2))
-            table_today.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("today", data[i][0], i<default_num_curves)+"</td><td>"+data[i][3].week+", "+data[i][3].month+"</td></tr>");
-        else
-            table_today2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("today", data[i][0], i<default_num_curves)+"</td><td>"+data[i][3].week+", "+data[i][3].month+"</td></tr>");
-    }
-    graph_options.series.graphNum=1;
-    plot_today = $.plot($("#graph-today"), to_plot, graph_options);
-    hide_unchecked_curves(plot_today, unchecked_label);
-    $("#graph-today").bind("plothover", function (event, pos, item) {
-        if (item) {
-            var date = new Date(item.datapoint[0]);
-            var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
-            var y = item.datapoint[1];
-
-            $("#tooltip").html(item.series.label + " for "+x + " = " + y)
-                .css({top: item.pageY+5, left: item.pageX+5})
-                .fadeIn(200);
-        } else {
-            $("#tooltip").hide();
-        }
-    });
-
-}));
-
-promises.push($.getJSON("{{ url_for('terms.terms_plot_top_data') }}", { set: set_week, num_day: 7, per_paste: per_paste }, function(data, status){
-    data.sort(function(a, b){return b[2]-a[2];});
-    // Sort data
-    var table = $("#table-week")
-    var table2 = $("#table-week2")
-    var to_plot = [];
-    var unchecked_label = [];
-    for(i=0; i<data.length; i++) {
-        var highlight = parseInt(data[i][3].day) > 20? " style=\"background-color: orange;\" " : "";
-        var curr_data = [];
-        for(j=0; j<data[i][1].length; j++) {
-            curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
-        }
-        if (i>=default_num_curves) {
-            unchecked_label.push(data[i][0]);
-        }
-        to_plot.push({ data: curr_data, label: data[i][0], idx: i});
-        if ( i < (data.length/2))
-            table.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("week", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].month+"</td></tr>");
-        else
-            table2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("week", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].month+"</td></tr>");
-    }
-    graph_options.series.graphNum=2;
-    plot_week = $.plot($("#graph-week"), to_plot, graph_options);
-    hide_unchecked_curves(plot_week, unchecked_label);
-    $("#graph-week").bind("plothover", function (event, pos, item) {
-        if (item) {
-            var date = new Date(item.datapoint[0]);
-            var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
-            var y = item.datapoint[1];
-
-            $("#tooltip").html(item.series.label + " for "+x + " = " + y)
-                .css({top: item.pageY+5, left: item.pageX+5})
-                .fadeIn(200);
-        } else {
-            $("#tooltip").hide();
-        }
-    });
-}));
-
-promises.push($.getJSON("{{ url_for('terms.terms_plot_top_data') }}", { set: set_month, num_day: 31, per_paste: per_paste }, function(data, status){
-    data.sort(function(a, b){return b[2]-a[2];});
-    // Sort data
-    var table = $("#table-month")
-    var table2 = $("#table-month2")
-    var to_plot = [];
-    var unchecked_label = [];
-    for(i=0; i<data.length; i++) {
-        var highlight = parseInt(data[i][3].day) > 20? " style=\"background-color: orange;\" " : "";
-        var curr_data = [];
-        for(j=0; j<data[i][1].length; j++) {
-            curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
-        }
-        if (i>=default_num_curves) {
-            unchecked_label.push(data[i][0]);
-        }
-        to_plot.push({ data: curr_data, label: data[i][0], idx: i});
-        if ( i < (data.length/2))
-            table.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("month", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].week+"</td></tr>");
-        else
-            table2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("month", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].week+"</td></tr>");
-    }
-    graph_options.series.graphNum=3;
-    plot_month = $.plot($("#graph-month"), to_plot, graph_options);
-    hide_unchecked_curves(plot_month, unchecked_label);
-    $("#graph-month").bind("plothover", function (event, pos, item) {
-        if (item) {
-            var date = new Date(item.datapoint[0]);
-            var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
-            var y = item.datapoint[1];
-
-            $("#tooltip").html(item.series.label + " for "+x + " = " + y)
-                .css({top: item.pageY+5, left: item.pageX+5})
-                .fadeIn(200);
-        } else {
-            $("#tooltip").hide();
-        }
-    });
-}));
-
-/* When everything has been received, start adding tooltip */
-$.when.apply($, promises).done( function (arg) {
-    $('[data-toggle="tooltip"]').tooltip();
-
-    $(".btn-interaction").unbind("click.interaction");
-    $(".btn-interaction").bind("click.interaction", perform_operation);
-    $(".check-interaction").unbind("click.interaction");
-    $(".check-interaction").bind("click.interaction", hide_or_show);
-});
-
-
-function addbuttons(term) {
-    return "<button class=\"fa fa-eye btn-interaction btn btn-success\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Add to tracked list\" style=\"margin-right: 5px;\""+
-        "data-section=\"followTerm\" data-term=\""+term+"\"></button>"+
-        "<button class=\"fa fa-eye-slash btn-interaction btn btn-danger\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Add to black-list\""+
-        "data-section=\"blacklistTerm\" data-term=\""+term+"\"></button>";
-}
-
-function addcheckbox(graph, term, checked) {
-    var checked_text = checked ? "checked" : "";
-    return "<input type=checkbox "+checked_text+" class=\"check-interaction\" data-term=\""+term+"\" data-graph=\""+graph+"\"></input>";
-}
-
-function perform_operation(){
-    var curr_section = $(this).attr('data-section');
-    var curr_term = $(this).attr('data-term');
-
-    var data_to_send = { section: curr_section, action:"add", term: curr_term};
-
-    $.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){
-        if(status == "success") {
-            location.reload();
-        }
-    });
-}
-
-
-function hide_unchecked_curves(plot, unchecked_label) {
-    var graphData = plot.getData();
-    var index;
-    for(i=0; i<graphData.length; i++) {
-        if($.inArray( graphData[i].label, unchecked_label ) != -1){
-            graphData[i].lines.show = false;
-        }
-    }
-    plot.setData(graphData);
-    plot.draw();
-}
-
-
-function hide_or_show() {
-    var curr_term = $(this).attr('data-term');
-    var graph = $(this).attr('data-graph');
-    var checked = $(this).prop('checked')
-
-    if(graph == "today") {
-        var graphData = plot_today.getData();
-        var index;
-        for(i=0; i<graphData.length; i++){
-            if(graphData[i].label == curr_term){
-                index = i;
-                break;
-            }
-        }
-        graphData[index].lines.show = checked;
-        plot_today.setData(graphData);
-        plot_today.draw();
-    } else if (graph == "week") {
-        var graphData = plot_week.getData();
-        var index;
-        for(i=0; i<graphData.length; i++){
-            if(graphData[i].label == curr_term){
-                index = i;
-                break;
-            }
-        }
-        graphData[index].lines.show = checked;
-        plot_week.setData(graphData);
-        plot_week.draw();
-    } else if (graph == "month") {
-          var graphData = plot_month.getData();
-        var index;
-        for(i=0; i<graphData.length; i++){
-            if(graphData[i].label == curr_term){
-                index = i;
-                break;
-            }
-        }
-        graphData[index].lines.show = checked;
-        plot_month.setData(graphData);
-        plot_month.draw();
-    }
-
-    // graph, hide curve
-}
-
-function hide_or_show2(index, graphNum) {
-    if (graphNum == 1)
-        var plot = plot_today;
-    else if (graphNum == 2)
-        var plot = plot_week;
-    else if (graphNum == 3)
-        var plot = plot_month;
-
-    var graphData = plot.getData();
-    graphData[index].lines.show = !graphData[index].lines.show;
-    plot.setData(graphData);
-    plot.draw();
-}
-</script>
-
-</body>
-</html>

var/www/templates/modals/create_hive_case.html

@@ -6,9 +6,9 @@
                 <img id="hive-logo" src="{{url_for('static', filename='image/thehive-logo.png') }}" width="500" >
             </div>
 
-            <form method="post" action="{{ url_for('PasteSubmit.create_hive_case') }}" target="_blank">
+            <form method="post" action="{{ url_for('import_export.create_thehive_case') }}" target="_blank">
 
-            <div class="modal-body">
+                <div class="modal-body">
 
                     <div class="input clear required">
                         <label for="EventThreatLevelId">Threat Level</label>
@@ -35,8 +35,9 @@
                         <label for="hive_description">Description</label>
                         <input name="hive_description" class="form-control span6" placeholder="Quick Case Description" type="text" id="hive_description"/>
                     </div>
+                    <input type="hidden" id="obj_type" name="obj_type" value="item">
                     <input type="hidden" id="obj_id" name="obj_id" value="{{ dict_item['id'] }}">
-            </div>
+                </div>
 
             <div class="modal-footer">
                 <button class="btn btn-primary" type="submit">