AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-30 01:37:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [cleanup] remove ARDB + fix hive case

Browse source
This commit is contained in:
Terrtia 2023-01-18 16:28:08 +01:00
parent 0e41c95b5c
commit 3365a054a8
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
42 changed files with 431 additions and 3083 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
bin/DB_KVROCKS_MIGRATION.py
View file

@ -252,6 +252,8 @@ def trackers_migration():
for id in old_Tracker.get_retro_hunt_items_by_daterange(task_uuid, meta['date_from'], meta['date_to']): for id in old_Tracker.get_retro_hunt_items_by_daterange(task_uuid, meta['date_from'], meta['date_to']):
Tracker.save_retro_hunt_match(task_uuid, id) Tracker.save_retro_hunt_match(task_uuid, id)
Tracker._fix_db_custom_tags()
def investigations_migration(): def investigations_migration():
print('INVESTIGATION MIGRATION...') print('INVESTIGATION MIGRATION...')

14
bin/Helper.py
View file

@ -140,12 +140,6 @@ class Process(object):
db=self.config.get('RedisPubSub', 'db'), db=self.config.get('RedisPubSub', 'db'),
decode_responses=True) decode_responses=True)
self.serv_statistics = redis.StrictRedis(
host=self.config.get('ARDB_Statistics', 'host'),
port=self.config.get('ARDB_Statistics', 'port'),
db=self.config.get('ARDB_Statistics', 'db'),
decode_responses=True)
self.moduleNum = os.getpid() self.moduleNum = os.getpid()
def populate_set_in(self): def populate_set_in(self):
@ -181,9 +175,9 @@ class Process(object):
try: try:
if '.gz' in message: if '.gz' in message:
path = message.split(".")[-2].split("/")[-1] path = message.split(".")[-2].split("/")[-1]
#find start of path with AIL_HOME # find start of path with AIL_HOME
index_s = message.find(os.environ['AIL_HOME']) index_s = message.find(os.environ['AIL_HOME'])
#Stop when .gz # Stop when .gz
index_e = message.find(".gz")+3 index_e = message.find(".gz")+3
if(index_s == -1): if(index_s == -1):
complete_path = message[0:index_e] complete_path = message[0:index_e]
@ -200,7 +194,6 @@ class Process(object):
self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum)) self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
curr_date = datetime.date.today() curr_date = datetime.date.today()
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'paste_by_modules_in:'+self.subscriber_name, 1)
return message return message
except: except:
@ -238,6 +231,3 @@ class Process(object):
continue continue
self.pubsub.publish(message) self.pubsub.publish(message)
def incr_module_timeout_statistic(self):
curr_date = datetime.date.today()
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'paste_by_modules_timeout:'+self.subscriber_name, 1)

4
bin/LAUNCH.sh
View file

@ -237,6 +237,8 @@ function launching_scripts {
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Telegram" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Telegram.py; read x" screen -S "Script_AIL" -X screen -t "Telegram" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Telegram.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Tools" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Tools.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Hosts" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Hosts.py; read x" screen -S "Script_AIL" -X screen -t "Hosts" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Hosts.py; read x"
sleep 0.1 sleep 0.1
@ -283,8 +285,6 @@ function launching_scripts {
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x" screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Tools" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tools.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x" screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./IPAddress.py; read x" screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./IPAddress.py; read x"

69
bin/Release.py
View file

@ -1,69 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import time
from lib.objects.Items import Item
from pubsublogger import publisher
from Helper import Process
import re
import signal
class TimeoutException(Exception):
pass
def timeout_handler(signum, frame):
raise TimeoutException
signal.signal(signal.SIGALRM, timeout_handler)
'''
This module takes its input from the global module.
It applies some regex and publish matched content
'''
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
config_section = "Release"
p = Process(config_section)
max_execution_time = p.config.getint("Curve", "max_execution_time")
publisher.info("Release scripts to find release names")
movie = "[a-zA-Z0-9.]+\.[0-9]{4}.[a-zA-Z0-9.]+\-[a-zA-Z]+"
tv = "[a-zA-Z0-9.]+\.S[0-9]{2}E[0-9]{2}.[a-zA-Z0-9.]+\.[a-zA-Z0-9.]+\-[a-zA-Z0-9]+"
xxx = "[a-zA-Z0-9._]+.XXX.[a-zA-Z0-9.]+\-[a-zA-Z0-9]+"
regexs = [movie, tv, xxx]
regex = '|'.join(regexs)
while True:
signal.alarm(max_execution_time)
filepath = p.get_from_set()
if filepath is None:
publisher.debug("Script Release is Idling 10s")
print('Sleeping')
time.sleep(10)
continue
item = Item(filepath)
content = item.get_content()
#signal.alarm(max_execution_time)
try:
releases = set(re.findall(regex, content))
if len(releases) == 0:
continue
to_print = f'Release;{item.get_source()};{item.get_date()};{item.get_basename()};{len(releases)} releases;{item.get_id()}'
print(to_print)
if len(releases) > 30:
publisher.warning(to_print)
else:
publisher.info(to_print)
except TimeoutException:
p.incr_module_timeout_statistic()
print(f"{item.get_id()} processing timeout")
continue
else:
signal.alarm(0)

55
bin/SourceCode.py
View file

@ -1,55 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import time
from lib.objects.Items import Item
from pubsublogger import publisher
from Helper import Process
import re
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
config_section = "SourceCode"
p = Process(config_section)
publisher.info("Finding Source Code")
critical = 0 # AS TO BE IMPORTANT, MIGHT BE REMOVED
# RELEVANT LANGUAGES
shell = r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+\:\~\$"
c = r"\#include\ \<[a-z\/]+.h\>"
php = r"\<\?php"
python = r"import\ [\w]+"
bash = r"#!\/[\w]*\/bash"
javascript = r"function\(\)"
ruby = r"require \ [\w]+"
adr = r"0x[a-f0-9]{2}"
# asm = r"\"((?s).{1}x[0-9a-f]{2}){3,}" ISSUES WITH FINDALL, pattern like \x54\xaf\x23\..
languages = [shell, c, php, bash, python, javascript, bash, ruby, adr]
regex = '|'.join(languages)
print(regex)
while True:
message = p.get_from_set()
if message is None:
publisher.debug("Script Source Code is Idling 10s")
print('Sleeping')
time.sleep(10)
continue
filepath, count = message.split()
item = Item(filepath)
content = item.get_content()
match_set = set(re.findall(regex, content))
if len(match_set) == 0:
continue
to_print = f'SourceCode;{item.get_source()};{item.get_date()};{item.get_basename()};{item.get_id()}'
if len(match_set) > critical:
publisher.warning(to_print)
else:
publisher.info(to_print)

19
bin/export/Export.py
View file

@ -72,6 +72,25 @@ def is_hive_connected():
HIVE_CLIENT = get_hive_client() HIVE_CLIENT = get_hive_client()
def sanitize_threat_level_hive(threat_level):
try:
int(threat_level)
if 1 <= threat_level <= 3:
return threat_level
else:
return 2
except:
return 2
def sanitize_tlp_hive(tlp):
try:
int(tlp)
if 0 <= tlp <= 3:
return tlp
else:
return 2
except:
return 2
def create_thehive_alert(item_id, tag_trigger): def create_thehive_alert(item_id, tag_trigger):
item = Item(item_id) item = Item(item_id)

76
bin/helper/dump_crawler_history.py
View file

@ -1,76 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import gzip
import datetime
import redis
import json
import time
import shutil
sys.path.append(os.environ['AIL_BIN'])
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
from HiddenServices import HiddenServices
from Helper import Process
def substract_date(date_from, date_to):
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
delta = date_to - date_from # timedelta
l_date = []
for i in range(delta.days + 1):
date = date_from + datetime.timedelta(i)
l_date.append( date.strftime('%Y%m%d') )
return l_date
config_section = 'Keys'
p = Process(config_section)
r_serv_onion = redis.StrictRedis(
host=p.config.get("ARDB_Onion", "host"),
port=p.config.getint("ARDB_Onion", "port"),
db=p.config.getint("ARDB_Onion", "db"),
decode_responses=True)
date_from = '20190614'
date_to = '20190615'
service_type = 'onion'
date_range = substract_date(date_from, date_to)
dir_path = os.path.join(os.environ['AIL_HOME'], 'temp')
domain_skipped = []
for date in date_range:
domains_up = list(r_serv_onion.smembers('{}_up:{}'.format(service_type, date)))
if domains_up:
save_path = os.path.join(dir_path, date[0:4], date[4:6], date[6:8])
try:
os.makedirs(save_path)
except FileExistsError:
pass
for domain in domains_up:
print(domain)
h = HiddenServices(domain, 'onion')
item_core = h.get_domain_crawled_core_item()
if 'root_item' in item_core:
l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item'])
try:
res = h.create_domain_basic_archive(l_pastes)
filename = os.path.join(save_path, '{}'.format(domain))
with open(filename, 'wb') as f:
shutil.copyfileobj(res, f)
print('done')
except Exception as e:
print('skipped')
domain_skipped.append(domain)
pass
print()
print()
print('DOMAINS SKIPPED: ')
for domain in domain_skipped:
print(domain)

50
bin/helper/reprocess_bitcoin.py
View file

@ -1,50 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import gzip
import base64
import uuid
import datetime
import base64
import redis
import json
import time
sys.path.append(os.environ['AIL_BIN'])
from Helper import Process
def substract_date(date_from, date_to):
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
delta = date_to - date_from # timedelta
l_date = []
for i in range(delta.days + 1):
date = date_from + datetime.timedelta(i)
l_date.append( date.strftime('%Y%m%d') )
return l_date
config_section = 'Global'
p = Process(config_section)
r_tags = redis.StrictRedis(
host=p.config.get("ARDB_Tags", "host"),
port=p.config.getint("ARDB_Tags", "port"),
db=p.config.getint("ARDB_Tags", "db"),
decode_responses=True)
tag = 'infoleak:automatic-detection="bitcoin-address"'
# get tag first/last seen
first_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'first_seen')
last_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
l_dates = substract_date(first_seen, last_seen)
# get all tagged items
for date in l_dates:
daily_tagged_items = r_tags.smembers('{}:{}'.format(tag, date))
for item in daily_tagged_items:
p.populate_set_out(item)

50
bin/helper/reprocess_pgp_message.py
View file

@ -1,50 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import sys
import gzip
import base64
import uuid
import datetime
import base64
import redis
import json
import time
sys.path.append(os.environ['AIL_BIN'])
from Helper import Process
def substract_date(date_from, date_to):
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
delta = date_to - date_from # timedelta
l_date = []
for i in range(delta.days + 1):
date = date_from + datetime.timedelta(i)
l_date.append( date.strftime('%Y%m%d') )
return l_date
config_section = 'Keys'
p = Process(config_section)
r_tags = redis.StrictRedis(
host=p.config.get("ARDB_Tags", "host"),
port=p.config.getint("ARDB_Tags", "port"),
db=p.config.getint("ARDB_Tags", "db"),
decode_responses=True)
tag = 'infoleak:automatic-detection="pgp-message"'
# get tag first/last seen
first_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'first_seen')
last_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
l_dates = substract_date(first_seen, last_seen)
# get all tagged items
for date in l_dates:
daily_tagged_items = r_tags.smembers('{}:{}'.format(tag, date))
for item in daily_tagged_items:
p.populate_set_out(item, 'PgpDump')

1
bin/import/importer.py
View file

@ -10,7 +10,6 @@ Recieve Json Items (example: Twitter feeder)
import os import os
import importlib import importlib
import json import json
import redis
import sys import sys
import time import time

2
bin/lib/Config_DB.py
View file

@ -15,7 +15,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
from lib import ConfigLoader from lib import ConfigLoader
config_loader = ConfigLoader.ConfigLoader() config_loader = ConfigLoader.ConfigLoader()
r_serv_db = config_loader.get_redis_conn("ARDB_DB") r_serv_db = config_loader.get_redis_conn("_DB")
config_loader = None config_loader = None
#### TO PUT IN CONFIG #### TO PUT IN CONFIG

4
bin/lib/MispModules.py
View file

@ -3,6 +3,7 @@
import os import os
import json import json
import requests import requests
import sys
import configparser import configparser
misp_module_url = 'http://localhost:6666' misp_module_url = 'http://localhost:6666'
@ -16,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader from lib import ConfigLoader
config_loader = ConfigLoader.ConfigLoader() config_loader = ConfigLoader.ConfigLoader()
r_serv = config_loader.get_redis_conn("ARDB_DB") r_serv = config_loader.get_redis_conn("_DB")
config_loader = None config_loader = None
def init_config(config_path=default_config_path): def init_config(config_path=default_config_path):
@ -114,6 +115,7 @@ def parse_module_enrichment_response(misp_module_response):
#print(response_types) #print(response_types)
return response_values return response_values
if __name__ == "__main__": if __name__ == "__main__":
load_modules_list() load_modules_list()

5
bin/lib/Statistics.py
View file

@ -12,8 +12,7 @@ sys.path.append(os.environ['AIL_BIN'])
from lib import ConfigLoader from lib import ConfigLoader
config_loader = ConfigLoader.ConfigLoader() config_loader = ConfigLoader.ConfigLoader()
r_statistics = config_loader.get_redis_conn("ARDB_Statistics") r_statistics = config_loader.get_db_conn("Kvrocks_Stats")
# r_serv_trend = ConfigLoader().get_redis_conn("ARDB_Trending")
config_loader = None config_loader = None
PIE_CHART_MAX_CARDINALITY = 8 PIE_CHART_MAX_CARDINALITY = 8
@ -117,7 +116,7 @@ def update_module_stats(module_name, num, keyword, date):
member_set = r_statistics.zrangebyscore(f'top_{module_name}_set_{date}', '-inf', '+inf', withscores=True, start=0, num=1) member_set = r_statistics.zrangebyscore(f'top_{module_name}_set_{date}', '-inf', '+inf', withscores=True, start=0, num=1)
# Member set is a list of (value, score) pairs # Member set is a list of (value, score) pairs
if int(member_set[0][1]) < keyword_total_sum: if int(member_set[0][1]) < keyword_total_sum:
#remove min from set and add the new one # remove min from set and add the new one
r_statistics.zrem(f'top_{module_name}_set_{date}', member_set[0][0]) r_statistics.zrem(f'top_{module_name}_set_{date}', member_set[0][0])
r_statistics.zadd(f'top_{module_name}_set_{date}', {keyword: float(keyword_total_sum)}) r_statistics.zadd(f'top_{module_name}_set_{date}', {keyword: float(keyword_total_sum)})

70
bin/lib/Tag.py
View file

@ -620,7 +620,7 @@ def update_tag_metadata(tag, date, delete=False): # # TODO: delete Tags
# old # old
# r_tags.smembers(f'{tag}:{date}') # r_tags.smembers(f'{tag}:{date}')
# r_tags.smembers(f'{obj_type}:{tag}') # r_tags.smembers(f'{obj_type}:{tag}')
def get_tag_objects(obj_type, subtype='', date=''): def get_tag_objects(tag, obj_type, subtype='', date=''):
if obj_type == 'item': if obj_type == 'item':
return r_tags.smembers(f'{obj_type}:{subtype}:{tag}:{date}') return r_tags.smembers(f'{obj_type}:{subtype}:{tag}:{date}')
else: else:
@ -1079,6 +1079,10 @@ def get_modal_add_tags(object_id, object_type='item', object_subtype=''):
return {"active_taxonomies": get_active_taxonomies(), "active_galaxies": get_active_galaxies(), return {"active_taxonomies": get_active_taxonomies(), "active_galaxies": get_active_galaxies(),
"object_id": object_id, "object_type": object_type, "object_subtype": object_subtype} "object_id": object_id, "object_type": object_type, "object_subtype": object_subtype}
#####################################################################################
#####################################################################################
#####################################################################################
######## NEW VERSION ######## ######## NEW VERSION ########
def create_custom_tag(tag): def create_custom_tag(tag):
r_tags.sadd('tags:custom', tag) r_tags.sadd('tags:custom', tag)
@ -1137,28 +1141,45 @@ def get_enabled_tags_with_synonyms_ui():
## Objects tags ## ## Objects tags ##
###################################################################################
###################################################################################
###################################################################################
###################################################################################
################################################################################### ###################################################################################
################################################################################### ###################################################################################
################################################################################### ###################################################################################
################################################################################### ###################################################################################
################################################################################### ###################################################################################
def add_global_tag(tag, object_type=None): # TYPE -> taxonomy/galaxy/custom
'''
Create a set of all tags used in AIL (all + by object)
:param tag: tag class Tag:
:type tag: str
:param object_type: object type def __int__(self, t_type, t_id, obj='item'):
:type object_type: str self.type = t_type
''' self.id = t_id
r_tags.sadd('list_tags', tag) self.obj = obj
if object_type:
r_tags.sadd('list_tags:{}'.format(object_type), tag) def get_first_seen(self):
pass
def get_last_seen(self):
pass
def get_color(self):
pass
def is_enabled(self):
pass
def get_meta(self):
meta = {'first_seen': self.get_first_seen(),
'last_seen': self.get_last_seen(),
'obj': self.obj,
'tag': self.id,
'type': self.type}
###################################################################################
###################################################################################
###################################################################################
###################################################################################
def add_obj_tags(object_id, object_subtype, object_type, tags=[], galaxy_tags=[]): def add_obj_tags(object_id, object_subtype, object_type, tags=[], galaxy_tags=[]):
for tag in tags: for tag in tags:
@ -1201,23 +1222,6 @@ def api_add_obj_tags(tags=[], galaxy_tags=[], object_id=None, object_type="item"
res_dict['type'] = object_type res_dict['type'] = object_type
return res_dict, 200 return res_dict, 200
# def add_tag(object_type, tag, object_id, obj_date=None):
# # new tag
# if not is_obj_tagged(object_id, tag):
# # # TODO: # FIXME: sanitize object_type
# if obj_date:
# try:
# obj_date = int(obj_date)
# except:
# obj_date = None
# if not obj_date:
# obj_date = get_obj_date(object_type, object_id)
# add_global_tag(tag, object_type=object_type)
# add_obj_tag(object_type, object_id, tag, obj_date=obj_date)
# update_tag_metadata(tag, obj_date, object_type=object_type)
#
# # create tags stats # # TODO: put me in cache
# r_tags.hincrby('daily_tags:{}'.format(datetime.date.today().strftime("%Y%m%d")), tag, 1)
# def delete_obj_tag(object_type, object_id, tag, obj_date): # def delete_obj_tag(object_type, object_id, tag, obj_date):
# if object_type=="item": # # TODO: # FIXME: # REVIEW: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! # if object_type=="item": # # TODO: # FIXME: # REVIEW: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

15
bin/lib/Tracker.py
View file

@ -551,9 +551,11 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti
# create tracker tags list # create tracker tags list
for tag in tags: for tag in tags:
r_serv_tracker.sadd(f'tracker:tags:{tracker_uuid}', escape(tag)) tag = escape(tag)
r_serv_tracker.sadd(f'tracker:tags:{tracker_uuid}', tag)
Tag.create_custom_tag(tag)
# create tracker tags mail notification list # create tracker mail notification list
for mail in mails: for mail in mails:
r_serv_tracker.sadd(f'tracker:mail:{tracker_uuid}', escape(mail)) r_serv_tracker.sadd(f'tracker:mail:{tracker_uuid}', escape(mail))
@ -1030,7 +1032,9 @@ def create_retro_hunt_task(name, rule, date_from, date_to, creator, sources=[],
for source in sources: for source in sources:
r_serv_tracker.sadd(f'tracker:retro_hunt:task:sources:{task_uuid}', escape(source)) r_serv_tracker.sadd(f'tracker:retro_hunt:task:sources:{task_uuid}', escape(source))
for tag in tags: for tag in tags:
r_serv_tracker.sadd(f'tracker:retro_hunt:task:tags:{task_uuid}', escape(tag)) tag = escape(tag)
r_serv_tracker.sadd(f'tracker:retro_hunt:task:tags:{task_uuid}', tag)
Tag.create_custom_tag(tag)
for mail in mails: for mail in mails:
r_serv_tracker.sadd(f'tracker:retro_hunt:task:mails:{task_uuid}', escape(mail)) r_serv_tracker.sadd(f'tracker:retro_hunt:task:mails:{task_uuid}', escape(mail))
@ -1364,14 +1368,13 @@ def get_trackers_tags():
def _fix_db_custom_tags(): def _fix_db_custom_tags():
for tag in get_trackers_tags(): for tag in get_trackers_tags():
if not Tag.is_taxonomie_tag(tag) and not Tag.is_galaxy_tag(tag): if not Tag.is_taxonomie_tag(tag) and not Tag.is_galaxy_tag(tag):
print(tag)
Tag.create_custom_tag(tag) Tag.create_custom_tag(tag)
#### -- #### #### -- ####
#if __name__ == '__main__': if __name__ == '__main__':
#_fix_db_custom_tags() _fix_db_custom_tags()
# fix_all_tracker_uuid_list() # fix_all_tracker_uuid_list()
# res = get_all_tracker_uuid() # res = get_all_tracker_uuid()
# print(len(res)) # print(len(res))

1
bin/lib/Users.py
View file

@ -17,7 +17,6 @@ from lib.ConfigLoader import ConfigLoader
# Config # Config
config_loader = ConfigLoader() config_loader = ConfigLoader()
# r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_serv_db = config_loader.get_db_conn("Kvrocks_DB") r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
config_loader = None config_loader = None

114
bin/lib/ail_updates.py
View file

@ -10,42 +10,42 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib import Users
config_loader = ConfigLoader() config_loader = ConfigLoader()
#r_serv_db = config_loader.get_redis_conn("Kvrocks_DB") r_db = config_loader.get_redis_conn("Kvrocks_DB")
r_serv = config_loader.get_redis_conn("ARDB_DB")
config_loader = None config_loader = None
BACKGROUND_UPDATES = { BACKGROUND_UPDATES = {
'v1.5':{ 'v1.5': {
'nb_updates': 5, 'nb_updates': 5,
'message': 'Tags and Screenshots' 'message': 'Tags and Screenshots'
}, },
'v2.4':{ 'v2.4': {
'nb_updates': 1, 'nb_updates': 1,
'message': ' Domains Tags and Correlations' 'message': ' Domains Tags and Correlations'
}, },
'v2.6':{ 'v2.6': {
'nb_updates': 1, 'nb_updates': 1,
'message': 'Domains Tags and Correlations' 'message': 'Domains Tags and Correlations'
}, },
'v2.7':{ 'v2.7': {
'nb_updates': 1, 'nb_updates': 1,
'message': 'Domains Tags' 'message': 'Domains Tags'
}, },
'v3.4':{ 'v3.4': {
'nb_updates': 1, 'nb_updates': 1,
'message': 'Domains Languages' 'message': 'Domains Languages'
}, },
'v3.7':{ 'v3.7': {
'nb_updates': 1, 'nb_updates': 1,
'message': 'Trackers first_seen/last_seen' 'message': 'Trackers first_seen/last_seen'
} }
} }
def get_ail_version(): def get_ail_version():
return r_serv.get('ail:version') return r_db.get('ail:version')
def get_ail_float_version(): def get_ail_float_version():
version = get_ail_version() version = get_ail_version()
@ -55,20 +55,23 @@ def get_ail_float_version():
version = 0 version = 0
return version return version
def get_ail_all_updates(date_separator='-'): def get_ail_all_updates(date_separator='-'):
dict_update = r_serv.hgetall('ail:update_date') dict_update = r_db.hgetall('ail:update_date')
if date_separator: if date_separator:
for version in dict_update: for version in dict_update:
u_date = dict_update[version] u_date = dict_update[version]
dict_update[version] = f'{u_date[0:4]}{date_separator}{u_date[4:6]}{date_separator}{u_date[6:8]}' dict_update[version] = f'{u_date[0:4]}{date_separator}{u_date[4:6]}{date_separator}{u_date[6:8]}'
return dict_update return dict_update
def add_ail_update(version): def add_ail_update(version):
#Add new AIL version # Add new AIL version
r_serv.hset('ail:update_date', version, datetime.datetime.now().strftime("%Y%m%d")) r_db.hset('ail:update_date', version, datetime.datetime.now().strftime("%Y%m%d"))
#Set current ail version # Set current ail version
if float(version[1:]) > get_ail_float_version(): if float(version[1:]) > get_ail_float_version():
r_serv.set('ail:version', version) r_db.set('ail:version', version)
def check_version(version): def check_version(version):
if version[0] != 'v': if version[0] != 'v':
@ -83,75 +86,94 @@ def check_version(version):
return False return False
return True return True
#### UPDATE BACKGROUND #### #### UPDATE BACKGROUND ####
def exits_background_update_to_launch(): def exits_background_update_to_launch():
return r_serv.scard('ail:update:to_update') != 0 return r_db.scard('ail:update:to_update') != 0
def is_version_in_background_update(version): def is_version_in_background_update(version):
return r_serv.sismember('ail:update:to_update', version) return r_db.sismember('ail:update:to_update', version)
def get_all_background_updates_to_launch(): def get_all_background_updates_to_launch():
return r_serv.smembers('ail:update:to_update') return r_db.smembers('ail:update:to_update')
def get_current_background_update(): def get_current_background_update():
return r_serv.get('ail:update:update_in_progress') return r_db.get('ail:update:update_in_progress')
def get_current_background_update_script(): def get_current_background_update_script():
return r_serv.get('ail:update:current_background_script') return r_db.get('ail:update:current_background_script')
def get_current_background_update_script_path(version, script_name): def get_current_background_update_script_path(version, script_name):
return os.path.join(os.environ['AIL_HOME'], 'update', version, script_name) return os.path.join(os.environ['AIL_HOME'], 'update', version, script_name)
def get_current_background_nb_update_completed(): def get_current_background_nb_update_completed():
return r_serv.scard('ail:update:update_in_progress:completed') return r_db.scard('ail:update:update_in_progress:completed')
def get_current_background_update_progress(): def get_current_background_update_progress():
progress = r_serv.get('ail:update:current_background_script_stat') progress = r_db.get('ail:update:current_background_script_stat')
if not progress: if not progress:
progress = 0 progress = 0
return int(progress) return int(progress)
def get_background_update_error(): def get_background_update_error():
return r_serv.get('ail:update:error') return r_db.get('ail:update:error')
def add_background_updates_to_launch(version): def add_background_updates_to_launch(version):
return r_serv.sadd('ail:update:to_update', version) return r_db.sadd('ail:update:to_update', version)
def start_background_update(version): def start_background_update(version):
r_serv.delete('ail:update:error') r_db.delete('ail:update:error')
r_serv.set('ail:update:update_in_progress', version) r_db.set('ail:update:update_in_progress', version)
def set_current_background_update_script(script_name): def set_current_background_update_script(script_name):
r_serv.set('ail:update:current_background_script', script_name) r_db.set('ail:update:current_background_script', script_name)
r_serv.set('ail:update:current_background_script_stat', 0) r_db.set('ail:update:current_background_script_stat', 0)
def set_current_background_update_progress(progress): def set_current_background_update_progress(progress):
r_serv.set('ail:update:current_background_script_stat', progress) r_db.set('ail:update:current_background_script_stat', progress)
def set_background_update_error(error): def set_background_update_error(error):
r_serv.set('ail:update:error', error) r_db.set('ail:update:error', error)
def end_background_update_script(): def end_background_update_script():
r_serv.sadd('ail:update:update_in_progress:completed') r_db.sadd('ail:update:update_in_progress:completed')
def end_background_update(version): def end_background_update(version):
r_serv.delete('ail:update:update_in_progress') r_db.delete('ail:update:update_in_progress')
r_serv.delete('ail:update:current_background_script') r_db.delete('ail:update:current_background_script')
r_serv.delete('ail:update:current_background_script_stat') r_db.delete('ail:update:current_background_script_stat')
r_serv.delete('ail:update:update_in_progress:completed') r_db.delete('ail:update:update_in_progress:completed')
r_serv.srem('ail:update:to_update', version) r_db.srem('ail:update:to_update', version)
def clear_background_update(): def clear_background_update():
r_serv.delete('ail:update:error') r_db.delete('ail:update:error')
r_serv.delete('ail:update:update_in_progress') r_db.delete('ail:update:update_in_progress')
r_serv.delete('ail:update:current_background_script') r_db.delete('ail:update:current_background_script')
r_serv.delete('ail:update:current_background_script_stat') r_db.delete('ail:update:current_background_script_stat')
r_serv.delete('ail:update:update_in_progress:completed') r_db.delete('ail:update:update_in_progress:completed')
def get_update_background_message(version): def get_update_background_message(version):
return BACKGROUND_UPDATES[version]['message'] return BACKGROUND_UPDATES[version]['message']
# TODO: Detect error in subprocess # TODO: Detect error in subprocess
def get_update_background_metadata(): def get_update_background_metadata():
dict_update = {} dict_update = {}
@ -162,7 +184,7 @@ def get_update_background_metadata():
dict_update['script_progress'] = get_current_background_update_progress() dict_update['script_progress'] = get_current_background_update_progress()
dict_update['nb_update'] = BACKGROUND_UPDATES[dict_update['version']]['nb_updates'] dict_update['nb_update'] = BACKGROUND_UPDATES[dict_update['version']]['nb_updates']
dict_update['nb_completed'] = get_current_background_nb_update_completed() dict_update['nb_completed'] = get_current_background_nb_update_completed()
dict_update['progress'] = int(dict_update['nb_completed']*100/dict_update['nb_update']) dict_update['progress'] = int(dict_update['nb_completed'] * 100 / dict_update['nb_update'])
dict_update['error'] = get_background_update_error() dict_update['error'] = get_background_update_error()
return dict_update return dict_update
@ -170,14 +192,6 @@ def get_update_background_metadata():
##-- UPDATE BACKGROUND --## ##-- UPDATE BACKGROUND --##
##########################
if __name__ == '__main__': if __name__ == '__main__':
res = check_version('v3.1..1') res = check_version('v3.1..1')
print(res) print(res)

1
bin/lib/objects/Decodeds.py
View file

@ -36,7 +36,6 @@ except:
config_loader = ConfigLoader() config_loader = ConfigLoader()
r_objects = config_loader.get_db_conn("Kvrocks_Objects") r_objects = config_loader.get_db_conn("Kvrocks_Objects")
r_metadata = config_loader.get_redis_conn("ARDB_Metadata")
HASH_DIR = config_loader.get_config_str('Directories', 'hash') HASH_DIR = config_loader.get_config_str('Directories', 'hash')
baseurl = config_loader.get_config_str("Notifications", "ail_domain") baseurl = config_loader.get_config_str("Notifications", "ail_domain")
config_loader = None config_loader = None

28
bin/lib/objects/abstract_subtype_object.py
View file

@ -8,7 +8,6 @@ Base Class for AIL Objects
################################## ##################################
import os import os
import sys import sys
from abc import abstractmethod
# from flask import url_for # from flask import url_for
@ -25,7 +24,6 @@ from packages import Date
# LOAD CONFIG # LOAD CONFIG
config_loader = ConfigLoader() config_loader = ConfigLoader()
r_metadata = config_loader.get_redis_conn("ARDB_Metadata")
r_object = config_loader.get_db_conn("Kvrocks_Objects") r_object = config_loader.get_db_conn("Kvrocks_Objects")
config_loader = None config_loader = None
@ -51,13 +49,6 @@ class AbstractSubtypeObject(AbstractObject):
def exists(self): def exists(self):
return r_object.exists(f'meta:{self.type}:{self.subtype}:{self.id}') return r_object.exists(f'meta:{self.type}:{self.subtype}:{self.id}')
# def exists(self):
# res = r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id)
# if res is not None:
# return True
# else:
# return False
def get_first_seen(self, r_int=False): def get_first_seen(self, r_int=False):
first_seen = r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'first_seen') first_seen = r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'first_seen')
if r_int: if r_int:
@ -79,11 +70,11 @@ class AbstractSubtypeObject(AbstractObject):
return last_seen return last_seen
def get_nb_seen(self): def get_nb_seen(self):
return int(r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id)) return int(r_object.zscore(f'{self.type}_all:{self.subtype}', self.id))
# # TODO: CHECK RESULT # # TODO: CHECK RESULT
def get_nb_seen_by_date(self, date_day): def get_nb_seen_by_date(self, date_day):
nb = r_metadata.hget(f'{self.type}:{self.subtype}:{date_day}', self.id) nb = r_object.hget(f'{self.type}:{self.subtype}:{date_day}', self.id)
if nb is None: if nb is None:
return 0 return 0
else: else:
@ -134,9 +125,9 @@ class AbstractSubtypeObject(AbstractObject):
self.update_daterange(date) self.update_daterange(date)
update_obj_date(date, self.type, self.subtype) update_obj_date(date, self.type, self.subtype)
# daily # daily
r_metadata.hincrby(f'{self.type}:{self.subtype}:{date}', self.id, 1) r_object.hincrby(f'{self.type}:{self.subtype}:{date}', self.id, 1)
# all subtypes # all subtypes
r_metadata.zincrby(f'{self.type}_all:{self.subtype}', 1, self.id) r_object.zincrby(f'{self.type}_all:{self.subtype}', 1, self.id)
####################################################################### #######################################################################
####################################################################### #######################################################################
@ -158,14 +149,5 @@ class AbstractSubtypeObject(AbstractObject):
def _delete(self): def _delete(self):
pass pass
####################################
#
# _get_items
# get_metadata
#
#
def get_all_id(obj_type, subtype): def get_all_id(obj_type, subtype):
return r_metadata.zrange(f'{obj_type}_all:{subtype}', 0, -1) return r_object.zrange(f'{obj_type}_all:{subtype}', 0, -1)

3
bin/modules/Credential.py
View file

@ -69,8 +69,7 @@ class Credential(AbstractModule):
# Database # Database
config_loader = ConfigLoader.ConfigLoader() config_loader = ConfigLoader.ConfigLoader()
# self.server_cred = config_loader.get_redis_conn("ARDB_TermCred") # self.server_cred = config_loader.get_redis_conn("_TermCred")
self.server_statistics = config_loader.get_redis_conn("ARDB_Statistics")
# Config values # Config values
self.minimumLengthThreshold = config_loader.get_config_int("Credential", "minimumLengthThreshold") self.minimumLengthThreshold = config_loader.get_config_int("Credential", "minimumLengthThreshold")

8
bin/modules/Global.py
View file

@ -43,6 +43,7 @@ from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib.data_retention_engine import update_obj_date from lib.data_retention_engine import update_obj_date
from lib import item_basic from lib import item_basic
# from lib import Statistics
class Global(AbstractModule): class Global(AbstractModule):
""" """
@ -52,8 +53,6 @@ class Global(AbstractModule):
def __init__(self): def __init__(self):
super(Global, self).__init__() super(Global, self).__init__()
self.r_stats = ConfigLoader().get_redis_conn("ARDB_Statistics")
self.processed_item = 0 self.processed_item = 0
self.time_last_stats = time.time() self.time_last_stats = time.time()
@ -197,12 +196,13 @@ class Global(AbstractModule):
self.redis_logger.warning(f'Global; Incomplete file: {filename}') self.redis_logger.warning(f'Global; Incomplete file: {filename}')
print(f'Global; Incomplete file: {filename}') print(f'Global; Incomplete file: {filename}')
# save daily stats # save daily stats
self.r_stats.zincrby('module:Global:incomplete_file', 1, datetime.datetime.now().strftime('%Y%m%d')) # self.r_stats.zincrby('module:Global:incomplete_file', 1, datetime.datetime.now().strftime('%Y%m%d'))
# Statistics.
except OSError: except OSError:
self.redis_logger.warning(f'Global; Not a gzipped file: {filename}') self.redis_logger.warning(f'Global; Not a gzipped file: {filename}')
print(f'Global; Not a gzipped file: {filename}') print(f'Global; Not a gzipped file: {filename}')
# save daily stats # save daily stats
self.r_stats.zincrby('module:Global:invalid_file', 1, datetime.datetime.now().strftime('%Y%m%d')) # self.r_stats.zincrby('module:Global:invalid_file', 1, datetime.datetime.now().strftime('%Y%m%d'))
return curr_file_content return curr_file_content

7
bin/modules/LibInjection.py
View file

@ -13,7 +13,6 @@ It tries to identify SQL Injections with libinjection.
import os import os
import sys import sys
import urllib.request
import pylibinjection import pylibinjection
from datetime import datetime from datetime import datetime
@ -28,6 +27,7 @@ sys.path.append(os.environ['AIL_BIN'])
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item from lib.objects.Items import Item
from lib import Statistics
class LibInjection(AbstractModule): class LibInjection(AbstractModule):
"""docstring for LibInjection module.""" """docstring for LibInjection module."""
@ -37,9 +37,6 @@ class LibInjection(AbstractModule):
self.faup = Faup() self.faup = Faup()
config_loader = ConfigLoader()
self.server_statistics = config_loader.get_redis_conn("ARDB_Statistics")
self.redis_logger.info(f"Module: {self.module_name} Launched") self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, message): def compute(self, message):
@ -94,7 +91,7 @@ class LibInjection(AbstractModule):
tld = url_parsed['tld'] tld = url_parsed['tld']
if tld is not None: if tld is not None:
date = datetime.now().strftime("%Y%m") date = datetime.now().strftime("%Y%m")
self.server_statistics.hincrby(f'SQLInjection_by_tld:{date}', tld, 1) Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
if __name__ == "__main__": if __name__ == "__main__":

6
bin/modules/Onion.py
View file

@ -34,7 +34,6 @@ class Onion(AbstractModule):
config_loader = ConfigLoader() config_loader = ConfigLoader()
self.r_cache = config_loader.get_redis_conn("Redis_Cache") self.r_cache = config_loader.get_redis_conn("Redis_Cache")
self.r_onion = config_loader.get_redis_conn("ARDB_Onion")
self.pending_seconds = config_loader.get_config_int("Onion", "max_execution_time") self.pending_seconds = config_loader.get_config_int("Onion", "max_execution_time")
# regex timeout # regex timeout
@ -91,7 +90,7 @@ class Onion(AbstractModule):
if onion_urls: if onion_urls:
if crawlers.is_crawler_activated(): if crawlers.is_crawler_activated():
for domain in domains:# TODO LOAD DEFAULT SCREENSHOT + HAR for domain in domains: # TODO LOAD DEFAULT SCREENSHOT + HAR
task_uuid = crawlers.add_crawler_task(domain, parent=item.get_id()) task_uuid = crawlers.add_crawler_task(domain, parent=item.get_id())
if task_uuid: if task_uuid:
print(f'{domain} added to crawler queue: {task_uuid}') print(f'{domain} added to crawler queue: {task_uuid}')
@ -109,6 +108,3 @@ if __name__ == "__main__":
module = Onion() module = Onion()
# module.compute('submitted/2022/10/10/submitted_705d1d92-7e9a-4a44-8c21-ccd167bfb7db.gz 9') # module.compute('submitted/2022/10/10/submitted_705d1d92-7e9a-4a44-8c21-ccd167bfb7db.gz 9')
module.run() module.run()
# 5ajw6aqf3ep7sijnscdzw77t7xq4xjpsy335yb2wiwgouo7yfxtjlmid.onion to debian.org

6
bin/modules/SQLInjectionDetection.py
View file

@ -27,6 +27,7 @@ sys.path.append(os.environ['AIL_BIN'])
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item from lib.objects.Items import Item
from lib import Statistics
class SQLInjectionDetection(AbstractModule): class SQLInjectionDetection(AbstractModule):
"""docstring for SQLInjectionDetection module.""" """docstring for SQLInjectionDetection module."""
@ -40,9 +41,6 @@ class SQLInjectionDetection(AbstractModule):
self.faup = Faup() self.faup = Faup()
config_loader = ConfigLoader()
self.server_statistics = config_loader.get_redis_conn("ARDB_Statistics")
self.redis_logger.info(f"Module: {self.module_name} Launched") self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, message): def compute(self, message):
@ -75,7 +73,7 @@ class SQLInjectionDetection(AbstractModule):
except: except:
pass pass
date = datetime.now().strftime("%Y%m") date = datetime.now().strftime("%Y%m")
self.server_statistics.hincrby(f'SQLInjection_by_tld:{date}', tld, 1) Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
# Try to detect if the url passed might be an sql injection by applying the regex # Try to detect if the url passed might be an sql injection by applying the regex
# defined above on it. # defined above on it.

2
bin/modules/SentimentAnalysis.py
View file

@ -62,7 +62,7 @@ class SentimentAnalysis(AbstractModule):
self.sentiment_lexicon_file = ConfigLoader.ConfigLoader().get_config_str("Directories", "sentiment_lexicon_file") self.sentiment_lexicon_file = ConfigLoader.ConfigLoader().get_config_str("Directories", "sentiment_lexicon_file")
# REDIS_LEVEL_DB # # REDIS_LEVEL_DB #
self.db = ConfigLoader.ConfigLoader().get_redis_conn("ARDB_Sentiment") self.db = ConfigLoader.ConfigLoader().get_redis_conn("_Sentiment")
self.time1 = time.time() self.time1 = time.time()

253
bin/packages/config.cfg.docker-compose-sample
View file

@ -1,253 +0,0 @@
[Directories]
bloomfilters = Blooms
dicofilters = Dicos
pastes = PASTES
hash = HASHS
crawled = crawled
crawled_screenshot = CRAWLED_SCREENSHOT
wordtrending_csv = var/www/static/csv/wordstrendingdata
wordsfile = files/wordfile
protocolstrending_csv = var/www/static/csv/protocolstrendingdata
protocolsfile = files/protocolsfile
tldstrending_csv = var/www/static/csv/tldstrendingdata
tldsfile = faup/src/data/mozilla.tlds
domainstrending_csv = var/www/static/csv/domainstrendingdata
pystemonpath = /opt/pystemon/
sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt
##### Notifications ######
[Notifications]
ail_domain = http://localhost:7000
sender = sender@example.com
sender_host = smtp.example.com
sender_port = 1337
sender_pw = None
# optional for using with authenticated SMTP over SSL
# sender_pw = securepassword
##### Flask #####
[Flask]
#Proxying requests to the app
baseUrl = /
#Number of logs to display in the dashboard
max_dashboard_logs = 15
#Maximum number of character to display in the toolip
max_preview_char = 250
#Maximum number of character to display in the modal
max_preview_modal = 800
#Default number of header to display in trending graphs
default_display = 10
#Number of minutes displayed for the number of processed pastes.
minute_processed_paste = 10
#Maximum line length authorized to make a diff between duplicates
DiffMaxLineLength = 10000
#### Modules ####
[BankAccount]
max_execution_time = 60
[Categ]
#Minimum number of match between the paste and the category file
matchingThreshold=1
[Credential]
#Minimum length that a credential must have to be considered as such
minimumLengthThreshold=3
#Will be pushed as alert if the number of credentials is greater to that number
criticalNumberToAlert=8
#Will be considered as false positive if less that X matches from the top password list
minTopPassList=5
[Curve]
max_execution_time = 90
[Onion]
max_execution_time = 180
[Base64]
path = Base64/
max_execution_time = 60
[Binary]
path = Base64/
max_execution_time = 60
[Hex]
path = Base64/
max_execution_time = 60
[Modules_Duplicates]
#Number of month to look back
maximum_month_range = 3
#The value where two pastes are considerate duplicate for ssdeep.
threshold_duplicate_ssdeep = 50
#The value where two pastes are considerate duplicate for tlsh.
threshold_duplicate_tlsh = 52
#Minimum size of the paste considered
min_paste_size = 0.3
[Module_ModuleInformation]
#Threshold to deduce if a module is stuck or not, in seconds.
threshold_stucked_module=600
[Module_Mixer]
#Define the configuration of the mixer, possible value: 1, 2 or 3
operation_mode = 3
#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
ttl_duplicate = 86400
default_unnamed_feed_name = unnamed_feeder
[RegexForTermsFrequency]
max_execution_time = 60
##### Redis #####
[Redis_Cache]
host = localhost
port = 6379
db = 0
[Redis_Log]
host = localhost
port = 6380
db = 0
[Redis_Log_submit]
host = localhost
port = 6380
db = 1
[Redis_Queues]
host = localhost
port = 6381
db = 0
[Redis_Data_Merging]
host = localhost
port = 6379
db = 1
[Redis_Paste_Name]
host = localhost
port = 6379
db = 2
[Redis_Mixer_Cache]
host = localhost
port = 6381
db = 1
##### ARDB #####
[ARDB_Curve]
host = localhost
port = 6382
db = 1
[ARDB_Sentiment]
host = localhost
port = 6382
db = 4
[ARDB_TermFreq]
host = localhost
port = 6382
db = 2
[ARDB_TermCred]
host = localhost
port = 6382
db = 5
[ARDB_DB]
host = localhost
port = 6382
db = 0
[ARDB_Trending]
host = localhost
port = 6382
db = 3
[ARDB_Hashs]
host = localhost
db = 1
[ARDB_Tags]
host = localhost
port = 6382
db = 6
[ARDB_Metadata]
host = localhost
port = 6382
db = 7
[ARDB_Statistics]
host = localhost
port = 6382
db = 8
[ARDB_Onion]
host = localhost
port = 6382
db = 9
[Url]
cc_critical = DE
[DomClassifier]
cc = DE
cc_tld = r'\.de$'
dns = 8.8.8.8
[Mail]
dns = 8.8.8.8
[Web]
dns = 149.13.33.69
# Indexer configuration
[Indexer]
type = whoosh
path = indexdir
register = indexdir/all_index.txt
#size in Mb
index_max_size = 2000
[ailleakObject]
maxDuplicateToPushToMISP=10
###############################################################################
# For multiple feed, add them with "," without space
# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
[ZMQ_Global]
#address = tcp://crf.circl.lu:5556
address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
channel = 102
bind = tcp://127.0.0.1:5556
[ZMQ_Url]
address = tcp://127.0.0.1:5004
channel = urls
[ZMQ_FetchedOnion]
address = tcp://127.0.0.1:5005
channel = FetchedOnion
[RedisPubSub]
host = localhost
port = 6381
db = 0
[Crawler]
activate_crawler = False
crawler_depth_limit = 1
splash_url_onion = http://172.17.0.1
splash_onion_port = 8050

39
bin/packages/modules.cfg
View file

@ -12,10 +12,6 @@ publish = Redis_Mixer,Redis_Tags
subscribe = Redis_Mixer subscribe = Redis_Mixer
publish = Redis_Global,Redis_ModuleStats publish = Redis_Global,Redis_ModuleStats
#[PreProcessFeed]
#subscribe = Redis_preProcess1
#publish = Redis_Mixer
[Duplicates] [Duplicates]
subscribe = Redis_Duplicate subscribe = Redis_Duplicate
@ -62,7 +58,7 @@ subscribe = Redis_Global
[Categ] [Categ]
subscribe = Redis_Global subscribe = Redis_Global
publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Urls,Redis_Credential,Redis_SourceCode,Redis_Cve,Redis_ApiKey,Redis_SyncModule publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Urls,Redis_Credential,Redis_Cve,Redis_ApiKey,Redis_SyncModule
[CreditCards] [CreditCards]
subscribe = Redis_CreditCards subscribe = Redis_CreditCards
@ -78,13 +74,9 @@ publish = Redis_ModuleStats,Redis_Tags
[Onion] [Onion]
subscribe = Redis_Onion subscribe = Redis_Onion
publish = Redis_ValidOnion,Redis_Tags,Redis_Crawler publish = Redis_Tags,Redis_Crawler
#publish = Redis_ValidOnion,ZMQ_FetchedOnion,Redis_Tags,Redis_Crawler #publish = Redis_ValidOnion,ZMQ_FetchedOnion,Redis_Tags,Redis_Crawler
# TODO remove me
[DumpValidOnion]
subscribe = Redis_ValidOnion
[Urls] [Urls]
subscribe = Redis_Urls subscribe = Redis_Urls
publish = Redis_Url publish = Redis_Url
@ -112,15 +104,8 @@ subscribe = Redis_SyncModule
[MISP_The_hive_feeder] [MISP_The_hive_feeder]
subscribe = Redis_Tags_feed subscribe = Redis_Tags_feed
#[send_to_queue] #[SentimentAnalysis]
#subscribe = Redis_Cve #subscribe = Redis_Global
#publish = Redis_Tags
[SentimentAnalysis]
subscribe = Redis_Global
[Release]
subscribe = Redis_Global
[Credential] [Credential]
subscribe = Redis_Credential subscribe = Redis_Credential
@ -130,9 +115,10 @@ publish = Redis_Duplicate,Redis_ModuleStats,Redis_Tags
subscribe = Redis_Cve subscribe = Redis_Cve
publish = Redis_Tags publish = Redis_Tags
[Phone] # Disabled
subscribe = Redis_Global #[Phone]
publish = Redis_Tags #subscribe = Redis_Global
#publish = Redis_Tags
[Keys] [Keys]
subscribe = Redis_Global subscribe = Redis_Global
@ -159,7 +145,6 @@ subscribe = Redis
publish = Redis_Mixer publish = Redis_Mixer
[Crawler] [Crawler]
subscribe = Redis_Crawler
publish = Redis_Mixer,Redis_Tags publish = Redis_Mixer,Redis_Tags
[IP] [IP]
@ -169,3 +154,11 @@ publish = Redis_Tags
[Zerobins] [Zerobins]
subscribe = Redis_Url subscribe = Redis_Url
#[PreProcessFeed]
#subscribe = Redis_preProcess1
#publish = Redis_Mixer
# [My_Module]
# subscribe = Redis_Global
# publish = Redis_Tags

10
bin/template.py
View file

@ -36,11 +36,11 @@ class Template(AbstractModule):
# Send module state to logs # Send module state to logs
self.redis_logger.info(f'Module {self.module_name} initialized') self.redis_logger.info(f'Module {self.module_name} initialized')
def computeNone(self): # def computeNone(self):
""" # """
Do something when there is no message in the queue # Do something when there is no message in the queue
""" # """
self.redis_logger.debug("No message in queue") # self.redis_logger.debug("No message in queue")
def compute(self, message): def compute(self, message):
""" """

31
var/www/blueprints/import_export.py
View file

@ -81,7 +81,7 @@ def import_object_file():
return render_template("import_object.html", all_imported_obj=all_imported_obj, error=error) return render_template("import_object.html", all_imported_obj=all_imported_obj, error=error)
@import_export.route("/objects/misp/export", methods=['GET']) @import_export.route("/misp/objects/export", methods=['GET'])
@login_required @login_required
@login_analyst @login_analyst
def objects_misp_export(): def objects_misp_export():
@ -91,7 +91,7 @@ def objects_misp_export():
return render_template("export_object.html", object_types=object_types, to_export=to_export) return render_template("export_object.html", object_types=object_types, to_export=to_export)
@import_export.route("/objects/misp/export/post", methods=['POST']) @import_export.route("/misp/objects/export/post", methods=['POST'])
@login_required @login_required
@login_analyst @login_analyst
def objects_misp_export_post(): def objects_misp_export_post():
@ -159,7 +159,7 @@ def objects_misp_export_post():
misp_url=event['url']) misp_url=event['url'])
@import_export.route("/objects/misp/export/add", methods=['GET']) @import_export.route("/misp/objects/export/add", methods=['GET'])
@login_required @login_required
@login_analyst @login_analyst
def add_object_id_to_export(): def add_object_id_to_export():
@ -181,7 +181,7 @@ def add_object_id_to_export():
return redirect(url_for('import_export.objects_misp_export')) return redirect(url_for('import_export.objects_misp_export'))
@import_export.route("/objects/misp/export/delete", methods=['GET']) @import_export.route("/misp/objects/export/delete", methods=['GET'])
@login_required @login_required
@login_analyst @login_analyst
def delete_object_id_to_export(): def delete_object_id_to_export():
@ -194,7 +194,7 @@ def delete_object_id_to_export():
return jsonify(success=True) return jsonify(success=True)
@import_export.route("/import_export/investigation", methods=['GET']) @import_export.route("/investigation/misp/export", methods=['GET'])
@login_required @login_required
@login_analyst @login_analyst
def export_investigation(): def export_investigation():
@ -206,3 +206,24 @@ def export_investigation():
return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True), return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True),
mimetype='application/json'), 400 mimetype='application/json'), 400
return redirect(url_for('investigations_b.show_investigation', uuid=investigation_uuid)) return redirect(url_for('investigations_b.show_investigation', uuid=investigation_uuid))
@import_export.route("/thehive/objects/case/export", methods=['POST'])
@login_required
@login_analyst
def create_thehive_case():
description = request.form['hive_description']
title = request.form['hive_case_title']
threat_level = Export.sanitize_threat_level_hive(request.form['threat_level_hive'])
tlp = Export.sanitize_tlp_hive(request.form['hive_tlp'])
item_id = request.form['obj_id']
item = Item(item_id)
if not item.exists():
abort(404)
case_id = Export.create_thehive_case(item_id, title=title, tlp=tlp, threat_level=threat_level, description=description)
if case_id:
return redirect(Export.get_case_url(case_id))
else:
return 'error'

79
var/www/modules/Flask_config.py
View file

@ -30,15 +30,9 @@ r_cache = config_loader.get_redis_conn("Redis_Cache")
r_serv_log = config_loader.get_redis_conn("Redis_Log") r_serv_log = config_loader.get_redis_conn("Redis_Log")
r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit") r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit")
r_serv_charts = config_loader.get_redis_conn("ARDB_Trending") # -> TODO MIGRATE Stats Graphs
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") # -> TODO MIGRATE /correlation/ subtypes objects
r_serv_charts = config_loader.get_redis_conn("ARDB_Trending") r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") # -> TODO MIGRATE AUTO CRAWLER
r_serv_sentiment = config_loader.get_redis_conn("ARDB_Sentiment")
r_serv_term = config_loader.get_redis_conn("ARDB_Tracker")
r_serv_cred = config_loader.get_redis_conn("ARDB_TermCred")
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
r_serv_statistics = config_loader.get_redis_conn("ARDB_Statistics")
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
# # # # # # # # # # # # # #
r_serv_db = config_loader.get_db_conn("Kvrocks_DB") r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
@ -51,70 +45,43 @@ redis_logger.port = 6380
# Channel name to publish logs # Channel name to publish logs
redis_logger.channel = 'Flask' redis_logger.channel = 'Flask'
sys.path.append('../../configs/keys') sys.path.append('../../configs/keys')
# MISP #
try:
from pymisp import PyMISP
from mispKEYS import misp_url, misp_key, misp_verifycert
pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
misp_event_url = misp_url + '/events/view/'
print('Misp connected')
except:
print('Misp not connected')
pymisp = False
misp_event_url = '#'
# The Hive #
try:
from thehive4py.api import TheHiveApi
import thehive4py.exceptions
from theHiveKEYS import the_hive_url, the_hive_key, the_hive_verifycert
if the_hive_url == '':
HiveApi = False
hive_case_url = '#'
print('The HIVE not connected')
else:
HiveApi = TheHiveApi(the_hive_url, the_hive_key, cert=the_hive_verifycert)
hive_case_url = the_hive_url+'/index.html#/case/id_here/details'
except:
print('The HIVE not connected')
HiveApi = False
hive_case_url = '#'
if HiveApi != False:
try:
HiveApi.get_alert(0)
print('The Hive connected')
except thehive4py.exceptions.AlertException:
HiveApi = False
print('The Hive not connected')
#### VARIABLES #### #### VARIABLES ####
baseUrl = config_loader.get_config_str("Flask", "baseurl") baseUrl = config_loader.get_config_str("Flask", "baseurl")
baseUrl = baseUrl.replace('/', '') baseUrl = baseUrl.replace('/', '')
if baseUrl != '': if baseUrl != '':
baseUrl = '/'+baseUrl baseUrl = '/' + baseUrl
max_preview_char = int(config_loader.get_config_str("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip max_preview_char = int(
max_preview_modal = int(config_loader.get_config_str("Flask", "max_preview_modal")) # Maximum number of character to display in the modal config_loader.get_config_str("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
max_preview_modal = int(
config_loader.get_config_str("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
max_tags_result = 50 max_tags_result = 50
DiffMaxLineLength = int(config_loader.get_config_str("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value DiffMaxLineLength = int(config_loader.get_config_str("Flask",
"DiffMaxLineLength")) # Use to display the estimated percentage instead of a raw value
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_message': 'An Update is running on the background. Some informations like Tags, screenshot can be', dict_update_description = {'v1.5': {'nb_background_update': 5,
'update_warning_message': 'An Update is running on the background. Some informations like Tags, screenshot can be',
'update_warning_message_notice_me': 'missing from the UI.'}, 'update_warning_message_notice_me': 'missing from the UI.'},
'v2.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be', 'v2.4': {'nb_background_update': 1,
'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
'update_warning_message_notice_me': 'missing from the UI.'}, 'update_warning_message_notice_me': 'missing from the UI.'},
'v2.6':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be', 'v2.6': {'nb_background_update': 1,
'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
'update_warning_message_notice_me': 'missing from the UI.'}, 'update_warning_message_notice_me': 'missing from the UI.'},
'v2.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be', 'v2.7': {'nb_background_update': 1,
'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be',
'update_warning_message_notice_me': 'missing from the UI.'}, 'update_warning_message_notice_me': 'missing from the UI.'},
'v3.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Languages can be', 'v3.4': {'nb_background_update': 1,
'update_warning_message': 'An Update is running on the background. Some informations like Domain Languages can be',
'update_warning_message_notice_me': 'missing from the UI.'}, 'update_warning_message_notice_me': 'missing from the UI.'},
'v3.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Tracker first_seen/last_seen can be', 'v3.7': {'nb_background_update': 1,
'update_warning_message': 'An Update is running on the background. Some informations like Tracker first_seen/last_seen can be',
'update_warning_message_notice_me': 'missing from the UI.'} 'update_warning_message_notice_me': 'missing from the UI.'}
} }
@ -137,10 +104,10 @@ SUBMIT_PASTE_TEXT_MAX_SIZE = int(config_loader.get_config_str("SubmitPaste", "TE
SUBMIT_PASTE_FILE_MAX_SIZE = int(config_loader.get_config_str("SubmitPaste", "FILE_MAX_SIZE")) SUBMIT_PASTE_FILE_MAX_SIZE = int(config_loader.get_config_str("SubmitPaste", "FILE_MAX_SIZE"))
SUBMIT_PASTE_FILE_ALLOWED_EXTENSIONS = [item.strip() for item in config_loader.get_config_str("SubmitPaste", "FILE_ALLOWED_EXTENSIONS").split(',')] SUBMIT_PASTE_FILE_ALLOWED_EXTENSIONS = [item.strip() for item in config_loader.get_config_str("SubmitPaste", "FILE_ALLOWED_EXTENSIONS").split(',')]
# VT # VT
try: try:
from virusTotalKEYS import vt_key from virusTotalKEYS import vt_key
if vt_key != '': if vt_key != '':
vt_auth = vt_key vt_auth = vt_key
vt_enabled = True vt_enabled = True

40
var/www/modules/PasteSubmit/Flask_PasteSubmit.py
View file

@ -11,10 +11,8 @@ import re
import os import os
import sys import sys
import string import string
import datetime
import unicodedata import unicodedata
import uuid import uuid
from io import BytesIO
from functools import wraps from functools import wraps
@ -42,9 +40,6 @@ import Flask_config
app = Flask_config.app app = Flask_config.app
baseUrl = Flask_config.baseUrl baseUrl = Flask_config.baseUrl
r_serv_metadata = Flask_config.r_serv_metadata # TODO REMOVE ME
r_serv_db = Flask_config.r_serv_db # TODO REMOVE ME r_serv_db = Flask_config.r_serv_db # TODO REMOVE ME
r_serv_log_submit = Flask_config.r_serv_log_submit # TODO REMOVE ME r_serv_log_submit = Flask_config.r_serv_log_submit # TODO REMOVE ME
@ -317,35 +312,16 @@ def submit_status():
else: else:
return 'INVALID UUID' return 'INVALID UUID'
# TODO MOVE ME IN import_export blueprint
@PasteSubmit.route("/PasteSubmit/create_hive_case", methods=['POST'])
@login_required
@login_analyst
def create_hive_case():
tlp = request.form['hive_tlp'] ######################################################################################################
if tlp: ######################################################################################################
tlp = int(tlp) ######################################################################################################
else: ######################################################################################################
tlp = 2
threat_level = request.form['threat_level_hive']
if threat_level:
threat_level = int(threat_level)
else:
threat_level = 2
description = request.form['hive_description']
title = request.form['hive_case_title']
item_id = request.form['obj_id']
if (0 <= tlp <= 3) and (1 <= threat_level <= 4): # TODO MIGRATE TAGS PUSH
case_id = Export.create_thehive_case(item_id, title=title, tlp=tlp, threat_level=threat_level, description=description) # TEMPORARY: 2 SET OF CUSTOM + infoleak tags ?????????
if case_id:
return redirect(get_case_url(case_id))
else:
return 'error'
return 'error'
@PasteSubmit.route("/PasteSubmit/edit_tag_export") @PasteSubmit.route("/PasteSubmit/edit_tag_export")
@login_required @login_required
@ -410,8 +386,8 @@ def edit_tag_export():
status_hive=status_hive, status_hive=status_hive,
nb_tags_whitelist_misp=nb_tags_whitelist_misp, nb_tags_whitelist_misp=nb_tags_whitelist_misp,
nb_tags_whitelist_hive=nb_tags_whitelist_hive, nb_tags_whitelist_hive=nb_tags_whitelist_hive,
flag_misp=flag_misp, flag_misp=True,
flag_hive=flag_hive) flag_hive=True)
@PasteSubmit.route("/PasteSubmit/tag_export_edited", methods=['POST']) @PasteSubmit.route("/PasteSubmit/tag_export_edited", methods=['POST'])
@login_required @login_required

1
var/www/modules/hiddenServices/Flask_hiddenServices.py
View file

@ -27,7 +27,6 @@ app = Flask_config.app
baseUrl = Flask_config.baseUrl baseUrl = Flask_config.baseUrl
r_cache = Flask_config.r_cache r_cache = Flask_config.r_cache
r_serv_onion = Flask_config.r_serv_onion r_serv_onion = Flask_config.r_serv_onion
r_serv_metadata = Flask_config.r_serv_metadata
bootstrap_label = Flask_config.bootstrap_label bootstrap_label = Flask_config.bootstrap_label
hiddenServices = Blueprint('hiddenServices', __name__, template_folder='templates') hiddenServices = Blueprint('hiddenServices', __name__, template_folder='templates')

5
var/www/modules/restApi/Flask_restApi.py
View file

@ -41,8 +41,6 @@ app = Flask_config.app
baseUrl = Flask_config.baseUrl baseUrl = Flask_config.baseUrl
r_cache = Flask_config.r_cache r_cache = Flask_config.r_cache
r_serv_db = Flask_config.r_serv_db r_serv_db = Flask_config.r_serv_db
r_serv_onion = Flask_config.r_serv_onion
r_serv_metadata = Flask_config.r_serv_metadata
restApi = Blueprint('restApi', __name__, template_folder='templates') restApi = Blueprint('restApi', __name__, template_folder='templates')
@ -258,7 +256,8 @@ def add_item_tags():
tags = data.get('tags', []) tags = data.get('tags', [])
galaxy = data.get('galaxy', []) galaxy = data.get('galaxy', [])
res = Tag.api_add_obj_tags(tags=tags, galaxy_tags=galaxy, object_id=object_id, object_type="item") # res = Tag.api_add_obj_tags(tags=tags, galaxy_tags=galaxy, object_id=object_id, object_type="item")
res = {'error': 'disabled endpoint'}, 500
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #

314
var/www/modules/sentiment/Flask_sentiment.py
View file

@ -1,160 +1,160 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
''' # '''
Flask functions and routes for the trending modules page # Flask functions and routes for the trending modules page
''' # '''
import os # import os
import sys # import sys
import datetime # import datetime
import calendar # import calendar
import flask # import flask
from flask import Flask, render_template, jsonify, request, Blueprint # from flask import Flask, render_template, jsonify, request, Blueprint
#
from Role_Manager import login_admin, login_analyst, login_read_only # from Role_Manager import login_admin, login_analyst, login_read_only
from flask_login import login_required # from flask_login import login_required
#
sys.path.append(os.environ['AIL_BIN']) # sys.path.append(os.environ['AIL_BIN'])
################################## # ##################################
# Import Project packages # # Import Project packages
################################## # ##################################
from packages.Date import Date # from packages.Date import Date
#
# ============ VARIABLES ============ # # ============ VARIABLES ============
import Flask_config # import Flask_config
#
app = Flask_config.app # app = Flask_config.app
baseUrl = Flask_config.baseUrl # baseUrl = Flask_config.baseUrl
r_serv_charts = Flask_config.r_serv_charts # r_serv_charts = Flask_config.r_serv_charts
r_serv_sentiment = Flask_config.r_serv_sentiment # r_serv_sentiment = Flask_config.r_serv_sentiment
#
sentiments = Blueprint('sentiments', __name__, template_folder='templates') # sentiments = Blueprint('sentiments', __name__, template_folder='templates')
#
# ============ FUNCTIONS ============ # # ============ FUNCTIONS ============
#
def get_date_range(num_day): # def get_date_range(num_day):
curr_date = datetime.date.today() # curr_date = datetime.date.today()
date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2)) # date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
date_list = [] # date_list = []
#
for i in range(0, num_day+1): # for i in range(0, num_day+1):
date_list.append(date.substract_day(i)) # date_list.append(date.substract_day(i))
return date_list # return date_list
#
#
# ============ ROUTES ============ # # ============ ROUTES ============
#
@sentiments.route("/sentiment_analysis_trending/") # @sentiments.route("/sentiment_analysis_trending/")
@login_required # @login_required
@login_read_only # @login_read_only
def sentiment_analysis_trending(): # def sentiment_analysis_trending():
return render_template("sentiment_analysis_trending.html") # return render_template("sentiment_analysis_trending.html")
#
#
@sentiments.route("/sentiment_analysis_getplotdata/", methods=['GET']) # @sentiments.route("/sentiment_analysis_getplotdata/", methods=['GET'])
@login_required # @login_required
@login_read_only # @login_read_only
def sentiment_analysis_getplotdata(): # def sentiment_analysis_getplotdata():
# Get the top providers based on number of pastes # # Get the top providers based on number of pastes
oneHour = 60*60 # oneHour = 60*60
sevenDays = oneHour*24*7 # sevenDays = oneHour*24*7
dateStart = datetime.datetime.now() # dateStart = datetime.datetime.now()
dateStart = dateStart.replace(minute=0, second=0, microsecond=0) # dateStart = dateStart.replace(minute=0, second=0, microsecond=0)
dateStart_timestamp = calendar.timegm(dateStart.timetuple()) # dateStart_timestamp = calendar.timegm(dateStart.timetuple())
#
getAllProviders = request.args.get('getProviders') # getAllProviders = request.args.get('getProviders')
provider = request.args.get('provider') # provider = request.args.get('provider')
allProvider = request.args.get('all') # allProvider = request.args.get('all')
if getAllProviders == 'True': # if getAllProviders == 'True':
if allProvider == "True": # if allProvider == "True":
range_providers = r_serv_charts.smembers('all_provider_set') # range_providers = r_serv_charts.smembers('all_provider_set')
#
return jsonify(list(range_providers)) # return jsonify(list(range_providers))
else: # else:
range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8) # range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8)
# if empty, get yesterday top providers # # if empty, get yesterday top providers
range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers # range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers
#
#
# if still empty, takes from all providers # # if still empty, takes from all providers
if range_providers == []: # if range_providers == []:
print('today provider empty') # print('today provider empty')
range_providers = r_serv_charts.smembers('all_provider_set') # range_providers = r_serv_charts.smembers('all_provider_set')
#
return jsonify(list(range_providers)) # return jsonify(list(range_providers))
#
elif provider is not None: # elif provider is not None:
to_return = {} # to_return = {}
#
cur_provider_name = provider + '_' # cur_provider_name = provider + '_'
list_date = {} # list_date = {}
for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour): # for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour):
cur_set_name = cur_provider_name + str(cur_timestamp) # cur_set_name = cur_provider_name + str(cur_timestamp)
#
list_value = [] # list_value = []
for cur_id in r_serv_sentiment.smembers(cur_set_name): # for cur_id in r_serv_sentiment.smembers(cur_set_name):
cur_value = (r_serv_sentiment.get(cur_id)) # cur_value = (r_serv_sentiment.get(cur_id))
list_value.append(cur_value) # list_value.append(cur_value)
list_date[cur_timestamp] = list_value # list_date[cur_timestamp] = list_value
to_return[provider] = list_date # to_return[provider] = list_date
#
return jsonify(to_return) # return jsonify(to_return)
return "Bad request" # return "Bad request"
#
#
#
@sentiments.route("/sentiment_analysis_plot_tool/") # @sentiments.route("/sentiment_analysis_plot_tool/")
@login_required # @login_required
@login_read_only # @login_read_only
def sentiment_analysis_plot_tool(): # def sentiment_analysis_plot_tool():
return render_template("sentiment_analysis_plot_tool.html") # return render_template("sentiment_analysis_plot_tool.html")
#
#
#
@sentiments.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET']) # @sentiments.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET'])
@login_required # @login_required
@login_read_only # @login_read_only
def sentiment_analysis_plot_tool_getdata(): # def sentiment_analysis_plot_tool_getdata():
getProviders = request.args.get('getProviders') # getProviders = request.args.get('getProviders')
#
if getProviders == 'True': # if getProviders == 'True':
providers = [] # providers = []
for cur_provider in r_serv_charts.smembers('all_provider_set'): # for cur_provider in r_serv_charts.smembers('all_provider_set'):
providers.append(cur_provider) # providers.append(cur_provider)
return jsonify(providers) # return jsonify(providers)
#
else: # else:
query = request.args.get('query') # query = request.args.get('query')
query = query.split(',') # query = query.split(',')
Qdate = request.args.get('Qdate') # Qdate = request.args.get('Qdate')
#
date1 = (Qdate.split('-')[0]).split('/') # date1 = (Qdate.split('-')[0]).split('/')
date1 = datetime.date(int(date1[2]), int(date1[0]), int(date1[1])) # date1 = datetime.date(int(date1[2]), int(date1[0]), int(date1[1]))
#
date2 = (Qdate.split('-')[1]).split('/') # date2 = (Qdate.split('-')[1]).split('/')
date2 = datetime.date(int(date2[2]), int(date2[0]), int(date2[1])) # date2 = datetime.date(int(date2[2]), int(date2[0]), int(date2[1]))
#
timestamp1 = calendar.timegm(date1.timetuple()) # timestamp1 = calendar.timegm(date1.timetuple())
timestamp2 = calendar.timegm(date2.timetuple()) # timestamp2 = calendar.timegm(date2.timetuple())
#
oneHour = 60*60 # oneHour = 60*60
oneDay = oneHour*24 # oneDay = oneHour*24
#
to_return = {} # to_return = {}
for cur_provider in query: # for cur_provider in query:
list_date = {} # list_date = {}
cur_provider_name = cur_provider + '_' # cur_provider_name = cur_provider + '_'
for cur_timestamp in range(int(timestamp1), int(timestamp2)+oneDay, oneHour): # for cur_timestamp in range(int(timestamp1), int(timestamp2)+oneDay, oneHour):
cur_set_name = cur_provider_name + str(cur_timestamp) # cur_set_name = cur_provider_name + str(cur_timestamp)
#
list_value = [] # list_value = []
for cur_id in r_serv_sentiment.smembers(cur_set_name): # for cur_id in r_serv_sentiment.smembers(cur_set_name):
cur_value = (r_serv_sentiment.get(cur_id)) # cur_value = (r_serv_sentiment.get(cur_id))
list_value.append(cur_value) # list_value.append(cur_value)
list_date[cur_timestamp] = list_value # list_date[cur_timestamp] = list_value
to_return[cur_provider] = list_date # to_return[cur_provider] = list_date
#
return jsonify(to_return) # return jsonify(to_return)
#
# ========= REGISTRATION ========= # # ========= REGISTRATION =========
app.register_blueprint(sentiments, url_prefix=baseUrl) # app.register_blueprint(sentiments, url_prefix=baseUrl)

14
var/www/modules/sentiment/templates/header_sentiment.html
View file

@ -1,7 +1,7 @@
<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-heart"></i> Sentiment Analysis {#<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-heart"></i> Sentiment Analysis#}
<span class="caret"></span></a> {# <span class="caret"></span></a>#}
<ul class="dropdown-menu"> {# <ul class="dropdown-menu">#}
<li><a href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-bar-chart-o"> </i> Sentiment trending</a></li> {# <li><a href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-bar-chart-o"> </i> Sentiment trending</a></li>#}
<li><a href="{{ url_for('sentiments.sentiment_analysis_plot_tool') }}"><i class="fa fa-wrench"> </i> Sentiment plot Tool</a></li> {# <li><a href="{{ url_for('sentiments.sentiment_analysis_plot_tool') }}"><i class="fa fa-wrench"> </i> Sentiment plot Tool</a></li>#}
</ul> {# </ul>#}
</li> {#</li>#}

356
var/www/modules/terms/Flask_terms.py
View file

@ -1,356 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Flask functions and routes for the trending modules page
note: The matching of credential against supplied credential is done using Levenshtein distance
'''
import os
import sys
import flask
from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, Response
from Role_Manager import login_admin, login_analyst, login_user_no_api, login_read_only
from flask_login import login_required, current_user
import Levenshtein
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from lib.objects.Items import Item
# ============ VARIABLES ============
import Flask_config
app = Flask_config.app
baseUrl = Flask_config.baseUrl
r_serv_term = Flask_config.r_serv_term
r_serv_cred = Flask_config.r_serv_cred
r_serv_db = Flask_config.r_serv_db
bootstrap_label = Flask_config.bootstrap_label
terms = Blueprint('terms', __name__, template_folder='templates')
'''TERM'''
DEFAULT_MATCH_PERCENT = 50
#tracked
TrackedTermsSet_Name = "TrackedSetTermSet"
TrackedTermsDate_Name = "TrackedTermDate"
#black
BlackListTermsDate_Name = "BlackListTermDate"
BlackListTermsSet_Name = "BlackListSetTermSet"
#regex
TrackedRegexSet_Name = "TrackedRegexSet"
TrackedRegexDate_Name = "TrackedRegexDate"
#set
TrackedSetSet_Name = "TrackedSetSet"
TrackedSetDate_Name = "TrackedSetDate"
# notifications enabled/disabled
# same value as in `bin/NotificationHelper.py`
TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
# associated notification email addresses for a specific term`
# same value as in `bin/NotificationHelper.py`
# Keys will be e.g. TrackedNotificationEmails_<TERMNAME>
TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
'''CRED'''
REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername'
REDIS_KEY_NUM_PATH = 'uniqNumForUsername'
REDIS_KEY_ALL_CRED_SET = 'AllCredentials'
REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
REDIS_KEY_ALL_PATH_SET = 'AllPath'
REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev'
REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
# ============ FUNCTIONS ============
def Term_getValueOverRange(word, startDate, num_day, per_paste=""):
passed_days = 0
oneDay = 60*60*24
to_return = []
curr_to_return = 0
for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay):
value = r_serv_term.hget(per_paste+str(timestamp), word)
curr_to_return += int(value) if value is not None else 0
for i in num_day:
if passed_days == i-1:
to_return.append(curr_to_return)
passed_days += 1
return to_return
#Mix suplied username, if extensive is set, slice username(s) with different windows
def mixUserName(supplied, extensive=False):
#e.g.: John Smith
terms = supplied.split()[:2]
usernames = []
if len(terms) == 1:
terms.append(' ')
#john, smith, John, Smith, JOHN, SMITH
usernames += [terms[0].lower()]
usernames += [terms[1].lower()]
usernames += [terms[0][0].upper() + terms[0][1:].lower()]
usernames += [terms[1][0].upper() + terms[1][1:].lower()]
usernames += [terms[0].upper()]
usernames += [terms[1].upper()]
#johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN
usernames += [(terms[0].lower() + terms[1].lower()).strip()]
usernames += [(terms[1].lower() + terms[0].lower()).strip()]
usernames += [(terms[0].upper() + terms[1].lower()).strip()]
usernames += [(terms[0].lower() + terms[1].upper()).strip()]
usernames += [(terms[1].upper() + terms[0].lower()).strip()]
usernames += [(terms[1].lower() + terms[0].upper()).strip()]
#Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith
usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
usernames += [(terms[0].lower() + terms[1][0].upper()).strip()]
usernames += [(terms[0].upper() + terms[1][0].lower()).strip()]
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
if not extensive:
return usernames
#Slice the supplied username(s)
mixedSupplied = supplied.replace(' ','')
minWindow = 3 if len(mixedSupplied)/2 < 4 else len(mixedSupplied)/2
for winSize in range(3,len(mixedSupplied)):
for startIndex in range(0, len(mixedSupplied)-winSize):
usernames += [mixedSupplied[startIndex:startIndex+winSize]]
filtered_usernames = []
for usr in usernames:
if len(usr) > 2:
filtered_usernames.append(usr)
return filtered_usernames
def save_tag_to_auto_push(list_tag):
for tag in set(list_tag):
#limit tag length
if len(tag) > 49:
tag = tag[0:48]
r_serv_db.sadd('list_export_tags', tag)
# ============ ROUTES ============
# TODO: remove + clean
# @terms.route("/terms_plot_tool/")
# @login_required
# @login_read_only
# def terms_plot_tool():
# term = request.args.get('term')
# if term is not None:
# return render_template("terms_plot_tool.html", term=term)
# else:
# return render_template("terms_plot_tool.html", term="")
#
#
# @terms.route("/terms_plot_tool_data/")
# @login_required
# @login_read_only
# def terms_plot_tool_data():
# oneDay = 60*60*24
# range_start = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
# range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0)
# range_start = calendar.timegm(range_start.timetuple())
# range_end = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0;
# range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0)
# range_end = calendar.timegm(range_end.timetuple())
# term = request.args.get('term')
#
# per_paste = request.args.get('per_paste')
# if per_paste == "1" or per_paste is None:
# per_paste = "per_paste_"
# else:
# per_paste = ""
#
# if term is None:
# return "None"
#
# else:
# value_range = []
# for timestamp in range(range_start, range_end+oneDay, oneDay):
# value = r_serv_term.hget(per_paste+str(timestamp), term)
# curr_value_range = int(value) if value is not None else 0
# value_range.append([timestamp, curr_value_range])
# value_range.insert(0,term)
# return jsonify(value_range)
#
# @terms.route("/terms_plot_top/"
# @login_required
# @login_read_only
# def terms_plot_top():
# per_paste = request.args.get('per_paste')
# per_paste = per_paste if per_paste is not None else 1
# return render_template("terms_plot_top.html", per_paste=per_paste)
#
#
# @terms.route("/terms_plot_top_data/")
# @login_required
# @login_read_only
# def terms_plot_top_data():
# oneDay = 60*60*24
# today = datetime.datetime.now()
# today = today.replace(hour=0, minute=0, second=0, microsecond=0)
# today_timestamp = calendar.timegm(today.timetuple())
#
# per_paste = request.args.get('per_paste')
# if per_paste == "1" or per_paste is None:
# per_paste = "per_paste_"
# else:
# per_paste = ""
#
# set_day = per_paste + "TopTermFreq_set_day_" + str(today_timestamp)
# set_week = per_paste + "TopTermFreq_set_week";
# set_month = per_paste + "TopTermFreq_set_month";
#
# the_set = per_paste + request.args.get('set')
# num_day = int(request.args.get('num_day'))
#
# if the_set is None:
# return "None"
# else:
# to_return = []
# if "TopTermFreq_set_day" in the_set:
# the_set += "_" + str(today_timestamp)
#
# for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20):
# position = {}
# position['day'] = r_serv_term.zrevrank(set_day, term)
# position['day'] = position['day']+1 if position['day'] is not None else "<20"
# position['week'] = r_serv_term.zrevrank(set_week, term)
# position['week'] = position['week']+1 if position['week'] is not None else "<20"
# position['month'] = r_serv_term.zrevrank(set_month, term)
# position['month'] = position['month']+1 if position['month'] is not None else "<20"
# value_range = []
# for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay):
# value = r_serv_term.hget(per_paste+str(timestamp), term)
# curr_value_range = int(value) if value is not None else 0
# value_range.append([timestamp, curr_value_range])
#
# to_return.append([term, value_range, tot_value, position])
#
# return jsonify(to_return)
@terms.route("/credentials_tracker/")
@login_required
@login_read_only
def credentials_tracker():
return render_template("credentials_tracker.html")
@terms.route("/credentials_management_query_paste/", methods=['GET', 'POST'])
@login_required
@login_user_no_api
def credentials_management_query_paste():
cred = request.args.get('cred')
allPath = request.json['allPath']
paste_info = []
for pathNum in allPath:
path = r_serv_cred.hget(REDIS_KEY_ALL_PATH_SET_REV, pathNum)
item = Item(path)
p_date = item.get_date(separator=True)
p_source = item.get_source()
p_content = item.get_content()
p_encoding = item.get_mimetype()
p_size = item.get_size()
p_mime = p_encoding
lineinfo = item.get_meta_lines(content=p_content)
p_lineinfo = lineinfo['nb'], lineinfo['max_length']
if p_content != 0:
p_content = p_content[0:400]
paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content})
return jsonify(paste_info)
@terms.route("/credentials_management_action/", methods=['GET'])
@login_required
@login_user_no_api
def cred_management_action():
supplied = request.args.get('term')
action = request.args.get('action')
section = request.args.get('section')
extensive = request.args.get('extensive')
extensive = True if extensive == "true" else False
if extensive:
#collectDico
AllUsernameInRedis = r_serv_cred.hgetall(REDIS_KEY_ALL_CRED_SET).keys()
uniq_num_set = set()
if action == "seek":
possibilities = mixUserName(supplied, extensive)
for poss in possibilities:
num = r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET, poss)
if num is not None:
uniq_num_set.add(num)
for num in r_serv_cred.smembers(poss):
uniq_num_set.add(num)
#Extensive /!\
if extensive:
iter_num = 0
tot_iter = len(AllUsernameInRedis)*len(possibilities)
for tempUsername in AllUsernameInRedis:
for poss in possibilities:
#FIXME print progress
if(iter_num % int(tot_iter/20) == 0):
#print("searching: {}% done".format(int(iter_num/tot_iter*100)), sep=' ', end='\r', flush=True)
print("searching: {}% done".format(float(iter_num)/float(tot_iter)*100))
iter_num += 1
if poss in tempUsername:
num = (r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET, tempUsername))
if num is not None:
uniq_num_set.add(num)
for num in r_serv_cred.smembers(tempUsername):
uniq_num_set.add(num)
data = {'usr': [], 'path': [], 'numPaste': [], 'simil': []}
for Unum in uniq_num_set:
levenRatio = 2.0
username = (r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum))
# Calculate Levenshtein distance, ignore negative ratio
supp_splitted = supplied.split()
supp_mixed = supplied.replace(' ','')
supp_splitted.append(supp_mixed)
for indiv_supplied in supp_splitted:
levenRatio = float(Levenshtein.ratio(indiv_supplied, username))
levenRatioStr = "{:.1%}".format(levenRatio)
data['usr'].append(username)
allPathNum = list(r_serv_cred.smembers(REDIS_KEY_MAP_CRED_TO_PATH+'_'+Unum))
data['path'].append(allPathNum)
data['numPaste'].append(len(allPathNum))
data['simil'].append(levenRatioStr)
to_return = {}
to_return["section"] = section
to_return["action"] = action
to_return["term"] = supplied
to_return["data"] = data
return jsonify(to_return)
# ========= REGISTRATION =========
app.register_blueprint(terms, url_prefix=baseUrl)

253
var/www/modules/terms/templates/credentials_tracker.html
View file

@ -1,253 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Credentials Tracker - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.pie.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
<style>
.btn-link {
color: #000000
}
</style>
</head>
<body>
<!-- Modal -->
<div id="mymodal" class="modal fade" role="dialog">
<div class="modal-dialog modal-lg">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div id="mymodalbody" class="modal-body" max-width="8500px">
<p>Loading paste information...</p>
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
</div>
<div class="modal-footer">
<a id="button_show_plot" target="_blank" href=""><button type="button" class="btn btn-info">Plot term</button></a>
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-termsfrequency" >Credential seeker</h1>
</div>
<!-- /.col-lg-12 -->
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-success">
<div class="panel-heading">
<strong>Credential seeker</strong>
</div>
<div class="panel-body">
<label><input type="checkbox" id="extensive" style="margin-bottom: 10px;"><strong>Extensive search (takes time)</strong></input></label>
<div class="form-group input-group" style="margin-bottom: 30px;">
<span class="input-group-addon"><span class="glyphicon glyphicon-screenshot"></span></span>
<input id="seekInput" class="form-control" placeholder="Credential to seek." type="text" style="max-width: 400px;">
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="seek" data-action="seek"> Seek</button>
<span id="nodata" class="alert alert-info" style="margin-left: 10px; display: none; padding: 6px;">No data</span>
</div>
<table class="table table-striped table-bordered table-hover" id="myTable">
<thead>
<tr>
<th style="max-width: 800px;">Username</th>
<th>Similarity</th>
<th># concerned paste(s)</th>
<th>Action</th>
</tr>
</thead>
<tbody>
</tbody>
</table>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
<script>
$(document).ready(function(){
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
$('[data-toggle="tooltip"]').tooltip();
table_track = $('#myTable').DataTable({ "order": [[ 1, "dec" ]] });
table_track.on( 'draw.dt', function () {
perform_binding();
});
$("#seekInput").on("keyup", function(event){
if(event.keyCode == 13){
$("#followTermBtn").click();
$("#seekInput").val("");
}
});
perform_binding();
$("#mymodal").on('hidden.bs.modal', function () {
$("#mymodalbody").html("<p>Loading paste information...</p>");
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
$("#mymodalbody").append(loading_gif); // Show the loading GIF
});
});
</script>
<script>
function perform_binding() {
$(".btn-interaction").off("click.interaction");
$(".btn-interaction").on("click.interaction", perform_operation);
}
function perform_modal_binding() {
// On click, get html content from url and update the corresponding modal
$("[data-toggle='modal']").on("click.openmodal", function (event) {
event.preventDefault();
var the_modal=$(this);
var url = "{{ url_for('terms.credentials_management_query_paste') }}?cred=" + encodeURIComponent($(this).attr('data-term'));
$.ajax({
type: 'POST',
url: url,
dataType: "json",
data: JSON.stringify({ 'allPath': JSON.parse($(this).attr('data-path')) }),
contentType : "application/json"
}).done(function (data) {
if (data.length != 0) {
var html_to_add = "";
html_to_add += "<table id=\"modal-table\" class=\"table table-striped\">";
html_to_add += "<thead>";
html_to_add += "<tr>";
html_to_add += "<th>Source</th>";
html_to_add += "<th>Date</th>";
html_to_add += "<th>Encoding</th>";
html_to_add += "<th>Size (Kb)</th>";
html_to_add += "<th># lines</th>";
html_to_add += "<th>Max length</th>";
html_to_add += "<th>Preview</th>";
html_to_add += "</tr>";
html_to_add += "</thead>";
html_to_add += "<tbody>";
for (i=0; i<data.length; i++) {
curr_data = data[i];
html_to_add += "<tr>";
html_to_add += "<td>"+curr_data.source+"</td>";
html_to_add += "<td>"+curr_data.date+"</td>";
html_to_add += "<td>"+curr_data.encoding+"</td>";
html_to_add += "<td>"+curr_data.size+"</td>";
html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";
html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content.replace(/\"/g, "\'")+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('objects_item.showItem') }}?id="+curr_data.path+"&num=0\"> <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
html_to_add += "</tr>";
}
html_to_add += "</tbody>";
html_to_add += "</table>";
$("#mymodalbody").html(html_to_add);
$("[data-toggle=popover]").popover();
$('#modal-table').DataTable();
} else {
$("#mymodalbody").html("No paste containing this term has been received yet.");
}
}
)});
}
function perform_operation(){
var curr_section = $(this).attr('data-section');
var curr_action = $(this).attr('data-action');
if (curr_action == "add") {
var curr_term = $('#'+curr_section+'Input').val();
} else if (curr_action == "seek") {
var curr_term = $('#'+curr_section+'Input').val();
} else {
var curr_term = $(this).attr('data-content');
}
var data_to_send = { section: curr_section, action:curr_action, term: curr_term, extensive: $("#extensive").is(":checked")};
if (curr_term != "") {
//console.log(data_to_send);
$.get("{{ url_for('terms.cred_management_action') }}", data_to_send, function(data, status){
if(status == "success") {
var json = data;
if(json.action == "add") {
//not used for the moment
} else if (json.action == "seek") {
table_track.clear().draw();
var rep = json.data;
//var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Track this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"add\">";
var action_button = "";
if (rep.usr.length == 0) {
$( "#nodata" ).removeClass('alert-info').addClass('alert-danger');
$( "#nodata" ).text("No data");
$( "#nodata" ).fadeIn( "fast", function() {
setTimeout(function() {
$( "#nodata" ).fadeOut("fast");
}, 2000);
});
} else {
for(i=0; i < rep.usr.length; i++) {
$( "#nodata" ).removeClass('alert-danger').addClass('alert-info');
$( "#nodata" ).text(curr_term);
$( "#nodata" ).fadeIn( "fast");
toAdd = "</button><span data-toggle=\"modal\" data-target=\"#mymodal\" data-term=\""+rep.usr[i]+"\" data-path=\"["+rep.path[i]+"]\" ><button class=\"btn-link\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Show concerned paste(s)\"><span class=\"glyphicon glyphicon-info-sign\"></span></button></span>";
table_track.row.add( [
rep.usr[i],
rep.simil[i],
rep.numPaste[i],
toAdd+action_button ] ).draw( false );
}
perform_binding();
perform_modal_binding();
}
}
}
});
}
}
</script>
</body>
</html>

6
var/www/modules/terms/templates/header_terms.html
View file

@ -1,6 +0,0 @@
<li id='page-termsfrequency'><a class="dropdown-toggle" data-toggle="dropdown" href="{{ url_for('terms.credentials_tracker') }}"><i class="fa fa-eye"></i> Terms frequency
<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials seeker</a></li>
</ul>
</li>

591
var/www/modules/terms/templates/terms_management.html
View file

@ -1,591 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Terms Management</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/switch_checkbox.css') }}" rel="stylesheet" type="text/css" />
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
<style>
.sparkLineStats ul {
padding-left:0;
list-style:none
}
.btn-link {
color: #000000
}
.popover-content {
white-space:pre-wrap;
word-wrap:break-word;
}
.mouse_pointer{
cursor: pointer;
}
.lb-md {
font-size: 16px;
}
</style>
</head>
<body>
<!-- Modal -->
<div id="mymodal" class="modal fade" role="dialog">
<div class="modal-dialog modal-lg">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div id="mymodalbody" class="modal-body" max-width="8500px">
<p>Loading paste information...</p>
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
</div>
<div class="modal-footer">
<a id="button_show_plot" target="_blank" href=""><button type="button" class="btn btn-info">Plot term</button></a>
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-termsfrequency" >Terms frequency: Management interface</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="row">
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
{% set uniq_id = namespace(modal_id=0)%}
<div class="col-lg-12">
<label class="switch">
<input id="per_paste" class="switch-input" value="per_paste" type="checkbox" onclick="reload_per_paste()">
<span class="switch-label" data-on="On" data-off="Off"></span>
<span class="switch-handle"></span>
</label>
<strong style="top: 3px; position: relative;">1 term per paste</strong>
<div id="panel-today" class="panel panel-success">
<div class="panel-heading">
<strong>Manage tracked terms</strong>
</div>
<div class="panel-body">
<div style="margin-bottom: 10px;">
<table>
<tr><td><b>Regex</b>: surround the term by '<b>/</b>'. </td> <td><b style="margin-left: 20px;">/([a-z])\w+([a-z])\n/</b></td></tr>
<tr><td><b>Set of terms</b>: surround the list by '<b>\</b>'. </td> <td><b style="margin-left: 20px;">\[term1, term2, ...]\</b></td></tr>
<tr><td> - To set a custom matching <b>threshold</b> (defaut=50), append it at the end as a inner list '<b>[thresh]</b>'. </td> <td><b style="margin-left: 20px;">\[term1, term2, ..., [75]]\</b></td></tr>
</table>
</div>
<div class="form-group input-group" style="margin-bottom: 30px;">
<span class="input-group-addon"><span class="fa fa-eye"></span></span>
<input id="followTermInput" class="form-control" placeholder="Term to track." type="text" style="max-width: 400px;">
<input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (optional, space separated)" type="text" style="max-width: 400px;">
<input id="followTermTag" class="form-control" placeholder="Tags (optional, space separated)" type="text" style="max-width: 400px;">
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button>
</div>
<table class="table table-striped table-bordered table-hover" id="myTable">
<thead>
<tr>
<th style="max-width: 800px;">Term</th>
<th>Added date</th>
<th>Day occurence</th>
<th>Week occurence</th>
<th>Month occurence</th>
<th># tracked paste</th>
<th>Action</th>
<th>Notification E-Mails</th>
</tr>
</thead>
<tbody>
<!-- SET -->
{% for set in trackSet_list %}
<tr style="background-color: #cdffca;">
<td>
<span class="term_name">{{ set }}</span>
<div>
{% for tag in notificationTagsTermMapping[set] %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
</a>
{% endfor %}
{% if notificationTagsTermMapping[set] %}
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
<div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
<div class="modal-dialog">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
<h2 class="text-center">Remove Custom Tag</h2>
</div>
<div class="modal-body">
<form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
{% for tag in notificationTagsTermMapping[set] %}
<div class="form-check">
<input type="hidden" class="form-control" name="term" value="{{ set }}">
<input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
<label class="form-check-label">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
</label>
<br>
</div>
{% endfor %}
</form>
</div>
<div class="modal-footer">
<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
<span class="glyphicon glyphicon-trash"></span>
<span class="label-icon">Remove Tags</span>
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
{% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
{% endif %}
</div>
</td>
<td>{{ trackSet_list_values[loop.index0][3] }}</td>
<td>{{ trackSet_list_values[loop.index0][0] }}</td>
<td>{{ trackSet_list_values[loop.index0][1] }}</td>
<td>{{ trackSet_list_values[loop.index0][2] }}</td>
<td>{{ trackSet_list_num_of_paste[loop.index0] }}</td>
<td><p style="margin: 0px; white-space: nowrap;">
<span data-toggle="modal" data-target="#mymodal" data-term="{{ set }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ set }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[set] %} checked {% endif %}>
</p></td>
<td>
{% for email in notificationEMailTermMapping[set] %}
<a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{set}}">
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
<span class="glyphicon glyphicon-trash" style="color:Red;" ></span>
</div>
</a>
{{ email }}
<br>
{% endfor %}
</td>
</tr>
{% endfor %}
<!-- REGEX -->
{% for regex in trackReg_list %}
<tr style="background-color: #fffdca;">
<td>
<span class="term_name">{{ regex }}</span>
<div>
{% for tag in notificationTagsTermMapping[regex] %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
</a>
{% endfor %}
{% if notificationTagsTermMapping[regex] %}
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
<div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
<div class="modal-dialog">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
<h2 class="text-center">Remove Custom Tag</h2>
</div>
<div class="modal-body">
<form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
{% for tag in notificationTagsTermMapping[regex] %}
<div class="form-check">
<input type="hidden" class="form-control" name="term" value="{{ regex }}">
<input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
<label class="form-check-label">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
</label>
<br>
</div>
{% endfor %}
</form>
</div>
<div class="modal-footer">
<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
<span class="glyphicon glyphicon-trash"></span>
<span class="label-icon">Remove Tags</span>
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
{% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
{% endif %}
</div>
</td>
<td>{{ trackReg_list_values[loop.index0][3] }}</td>
<td>{{ trackReg_list_values[loop.index0][0] }}</td>
<td>{{ trackReg_list_values[loop.index0][1] }}</td>
<td>{{ trackReg_list_values[loop.index0][2] }}</td>
<td>{{ trackReg_list_num_of_paste[loop.index0] }}</td>
<td><p style="margin: 0px; white-space: nowrap;">
<span data-toggle="modal" data-target="#mymodal" data-term="{{ regex }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ regex }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ regex }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[regex] %} checked {% endif %}>
</p></td>
<td>
{% for email in notificationEMailTermMapping[regex] %}
<a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{regex}}">
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
<span class="glyphicon glyphicon-trash" style="color:Red;"></span>
</div>
</a>
{{ email }}
<br>
{% endfor %}
</td>
</tr>
{% endfor %}
<!-- Normal term -->
{% for term in track_list %}
<tr>
<td>
<span class="term_name">{{ term }}</span>
<div>
{% for tag in notificationTagsTermMapping[term] %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
</a>
{% endfor %}
{% if notificationTagsTermMapping[term] %}
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
<div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
<div class="modal-dialog">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
<h2 class="text-center">Remove Custom Tag</h2>
</div>
<div class="modal-body">
<form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
{% for tag in notificationTagsTermMapping[term] %}
<div class="form-check">
<input type="hidden" class="form-control" name="term" value="{{ term }}">
<input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
<label class="form-check-label">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
</label>
<br>
</div>
{% endfor %}
</form>
</div>
<div class="modal-footer">
<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
<span class="glyphicon glyphicon-trash"></span>
<span class="label-icon">Remove Tags</span>
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
{% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
{% endif %}
</div>
</td>
<td>{{ track_list_values[loop.index0][3] }}</td>
<td>{{ track_list_values[loop.index0][0] }}</td>
<td>{{ track_list_values[loop.index0][1] }}</td>
<td>{{ track_list_values[loop.index0][2] }}</td>
<td>{{ track_list_num_of_paste[loop.index0] }}</td>
<td><p style="margin: 0px; white-space: nowrap;">
<span data-toggle="modal" data-target="#mymodal" data-term="{{ term }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ term }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ term }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[term] %} checked {% endif %}>
</p></td>
<td>
{% for email in notificationEMailTermMapping[term] %}
<a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{term}}">
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
<span class="glyphicon glyphicon-trash" style="color:Red;"></span>
</div>
</a>
{{ email }}
<br>
{% endfor %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- Panel OPTIONS -->
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-danger">
<div class="panel-heading">
<strong>Manage blacklisted terms</strong>
</div>
<div class="panel-body">
<div class="form-group input-group" style="margin-bottom: 30px;">
<span class="input-group-addon"><span class="fa fa-eye-slash "></span></span>
<input id="blacklistTermInput" class="form-control" placeholder="Term to track" type="text" style="max-width: 400px;">
<button id="blacklistTermBtn" class="btn btn-danger btn-interaction" style="margin-left: 10px;" data-section="blacklistTerm" data-action="add"> Black list a term</button>
</div>
<table class="table table-striped table-bordered table-hover" id="myTable2">
<thead>
<tr>
<th style="max-width: 800px;">Term</th>
<th>Added date</th>
<th>Action</th>
</tr>
</thead>
<tbody>
{% for term, date in black_list %}
<tr>
<td>{{ term }}</td>
<td>{{ date }}</td>
<td><p style="margin: 0px;">
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="right" title="Remove this term" data-content="{{ term }}" data-section="blacklistTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
</p></td>
</tr>
{% endfor %}
</tbody>
</table>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
</div>
<!-- import graph function -->
<script>
function reload_per_paste() {
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
window.location.href = {{ url_for('terms.terms_management') }}+"?per_paste="+checked;
}
var table_track;
var table_black;
function bindEventsForCurrentPage() {
// On click, get html content from url and update the corresponding modal
$("[data-toggle='modal']").unbind().on("click.openmodal", function (event) {
//console.log(data);
event.preventDefault();
var the_modal=$(this);
var url = "{{ url_for('terms.terms_management_query_paste') }}?term=" + encodeURIComponent($(this).attr('data-term'));
$.getJSON(url, function (data) {
if (data.length != 0) {
var html_to_add = "";
html_to_add += "<table id=\"modal-table\" class=\"table table-striped\">";
html_to_add += "<thead>";
html_to_add += "<tr>";
html_to_add += "<th>Source</th>";
html_to_add += "<th>Date</th>";
html_to_add += "<th>Mime</th>";
html_to_add += "<th>Size (Kb)</th>";
html_to_add += "<th># lines</th>";
html_to_add += "<th>Max length</th>";
html_to_add += "<th>Preview</th>";
html_to_add += "</tr>";
html_to_add += "</thead>";
html_to_add += "<tbody>";
for (i=0; i<data.length; i++) {
curr_data = data[i];
html_to_add += "<tr>";
html_to_add += "<td>"+curr_data.source+"</td>";
html_to_add += "<td>"+curr_data.date+"</td>";
html_to_add += "<td>"+curr_data.mime+"</td>";
html_to_add += "<td>"+curr_data.size+"</td>";
html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";
html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content.replace(/\"/g, "\'")+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('objects_item.showItem') }}?id="+curr_data.path+"&num=0\"> <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
html_to_add += "</tr>";
}
html_to_add += "</tbody>";
html_to_add += "</table>";
$("#mymodalbody").html(html_to_add);
$("[data-toggle=popover]").popover();
$("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
$('#modal-table').DataTable();
} else {
$("#mymodalbody").html("No paste containing this term has been received yet.");
$("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
}
});
});
}
$(document).ready(function(){
bindEventsForCurrentPage();
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
if({{ per_paste }} == 1) {
$("#per_paste").attr('checked', true)
}
$('[data-toggle="tooltip"]').tooltip();
table_track = $('#myTable').DataTable();
table_black = $('#myTable2').DataTable();
table_track.on( 'draw.dt', function () {
perform_binding();
});
table_black.on( 'draw.dt', function () {
perform_binding();
});
$("#followTermInput").keyup(function(event){
if(event.keyCode == 13){
$("#followTermBtn").click();
$("#followTermInput").val("");
}
});
$("#blacklistTermInput").keyup(function(event){
if(event.keyCode == 13){
$("#blacklistTermBtn").click();
$("#blacklistTermInput").val("");
}
});
perform_binding();
$("#mymodal").on('hidden.bs.modal', function () {
$("#mymodalbody").html("<p>Loading paste information...</p>");
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
$("#mymodalbody").append(loading_gif); // Show the loading GIF
});
});
</script>
<script>
function perform_binding() {
$(".btn-interaction").unbind("click.interaction");
$(".btn-interaction").bind("click.interaction", perform_operation);
}
function perform_operation(){
var curr_section = $(this).attr('data-section');
var curr_action = $(this).attr('data-action');
var row_tr = $(this).closest("tr");
if (curr_action == "add") {
var curr_term = $('#'+curr_section+'Input').val();
var email_addresses = $('#followTermEMailNotificationReceiversInput').val();
var tags = $('#followTermTag').val();
} else {
var curr_term = $(this).attr('data-content');
var email_addresses = "";
}
var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses, tags: tags};
if (curr_term != "") {
//console.log(data_to_send);
$.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){
if(status == "success") {
var json = data;
if(json.section == "followTerm") {
if(json.action == "add") {
// query data
$.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
reload_per_paste();
});
} else if (json.action == "delete") {
row_tr.remove()
}
} else if(json.section == "blacklistTerm"){
if(json.action == "add") {
$.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
console.log(data2);
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"blacklistTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
table_black.row.add( [ json.term, data2[3], action_button ] ).draw( false );
perform_binding();
});
} else if (json.action == "delete") {
// Find indexes of row which have the term in the first column
var index = table_black.rows().eq( 0 ).filter( function (rowIdx) {
return table_black.cell( rowIdx, 0 ).data() === json.term;
} );
table_black.rows(index).remove().draw( false );
}
}
}
});
}
}
</script>

275
var/www/modules/terms/templates/terms_plot_tool.html
View file

@ -1,275 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Terms Plot Tool - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/switch_checkbox.css') }}" rel="stylesheet" type="text/css" />
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-ui.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
<style>
.sparkLineStats ul {
padding-left:0;
list-style:none
}
</style>
</head>
<body>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-termsfrequency" >Terms plot tool</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="row">
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-default">
<div class="panel-heading">
<strong>Select options</strong>
</div>
<div class="panel-body">
<div aria-disabled="false" class="slider sliderRange sliderBlue ui-slider ui-slider-horizontal ui-widget ui-widget-content ui-corner-all" style="margin-bottom: 5px;"></div>
<strong>Date:</strong> <input type="text" id="amount" readonly style="border:0; color:#f6931f; font-weight:bold;"> <button id="plot-month" onclick="replot();" class="btn btn-info" style="float: right;">Replot</button>
<div class="form-group input-group" style="margin-top: 30px;">
<span class="input-group-addon"><span class="glyphicon glyphicon-stats"></span></span>
<input id="TermInput" class="form-control" placeholder="Term to plot" type="text" style="max-width: 400px;" data-init-plot="{{ term }}">
<button id="plot-btn" class="btn btn-info" style="margin-left: 10px;"><span class="fa fa-caret-down"> Plot a term</button>
<button id="plot-btn-add" class="btn btn-success" style="margin-left: 6px;"><span class="fa fa-plus"></span> Add the term to the chart</button>
</div>
<div class='pull_right' style="float: right;">
<label class="switch">
<input id="per_paste" class="switch-input" value="per_paste" type="checkbox" checked>
<span class="switch-label" data-on="On" data-off="Off"></span>
<span class="switch-handle"></span>
</label>
<strong style="top: 3px; position: relative;">1 term per paste</strong>
</div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- Panel PLOT -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-default">
<div class="panel-heading">
<strong>Graph</strong>
</div>
<div class="panel-body">
<div id="graph" style="height: 300px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
</div>
<div style="position: absolute; border: 1px solid rgb(255, 221, 221); padding: 2px; background-color: #333; color:white; opacity: 0.8; top: 423px; left: 616px; display: none;" id="tooltip"></div>
<!-- import graph function -->
<script>
$(document).ready(function(){
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
/* Create the slider and button*/
$('#plot-btn-add').hide();
var today = Date.now();
var old_day = today - (12*31*24*60*60)*1000;
$( ".sliderRange" ).slider({
range: true,
min: old_day,
max: today,
values: [ today - (7*24*60*60)*1000, today ],
step: 24*60*60*1000,
slide: function( event, ui ) {
$( "#amount" ).val( new Date(ui.values[ 0 ]).toLocaleDateString() + " - " + new Date(ui.values[ 1 ]).toLocaleDateString() );
}
});
$( "#amount" ).val( new Date($( ".sliderRange" ).slider( "values", 0 )).toLocaleDateString() +
" - " + new Date($( ".sliderRange" ).slider( "values", 1 )).toLocaleDateString() );
$('#plot-btn').click(plotData);
$('#plot-btn-add').click(addData);
$("#TermInput").val($("#TermInput").attr("data-init-plot"));
if($("#TermInput").attr("data-init-plot") != "") {
$("#plot-btn").click();
}
$("#TermInput").keyup(function(event){
if(event.keyCode == 13){
$("#plot-btn").click();
$("#TermInput").val("");
}
});
});
</script>
<script>
var plot;
var graph_data = [];
var plotted_terms = [];
var graph_options = {
series: {
lines: {
show: true,
lineWidth: 2
},
bars: {show: false, barWidth: 60*60*1000},
shadowSize: 0
},
grid: {
hoverable: true,
clickable: true,
tickColor: "#f9f9f9",
borderWidth: 0
},
xaxis: {
mode: "time",
timeformat: "%m/%d",
minTickSize: [1, "day"]
},
yaxis: {
autoscaleMargin: 0.1,
},
}
function plotData() {
$('#plot-btn-add').show("fast");
var curthis = $(this);
var term = $('#TermInput').val();
plotted_terms = [term]
var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
$.getJSON("{{ url_for('terms.terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
graph_data = [];
var to_plot = [];
var curr_data = [];
for(i=1; i<data.length; i++) {
curr_data.push([data[i][0]*1000, data[i][1]]);
}
to_plot.push({ data: curr_data, label: term});
graph_data.push({ data: curr_data, label: term});
plot = $.plot($("#graph"), to_plot, graph_options);
$("#graph").bind("plothover", function (event, pos, item) {
if (item) {
var date = new Date(item.datapoint[0]);
var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
var y = item.datapoint[1];
$("#tooltip").html(item.series.label + " for "+x + " = " + y)
.css({top: item.pageY-15, left: item.pageX+5})
.fadeIn(200);
} else {
$("#tooltip").hide();
}
});
$("#TermInput").val("");
})
}
function addData() {
var curthis = $(this);
var term = $('#TermInput').val();
plotted_terms.push(term)
var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
$.getJSON("{{ url_for('terms.terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
var curr_data = [];
for(i=1; i<data.length; i++) {
curr_data.push([data[i][0]*1000, data[i][1]]);
}
graph_data.push({ data: curr_data, label: term});
plot = $.plot($("#graph"), graph_data, graph_options);
$("#TermInput").val("");
})
}
function replot() {
graph_data = [];
promises = [];
for(i=0; i<plotted_terms.length; i++) {
var term = plotted_terms[i];
var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
promises.push($.getJSON("{{ url_for('terms.terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
var curr_data = [];
for(i=1; i<data.length; i++) {
curr_data.push([data[i][0]*1000, data[i][1]]);
}
graph_data.push({ data: curr_data, label: data[0]});
$("#TermInput").val("");
}))
}
$.when.apply($, promises).done( function () {
plot = $.plot($("#graph"), graph_data, graph_options);
});
}
</script>

576
var/www/modules/terms/templates/terms_plot_top.html
View file

@ -1,576 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Terms Plot Top - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/switch_checkbox.css') }}" rel="stylesheet" type="text/css" />
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
<style>
.sparkLineStats ul {
padding-left:0;
list-style:none
}
.table > tbody > tr > td {
padding: 5px;
}
</style>
</head>
<body>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-termsfrequency" >Terms frequency: Top set information</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="row">
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div class='pull_right'>
<label class="switch">
<input id="per_paste" class="switch-input" value="per_paste" type="checkbox" onclick="reload_per_paste()">
<span class="switch-label" data-on="On" data-off="Off"></span>
<span class="switch-handle"></span>
</label>
<strong style="top: 3px; position: relative;">1 term per paste</strong>
<div>
<div id="panel-today" class="panel panel-info">
<div class="panel-heading">
<strong>Today</strong>
</div>
<div class="panel-body">
<div id="panel-today" class="panel panel-default">
<div class="panel-heading">
<strong>Today top word</strong>
</div>
<div class="panel-body">
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the week and month set">Position</th>
</tr>
</thead>
<tbody id="table-today">
</tbody>
</table>
</div>
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the week and month set">Position</th>
</tr>
</thead>
<tbody id="table-today2">
</tbody>
</table>
</div>
</div>
<!-- /.panel-body -->
</div>
<div id="panel-today" class="panel panel-default">
<div class="panel-heading">
<strong>Graph</strong>
</div>
<div class="panel-body">
<div id="graph-today" style="height: 400px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-info">
<div class="panel-heading">
<strong>Week</strong>
</div>
<div class="panel-body">
<div id="panel-week" class="panel panel-default">
<div class="panel-heading">
<strong>Week top word</strong>
</div>
<div class="panel-body">
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and month set">Position</th>
</tr>
</thead>
<tbody id="table-week">
</tbody>
</table>
</div>
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and month set">Position</th>
</tr>
</thead>
<tbody id="table-week2">
</tbody>
</table>
</div>
</div>
<!-- /.panel-body -->
</div>
<div id="panel-week" class="panel panel-default">
<div class="panel-heading">
<strong>Graph</strong>
</div>
<div class="panel-body">
<div id="graph-week" style="height: 400px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- Panel OPTIONS -->
<div class="row">
<div class="col-lg-12">
<div class="row">
<div class="col-lg-12">
<div id="panel-today" class="panel panel-info">
<div class="panel-heading">
<strong>Month</strong>
</div>
<div class="panel-body">
<div id="panel-month" class="panel panel-default">
<div class="panel-heading">
<strong>Month top word</strong>
</div>
<div class="panel-body">
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and week set">Position</th>
</tr>
</thead>
<tbody id="table-month">
</tbody>
</table>
</div>
<div class="col-lg-6">
<table class="table table-striped">
<thead>
<tr>
<th>Term</th>
<th>Value</th>
<th>Action</th>
<th>Show</th>
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and week set">Position</th>
</tr>
</thead>
<tbody id="table-month2">
</tbody>
</table>
</div>
</div>
<!-- /.panel-body -->
</div>
<div id="panel-month" class="panel panel-default">
<div class="panel-heading">
<strong>Graph</strong>
</div>
<div class="panel-body">
<div id="graph-month" style="height: 400px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.panel -->
</div>
<!-- /.panel -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
</div>
<div style="position: absolute; border: 1px solid rgb(255, 221, 221); padding: 2px; background-color: #333; color:white; opacity: 0.8; top: 423px; left: 616px; display: none;" id="tooltip"></div>
<!-- import graph function -->
<script>
$(document).ready(function(){
activePage = $('h1.page-header').attr('data-page');
$("#"+activePage).addClass("active");
if({{ per_paste }} == 1) {
$("#per_paste").attr('checked', true)
}
});
function reload_per_paste() {
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
window.location.href = {{ url_for('terms.terms_plot_top') }}+"?per_paste="+checked;
}
</script>
<script>
var graph_options = {
series: {
lines: {
show: true,
lineWidth: 2
},
bars: {show: false, barWidth: 60*60*1000},
shadowSize: 0
},
grid: {
hoverable: true,
clickable: true,
tickColor: "#f9f9f9",
borderWidth: 0
},
legend: { show: true,
noColumns: 1,
position: "nw",
labelFormatter: function(label, series) {
return "<a href=\"#\" onClick=\"hide_or_show2("+series.idx+", "+series.graphNum+"); return false;\" >" + label + "</a>";
}
},
xaxis: {
mode: "time",
timeformat: "%m/%d",
minTickSize: [1, "day"]
},
yaxis: {
//transform: function (v) { return v < 1 ? v : Math.log(v); }
autoscaleMargin: 0.1,
},
tooltip: true,
tooltipOpts: {
content: " %s (%x.1 is %y.4) ",
shifts: {
x: -60,
y: 25
}
}
}
set_today = "TopTermFreq_set_day";
set_week = "TopTermFreq_set_week";
set_month = "TopTermFreq_set_month";
default_num_curves = 8;
per_paste = {{ per_paste }}
var plot_today;
var plot_week;
var plot_month;
var promises = []; // Used to know when everything has been received
promises.push($.getJSON("{{ url_for('terms.terms_plot_top_data') }}", { set: set_today, num_day: 5, per_paste: per_paste }, function(data, status){
data.sort(function(a, b){return b[2]-a[2];});
// Sort data
var table_today = $("#table-today")
var table_today2 = $("#table-today2")
var to_plot = [];
var unchecked_label = [];
for(i=0; i<data.length; i++) {
var highlight = data[i][3].week == "<20"? " style=\"background-color: lightgreen;\" " : "";
var curr_data = [];
for(j=0; j<data[i][1].length; j++) {
curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
}
if (i>=default_num_curves) {
unchecked_label.push(data[i][0]);
}
to_plot.push({ data: curr_data, label: data[i][0], idx: i});
if ( i < (data.length/2))
table_today.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("today", data[i][0], i<default_num_curves)+"</td><td>"+data[i][3].week+", "+data[i][3].month+"</td></tr>");
else
table_today2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("today", data[i][0], i<default_num_curves)+"</td><td>"+data[i][3].week+", "+data[i][3].month+"</td></tr>");
}
graph_options.series.graphNum=1;
plot_today = $.plot($("#graph-today"), to_plot, graph_options);
hide_unchecked_curves(plot_today, unchecked_label);
$("#graph-today").bind("plothover", function (event, pos, item) {
if (item) {
var date = new Date(item.datapoint[0]);
var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
var y = item.datapoint[1];
$("#tooltip").html(item.series.label + " for "+x + " = " + y)
.css({top: item.pageY+5, left: item.pageX+5})
.fadeIn(200);
} else {
$("#tooltip").hide();
}
});
}));
promises.push($.getJSON("{{ url_for('terms.terms_plot_top_data') }}", { set: set_week, num_day: 7, per_paste: per_paste }, function(data, status){
data.sort(function(a, b){return b[2]-a[2];});
// Sort data
var table = $("#table-week")
var table2 = $("#table-week2")
var to_plot = [];
var unchecked_label = [];
for(i=0; i<data.length; i++) {
var highlight = parseInt(data[i][3].day) > 20? " style=\"background-color: orange;\" " : "";
var curr_data = [];
for(j=0; j<data[i][1].length; j++) {
curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
}
if (i>=default_num_curves) {
unchecked_label.push(data[i][0]);
}
to_plot.push({ data: curr_data, label: data[i][0], idx: i});
if ( i < (data.length/2))
table.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("week", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].month+"</td></tr>");
else
table2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("week", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].month+"</td></tr>");
}
graph_options.series.graphNum=2;
plot_week = $.plot($("#graph-week"), to_plot, graph_options);
hide_unchecked_curves(plot_week, unchecked_label);
$("#graph-week").bind("plothover", function (event, pos, item) {
if (item) {
var date = new Date(item.datapoint[0]);
var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
var y = item.datapoint[1];
$("#tooltip").html(item.series.label + " for "+x + " = " + y)
.css({top: item.pageY+5, left: item.pageX+5})
.fadeIn(200);
} else {
$("#tooltip").hide();
}
});
}));
promises.push($.getJSON("{{ url_for('terms.terms_plot_top_data') }}", { set: set_month, num_day: 31, per_paste: per_paste }, function(data, status){
data.sort(function(a, b){return b[2]-a[2];});
// Sort data
var table = $("#table-month")
var table2 = $("#table-month2")
var to_plot = [];
var unchecked_label = [];
for(i=0; i<data.length; i++) {
var highlight = parseInt(data[i][3].day) > 20? " style=\"background-color: orange;\" " : "";
var curr_data = [];
for(j=0; j<data[i][1].length; j++) {
curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
}
if (i>=default_num_curves) {
unchecked_label.push(data[i][0]);
}
to_plot.push({ data: curr_data, label: data[i][0], idx: i});
if ( i < (data.length/2))
table.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("month", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].week+"</td></tr>");
else
table2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("month", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].week+"</td></tr>");
}
graph_options.series.graphNum=3;
plot_month = $.plot($("#graph-month"), to_plot, graph_options);
hide_unchecked_curves(plot_month, unchecked_label);
$("#graph-month").bind("plothover", function (event, pos, item) {
if (item) {
var date = new Date(item.datapoint[0]);
var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
var y = item.datapoint[1];
$("#tooltip").html(item.series.label + " for "+x + " = " + y)
.css({top: item.pageY+5, left: item.pageX+5})
.fadeIn(200);
} else {
$("#tooltip").hide();
}
});
}));
/* When everything has been received, start adding tooltip */
$.when.apply($, promises).done( function (arg) {
$('[data-toggle="tooltip"]').tooltip();
$(".btn-interaction").unbind("click.interaction");
$(".btn-interaction").bind("click.interaction", perform_operation);
$(".check-interaction").unbind("click.interaction");
$(".check-interaction").bind("click.interaction", hide_or_show);
});
function addbuttons(term) {
return "<button class=\"fa fa-eye btn-interaction btn btn-success\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Add to tracked list\" style=\"margin-right: 5px;\""+
"data-section=\"followTerm\" data-term=\""+term+"\"></button>"+
"<button class=\"fa fa-eye-slash btn-interaction btn btn-danger\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Add to black-list\""+
"data-section=\"blacklistTerm\" data-term=\""+term+"\"></button>";
}
function addcheckbox(graph, term, checked) {
var checked_text = checked ? "checked" : "";
return "<input type=checkbox "+checked_text+" class=\"check-interaction\" data-term=\""+term+"\" data-graph=\""+graph+"\"></input>";
}
function perform_operation(){
var curr_section = $(this).attr('data-section');
var curr_term = $(this).attr('data-term');
var data_to_send = { section: curr_section, action:"add", term: curr_term};
$.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){
if(status == "success") {
location.reload();
}
});
}
function hide_unchecked_curves(plot, unchecked_label) {
var graphData = plot.getData();
var index;
for(i=0; i<graphData.length; i++) {
if($.inArray( graphData[i].label, unchecked_label ) != -1){
graphData[i].lines.show = false;
}
}
plot.setData(graphData);
plot.draw();
}
function hide_or_show() {
var curr_term = $(this).attr('data-term');
var graph = $(this).attr('data-graph');
var checked = $(this).prop('checked')
if(graph == "today") {
var graphData = plot_today.getData();
var index;
for(i=0; i<graphData.length; i++){
if(graphData[i].label == curr_term){
index = i;
break;
}
}
graphData[index].lines.show = checked;
plot_today.setData(graphData);
plot_today.draw();
} else if (graph == "week") {
var graphData = plot_week.getData();
var index;
for(i=0; i<graphData.length; i++){
if(graphData[i].label == curr_term){
index = i;
break;
}
}
graphData[index].lines.show = checked;
plot_week.setData(graphData);
plot_week.draw();
} else if (graph == "month") {
var graphData = plot_month.getData();
var index;
for(i=0; i<graphData.length; i++){
if(graphData[i].label == curr_term){
index = i;
break;
}
}
graphData[index].lines.show = checked;
plot_month.setData(graphData);
plot_month.draw();
}
// graph, hide curve
}
function hide_or_show2(index, graphNum) {
if (graphNum == 1)
var plot = plot_today;
else if (graphNum == 2)
var plot = plot_week;
else if (graphNum == 3)
var plot = plot_month;
var graphData = plot.getData();
graphData[index].lines.show = !graphData[index].lines.show;
plot.setData(graphData);
plot.draw();
}
</script>
</body>
</html>

3
var/www/templates/modals/create_hive_case.html
View file

@ -6,7 +6,7 @@
<img id="hive-logo" src="{{url_for('static', filename='image/thehive-logo.png') }}" width="500" > <img id="hive-logo" src="{{url_for('static', filename='image/thehive-logo.png') }}" width="500" >
</div> </div>
<form method="post" action="{{ url_for('PasteSubmit.create_hive_case') }}" target="_blank"> <form method="post" action="{{ url_for('import_export.create_thehive_case') }}" target="_blank">
<div class="modal-body"> <div class="modal-body">
@ -35,6 +35,7 @@
<label for="hive_description">Description</label> <label for="hive_description">Description</label>
<input name="hive_description" class="form-control span6" placeholder="Quick Case Description" type="text" id="hive_description"/> <input name="hive_description" class="form-control span6" placeholder="Quick Case Description" type="text" id="hive_description"/>
</div> </div>
<input type="hidden" id="obj_type" name="obj_type" value="item">
<input type="hidden" id="obj_id" name="obj_id" value="{{ dict_item['id'] }}"> <input type="hidden" id="obj_id" name="obj_id" value="{{ dict_item['id'] }}">
</div> </div>