mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-26 07:47:17 +00:00
chg: [cleanup] remove ARDB + fix hive case
This commit is contained in:
parent
0e41c95b5c
commit
3365a054a8
42 changed files with 431 additions and 3083 deletions
|
@ -252,6 +252,8 @@ def trackers_migration():
|
|||
for id in old_Tracker.get_retro_hunt_items_by_daterange(task_uuid, meta['date_from'], meta['date_to']):
|
||||
Tracker.save_retro_hunt_match(task_uuid, id)
|
||||
|
||||
Tracker._fix_db_custom_tags()
|
||||
|
||||
|
||||
def investigations_migration():
|
||||
print('INVESTIGATION MIGRATION...')
|
||||
|
|
|
@ -140,12 +140,6 @@ class Process(object):
|
|||
db=self.config.get('RedisPubSub', 'db'),
|
||||
decode_responses=True)
|
||||
|
||||
self.serv_statistics = redis.StrictRedis(
|
||||
host=self.config.get('ARDB_Statistics', 'host'),
|
||||
port=self.config.get('ARDB_Statistics', 'port'),
|
||||
db=self.config.get('ARDB_Statistics', 'db'),
|
||||
decode_responses=True)
|
||||
|
||||
self.moduleNum = os.getpid()
|
||||
|
||||
def populate_set_in(self):
|
||||
|
@ -181,9 +175,9 @@ class Process(object):
|
|||
try:
|
||||
if '.gz' in message:
|
||||
path = message.split(".")[-2].split("/")[-1]
|
||||
#find start of path with AIL_HOME
|
||||
# find start of path with AIL_HOME
|
||||
index_s = message.find(os.environ['AIL_HOME'])
|
||||
#Stop when .gz
|
||||
# Stop when .gz
|
||||
index_e = message.find(".gz")+3
|
||||
if(index_s == -1):
|
||||
complete_path = message[0:index_e]
|
||||
|
@ -200,7 +194,6 @@ class Process(object):
|
|||
self.r_temp.sadd("MODULE_TYPE_"+self.subscriber_name, str(self.moduleNum))
|
||||
|
||||
curr_date = datetime.date.today()
|
||||
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'paste_by_modules_in:'+self.subscriber_name, 1)
|
||||
return message
|
||||
|
||||
except:
|
||||
|
@ -238,6 +231,3 @@ class Process(object):
|
|||
continue
|
||||
self.pubsub.publish(message)
|
||||
|
||||
def incr_module_timeout_statistic(self):
|
||||
curr_date = datetime.date.today()
|
||||
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'paste_by_modules_timeout:'+self.subscriber_name, 1)
|
||||
|
|
|
@ -237,6 +237,8 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Telegram" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Telegram.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tools" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Tools.py; read x"
|
||||
sleep 0.1
|
||||
|
||||
screen -S "Script_AIL" -X screen -t "Hosts" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Hosts.py; read x"
|
||||
sleep 0.1
|
||||
|
@ -283,8 +285,6 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tools" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tools.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./IPAddress.py; read x"
|
||||
|
|
|
@ -1,69 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
import time
|
||||
from lib.objects.Items import Item
|
||||
from pubsublogger import publisher
|
||||
from Helper import Process
|
||||
import re
|
||||
|
||||
import signal
|
||||
|
||||
class TimeoutException(Exception):
|
||||
pass
|
||||
|
||||
def timeout_handler(signum, frame):
|
||||
raise TimeoutException
|
||||
|
||||
signal.signal(signal.SIGALRM, timeout_handler)
|
||||
|
||||
'''
|
||||
This module takes its input from the global module.
|
||||
It applies some regex and publish matched content
|
||||
'''
|
||||
|
||||
if __name__ == "__main__":
|
||||
publisher.port = 6380
|
||||
publisher.channel = "Script"
|
||||
config_section = "Release"
|
||||
p = Process(config_section)
|
||||
max_execution_time = p.config.getint("Curve", "max_execution_time")
|
||||
publisher.info("Release scripts to find release names")
|
||||
|
||||
movie = "[a-zA-Z0-9.]+\.[0-9]{4}.[a-zA-Z0-9.]+\-[a-zA-Z]+"
|
||||
tv = "[a-zA-Z0-9.]+\.S[0-9]{2}E[0-9]{2}.[a-zA-Z0-9.]+\.[a-zA-Z0-9.]+\-[a-zA-Z0-9]+"
|
||||
xxx = "[a-zA-Z0-9._]+.XXX.[a-zA-Z0-9.]+\-[a-zA-Z0-9]+"
|
||||
|
||||
regexs = [movie, tv, xxx]
|
||||
|
||||
regex = '|'.join(regexs)
|
||||
while True:
|
||||
signal.alarm(max_execution_time)
|
||||
filepath = p.get_from_set()
|
||||
if filepath is None:
|
||||
publisher.debug("Script Release is Idling 10s")
|
||||
print('Sleeping')
|
||||
time.sleep(10)
|
||||
continue
|
||||
|
||||
item = Item(filepath)
|
||||
content = item.get_content()
|
||||
|
||||
#signal.alarm(max_execution_time)
|
||||
try:
|
||||
releases = set(re.findall(regex, content))
|
||||
if len(releases) == 0:
|
||||
continue
|
||||
|
||||
to_print = f'Release;{item.get_source()};{item.get_date()};{item.get_basename()};{len(releases)} releases;{item.get_id()}'
|
||||
print(to_print)
|
||||
if len(releases) > 30:
|
||||
publisher.warning(to_print)
|
||||
else:
|
||||
publisher.info(to_print)
|
||||
|
||||
except TimeoutException:
|
||||
p.incr_module_timeout_statistic()
|
||||
print(f"{item.get_id()} processing timeout")
|
||||
continue
|
||||
else:
|
||||
signal.alarm(0)
|
|
@ -1,55 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
import time
|
||||
from lib.objects.Items import Item
|
||||
from pubsublogger import publisher
|
||||
from Helper import Process
|
||||
import re
|
||||
|
||||
if __name__ == "__main__":
|
||||
publisher.port = 6380
|
||||
publisher.channel = "Script"
|
||||
config_section = "SourceCode"
|
||||
p = Process(config_section)
|
||||
publisher.info("Finding Source Code")
|
||||
|
||||
critical = 0 # AS TO BE IMPORTANT, MIGHT BE REMOVED
|
||||
|
||||
# RELEVANT LANGUAGES
|
||||
shell = r"[a-zA-Z0-9]+@[a-zA-Z0-9\-]+\:\~\$"
|
||||
c = r"\#include\ \<[a-z\/]+.h\>"
|
||||
php = r"\<\?php"
|
||||
python = r"import\ [\w]+"
|
||||
bash = r"#!\/[\w]*\/bash"
|
||||
javascript = r"function\(\)"
|
||||
ruby = r"require \ [\w]+"
|
||||
adr = r"0x[a-f0-9]{2}"
|
||||
|
||||
# asm = r"\"((?s).{1}x[0-9a-f]{2}){3,}" ISSUES WITH FINDALL, pattern like \x54\xaf\x23\..
|
||||
|
||||
languages = [shell, c, php, bash, python, javascript, bash, ruby, adr]
|
||||
regex = '|'.join(languages)
|
||||
print(regex)
|
||||
|
||||
while True:
|
||||
message = p.get_from_set()
|
||||
if message is None:
|
||||
publisher.debug("Script Source Code is Idling 10s")
|
||||
print('Sleeping')
|
||||
time.sleep(10)
|
||||
continue
|
||||
|
||||
filepath, count = message.split()
|
||||
|
||||
item = Item(filepath)
|
||||
content = item.get_content()
|
||||
match_set = set(re.findall(regex, content))
|
||||
if len(match_set) == 0:
|
||||
continue
|
||||
|
||||
to_print = f'SourceCode;{item.get_source()};{item.get_date()};{item.get_basename()};{item.get_id()}'
|
||||
|
||||
if len(match_set) > critical:
|
||||
publisher.warning(to_print)
|
||||
else:
|
||||
publisher.info(to_print)
|
|
@ -72,6 +72,25 @@ def is_hive_connected():
|
|||
|
||||
HIVE_CLIENT = get_hive_client()
|
||||
|
||||
def sanitize_threat_level_hive(threat_level):
|
||||
try:
|
||||
int(threat_level)
|
||||
if 1 <= threat_level <= 3:
|
||||
return threat_level
|
||||
else:
|
||||
return 2
|
||||
except:
|
||||
return 2
|
||||
|
||||
def sanitize_tlp_hive(tlp):
|
||||
try:
|
||||
int(tlp)
|
||||
if 0 <= tlp <= 3:
|
||||
return tlp
|
||||
else:
|
||||
return 2
|
||||
except:
|
||||
return 2
|
||||
|
||||
def create_thehive_alert(item_id, tag_trigger):
|
||||
item = Item(item_id)
|
||||
|
|
|
@ -1,76 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import gzip
|
||||
import datetime
|
||||
import redis
|
||||
import json
|
||||
import time
|
||||
|
||||
import shutil
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||
from HiddenServices import HiddenServices
|
||||
from Helper import Process
|
||||
|
||||
def substract_date(date_from, date_to):
|
||||
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
|
||||
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
|
||||
delta = date_to - date_from # timedelta
|
||||
l_date = []
|
||||
for i in range(delta.days + 1):
|
||||
date = date_from + datetime.timedelta(i)
|
||||
l_date.append( date.strftime('%Y%m%d') )
|
||||
return l_date
|
||||
|
||||
config_section = 'Keys'
|
||||
p = Process(config_section)
|
||||
|
||||
r_serv_onion = redis.StrictRedis(
|
||||
host=p.config.get("ARDB_Onion", "host"),
|
||||
port=p.config.getint("ARDB_Onion", "port"),
|
||||
db=p.config.getint("ARDB_Onion", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
date_from = '20190614'
|
||||
date_to = '20190615'
|
||||
service_type = 'onion'
|
||||
date_range = substract_date(date_from, date_to)
|
||||
|
||||
dir_path = os.path.join(os.environ['AIL_HOME'], 'temp')
|
||||
|
||||
domain_skipped = []
|
||||
|
||||
for date in date_range:
|
||||
domains_up = list(r_serv_onion.smembers('{}_up:{}'.format(service_type, date)))
|
||||
if domains_up:
|
||||
save_path = os.path.join(dir_path, date[0:4], date[4:6], date[6:8])
|
||||
try:
|
||||
os.makedirs(save_path)
|
||||
except FileExistsError:
|
||||
pass
|
||||
for domain in domains_up:
|
||||
print(domain)
|
||||
h = HiddenServices(domain, 'onion')
|
||||
item_core = h.get_domain_crawled_core_item()
|
||||
if 'root_item' in item_core:
|
||||
l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item'])
|
||||
try:
|
||||
res = h.create_domain_basic_archive(l_pastes)
|
||||
filename = os.path.join(save_path, '{}'.format(domain))
|
||||
with open(filename, 'wb') as f:
|
||||
shutil.copyfileobj(res, f)
|
||||
print('done')
|
||||
except Exception as e:
|
||||
print('skipped')
|
||||
domain_skipped.append(domain)
|
||||
pass
|
||||
|
||||
print()
|
||||
print()
|
||||
print('DOMAINS SKIPPED: ')
|
||||
for domain in domain_skipped:
|
||||
print(domain)
|
|
@ -1,50 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import gzip
|
||||
import base64
|
||||
import uuid
|
||||
import datetime
|
||||
import base64
|
||||
import redis
|
||||
import json
|
||||
import time
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from Helper import Process
|
||||
|
||||
def substract_date(date_from, date_to):
|
||||
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
|
||||
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
|
||||
delta = date_to - date_from # timedelta
|
||||
l_date = []
|
||||
for i in range(delta.days + 1):
|
||||
date = date_from + datetime.timedelta(i)
|
||||
l_date.append( date.strftime('%Y%m%d') )
|
||||
return l_date
|
||||
|
||||
config_section = 'Global'
|
||||
p = Process(config_section)
|
||||
|
||||
r_tags = redis.StrictRedis(
|
||||
host=p.config.get("ARDB_Tags", "host"),
|
||||
port=p.config.getint("ARDB_Tags", "port"),
|
||||
db=p.config.getint("ARDB_Tags", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
tag = 'infoleak:automatic-detection="bitcoin-address"'
|
||||
|
||||
# get tag first/last seen
|
||||
first_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'first_seen')
|
||||
last_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
|
||||
|
||||
l_dates = substract_date(first_seen, last_seen)
|
||||
|
||||
# get all tagged items
|
||||
for date in l_dates:
|
||||
daily_tagged_items = r_tags.smembers('{}:{}'.format(tag, date))
|
||||
|
||||
for item in daily_tagged_items:
|
||||
p.populate_set_out(item)
|
|
@ -1,50 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
import gzip
|
||||
import base64
|
||||
import uuid
|
||||
import datetime
|
||||
import base64
|
||||
import redis
|
||||
import json
|
||||
import time
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
from Helper import Process
|
||||
|
||||
def substract_date(date_from, date_to):
|
||||
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
|
||||
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
|
||||
delta = date_to - date_from # timedelta
|
||||
l_date = []
|
||||
for i in range(delta.days + 1):
|
||||
date = date_from + datetime.timedelta(i)
|
||||
l_date.append( date.strftime('%Y%m%d') )
|
||||
return l_date
|
||||
|
||||
config_section = 'Keys'
|
||||
p = Process(config_section)
|
||||
|
||||
r_tags = redis.StrictRedis(
|
||||
host=p.config.get("ARDB_Tags", "host"),
|
||||
port=p.config.getint("ARDB_Tags", "port"),
|
||||
db=p.config.getint("ARDB_Tags", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
tag = 'infoleak:automatic-detection="pgp-message"'
|
||||
|
||||
# get tag first/last seen
|
||||
first_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'first_seen')
|
||||
last_seen = r_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
|
||||
|
||||
l_dates = substract_date(first_seen, last_seen)
|
||||
|
||||
# get all tagged items
|
||||
for date in l_dates:
|
||||
daily_tagged_items = r_tags.smembers('{}:{}'.format(tag, date))
|
||||
|
||||
for item in daily_tagged_items:
|
||||
p.populate_set_out(item, 'PgpDump')
|
|
@ -10,7 +10,6 @@ Recieve Json Items (example: Twitter feeder)
|
|||
import os
|
||||
import importlib
|
||||
import json
|
||||
import redis
|
||||
import sys
|
||||
import time
|
||||
|
||||
|
|
|
@ -15,7 +15,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
|||
from lib import ConfigLoader
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_db = config_loader.get_redis_conn("_DB")
|
||||
config_loader = None
|
||||
|
||||
#### TO PUT IN CONFIG
|
||||
|
|
|
@ -3,6 +3,7 @@
|
|||
import os
|
||||
import json
|
||||
import requests
|
||||
import sys
|
||||
import configparser
|
||||
|
||||
misp_module_url = 'http://localhost:6666'
|
||||
|
@ -16,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
from lib import ConfigLoader
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv = config_loader.get_redis_conn("_DB")
|
||||
config_loader = None
|
||||
|
||||
def init_config(config_path=default_config_path):
|
||||
|
@ -114,6 +115,7 @@ def parse_module_enrichment_response(misp_module_response):
|
|||
#print(response_types)
|
||||
return response_values
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
load_modules_list()
|
||||
|
|
|
@ -12,8 +12,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
from lib import ConfigLoader
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_statistics = config_loader.get_redis_conn("ARDB_Statistics")
|
||||
# r_serv_trend = ConfigLoader().get_redis_conn("ARDB_Trending")
|
||||
r_statistics = config_loader.get_db_conn("Kvrocks_Stats")
|
||||
config_loader = None
|
||||
|
||||
PIE_CHART_MAX_CARDINALITY = 8
|
||||
|
@ -113,11 +112,11 @@ def update_module_stats(module_name, num, keyword, date):
|
|||
|
||||
if r_statistics.zcard(f'top_{module_name}_set_{date}') < PIE_CHART_MAX_CARDINALITY:
|
||||
r_statistics.zadd(f'top_{module_name}_set_{date}', {keyword: float(keyword_total_sum)})
|
||||
else: # zset at full capacity
|
||||
else: # zset at full capacity
|
||||
member_set = r_statistics.zrangebyscore(f'top_{module_name}_set_{date}', '-inf', '+inf', withscores=True, start=0, num=1)
|
||||
# Member set is a list of (value, score) pairs
|
||||
if int(member_set[0][1]) < keyword_total_sum:
|
||||
#remove min from set and add the new one
|
||||
# remove min from set and add the new one
|
||||
r_statistics.zrem(f'top_{module_name}_set_{date}', member_set[0][0])
|
||||
r_statistics.zadd(f'top_{module_name}_set_{date}', {keyword: float(keyword_total_sum)})
|
||||
|
||||
|
|
|
@ -620,7 +620,7 @@ def update_tag_metadata(tag, date, delete=False): # # TODO: delete Tags
|
|||
# old
|
||||
# r_tags.smembers(f'{tag}:{date}')
|
||||
# r_tags.smembers(f'{obj_type}:{tag}')
|
||||
def get_tag_objects(obj_type, subtype='', date=''):
|
||||
def get_tag_objects(tag, obj_type, subtype='', date=''):
|
||||
if obj_type == 'item':
|
||||
return r_tags.smembers(f'{obj_type}:{subtype}:{tag}:{date}')
|
||||
else:
|
||||
|
@ -1079,6 +1079,10 @@ def get_modal_add_tags(object_id, object_type='item', object_subtype=''):
|
|||
return {"active_taxonomies": get_active_taxonomies(), "active_galaxies": get_active_galaxies(),
|
||||
"object_id": object_id, "object_type": object_type, "object_subtype": object_subtype}
|
||||
|
||||
#####################################################################################
|
||||
#####################################################################################
|
||||
#####################################################################################
|
||||
|
||||
######## NEW VERSION ########
|
||||
def create_custom_tag(tag):
|
||||
r_tags.sadd('tags:custom', tag)
|
||||
|
@ -1137,28 +1141,45 @@ def get_enabled_tags_with_synonyms_ui():
|
|||
|
||||
## Objects tags ##
|
||||
|
||||
###################################################################################
|
||||
###################################################################################
|
||||
###################################################################################
|
||||
###################################################################################
|
||||
###################################################################################
|
||||
###################################################################################
|
||||
###################################################################################
|
||||
###################################################################################
|
||||
###################################################################################
|
||||
|
||||
def add_global_tag(tag, object_type=None):
|
||||
'''
|
||||
Create a set of all tags used in AIL (all + by object)
|
||||
# TYPE -> taxonomy/galaxy/custom
|
||||
|
||||
:param tag: tag
|
||||
:type tag: str
|
||||
:param object_type: object type
|
||||
:type object_type: str
|
||||
'''
|
||||
r_tags.sadd('list_tags', tag)
|
||||
if object_type:
|
||||
r_tags.sadd('list_tags:{}'.format(object_type), tag)
|
||||
class Tag:
|
||||
|
||||
def __int__(self, t_type, t_id, obj='item'):
|
||||
self.type = t_type
|
||||
self.id = t_id
|
||||
self.obj = obj
|
||||
|
||||
def get_first_seen(self):
|
||||
pass
|
||||
|
||||
def get_last_seen(self):
|
||||
pass
|
||||
|
||||
def get_color(self):
|
||||
pass
|
||||
|
||||
def is_enabled(self):
|
||||
pass
|
||||
|
||||
def get_meta(self):
|
||||
meta = {'first_seen': self.get_first_seen(),
|
||||
'last_seen': self.get_last_seen(),
|
||||
'obj': self.obj,
|
||||
'tag': self.id,
|
||||
'type': self.type}
|
||||
|
||||
|
||||
###################################################################################
|
||||
###################################################################################
|
||||
###################################################################################
|
||||
###################################################################################
|
||||
|
||||
def add_obj_tags(object_id, object_subtype, object_type, tags=[], galaxy_tags=[]):
|
||||
for tag in tags:
|
||||
|
@ -1201,23 +1222,6 @@ def api_add_obj_tags(tags=[], galaxy_tags=[], object_id=None, object_type="item"
|
|||
res_dict['type'] = object_type
|
||||
return res_dict, 200
|
||||
|
||||
# def add_tag(object_type, tag, object_id, obj_date=None):
|
||||
# # new tag
|
||||
# if not is_obj_tagged(object_id, tag):
|
||||
# # # TODO: # FIXME: sanitize object_type
|
||||
# if obj_date:
|
||||
# try:
|
||||
# obj_date = int(obj_date)
|
||||
# except:
|
||||
# obj_date = None
|
||||
# if not obj_date:
|
||||
# obj_date = get_obj_date(object_type, object_id)
|
||||
# add_global_tag(tag, object_type=object_type)
|
||||
# add_obj_tag(object_type, object_id, tag, obj_date=obj_date)
|
||||
# update_tag_metadata(tag, obj_date, object_type=object_type)
|
||||
#
|
||||
# # create tags stats # # TODO: put me in cache
|
||||
# r_tags.hincrby('daily_tags:{}'.format(datetime.date.today().strftime("%Y%m%d")), tag, 1)
|
||||
|
||||
# def delete_obj_tag(object_type, object_id, tag, obj_date):
|
||||
# if object_type=="item": # # TODO: # FIXME: # REVIEW: !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
|
||||
|
|
|
@ -551,9 +551,11 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti
|
|||
|
||||
# create tracker tags list
|
||||
for tag in tags:
|
||||
r_serv_tracker.sadd(f'tracker:tags:{tracker_uuid}', escape(tag))
|
||||
tag = escape(tag)
|
||||
r_serv_tracker.sadd(f'tracker:tags:{tracker_uuid}', tag)
|
||||
Tag.create_custom_tag(tag)
|
||||
|
||||
# create tracker tags mail notification list
|
||||
# create tracker mail notification list
|
||||
for mail in mails:
|
||||
r_serv_tracker.sadd(f'tracker:mail:{tracker_uuid}', escape(mail))
|
||||
|
||||
|
@ -1030,7 +1032,9 @@ def create_retro_hunt_task(name, rule, date_from, date_to, creator, sources=[],
|
|||
for source in sources:
|
||||
r_serv_tracker.sadd(f'tracker:retro_hunt:task:sources:{task_uuid}', escape(source))
|
||||
for tag in tags:
|
||||
r_serv_tracker.sadd(f'tracker:retro_hunt:task:tags:{task_uuid}', escape(tag))
|
||||
tag = escape(tag)
|
||||
r_serv_tracker.sadd(f'tracker:retro_hunt:task:tags:{task_uuid}', tag)
|
||||
Tag.create_custom_tag(tag)
|
||||
for mail in mails:
|
||||
r_serv_tracker.sadd(f'tracker:retro_hunt:task:mails:{task_uuid}', escape(mail))
|
||||
|
||||
|
@ -1364,14 +1368,13 @@ def get_trackers_tags():
|
|||
def _fix_db_custom_tags():
|
||||
for tag in get_trackers_tags():
|
||||
if not Tag.is_taxonomie_tag(tag) and not Tag.is_galaxy_tag(tag):
|
||||
print(tag)
|
||||
Tag.create_custom_tag(tag)
|
||||
|
||||
#### -- ####
|
||||
|
||||
#if __name__ == '__main__':
|
||||
if __name__ == '__main__':
|
||||
|
||||
#_fix_db_custom_tags()
|
||||
_fix_db_custom_tags()
|
||||
# fix_all_tracker_uuid_list()
|
||||
# res = get_all_tracker_uuid()
|
||||
# print(len(res))
|
||||
|
|
|
@ -17,7 +17,6 @@ from lib.ConfigLoader import ConfigLoader
|
|||
|
||||
# Config
|
||||
config_loader = ConfigLoader()
|
||||
# r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
|
||||
config_loader = None
|
||||
|
||||
|
|
|
@ -10,42 +10,42 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib import Users
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
#r_serv_db = config_loader.get_redis_conn("Kvrocks_DB")
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_db = config_loader.get_redis_conn("Kvrocks_DB")
|
||||
config_loader = None
|
||||
|
||||
BACKGROUND_UPDATES = {
|
||||
'v1.5':{
|
||||
'nb_updates': 5,
|
||||
'message': 'Tags and Screenshots'
|
||||
},
|
||||
'v2.4':{
|
||||
'nb_updates': 1,
|
||||
'message': ' Domains Tags and Correlations'
|
||||
},
|
||||
'v2.6':{
|
||||
'nb_updates': 1,
|
||||
'message': 'Domains Tags and Correlations'
|
||||
},
|
||||
'v2.7':{
|
||||
'nb_updates': 1,
|
||||
'message': 'Domains Tags'
|
||||
},
|
||||
'v3.4':{
|
||||
'nb_updates': 1,
|
||||
'message': 'Domains Languages'
|
||||
},
|
||||
'v3.7':{
|
||||
'nb_updates': 1,
|
||||
'message': 'Trackers first_seen/last_seen'
|
||||
}
|
||||
'v1.5': {
|
||||
'nb_updates': 5,
|
||||
'message': 'Tags and Screenshots'
|
||||
},
|
||||
'v2.4': {
|
||||
'nb_updates': 1,
|
||||
'message': ' Domains Tags and Correlations'
|
||||
},
|
||||
'v2.6': {
|
||||
'nb_updates': 1,
|
||||
'message': 'Domains Tags and Correlations'
|
||||
},
|
||||
'v2.7': {
|
||||
'nb_updates': 1,
|
||||
'message': 'Domains Tags'
|
||||
},
|
||||
'v3.4': {
|
||||
'nb_updates': 1,
|
||||
'message': 'Domains Languages'
|
||||
},
|
||||
'v3.7': {
|
||||
'nb_updates': 1,
|
||||
'message': 'Trackers first_seen/last_seen'
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
def get_ail_version():
|
||||
return r_serv.get('ail:version')
|
||||
return r_db.get('ail:version')
|
||||
|
||||
|
||||
def get_ail_float_version():
|
||||
version = get_ail_version()
|
||||
|
@ -55,20 +55,23 @@ def get_ail_float_version():
|
|||
version = 0
|
||||
return version
|
||||
|
||||
|
||||
def get_ail_all_updates(date_separator='-'):
|
||||
dict_update = r_serv.hgetall('ail:update_date')
|
||||
dict_update = r_db.hgetall('ail:update_date')
|
||||
if date_separator:
|
||||
for version in dict_update:
|
||||
u_date = dict_update[version]
|
||||
dict_update[version] = f'{u_date[0:4]}{date_separator}{u_date[4:6]}{date_separator}{u_date[6:8]}'
|
||||
return dict_update
|
||||
|
||||
|
||||
def add_ail_update(version):
|
||||
#Add new AIL version
|
||||
r_serv.hset('ail:update_date', version, datetime.datetime.now().strftime("%Y%m%d"))
|
||||
#Set current ail version
|
||||
# Add new AIL version
|
||||
r_db.hset('ail:update_date', version, datetime.datetime.now().strftime("%Y%m%d"))
|
||||
# Set current ail version
|
||||
if float(version[1:]) > get_ail_float_version():
|
||||
r_serv.set('ail:version', version)
|
||||
r_db.set('ail:version', version)
|
||||
|
||||
|
||||
def check_version(version):
|
||||
if version[0] != 'v':
|
||||
|
@ -83,75 +86,94 @@ def check_version(version):
|
|||
return False
|
||||
return True
|
||||
|
||||
|
||||
#### UPDATE BACKGROUND ####
|
||||
|
||||
def exits_background_update_to_launch():
|
||||
return r_serv.scard('ail:update:to_update') != 0
|
||||
return r_db.scard('ail:update:to_update') != 0
|
||||
|
||||
|
||||
def is_version_in_background_update(version):
|
||||
return r_serv.sismember('ail:update:to_update', version)
|
||||
return r_db.sismember('ail:update:to_update', version)
|
||||
|
||||
|
||||
def get_all_background_updates_to_launch():
|
||||
return r_serv.smembers('ail:update:to_update')
|
||||
return r_db.smembers('ail:update:to_update')
|
||||
|
||||
|
||||
def get_current_background_update():
|
||||
return r_serv.get('ail:update:update_in_progress')
|
||||
return r_db.get('ail:update:update_in_progress')
|
||||
|
||||
|
||||
def get_current_background_update_script():
|
||||
return r_serv.get('ail:update:current_background_script')
|
||||
return r_db.get('ail:update:current_background_script')
|
||||
|
||||
|
||||
def get_current_background_update_script_path(version, script_name):
|
||||
return os.path.join(os.environ['AIL_HOME'], 'update', version, script_name)
|
||||
|
||||
|
||||
def get_current_background_nb_update_completed():
|
||||
return r_serv.scard('ail:update:update_in_progress:completed')
|
||||
return r_db.scard('ail:update:update_in_progress:completed')
|
||||
|
||||
|
||||
def get_current_background_update_progress():
|
||||
progress = r_serv.get('ail:update:current_background_script_stat')
|
||||
progress = r_db.get('ail:update:current_background_script_stat')
|
||||
if not progress:
|
||||
progress = 0
|
||||
return int(progress)
|
||||
|
||||
|
||||
def get_background_update_error():
|
||||
return r_serv.get('ail:update:error')
|
||||
return r_db.get('ail:update:error')
|
||||
|
||||
|
||||
def add_background_updates_to_launch(version):
|
||||
return r_serv.sadd('ail:update:to_update', version)
|
||||
return r_db.sadd('ail:update:to_update', version)
|
||||
|
||||
|
||||
def start_background_update(version):
|
||||
r_serv.delete('ail:update:error')
|
||||
r_serv.set('ail:update:update_in_progress', version)
|
||||
r_db.delete('ail:update:error')
|
||||
r_db.set('ail:update:update_in_progress', version)
|
||||
|
||||
|
||||
def set_current_background_update_script(script_name):
|
||||
r_serv.set('ail:update:current_background_script', script_name)
|
||||
r_serv.set('ail:update:current_background_script_stat', 0)
|
||||
r_db.set('ail:update:current_background_script', script_name)
|
||||
r_db.set('ail:update:current_background_script_stat', 0)
|
||||
|
||||
|
||||
def set_current_background_update_progress(progress):
|
||||
r_serv.set('ail:update:current_background_script_stat', progress)
|
||||
r_db.set('ail:update:current_background_script_stat', progress)
|
||||
|
||||
|
||||
def set_background_update_error(error):
|
||||
r_serv.set('ail:update:error', error)
|
||||
r_db.set('ail:update:error', error)
|
||||
|
||||
|
||||
def end_background_update_script():
|
||||
r_serv.sadd('ail:update:update_in_progress:completed')
|
||||
r_db.sadd('ail:update:update_in_progress:completed')
|
||||
|
||||
|
||||
def end_background_update(version):
|
||||
r_serv.delete('ail:update:update_in_progress')
|
||||
r_serv.delete('ail:update:current_background_script')
|
||||
r_serv.delete('ail:update:current_background_script_stat')
|
||||
r_serv.delete('ail:update:update_in_progress:completed')
|
||||
r_serv.srem('ail:update:to_update', version)
|
||||
r_db.delete('ail:update:update_in_progress')
|
||||
r_db.delete('ail:update:current_background_script')
|
||||
r_db.delete('ail:update:current_background_script_stat')
|
||||
r_db.delete('ail:update:update_in_progress:completed')
|
||||
r_db.srem('ail:update:to_update', version)
|
||||
|
||||
|
||||
def clear_background_update():
|
||||
r_serv.delete('ail:update:error')
|
||||
r_serv.delete('ail:update:update_in_progress')
|
||||
r_serv.delete('ail:update:current_background_script')
|
||||
r_serv.delete('ail:update:current_background_script_stat')
|
||||
r_serv.delete('ail:update:update_in_progress:completed')
|
||||
r_db.delete('ail:update:error')
|
||||
r_db.delete('ail:update:update_in_progress')
|
||||
r_db.delete('ail:update:current_background_script')
|
||||
r_db.delete('ail:update:current_background_script_stat')
|
||||
r_db.delete('ail:update:update_in_progress:completed')
|
||||
|
||||
|
||||
def get_update_background_message(version):
|
||||
return BACKGROUND_UPDATES[version]['message']
|
||||
|
||||
|
||||
# TODO: Detect error in subprocess
|
||||
def get_update_background_metadata():
|
||||
dict_update = {}
|
||||
|
@ -162,7 +184,7 @@ def get_update_background_metadata():
|
|||
dict_update['script_progress'] = get_current_background_update_progress()
|
||||
dict_update['nb_update'] = BACKGROUND_UPDATES[dict_update['version']]['nb_updates']
|
||||
dict_update['nb_completed'] = get_current_background_nb_update_completed()
|
||||
dict_update['progress'] = int(dict_update['nb_completed']*100/dict_update['nb_update'])
|
||||
dict_update['progress'] = int(dict_update['nb_completed'] * 100 / dict_update['nb_update'])
|
||||
dict_update['error'] = get_background_update_error()
|
||||
return dict_update
|
||||
|
||||
|
@ -170,14 +192,6 @@ def get_update_background_metadata():
|
|||
##-- UPDATE BACKGROUND --##
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
##########################
|
||||
|
||||
if __name__ == '__main__':
|
||||
res = check_version('v3.1..1')
|
||||
print(res)
|
||||
|
|
|
@ -36,7 +36,6 @@ except:
|
|||
config_loader = ConfigLoader()
|
||||
r_objects = config_loader.get_db_conn("Kvrocks_Objects")
|
||||
|
||||
r_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
HASH_DIR = config_loader.get_config_str('Directories', 'hash')
|
||||
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
|
||||
config_loader = None
|
||||
|
|
|
@ -8,7 +8,6 @@ Base Class for AIL Objects
|
|||
##################################
|
||||
import os
|
||||
import sys
|
||||
from abc import abstractmethod
|
||||
|
||||
# from flask import url_for
|
||||
|
||||
|
@ -25,7 +24,6 @@ from packages import Date
|
|||
|
||||
# LOAD CONFIG
|
||||
config_loader = ConfigLoader()
|
||||
r_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
r_object = config_loader.get_db_conn("Kvrocks_Objects")
|
||||
config_loader = None
|
||||
|
||||
|
@ -51,13 +49,6 @@ class AbstractSubtypeObject(AbstractObject):
|
|||
def exists(self):
|
||||
return r_object.exists(f'meta:{self.type}:{self.subtype}:{self.id}')
|
||||
|
||||
# def exists(self):
|
||||
# res = r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id)
|
||||
# if res is not None:
|
||||
# return True
|
||||
# else:
|
||||
# return False
|
||||
|
||||
def get_first_seen(self, r_int=False):
|
||||
first_seen = r_object.hget(f'meta:{self.type}:{self.subtype}:{self.id}', 'first_seen')
|
||||
if r_int:
|
||||
|
@ -79,11 +70,11 @@ class AbstractSubtypeObject(AbstractObject):
|
|||
return last_seen
|
||||
|
||||
def get_nb_seen(self):
|
||||
return int(r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id))
|
||||
return int(r_object.zscore(f'{self.type}_all:{self.subtype}', self.id))
|
||||
|
||||
# # TODO: CHECK RESULT
|
||||
def get_nb_seen_by_date(self, date_day):
|
||||
nb = r_metadata.hget(f'{self.type}:{self.subtype}:{date_day}', self.id)
|
||||
nb = r_object.hget(f'{self.type}:{self.subtype}:{date_day}', self.id)
|
||||
if nb is None:
|
||||
return 0
|
||||
else:
|
||||
|
@ -134,9 +125,9 @@ class AbstractSubtypeObject(AbstractObject):
|
|||
self.update_daterange(date)
|
||||
update_obj_date(date, self.type, self.subtype)
|
||||
# daily
|
||||
r_metadata.hincrby(f'{self.type}:{self.subtype}:{date}', self.id, 1)
|
||||
r_object.hincrby(f'{self.type}:{self.subtype}:{date}', self.id, 1)
|
||||
# all subtypes
|
||||
r_metadata.zincrby(f'{self.type}_all:{self.subtype}', 1, self.id)
|
||||
r_object.zincrby(f'{self.type}_all:{self.subtype}', 1, self.id)
|
||||
|
||||
#######################################################################
|
||||
#######################################################################
|
||||
|
@ -158,14 +149,5 @@ class AbstractSubtypeObject(AbstractObject):
|
|||
def _delete(self):
|
||||
pass
|
||||
|
||||
|
||||
####################################
|
||||
#
|
||||
# _get_items
|
||||
# get_metadata
|
||||
#
|
||||
#
|
||||
|
||||
|
||||
def get_all_id(obj_type, subtype):
|
||||
return r_metadata.zrange(f'{obj_type}_all:{subtype}', 0, -1)
|
||||
return r_object.zrange(f'{obj_type}_all:{subtype}', 0, -1)
|
||||
|
|
|
@ -69,8 +69,7 @@ class Credential(AbstractModule):
|
|||
|
||||
# Database
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
# self.server_cred = config_loader.get_redis_conn("ARDB_TermCred")
|
||||
self.server_statistics = config_loader.get_redis_conn("ARDB_Statistics")
|
||||
# self.server_cred = config_loader.get_redis_conn("_TermCred")
|
||||
|
||||
# Config values
|
||||
self.minimumLengthThreshold = config_loader.get_config_int("Credential", "minimumLengthThreshold")
|
||||
|
|
|
@ -43,6 +43,7 @@ from modules.abstract_module import AbstractModule
|
|||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.data_retention_engine import update_obj_date
|
||||
from lib import item_basic
|
||||
# from lib import Statistics
|
||||
|
||||
class Global(AbstractModule):
|
||||
"""
|
||||
|
@ -52,8 +53,6 @@ class Global(AbstractModule):
|
|||
def __init__(self):
|
||||
super(Global, self).__init__()
|
||||
|
||||
self.r_stats = ConfigLoader().get_redis_conn("ARDB_Statistics")
|
||||
|
||||
self.processed_item = 0
|
||||
self.time_last_stats = time.time()
|
||||
|
||||
|
@ -197,12 +196,13 @@ class Global(AbstractModule):
|
|||
self.redis_logger.warning(f'Global; Incomplete file: {filename}')
|
||||
print(f'Global; Incomplete file: {filename}')
|
||||
# save daily stats
|
||||
self.r_stats.zincrby('module:Global:incomplete_file', 1, datetime.datetime.now().strftime('%Y%m%d'))
|
||||
# self.r_stats.zincrby('module:Global:incomplete_file', 1, datetime.datetime.now().strftime('%Y%m%d'))
|
||||
# Statistics.
|
||||
except OSError:
|
||||
self.redis_logger.warning(f'Global; Not a gzipped file: {filename}')
|
||||
print(f'Global; Not a gzipped file: {filename}')
|
||||
# save daily stats
|
||||
self.r_stats.zincrby('module:Global:invalid_file', 1, datetime.datetime.now().strftime('%Y%m%d'))
|
||||
# self.r_stats.zincrby('module:Global:invalid_file', 1, datetime.datetime.now().strftime('%Y%m%d'))
|
||||
|
||||
return curr_file_content
|
||||
|
||||
|
|
|
@ -13,7 +13,6 @@ It tries to identify SQL Injections with libinjection.
|
|||
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
import pylibinjection
|
||||
|
||||
from datetime import datetime
|
||||
|
@ -28,6 +27,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
from modules.abstract_module import AbstractModule
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.Items import Item
|
||||
from lib import Statistics
|
||||
|
||||
class LibInjection(AbstractModule):
|
||||
"""docstring for LibInjection module."""
|
||||
|
@ -37,9 +37,6 @@ class LibInjection(AbstractModule):
|
|||
|
||||
self.faup = Faup()
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
self.server_statistics = config_loader.get_redis_conn("ARDB_Statistics")
|
||||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, message):
|
||||
|
@ -94,7 +91,7 @@ class LibInjection(AbstractModule):
|
|||
tld = url_parsed['tld']
|
||||
if tld is not None:
|
||||
date = datetime.now().strftime("%Y%m")
|
||||
self.server_statistics.hincrby(f'SQLInjection_by_tld:{date}', tld, 1)
|
||||
Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
@ -34,7 +34,6 @@ class Onion(AbstractModule):
|
|||
|
||||
config_loader = ConfigLoader()
|
||||
self.r_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||
self.r_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
|
||||
self.pending_seconds = config_loader.get_config_int("Onion", "max_execution_time")
|
||||
# regex timeout
|
||||
|
@ -91,7 +90,7 @@ class Onion(AbstractModule):
|
|||
|
||||
if onion_urls:
|
||||
if crawlers.is_crawler_activated():
|
||||
for domain in domains:# TODO LOAD DEFAULT SCREENSHOT + HAR
|
||||
for domain in domains: # TODO LOAD DEFAULT SCREENSHOT + HAR
|
||||
task_uuid = crawlers.add_crawler_task(domain, parent=item.get_id())
|
||||
if task_uuid:
|
||||
print(f'{domain} added to crawler queue: {task_uuid}')
|
||||
|
@ -109,6 +108,3 @@ if __name__ == "__main__":
|
|||
module = Onion()
|
||||
# module.compute('submitted/2022/10/10/submitted_705d1d92-7e9a-4a44-8c21-ccd167bfb7db.gz 9')
|
||||
module.run()
|
||||
|
||||
|
||||
# 5ajw6aqf3ep7sijnscdzw77t7xq4xjpsy335yb2wiwgouo7yfxtjlmid.onion to debian.org
|
|
@ -27,6 +27,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
from modules.abstract_module import AbstractModule
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.Items import Item
|
||||
from lib import Statistics
|
||||
|
||||
class SQLInjectionDetection(AbstractModule):
|
||||
"""docstring for SQLInjectionDetection module."""
|
||||
|
@ -40,9 +41,6 @@ class SQLInjectionDetection(AbstractModule):
|
|||
|
||||
self.faup = Faup()
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
self.server_statistics = config_loader.get_redis_conn("ARDB_Statistics")
|
||||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, message):
|
||||
|
@ -75,7 +73,7 @@ class SQLInjectionDetection(AbstractModule):
|
|||
except:
|
||||
pass
|
||||
date = datetime.now().strftime("%Y%m")
|
||||
self.server_statistics.hincrby(f'SQLInjection_by_tld:{date}', tld, 1)
|
||||
Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
|
||||
|
||||
# Try to detect if the url passed might be an sql injection by applying the regex
|
||||
# defined above on it.
|
||||
|
|
|
@ -62,7 +62,7 @@ class SentimentAnalysis(AbstractModule):
|
|||
self.sentiment_lexicon_file = ConfigLoader.ConfigLoader().get_config_str("Directories", "sentiment_lexicon_file")
|
||||
|
||||
# REDIS_LEVEL_DB #
|
||||
self.db = ConfigLoader.ConfigLoader().get_redis_conn("ARDB_Sentiment")
|
||||
self.db = ConfigLoader.ConfigLoader().get_redis_conn("_Sentiment")
|
||||
|
||||
self.time1 = time.time()
|
||||
|
||||
|
|
|
@ -1,253 +0,0 @@
|
|||
[Directories]
|
||||
bloomfilters = Blooms
|
||||
dicofilters = Dicos
|
||||
pastes = PASTES
|
||||
hash = HASHS
|
||||
crawled = crawled
|
||||
crawled_screenshot = CRAWLED_SCREENSHOT
|
||||
|
||||
wordtrending_csv = var/www/static/csv/wordstrendingdata
|
||||
wordsfile = files/wordfile
|
||||
|
||||
protocolstrending_csv = var/www/static/csv/protocolstrendingdata
|
||||
protocolsfile = files/protocolsfile
|
||||
|
||||
tldstrending_csv = var/www/static/csv/tldstrendingdata
|
||||
tldsfile = faup/src/data/mozilla.tlds
|
||||
|
||||
domainstrending_csv = var/www/static/csv/domainstrendingdata
|
||||
|
||||
pystemonpath = /opt/pystemon/
|
||||
|
||||
sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt
|
||||
|
||||
##### Notifications ######
|
||||
[Notifications]
|
||||
ail_domain = http://localhost:7000
|
||||
sender = sender@example.com
|
||||
sender_host = smtp.example.com
|
||||
sender_port = 1337
|
||||
sender_pw = None
|
||||
|
||||
# optional for using with authenticated SMTP over SSL
|
||||
# sender_pw = securepassword
|
||||
|
||||
##### Flask #####
|
||||
[Flask]
|
||||
#Proxying requests to the app
|
||||
baseUrl = /
|
||||
#Number of logs to display in the dashboard
|
||||
max_dashboard_logs = 15
|
||||
#Maximum number of character to display in the toolip
|
||||
max_preview_char = 250
|
||||
#Maximum number of character to display in the modal
|
||||
max_preview_modal = 800
|
||||
#Default number of header to display in trending graphs
|
||||
default_display = 10
|
||||
#Number of minutes displayed for the number of processed pastes.
|
||||
minute_processed_paste = 10
|
||||
#Maximum line length authorized to make a diff between duplicates
|
||||
DiffMaxLineLength = 10000
|
||||
|
||||
#### Modules ####
|
||||
[BankAccount]
|
||||
max_execution_time = 60
|
||||
|
||||
[Categ]
|
||||
#Minimum number of match between the paste and the category file
|
||||
matchingThreshold=1
|
||||
|
||||
[Credential]
|
||||
#Minimum length that a credential must have to be considered as such
|
||||
minimumLengthThreshold=3
|
||||
#Will be pushed as alert if the number of credentials is greater to that number
|
||||
criticalNumberToAlert=8
|
||||
#Will be considered as false positive if less that X matches from the top password list
|
||||
minTopPassList=5
|
||||
|
||||
[Curve]
|
||||
max_execution_time = 90
|
||||
|
||||
[Onion]
|
||||
max_execution_time = 180
|
||||
|
||||
[Base64]
|
||||
path = Base64/
|
||||
max_execution_time = 60
|
||||
|
||||
[Binary]
|
||||
path = Base64/
|
||||
max_execution_time = 60
|
||||
|
||||
[Hex]
|
||||
path = Base64/
|
||||
max_execution_time = 60
|
||||
|
||||
[Modules_Duplicates]
|
||||
#Number of month to look back
|
||||
maximum_month_range = 3
|
||||
#The value where two pastes are considerate duplicate for ssdeep.
|
||||
threshold_duplicate_ssdeep = 50
|
||||
#The value where two pastes are considerate duplicate for tlsh.
|
||||
threshold_duplicate_tlsh = 52
|
||||
#Minimum size of the paste considered
|
||||
min_paste_size = 0.3
|
||||
|
||||
[Module_ModuleInformation]
|
||||
#Threshold to deduce if a module is stuck or not, in seconds.
|
||||
threshold_stucked_module=600
|
||||
|
||||
[Module_Mixer]
|
||||
#Define the configuration of the mixer, possible value: 1, 2 or 3
|
||||
operation_mode = 3
|
||||
#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
|
||||
ttl_duplicate = 86400
|
||||
default_unnamed_feed_name = unnamed_feeder
|
||||
|
||||
[RegexForTermsFrequency]
|
||||
max_execution_time = 60
|
||||
|
||||
##### Redis #####
|
||||
[Redis_Cache]
|
||||
host = localhost
|
||||
port = 6379
|
||||
db = 0
|
||||
|
||||
[Redis_Log]
|
||||
host = localhost
|
||||
port = 6380
|
||||
db = 0
|
||||
|
||||
[Redis_Log_submit]
|
||||
host = localhost
|
||||
port = 6380
|
||||
db = 1
|
||||
|
||||
[Redis_Queues]
|
||||
host = localhost
|
||||
port = 6381
|
||||
db = 0
|
||||
|
||||
[Redis_Data_Merging]
|
||||
host = localhost
|
||||
port = 6379
|
||||
db = 1
|
||||
|
||||
[Redis_Paste_Name]
|
||||
host = localhost
|
||||
port = 6379
|
||||
db = 2
|
||||
|
||||
[Redis_Mixer_Cache]
|
||||
host = localhost
|
||||
port = 6381
|
||||
db = 1
|
||||
|
||||
##### ARDB #####
|
||||
[ARDB_Curve]
|
||||
host = localhost
|
||||
port = 6382
|
||||
db = 1
|
||||
|
||||
[ARDB_Sentiment]
|
||||
host = localhost
|
||||
port = 6382
|
||||
db = 4
|
||||
|
||||
[ARDB_TermFreq]
|
||||
host = localhost
|
||||
port = 6382
|
||||
db = 2
|
||||
|
||||
[ARDB_TermCred]
|
||||
host = localhost
|
||||
port = 6382
|
||||
db = 5
|
||||
|
||||
[ARDB_DB]
|
||||
host = localhost
|
||||
port = 6382
|
||||
db = 0
|
||||
|
||||
[ARDB_Trending]
|
||||
host = localhost
|
||||
port = 6382
|
||||
db = 3
|
||||
|
||||
[ARDB_Hashs]
|
||||
host = localhost
|
||||
db = 1
|
||||
|
||||
[ARDB_Tags]
|
||||
host = localhost
|
||||
port = 6382
|
||||
db = 6
|
||||
|
||||
[ARDB_Metadata]
|
||||
host = localhost
|
||||
port = 6382
|
||||
db = 7
|
||||
|
||||
[ARDB_Statistics]
|
||||
host = localhost
|
||||
port = 6382
|
||||
db = 8
|
||||
|
||||
[ARDB_Onion]
|
||||
host = localhost
|
||||
port = 6382
|
||||
db = 9
|
||||
|
||||
[Url]
|
||||
cc_critical = DE
|
||||
|
||||
[DomClassifier]
|
||||
cc = DE
|
||||
cc_tld = r'\.de$'
|
||||
dns = 8.8.8.8
|
||||
|
||||
[Mail]
|
||||
dns = 8.8.8.8
|
||||
|
||||
[Web]
|
||||
dns = 149.13.33.69
|
||||
|
||||
# Indexer configuration
|
||||
[Indexer]
|
||||
type = whoosh
|
||||
path = indexdir
|
||||
register = indexdir/all_index.txt
|
||||
#size in Mb
|
||||
index_max_size = 2000
|
||||
|
||||
[ailleakObject]
|
||||
maxDuplicateToPushToMISP=10
|
||||
|
||||
###############################################################################
|
||||
|
||||
# For multiple feed, add them with "," without space
|
||||
# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
|
||||
[ZMQ_Global]
|
||||
#address = tcp://crf.circl.lu:5556
|
||||
address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
|
||||
channel = 102
|
||||
bind = tcp://127.0.0.1:5556
|
||||
|
||||
[ZMQ_Url]
|
||||
address = tcp://127.0.0.1:5004
|
||||
channel = urls
|
||||
|
||||
[ZMQ_FetchedOnion]
|
||||
address = tcp://127.0.0.1:5005
|
||||
channel = FetchedOnion
|
||||
|
||||
[RedisPubSub]
|
||||
host = localhost
|
||||
port = 6381
|
||||
db = 0
|
||||
|
||||
[Crawler]
|
||||
activate_crawler = False
|
||||
crawler_depth_limit = 1
|
||||
splash_url_onion = http://172.17.0.1
|
||||
splash_onion_port = 8050
|
|
@ -12,10 +12,6 @@ publish = Redis_Mixer,Redis_Tags
|
|||
subscribe = Redis_Mixer
|
||||
publish = Redis_Global,Redis_ModuleStats
|
||||
|
||||
#[PreProcessFeed]
|
||||
#subscribe = Redis_preProcess1
|
||||
#publish = Redis_Mixer
|
||||
|
||||
[Duplicates]
|
||||
subscribe = Redis_Duplicate
|
||||
|
||||
|
@ -62,7 +58,7 @@ subscribe = Redis_Global
|
|||
|
||||
[Categ]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Urls,Redis_Credential,Redis_SourceCode,Redis_Cve,Redis_ApiKey,Redis_SyncModule
|
||||
publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Urls,Redis_Credential,Redis_Cve,Redis_ApiKey,Redis_SyncModule
|
||||
|
||||
[CreditCards]
|
||||
subscribe = Redis_CreditCards
|
||||
|
@ -78,13 +74,9 @@ publish = Redis_ModuleStats,Redis_Tags
|
|||
|
||||
[Onion]
|
||||
subscribe = Redis_Onion
|
||||
publish = Redis_ValidOnion,Redis_Tags,Redis_Crawler
|
||||
publish = Redis_Tags,Redis_Crawler
|
||||
#publish = Redis_ValidOnion,ZMQ_FetchedOnion,Redis_Tags,Redis_Crawler
|
||||
|
||||
# TODO remove me
|
||||
[DumpValidOnion]
|
||||
subscribe = Redis_ValidOnion
|
||||
|
||||
[Urls]
|
||||
subscribe = Redis_Urls
|
||||
publish = Redis_Url
|
||||
|
@ -112,15 +104,8 @@ subscribe = Redis_SyncModule
|
|||
[MISP_The_hive_feeder]
|
||||
subscribe = Redis_Tags_feed
|
||||
|
||||
#[send_to_queue]
|
||||
#subscribe = Redis_Cve
|
||||
#publish = Redis_Tags
|
||||
|
||||
[SentimentAnalysis]
|
||||
subscribe = Redis_Global
|
||||
|
||||
[Release]
|
||||
subscribe = Redis_Global
|
||||
#[SentimentAnalysis]
|
||||
#subscribe = Redis_Global
|
||||
|
||||
[Credential]
|
||||
subscribe = Redis_Credential
|
||||
|
@ -130,9 +115,10 @@ publish = Redis_Duplicate,Redis_ModuleStats,Redis_Tags
|
|||
subscribe = Redis_Cve
|
||||
publish = Redis_Tags
|
||||
|
||||
[Phone]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Tags
|
||||
# Disabled
|
||||
#[Phone]
|
||||
#subscribe = Redis_Global
|
||||
#publish = Redis_Tags
|
||||
|
||||
[Keys]
|
||||
subscribe = Redis_Global
|
||||
|
@ -159,7 +145,6 @@ subscribe = Redis
|
|||
publish = Redis_Mixer
|
||||
|
||||
[Crawler]
|
||||
subscribe = Redis_Crawler
|
||||
publish = Redis_Mixer,Redis_Tags
|
||||
|
||||
[IP]
|
||||
|
@ -169,3 +154,11 @@ publish = Redis_Tags
|
|||
[Zerobins]
|
||||
subscribe = Redis_Url
|
||||
|
||||
#[PreProcessFeed]
|
||||
#subscribe = Redis_preProcess1
|
||||
#publish = Redis_Mixer
|
||||
|
||||
# [My_Module]
|
||||
# subscribe = Redis_Global
|
||||
# publish = Redis_Tags
|
||||
|
||||
|
|
|
@ -36,11 +36,11 @@ class Template(AbstractModule):
|
|||
# Send module state to logs
|
||||
self.redis_logger.info(f'Module {self.module_name} initialized')
|
||||
|
||||
def computeNone(self):
|
||||
"""
|
||||
Do something when there is no message in the queue
|
||||
"""
|
||||
self.redis_logger.debug("No message in queue")
|
||||
# def computeNone(self):
|
||||
# """
|
||||
# Do something when there is no message in the queue
|
||||
# """
|
||||
# self.redis_logger.debug("No message in queue")
|
||||
|
||||
def compute(self, message):
|
||||
"""
|
||||
|
|
|
@ -81,7 +81,7 @@ def import_object_file():
|
|||
return render_template("import_object.html", all_imported_obj=all_imported_obj, error=error)
|
||||
|
||||
|
||||
@import_export.route("/objects/misp/export", methods=['GET'])
|
||||
@import_export.route("/misp/objects/export", methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def objects_misp_export():
|
||||
|
@ -91,7 +91,7 @@ def objects_misp_export():
|
|||
return render_template("export_object.html", object_types=object_types, to_export=to_export)
|
||||
|
||||
|
||||
@import_export.route("/objects/misp/export/post", methods=['POST'])
|
||||
@import_export.route("/misp/objects/export/post", methods=['POST'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def objects_misp_export_post():
|
||||
|
@ -159,7 +159,7 @@ def objects_misp_export_post():
|
|||
misp_url=event['url'])
|
||||
|
||||
|
||||
@import_export.route("/objects/misp/export/add", methods=['GET'])
|
||||
@import_export.route("/misp/objects/export/add", methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def add_object_id_to_export():
|
||||
|
@ -181,7 +181,7 @@ def add_object_id_to_export():
|
|||
return redirect(url_for('import_export.objects_misp_export'))
|
||||
|
||||
|
||||
@import_export.route("/objects/misp/export/delete", methods=['GET'])
|
||||
@import_export.route("/misp/objects/export/delete", methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def delete_object_id_to_export():
|
||||
|
@ -194,7 +194,7 @@ def delete_object_id_to_export():
|
|||
return jsonify(success=True)
|
||||
|
||||
|
||||
@import_export.route("/import_export/investigation", methods=['GET'])
|
||||
@import_export.route("/investigation/misp/export", methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def export_investigation():
|
||||
|
@ -206,3 +206,24 @@ def export_investigation():
|
|||
return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True),
|
||||
mimetype='application/json'), 400
|
||||
return redirect(url_for('investigations_b.show_investigation', uuid=investigation_uuid))
|
||||
|
||||
|
||||
@import_export.route("/thehive/objects/case/export", methods=['POST'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def create_thehive_case():
|
||||
description = request.form['hive_description']
|
||||
title = request.form['hive_case_title']
|
||||
threat_level = Export.sanitize_threat_level_hive(request.form['threat_level_hive'])
|
||||
tlp = Export.sanitize_tlp_hive(request.form['hive_tlp'])
|
||||
item_id = request.form['obj_id']
|
||||
|
||||
item = Item(item_id)
|
||||
if not item.exists():
|
||||
abort(404)
|
||||
|
||||
case_id = Export.create_thehive_case(item_id, title=title, tlp=tlp, threat_level=threat_level, description=description)
|
||||
if case_id:
|
||||
return redirect(Export.get_case_url(case_id))
|
||||
else:
|
||||
return 'error'
|
||||
|
|
|
@ -30,15 +30,9 @@ r_cache = config_loader.get_redis_conn("Redis_Cache")
|
|||
r_serv_log = config_loader.get_redis_conn("Redis_Log")
|
||||
r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit")
|
||||
|
||||
|
||||
|
||||
r_serv_charts = config_loader.get_redis_conn("ARDB_Trending")
|
||||
r_serv_sentiment = config_loader.get_redis_conn("ARDB_Sentiment")
|
||||
r_serv_term = config_loader.get_redis_conn("ARDB_Tracker")
|
||||
r_serv_cred = config_loader.get_redis_conn("ARDB_TermCred")
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
r_serv_statistics = config_loader.get_redis_conn("ARDB_Statistics")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
r_serv_charts = config_loader.get_redis_conn("ARDB_Trending") # -> TODO MIGRATE Stats Graphs
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") # -> TODO MIGRATE /correlation/ subtypes objects
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") # -> TODO MIGRATE AUTO CRAWLER
|
||||
|
||||
# # # # # # #
|
||||
r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
|
||||
|
@ -51,72 +45,45 @@ redis_logger.port = 6380
|
|||
# Channel name to publish logs
|
||||
redis_logger.channel = 'Flask'
|
||||
|
||||
|
||||
sys.path.append('../../configs/keys')
|
||||
# MISP #
|
||||
try:
|
||||
from pymisp import PyMISP
|
||||
from mispKEYS import misp_url, misp_key, misp_verifycert
|
||||
pymisp = PyMISP(misp_url, misp_key, misp_verifycert)
|
||||
misp_event_url = misp_url + '/events/view/'
|
||||
print('Misp connected')
|
||||
except:
|
||||
print('Misp not connected')
|
||||
pymisp = False
|
||||
misp_event_url = '#'
|
||||
# The Hive #
|
||||
try:
|
||||
from thehive4py.api import TheHiveApi
|
||||
import thehive4py.exceptions
|
||||
from theHiveKEYS import the_hive_url, the_hive_key, the_hive_verifycert
|
||||
if the_hive_url == '':
|
||||
HiveApi = False
|
||||
hive_case_url = '#'
|
||||
print('The HIVE not connected')
|
||||
else:
|
||||
HiveApi = TheHiveApi(the_hive_url, the_hive_key, cert=the_hive_verifycert)
|
||||
hive_case_url = the_hive_url+'/index.html#/case/id_here/details'
|
||||
except:
|
||||
print('The HIVE not connected')
|
||||
HiveApi = False
|
||||
hive_case_url = '#'
|
||||
|
||||
if HiveApi != False:
|
||||
try:
|
||||
HiveApi.get_alert(0)
|
||||
print('The Hive connected')
|
||||
except thehive4py.exceptions.AlertException:
|
||||
HiveApi = False
|
||||
print('The Hive not connected')
|
||||
|
||||
#### VARIABLES ####
|
||||
baseUrl = config_loader.get_config_str("Flask", "baseurl")
|
||||
baseUrl = baseUrl.replace('/', '')
|
||||
if baseUrl != '':
|
||||
baseUrl = '/'+baseUrl
|
||||
baseUrl = '/' + baseUrl
|
||||
|
||||
max_preview_char = int(config_loader.get_config_str("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
|
||||
max_preview_modal = int(config_loader.get_config_str("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
|
||||
max_preview_char = int(
|
||||
config_loader.get_config_str("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
|
||||
max_preview_modal = int(
|
||||
config_loader.get_config_str("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
|
||||
|
||||
max_tags_result = 50
|
||||
|
||||
DiffMaxLineLength = int(config_loader.get_config_str("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value
|
||||
DiffMaxLineLength = int(config_loader.get_config_str("Flask",
|
||||
"DiffMaxLineLength")) # Use to display the estimated percentage instead of a raw value
|
||||
|
||||
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
|
||||
|
||||
dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_message': 'An Update is running on the background. Some informations like Tags, screenshot can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||
'v2.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||
'v2.6':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||
'v2.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||
'v3.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Languages can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||
'v3.7':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Tracker first_seen/last_seen can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'}
|
||||
}
|
||||
dict_update_description = {'v1.5': {'nb_background_update': 5,
|
||||
'update_warning_message': 'An Update is running on the background. Some informations like Tags, screenshot can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||
'v2.4': {'nb_background_update': 1,
|
||||
'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||
'v2.6': {'nb_background_update': 1,
|
||||
'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||
'v2.7': {'nb_background_update': 1,
|
||||
'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||
'v3.4': {'nb_background_update': 1,
|
||||
'update_warning_message': 'An Update is running on the background. Some informations like Domain Languages can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||
'v3.7': {'nb_background_update': 1,
|
||||
'update_warning_message': 'An Update is running on the background. Some informations like Tracker first_seen/last_seen can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'}
|
||||
}
|
||||
|
||||
UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')
|
||||
|
||||
|
@ -137,10 +104,10 @@ SUBMIT_PASTE_TEXT_MAX_SIZE = int(config_loader.get_config_str("SubmitPaste", "TE
|
|||
SUBMIT_PASTE_FILE_MAX_SIZE = int(config_loader.get_config_str("SubmitPaste", "FILE_MAX_SIZE"))
|
||||
SUBMIT_PASTE_FILE_ALLOWED_EXTENSIONS = [item.strip() for item in config_loader.get_config_str("SubmitPaste", "FILE_ALLOWED_EXTENSIONS").split(',')]
|
||||
|
||||
|
||||
# VT
|
||||
try:
|
||||
from virusTotalKEYS import vt_key
|
||||
|
||||
if vt_key != '':
|
||||
vt_auth = vt_key
|
||||
vt_enabled = True
|
||||
|
|
|
@ -11,10 +11,8 @@ import re
|
|||
import os
|
||||
import sys
|
||||
import string
|
||||
import datetime
|
||||
import unicodedata
|
||||
import uuid
|
||||
from io import BytesIO
|
||||
|
||||
from functools import wraps
|
||||
|
||||
|
@ -42,9 +40,6 @@ import Flask_config
|
|||
app = Flask_config.app
|
||||
baseUrl = Flask_config.baseUrl
|
||||
|
||||
|
||||
|
||||
r_serv_metadata = Flask_config.r_serv_metadata # TODO REMOVE ME
|
||||
r_serv_db = Flask_config.r_serv_db # TODO REMOVE ME
|
||||
r_serv_log_submit = Flask_config.r_serv_log_submit # TODO REMOVE ME
|
||||
|
||||
|
@ -317,35 +312,16 @@ def submit_status():
|
|||
else:
|
||||
return 'INVALID UUID'
|
||||
|
||||
# TODO MOVE ME IN import_export blueprint
|
||||
@PasteSubmit.route("/PasteSubmit/create_hive_case", methods=['POST'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def create_hive_case():
|
||||
|
||||
tlp = request.form['hive_tlp']
|
||||
if tlp:
|
||||
tlp = int(tlp)
|
||||
else:
|
||||
tlp = 2
|
||||
threat_level = request.form['threat_level_hive']
|
||||
if threat_level:
|
||||
threat_level = int(threat_level)
|
||||
else:
|
||||
threat_level = 2
|
||||
description = request.form['hive_description']
|
||||
title = request.form['hive_case_title']
|
||||
item_id = request.form['obj_id']
|
||||
######################################################################################################
|
||||
######################################################################################################
|
||||
######################################################################################################
|
||||
######################################################################################################
|
||||
|
||||
if (0 <= tlp <= 3) and (1 <= threat_level <= 4):
|
||||
# TODO MIGRATE TAGS PUSH
|
||||
|
||||
case_id = Export.create_thehive_case(item_id, title=title, tlp=tlp, threat_level=threat_level, description=description)
|
||||
if case_id:
|
||||
return redirect(get_case_url(case_id))
|
||||
else:
|
||||
return 'error'
|
||||
# TEMPORARY: 2 SET OF CUSTOM + infoleak tags ?????????
|
||||
|
||||
return 'error'
|
||||
|
||||
@PasteSubmit.route("/PasteSubmit/edit_tag_export")
|
||||
@login_required
|
||||
|
@ -410,8 +386,8 @@ def edit_tag_export():
|
|||
status_hive=status_hive,
|
||||
nb_tags_whitelist_misp=nb_tags_whitelist_misp,
|
||||
nb_tags_whitelist_hive=nb_tags_whitelist_hive,
|
||||
flag_misp=flag_misp,
|
||||
flag_hive=flag_hive)
|
||||
flag_misp=True,
|
||||
flag_hive=True)
|
||||
|
||||
@PasteSubmit.route("/PasteSubmit/tag_export_edited", methods=['POST'])
|
||||
@login_required
|
||||
|
|
|
@ -27,7 +27,6 @@ app = Flask_config.app
|
|||
baseUrl = Flask_config.baseUrl
|
||||
r_cache = Flask_config.r_cache
|
||||
r_serv_onion = Flask_config.r_serv_onion
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
bootstrap_label = Flask_config.bootstrap_label
|
||||
|
||||
hiddenServices = Blueprint('hiddenServices', __name__, template_folder='templates')
|
||||
|
|
|
@ -41,8 +41,6 @@ app = Flask_config.app
|
|||
baseUrl = Flask_config.baseUrl
|
||||
r_cache = Flask_config.r_cache
|
||||
r_serv_db = Flask_config.r_serv_db
|
||||
r_serv_onion = Flask_config.r_serv_onion
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
|
||||
|
||||
restApi = Blueprint('restApi', __name__, template_folder='templates')
|
||||
|
@ -258,7 +256,8 @@ def add_item_tags():
|
|||
tags = data.get('tags', [])
|
||||
galaxy = data.get('galaxy', [])
|
||||
|
||||
res = Tag.api_add_obj_tags(tags=tags, galaxy_tags=galaxy, object_id=object_id, object_type="item")
|
||||
# res = Tag.api_add_obj_tags(tags=tags, galaxy_tags=galaxy, object_id=object_id, object_type="item")
|
||||
res = {'error': 'disabled endpoint'}, 500
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
|
|
|
@ -1,160 +1,160 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
Flask functions and routes for the trending modules page
|
||||
'''
|
||||
import os
|
||||
import sys
|
||||
import datetime
|
||||
import calendar
|
||||
import flask
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint
|
||||
|
||||
from Role_Manager import login_admin, login_analyst, login_read_only
|
||||
from flask_login import login_required
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from packages.Date import Date
|
||||
|
||||
# ============ VARIABLES ============
|
||||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_charts = Flask_config.r_serv_charts
|
||||
r_serv_sentiment = Flask_config.r_serv_sentiment
|
||||
|
||||
sentiments = Blueprint('sentiments', __name__, template_folder='templates')
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
def get_date_range(num_day):
|
||||
curr_date = datetime.date.today()
|
||||
date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
|
||||
date_list = []
|
||||
|
||||
for i in range(0, num_day+1):
|
||||
date_list.append(date.substract_day(i))
|
||||
return date_list
|
||||
|
||||
|
||||
# ============ ROUTES ============
|
||||
|
||||
@sentiments.route("/sentiment_analysis_trending/")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def sentiment_analysis_trending():
|
||||
return render_template("sentiment_analysis_trending.html")
|
||||
|
||||
|
||||
@sentiments.route("/sentiment_analysis_getplotdata/", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def sentiment_analysis_getplotdata():
|
||||
# Get the top providers based on number of pastes
|
||||
oneHour = 60*60
|
||||
sevenDays = oneHour*24*7
|
||||
dateStart = datetime.datetime.now()
|
||||
dateStart = dateStart.replace(minute=0, second=0, microsecond=0)
|
||||
dateStart_timestamp = calendar.timegm(dateStart.timetuple())
|
||||
|
||||
getAllProviders = request.args.get('getProviders')
|
||||
provider = request.args.get('provider')
|
||||
allProvider = request.args.get('all')
|
||||
if getAllProviders == 'True':
|
||||
if allProvider == "True":
|
||||
range_providers = r_serv_charts.smembers('all_provider_set')
|
||||
|
||||
return jsonify(list(range_providers))
|
||||
else:
|
||||
range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8)
|
||||
# if empty, get yesterday top providers
|
||||
range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers
|
||||
|
||||
|
||||
# if still empty, takes from all providers
|
||||
if range_providers == []:
|
||||
print('today provider empty')
|
||||
range_providers = r_serv_charts.smembers('all_provider_set')
|
||||
|
||||
return jsonify(list(range_providers))
|
||||
|
||||
elif provider is not None:
|
||||
to_return = {}
|
||||
|
||||
cur_provider_name = provider + '_'
|
||||
list_date = {}
|
||||
for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour):
|
||||
cur_set_name = cur_provider_name + str(cur_timestamp)
|
||||
|
||||
list_value = []
|
||||
for cur_id in r_serv_sentiment.smembers(cur_set_name):
|
||||
cur_value = (r_serv_sentiment.get(cur_id))
|
||||
list_value.append(cur_value)
|
||||
list_date[cur_timestamp] = list_value
|
||||
to_return[provider] = list_date
|
||||
|
||||
return jsonify(to_return)
|
||||
return "Bad request"
|
||||
|
||||
|
||||
|
||||
@sentiments.route("/sentiment_analysis_plot_tool/")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def sentiment_analysis_plot_tool():
|
||||
return render_template("sentiment_analysis_plot_tool.html")
|
||||
|
||||
|
||||
|
||||
@sentiments.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def sentiment_analysis_plot_tool_getdata():
|
||||
getProviders = request.args.get('getProviders')
|
||||
|
||||
if getProviders == 'True':
|
||||
providers = []
|
||||
for cur_provider in r_serv_charts.smembers('all_provider_set'):
|
||||
providers.append(cur_provider)
|
||||
return jsonify(providers)
|
||||
|
||||
else:
|
||||
query = request.args.get('query')
|
||||
query = query.split(',')
|
||||
Qdate = request.args.get('Qdate')
|
||||
|
||||
date1 = (Qdate.split('-')[0]).split('/')
|
||||
date1 = datetime.date(int(date1[2]), int(date1[0]), int(date1[1]))
|
||||
|
||||
date2 = (Qdate.split('-')[1]).split('/')
|
||||
date2 = datetime.date(int(date2[2]), int(date2[0]), int(date2[1]))
|
||||
|
||||
timestamp1 = calendar.timegm(date1.timetuple())
|
||||
timestamp2 = calendar.timegm(date2.timetuple())
|
||||
|
||||
oneHour = 60*60
|
||||
oneDay = oneHour*24
|
||||
|
||||
to_return = {}
|
||||
for cur_provider in query:
|
||||
list_date = {}
|
||||
cur_provider_name = cur_provider + '_'
|
||||
for cur_timestamp in range(int(timestamp1), int(timestamp2)+oneDay, oneHour):
|
||||
cur_set_name = cur_provider_name + str(cur_timestamp)
|
||||
|
||||
list_value = []
|
||||
for cur_id in r_serv_sentiment.smembers(cur_set_name):
|
||||
cur_value = (r_serv_sentiment.get(cur_id))
|
||||
list_value.append(cur_value)
|
||||
list_date[cur_timestamp] = list_value
|
||||
to_return[cur_provider] = list_date
|
||||
|
||||
return jsonify(to_return)
|
||||
|
||||
# ========= REGISTRATION =========
|
||||
app.register_blueprint(sentiments, url_prefix=baseUrl)
|
||||
# '''
|
||||
# Flask functions and routes for the trending modules page
|
||||
# '''
|
||||
# import os
|
||||
# import sys
|
||||
# import datetime
|
||||
# import calendar
|
||||
# import flask
|
||||
# from flask import Flask, render_template, jsonify, request, Blueprint
|
||||
#
|
||||
# from Role_Manager import login_admin, login_analyst, login_read_only
|
||||
# from flask_login import login_required
|
||||
#
|
||||
# sys.path.append(os.environ['AIL_BIN'])
|
||||
# ##################################
|
||||
# # Import Project packages
|
||||
# ##################################
|
||||
# from packages.Date import Date
|
||||
#
|
||||
# # ============ VARIABLES ============
|
||||
# import Flask_config
|
||||
#
|
||||
# app = Flask_config.app
|
||||
# baseUrl = Flask_config.baseUrl
|
||||
# r_serv_charts = Flask_config.r_serv_charts
|
||||
# r_serv_sentiment = Flask_config.r_serv_sentiment
|
||||
#
|
||||
# sentiments = Blueprint('sentiments', __name__, template_folder='templates')
|
||||
#
|
||||
# # ============ FUNCTIONS ============
|
||||
#
|
||||
# def get_date_range(num_day):
|
||||
# curr_date = datetime.date.today()
|
||||
# date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
|
||||
# date_list = []
|
||||
#
|
||||
# for i in range(0, num_day+1):
|
||||
# date_list.append(date.substract_day(i))
|
||||
# return date_list
|
||||
#
|
||||
#
|
||||
# # ============ ROUTES ============
|
||||
#
|
||||
# @sentiments.route("/sentiment_analysis_trending/")
|
||||
# @login_required
|
||||
# @login_read_only
|
||||
# def sentiment_analysis_trending():
|
||||
# return render_template("sentiment_analysis_trending.html")
|
||||
#
|
||||
#
|
||||
# @sentiments.route("/sentiment_analysis_getplotdata/", methods=['GET'])
|
||||
# @login_required
|
||||
# @login_read_only
|
||||
# def sentiment_analysis_getplotdata():
|
||||
# # Get the top providers based on number of pastes
|
||||
# oneHour = 60*60
|
||||
# sevenDays = oneHour*24*7
|
||||
# dateStart = datetime.datetime.now()
|
||||
# dateStart = dateStart.replace(minute=0, second=0, microsecond=0)
|
||||
# dateStart_timestamp = calendar.timegm(dateStart.timetuple())
|
||||
#
|
||||
# getAllProviders = request.args.get('getProviders')
|
||||
# provider = request.args.get('provider')
|
||||
# allProvider = request.args.get('all')
|
||||
# if getAllProviders == 'True':
|
||||
# if allProvider == "True":
|
||||
# range_providers = r_serv_charts.smembers('all_provider_set')
|
||||
#
|
||||
# return jsonify(list(range_providers))
|
||||
# else:
|
||||
# range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(0)[0], '+inf', '-inf', start=0, num=8)
|
||||
# # if empty, get yesterday top providers
|
||||
# range_providers = r_serv_charts.zrevrangebyscore('providers_set_'+ get_date_range(1)[1], '+inf', '-inf', start=0, num=8) if range_providers == [] else range_providers
|
||||
#
|
||||
#
|
||||
# # if still empty, takes from all providers
|
||||
# if range_providers == []:
|
||||
# print('today provider empty')
|
||||
# range_providers = r_serv_charts.smembers('all_provider_set')
|
||||
#
|
||||
# return jsonify(list(range_providers))
|
||||
#
|
||||
# elif provider is not None:
|
||||
# to_return = {}
|
||||
#
|
||||
# cur_provider_name = provider + '_'
|
||||
# list_date = {}
|
||||
# for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour):
|
||||
# cur_set_name = cur_provider_name + str(cur_timestamp)
|
||||
#
|
||||
# list_value = []
|
||||
# for cur_id in r_serv_sentiment.smembers(cur_set_name):
|
||||
# cur_value = (r_serv_sentiment.get(cur_id))
|
||||
# list_value.append(cur_value)
|
||||
# list_date[cur_timestamp] = list_value
|
||||
# to_return[provider] = list_date
|
||||
#
|
||||
# return jsonify(to_return)
|
||||
# return "Bad request"
|
||||
#
|
||||
#
|
||||
#
|
||||
# @sentiments.route("/sentiment_analysis_plot_tool/")
|
||||
# @login_required
|
||||
# @login_read_only
|
||||
# def sentiment_analysis_plot_tool():
|
||||
# return render_template("sentiment_analysis_plot_tool.html")
|
||||
#
|
||||
#
|
||||
#
|
||||
# @sentiments.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET'])
|
||||
# @login_required
|
||||
# @login_read_only
|
||||
# def sentiment_analysis_plot_tool_getdata():
|
||||
# getProviders = request.args.get('getProviders')
|
||||
#
|
||||
# if getProviders == 'True':
|
||||
# providers = []
|
||||
# for cur_provider in r_serv_charts.smembers('all_provider_set'):
|
||||
# providers.append(cur_provider)
|
||||
# return jsonify(providers)
|
||||
#
|
||||
# else:
|
||||
# query = request.args.get('query')
|
||||
# query = query.split(',')
|
||||
# Qdate = request.args.get('Qdate')
|
||||
#
|
||||
# date1 = (Qdate.split('-')[0]).split('/')
|
||||
# date1 = datetime.date(int(date1[2]), int(date1[0]), int(date1[1]))
|
||||
#
|
||||
# date2 = (Qdate.split('-')[1]).split('/')
|
||||
# date2 = datetime.date(int(date2[2]), int(date2[0]), int(date2[1]))
|
||||
#
|
||||
# timestamp1 = calendar.timegm(date1.timetuple())
|
||||
# timestamp2 = calendar.timegm(date2.timetuple())
|
||||
#
|
||||
# oneHour = 60*60
|
||||
# oneDay = oneHour*24
|
||||
#
|
||||
# to_return = {}
|
||||
# for cur_provider in query:
|
||||
# list_date = {}
|
||||
# cur_provider_name = cur_provider + '_'
|
||||
# for cur_timestamp in range(int(timestamp1), int(timestamp2)+oneDay, oneHour):
|
||||
# cur_set_name = cur_provider_name + str(cur_timestamp)
|
||||
#
|
||||
# list_value = []
|
||||
# for cur_id in r_serv_sentiment.smembers(cur_set_name):
|
||||
# cur_value = (r_serv_sentiment.get(cur_id))
|
||||
# list_value.append(cur_value)
|
||||
# list_date[cur_timestamp] = list_value
|
||||
# to_return[cur_provider] = list_date
|
||||
#
|
||||
# return jsonify(to_return)
|
||||
#
|
||||
# # ========= REGISTRATION =========
|
||||
# app.register_blueprint(sentiments, url_prefix=baseUrl)
|
||||
|
|
|
@ -1,7 +1,7 @@
|
|||
<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-heart"></i> Sentiment Analysis
|
||||
<span class="caret"></span></a>
|
||||
<ul class="dropdown-menu">
|
||||
<li><a href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-bar-chart-o"> </i> Sentiment trending</a></li>
|
||||
<li><a href="{{ url_for('sentiments.sentiment_analysis_plot_tool') }}"><i class="fa fa-wrench"> </i> Sentiment plot Tool</a></li>
|
||||
</ul>
|
||||
</li>
|
||||
{#<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-heart"></i> Sentiment Analysis#}
|
||||
{# <span class="caret"></span></a>#}
|
||||
{# <ul class="dropdown-menu">#}
|
||||
{# <li><a href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-bar-chart-o"> </i> Sentiment trending</a></li>#}
|
||||
{# <li><a href="{{ url_for('sentiments.sentiment_analysis_plot_tool') }}"><i class="fa fa-wrench"> </i> Sentiment plot Tool</a></li>#}
|
||||
{# </ul>#}
|
||||
{#</li>#}
|
||||
|
|
|
@ -1,356 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
Flask functions and routes for the trending modules page
|
||||
|
||||
note: The matching of credential against supplied credential is done using Levenshtein distance
|
||||
'''
|
||||
import os
|
||||
import sys
|
||||
import flask
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, Response
|
||||
|
||||
from Role_Manager import login_admin, login_analyst, login_user_no_api, login_read_only
|
||||
from flask_login import login_required, current_user
|
||||
import Levenshtein
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib.objects.Items import Item
|
||||
|
||||
# ============ VARIABLES ============
|
||||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_term = Flask_config.r_serv_term
|
||||
r_serv_cred = Flask_config.r_serv_cred
|
||||
r_serv_db = Flask_config.r_serv_db
|
||||
bootstrap_label = Flask_config.bootstrap_label
|
||||
|
||||
terms = Blueprint('terms', __name__, template_folder='templates')
|
||||
|
||||
'''TERM'''
|
||||
DEFAULT_MATCH_PERCENT = 50
|
||||
|
||||
#tracked
|
||||
TrackedTermsSet_Name = "TrackedSetTermSet"
|
||||
TrackedTermsDate_Name = "TrackedTermDate"
|
||||
#black
|
||||
BlackListTermsDate_Name = "BlackListTermDate"
|
||||
BlackListTermsSet_Name = "BlackListSetTermSet"
|
||||
#regex
|
||||
TrackedRegexSet_Name = "TrackedRegexSet"
|
||||
TrackedRegexDate_Name = "TrackedRegexDate"
|
||||
#set
|
||||
TrackedSetSet_Name = "TrackedSetSet"
|
||||
TrackedSetDate_Name = "TrackedSetDate"
|
||||
|
||||
# notifications enabled/disabled
|
||||
# same value as in `bin/NotificationHelper.py`
|
||||
TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
|
||||
|
||||
# associated notification email addresses for a specific term`
|
||||
# same value as in `bin/NotificationHelper.py`
|
||||
# Keys will be e.g. TrackedNotificationEmails_<TERMNAME>
|
||||
TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
|
||||
TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
|
||||
|
||||
'''CRED'''
|
||||
REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
|
||||
REDIS_KEY_NUM_USERNAME = 'uniqNumForUsername'
|
||||
REDIS_KEY_NUM_PATH = 'uniqNumForUsername'
|
||||
REDIS_KEY_ALL_CRED_SET = 'AllCredentials'
|
||||
REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
|
||||
REDIS_KEY_ALL_PATH_SET = 'AllPath'
|
||||
REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev'
|
||||
REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
|
||||
|
||||
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
def Term_getValueOverRange(word, startDate, num_day, per_paste=""):
|
||||
passed_days = 0
|
||||
oneDay = 60*60*24
|
||||
to_return = []
|
||||
curr_to_return = 0
|
||||
for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay):
|
||||
value = r_serv_term.hget(per_paste+str(timestamp), word)
|
||||
curr_to_return += int(value) if value is not None else 0
|
||||
for i in num_day:
|
||||
if passed_days == i-1:
|
||||
to_return.append(curr_to_return)
|
||||
passed_days += 1
|
||||
return to_return
|
||||
|
||||
#Mix suplied username, if extensive is set, slice username(s) with different windows
|
||||
def mixUserName(supplied, extensive=False):
|
||||
#e.g.: John Smith
|
||||
terms = supplied.split()[:2]
|
||||
usernames = []
|
||||
if len(terms) == 1:
|
||||
terms.append(' ')
|
||||
|
||||
#john, smith, John, Smith, JOHN, SMITH
|
||||
usernames += [terms[0].lower()]
|
||||
usernames += [terms[1].lower()]
|
||||
usernames += [terms[0][0].upper() + terms[0][1:].lower()]
|
||||
usernames += [terms[1][0].upper() + terms[1][1:].lower()]
|
||||
usernames += [terms[0].upper()]
|
||||
usernames += [terms[1].upper()]
|
||||
|
||||
#johnsmith, smithjohn, JOHNsmith, johnSMITH, SMITHjohn, smithJOHN
|
||||
usernames += [(terms[0].lower() + terms[1].lower()).strip()]
|
||||
usernames += [(terms[1].lower() + terms[0].lower()).strip()]
|
||||
usernames += [(terms[0].upper() + terms[1].lower()).strip()]
|
||||
usernames += [(terms[0].lower() + terms[1].upper()).strip()]
|
||||
usernames += [(terms[1].upper() + terms[0].lower()).strip()]
|
||||
usernames += [(terms[1].lower() + terms[0].upper()).strip()]
|
||||
#Jsmith, JSmith, jsmith, jSmith, johnS, Js, JohnSmith, Johnsmith, johnSmith
|
||||
usernames += [(terms[0][0].upper() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].upper() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0].lower() + terms[1][0].upper()).strip()]
|
||||
usernames += [(terms[0].upper() + terms[1][0].lower()).strip()]
|
||||
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].upper() + terms[0][1:].lower() + terms[1][0].lower() + terms[1][1:].lower()).strip()]
|
||||
usernames += [(terms[0][0].lower() + terms[0][1:].lower() + terms[1][0].upper() + terms[1][1:].lower()).strip()]
|
||||
|
||||
if not extensive:
|
||||
return usernames
|
||||
|
||||
#Slice the supplied username(s)
|
||||
mixedSupplied = supplied.replace(' ','')
|
||||
minWindow = 3 if len(mixedSupplied)/2 < 4 else len(mixedSupplied)/2
|
||||
for winSize in range(3,len(mixedSupplied)):
|
||||
for startIndex in range(0, len(mixedSupplied)-winSize):
|
||||
usernames += [mixedSupplied[startIndex:startIndex+winSize]]
|
||||
|
||||
filtered_usernames = []
|
||||
for usr in usernames:
|
||||
if len(usr) > 2:
|
||||
filtered_usernames.append(usr)
|
||||
return filtered_usernames
|
||||
|
||||
def save_tag_to_auto_push(list_tag):
|
||||
for tag in set(list_tag):
|
||||
#limit tag length
|
||||
if len(tag) > 49:
|
||||
tag = tag[0:48]
|
||||
r_serv_db.sadd('list_export_tags', tag)
|
||||
|
||||
# ============ ROUTES ============
|
||||
|
||||
# TODO: remove + clean
|
||||
|
||||
# @terms.route("/terms_plot_tool/")
|
||||
# @login_required
|
||||
# @login_read_only
|
||||
# def terms_plot_tool():
|
||||
# term = request.args.get('term')
|
||||
# if term is not None:
|
||||
# return render_template("terms_plot_tool.html", term=term)
|
||||
# else:
|
||||
# return render_template("terms_plot_tool.html", term="")
|
||||
#
|
||||
#
|
||||
# @terms.route("/terms_plot_tool_data/")
|
||||
# @login_required
|
||||
# @login_read_only
|
||||
# def terms_plot_tool_data():
|
||||
# oneDay = 60*60*24
|
||||
# range_start = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
|
||||
# range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
# range_start = calendar.timegm(range_start.timetuple())
|
||||
# range_end = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0;
|
||||
# range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
# range_end = calendar.timegm(range_end.timetuple())
|
||||
# term = request.args.get('term')
|
||||
#
|
||||
# per_paste = request.args.get('per_paste')
|
||||
# if per_paste == "1" or per_paste is None:
|
||||
# per_paste = "per_paste_"
|
||||
# else:
|
||||
# per_paste = ""
|
||||
#
|
||||
# if term is None:
|
||||
# return "None"
|
||||
#
|
||||
# else:
|
||||
# value_range = []
|
||||
# for timestamp in range(range_start, range_end+oneDay, oneDay):
|
||||
# value = r_serv_term.hget(per_paste+str(timestamp), term)
|
||||
# curr_value_range = int(value) if value is not None else 0
|
||||
# value_range.append([timestamp, curr_value_range])
|
||||
# value_range.insert(0,term)
|
||||
# return jsonify(value_range)
|
||||
#
|
||||
|
||||
# @terms.route("/terms_plot_top/"
|
||||
# @login_required
|
||||
# @login_read_only
|
||||
# def terms_plot_top():
|
||||
# per_paste = request.args.get('per_paste')
|
||||
# per_paste = per_paste if per_paste is not None else 1
|
||||
# return render_template("terms_plot_top.html", per_paste=per_paste)
|
||||
#
|
||||
#
|
||||
# @terms.route("/terms_plot_top_data/")
|
||||
# @login_required
|
||||
# @login_read_only
|
||||
# def terms_plot_top_data():
|
||||
# oneDay = 60*60*24
|
||||
# today = datetime.datetime.now()
|
||||
# today = today.replace(hour=0, minute=0, second=0, microsecond=0)
|
||||
# today_timestamp = calendar.timegm(today.timetuple())
|
||||
#
|
||||
# per_paste = request.args.get('per_paste')
|
||||
# if per_paste == "1" or per_paste is None:
|
||||
# per_paste = "per_paste_"
|
||||
# else:
|
||||
# per_paste = ""
|
||||
#
|
||||
# set_day = per_paste + "TopTermFreq_set_day_" + str(today_timestamp)
|
||||
# set_week = per_paste + "TopTermFreq_set_week";
|
||||
# set_month = per_paste + "TopTermFreq_set_month";
|
||||
#
|
||||
# the_set = per_paste + request.args.get('set')
|
||||
# num_day = int(request.args.get('num_day'))
|
||||
#
|
||||
# if the_set is None:
|
||||
# return "None"
|
||||
# else:
|
||||
# to_return = []
|
||||
# if "TopTermFreq_set_day" in the_set:
|
||||
# the_set += "_" + str(today_timestamp)
|
||||
#
|
||||
# for term, tot_value in r_serv_term.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20):
|
||||
# position = {}
|
||||
# position['day'] = r_serv_term.zrevrank(set_day, term)
|
||||
# position['day'] = position['day']+1 if position['day'] is not None else "<20"
|
||||
# position['week'] = r_serv_term.zrevrank(set_week, term)
|
||||
# position['week'] = position['week']+1 if position['week'] is not None else "<20"
|
||||
# position['month'] = r_serv_term.zrevrank(set_month, term)
|
||||
# position['month'] = position['month']+1 if position['month'] is not None else "<20"
|
||||
# value_range = []
|
||||
# for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay):
|
||||
# value = r_serv_term.hget(per_paste+str(timestamp), term)
|
||||
# curr_value_range = int(value) if value is not None else 0
|
||||
# value_range.append([timestamp, curr_value_range])
|
||||
#
|
||||
# to_return.append([term, value_range, tot_value, position])
|
||||
#
|
||||
# return jsonify(to_return)
|
||||
|
||||
|
||||
@terms.route("/credentials_tracker/")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def credentials_tracker():
|
||||
return render_template("credentials_tracker.html")
|
||||
|
||||
@terms.route("/credentials_management_query_paste/", methods=['GET', 'POST'])
|
||||
@login_required
|
||||
@login_user_no_api
|
||||
def credentials_management_query_paste():
|
||||
cred = request.args.get('cred')
|
||||
allPath = request.json['allPath']
|
||||
|
||||
paste_info = []
|
||||
for pathNum in allPath:
|
||||
path = r_serv_cred.hget(REDIS_KEY_ALL_PATH_SET_REV, pathNum)
|
||||
item = Item(path)
|
||||
p_date = item.get_date(separator=True)
|
||||
p_source = item.get_source()
|
||||
p_content = item.get_content()
|
||||
p_encoding = item.get_mimetype()
|
||||
p_size = item.get_size()
|
||||
p_mime = p_encoding
|
||||
lineinfo = item.get_meta_lines(content=p_content)
|
||||
p_lineinfo = lineinfo['nb'], lineinfo['max_length']
|
||||
if p_content != 0:
|
||||
p_content = p_content[0:400]
|
||||
paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content})
|
||||
|
||||
return jsonify(paste_info)
|
||||
|
||||
@terms.route("/credentials_management_action/", methods=['GET'])
|
||||
@login_required
|
||||
@login_user_no_api
|
||||
def cred_management_action():
|
||||
|
||||
supplied = request.args.get('term')
|
||||
action = request.args.get('action')
|
||||
section = request.args.get('section')
|
||||
extensive = request.args.get('extensive')
|
||||
extensive = True if extensive == "true" else False
|
||||
|
||||
if extensive:
|
||||
#collectDico
|
||||
AllUsernameInRedis = r_serv_cred.hgetall(REDIS_KEY_ALL_CRED_SET).keys()
|
||||
uniq_num_set = set()
|
||||
if action == "seek":
|
||||
possibilities = mixUserName(supplied, extensive)
|
||||
for poss in possibilities:
|
||||
num = r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET, poss)
|
||||
if num is not None:
|
||||
uniq_num_set.add(num)
|
||||
for num in r_serv_cred.smembers(poss):
|
||||
uniq_num_set.add(num)
|
||||
#Extensive /!\
|
||||
if extensive:
|
||||
iter_num = 0
|
||||
tot_iter = len(AllUsernameInRedis)*len(possibilities)
|
||||
for tempUsername in AllUsernameInRedis:
|
||||
for poss in possibilities:
|
||||
#FIXME print progress
|
||||
if(iter_num % int(tot_iter/20) == 0):
|
||||
#print("searching: {}% done".format(int(iter_num/tot_iter*100)), sep=' ', end='\r', flush=True)
|
||||
print("searching: {}% done".format(float(iter_num)/float(tot_iter)*100))
|
||||
iter_num += 1
|
||||
|
||||
if poss in tempUsername:
|
||||
num = (r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET, tempUsername))
|
||||
if num is not None:
|
||||
uniq_num_set.add(num)
|
||||
for num in r_serv_cred.smembers(tempUsername):
|
||||
uniq_num_set.add(num)
|
||||
|
||||
data = {'usr': [], 'path': [], 'numPaste': [], 'simil': []}
|
||||
for Unum in uniq_num_set:
|
||||
levenRatio = 2.0
|
||||
username = (r_serv_cred.hget(REDIS_KEY_ALL_CRED_SET_REV, Unum))
|
||||
|
||||
# Calculate Levenshtein distance, ignore negative ratio
|
||||
supp_splitted = supplied.split()
|
||||
supp_mixed = supplied.replace(' ','')
|
||||
supp_splitted.append(supp_mixed)
|
||||
for indiv_supplied in supp_splitted:
|
||||
levenRatio = float(Levenshtein.ratio(indiv_supplied, username))
|
||||
levenRatioStr = "{:.1%}".format(levenRatio)
|
||||
|
||||
data['usr'].append(username)
|
||||
|
||||
|
||||
allPathNum = list(r_serv_cred.smembers(REDIS_KEY_MAP_CRED_TO_PATH+'_'+Unum))
|
||||
|
||||
data['path'].append(allPathNum)
|
||||
data['numPaste'].append(len(allPathNum))
|
||||
data['simil'].append(levenRatioStr)
|
||||
|
||||
to_return = {}
|
||||
to_return["section"] = section
|
||||
to_return["action"] = action
|
||||
to_return["term"] = supplied
|
||||
to_return["data"] = data
|
||||
|
||||
return jsonify(to_return)
|
||||
|
||||
|
||||
# ========= REGISTRATION =========
|
||||
app.register_blueprint(terms, url_prefix=baseUrl)
|
|
@ -1,253 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Credentials Tracker - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
|
||||
<!-- JS -->
|
||||
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.pie.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
|
||||
|
||||
<style>
|
||||
.btn-link {
|
||||
color: #000000
|
||||
}
|
||||
</style>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
<!-- Modal -->
|
||||
<div id="mymodal" class="modal fade" role="dialog">
|
||||
<div class="modal-dialog modal-lg">
|
||||
|
||||
<!-- Modal content-->
|
||||
<div id="mymodalcontent" class="modal-content">
|
||||
<div id="mymodalbody" class="modal-body" max-width="8500px">
|
||||
<p>Loading paste information...</p>
|
||||
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<a id="button_show_plot" target="_blank" href=""><button type="button" class="btn btn-info">Plot term</button></a>
|
||||
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% include 'navbar.html' %}
|
||||
|
||||
<div id="page-wrapper">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<h1 class="page-header" data-page="page-termsfrequency" >Credential seeker</h1>
|
||||
</div>
|
||||
<!-- /.col-lg-12 -->
|
||||
<!-- Panel OPTIONS -->
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div id="panel-today" class="panel panel-success">
|
||||
<div class="panel-heading">
|
||||
<strong>Credential seeker</strong>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
|
||||
<label><input type="checkbox" id="extensive" style="margin-bottom: 10px;"><strong>Extensive search (takes time)</strong></input></label>
|
||||
<div class="form-group input-group" style="margin-bottom: 30px;">
|
||||
<span class="input-group-addon"><span class="glyphicon glyphicon-screenshot"></span></span>
|
||||
<input id="seekInput" class="form-control" placeholder="Credential to seek." type="text" style="max-width: 400px;">
|
||||
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="seek" data-action="seek"> Seek</button>
|
||||
<span id="nodata" class="alert alert-info" style="margin-left: 10px; display: none; padding: 6px;">No data</span>
|
||||
</div>
|
||||
|
||||
<table class="table table-striped table-bordered table-hover" id="myTable">
|
||||
<thead>
|
||||
<tr>
|
||||
<th style="max-width: 800px;">Username</th>
|
||||
<th>Similarity</th>
|
||||
<th># concerned paste(s)</th>
|
||||
<th>Action</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
</tbody>
|
||||
</table>
|
||||
<!-- /.panel-body -->
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.panel -->
|
||||
</div>
|
||||
<!-- /.panel -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
</div>
|
||||
<!-- /#page-wrapper -->
|
||||
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
activePage = $('h1.page-header').attr('data-page');
|
||||
$("#"+activePage).addClass("active");
|
||||
|
||||
$('[data-toggle="tooltip"]').tooltip();
|
||||
table_track = $('#myTable').DataTable({ "order": [[ 1, "dec" ]] });
|
||||
|
||||
table_track.on( 'draw.dt', function () {
|
||||
perform_binding();
|
||||
});
|
||||
|
||||
$("#seekInput").on("keyup", function(event){
|
||||
if(event.keyCode == 13){
|
||||
$("#followTermBtn").click();
|
||||
$("#seekInput").val("");
|
||||
}
|
||||
});
|
||||
|
||||
perform_binding();
|
||||
|
||||
$("#mymodal").on('hidden.bs.modal', function () {
|
||||
$("#mymodalbody").html("<p>Loading paste information...</p>");
|
||||
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
|
||||
$("#mymodalbody").append(loading_gif); // Show the loading GIF
|
||||
});
|
||||
});
|
||||
</script>
|
||||
|
||||
<script>
|
||||
function perform_binding() {
|
||||
$(".btn-interaction").off("click.interaction");
|
||||
$(".btn-interaction").on("click.interaction", perform_operation);
|
||||
}
|
||||
|
||||
function perform_modal_binding() {
|
||||
// On click, get html content from url and update the corresponding modal
|
||||
$("[data-toggle='modal']").on("click.openmodal", function (event) {
|
||||
event.preventDefault();
|
||||
var the_modal=$(this);
|
||||
var url = "{{ url_for('terms.credentials_management_query_paste') }}?cred=" + encodeURIComponent($(this).attr('data-term'));
|
||||
$.ajax({
|
||||
type: 'POST',
|
||||
url: url,
|
||||
dataType: "json",
|
||||
data: JSON.stringify({ 'allPath': JSON.parse($(this).attr('data-path')) }),
|
||||
contentType : "application/json"
|
||||
}).done(function (data) {
|
||||
if (data.length != 0) {
|
||||
var html_to_add = "";
|
||||
html_to_add += "<table id=\"modal-table\" class=\"table table-striped\">";
|
||||
html_to_add += "<thead>";
|
||||
html_to_add += "<tr>";
|
||||
html_to_add += "<th>Source</th>";
|
||||
html_to_add += "<th>Date</th>";
|
||||
html_to_add += "<th>Encoding</th>";
|
||||
html_to_add += "<th>Size (Kb)</th>";
|
||||
html_to_add += "<th># lines</th>";
|
||||
html_to_add += "<th>Max length</th>";
|
||||
html_to_add += "<th>Preview</th>";
|
||||
html_to_add += "</tr>";
|
||||
html_to_add += "</thead>";
|
||||
html_to_add += "<tbody>";
|
||||
for (i=0; i<data.length; i++) {
|
||||
curr_data = data[i];
|
||||
html_to_add += "<tr>";
|
||||
html_to_add += "<td>"+curr_data.source+"</td>";
|
||||
html_to_add += "<td>"+curr_data.date+"</td>";
|
||||
html_to_add += "<td>"+curr_data.encoding+"</td>";
|
||||
html_to_add += "<td>"+curr_data.size+"</td>";
|
||||
html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
|
||||
html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";
|
||||
html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content.replace(/\"/g, "\'")+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('objects_item.showItem') }}?id="+curr_data.path+"&num=0\"> <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
|
||||
|
||||
html_to_add += "</tr>";
|
||||
}
|
||||
html_to_add += "</tbody>";
|
||||
html_to_add += "</table>";
|
||||
$("#mymodalbody").html(html_to_add);
|
||||
$("[data-toggle=popover]").popover();
|
||||
$('#modal-table').DataTable();
|
||||
} else {
|
||||
$("#mymodalbody").html("No paste containing this term has been received yet.");
|
||||
}
|
||||
}
|
||||
)});
|
||||
}
|
||||
|
||||
function perform_operation(){
|
||||
var curr_section = $(this).attr('data-section');
|
||||
var curr_action = $(this).attr('data-action');
|
||||
if (curr_action == "add") {
|
||||
var curr_term = $('#'+curr_section+'Input').val();
|
||||
} else if (curr_action == "seek") {
|
||||
var curr_term = $('#'+curr_section+'Input').val();
|
||||
} else {
|
||||
var curr_term = $(this).attr('data-content');
|
||||
}
|
||||
var data_to_send = { section: curr_section, action:curr_action, term: curr_term, extensive: $("#extensive").is(":checked")};
|
||||
|
||||
if (curr_term != "") {
|
||||
//console.log(data_to_send);
|
||||
$.get("{{ url_for('terms.cred_management_action') }}", data_to_send, function(data, status){
|
||||
if(status == "success") {
|
||||
var json = data;
|
||||
|
||||
if(json.action == "add") {
|
||||
//not used for the moment
|
||||
|
||||
} else if (json.action == "seek") {
|
||||
table_track.clear().draw();
|
||||
var rep = json.data;
|
||||
//var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Track this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"add\">";
|
||||
var action_button = "";
|
||||
if (rep.usr.length == 0) {
|
||||
$( "#nodata" ).removeClass('alert-info').addClass('alert-danger');
|
||||
$( "#nodata" ).text("No data");
|
||||
$( "#nodata" ).fadeIn( "fast", function() {
|
||||
setTimeout(function() {
|
||||
$( "#nodata" ).fadeOut("fast");
|
||||
}, 2000);
|
||||
});
|
||||
} else {
|
||||
for(i=0; i < rep.usr.length; i++) {
|
||||
$( "#nodata" ).removeClass('alert-danger').addClass('alert-info');
|
||||
$( "#nodata" ).text(curr_term);
|
||||
$( "#nodata" ).fadeIn( "fast");
|
||||
toAdd = "</button><span data-toggle=\"modal\" data-target=\"#mymodal\" data-term=\""+rep.usr[i]+"\" data-path=\"["+rep.path[i]+"]\" ><button class=\"btn-link\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Show concerned paste(s)\"><span class=\"glyphicon glyphicon-info-sign\"></span></button></span>";
|
||||
table_track.row.add( [
|
||||
rep.usr[i],
|
||||
rep.simil[i],
|
||||
rep.numPaste[i],
|
||||
toAdd+action_button ] ).draw( false );
|
||||
}
|
||||
perform_binding();
|
||||
perform_modal_binding();
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
|
@ -1,6 +0,0 @@
|
|||
<li id='page-termsfrequency'><a class="dropdown-toggle" data-toggle="dropdown" href="{{ url_for('terms.credentials_tracker') }}"><i class="fa fa-eye"></i> Terms frequency
|
||||
<span class="caret"></span></a>
|
||||
<ul class="dropdown-menu">
|
||||
<li><a href="{{ url_for('terms.credentials_tracker') }}"><i class="glyphicon glyphicon-screenshot"> </i> Credentials seeker</a></li>
|
||||
</ul>
|
||||
</li>
|
|
@ -1,591 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Terms Management</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
|
||||
<link href="{{ url_for('static', filename='css/switch_checkbox.css') }}" rel="stylesheet" type="text/css" />
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
|
||||
|
||||
<style>
|
||||
.sparkLineStats ul {
|
||||
padding-left:0;
|
||||
list-style:none
|
||||
}
|
||||
|
||||
.btn-link {
|
||||
color: #000000
|
||||
}
|
||||
|
||||
.popover-content {
|
||||
white-space:pre-wrap;
|
||||
word-wrap:break-word;
|
||||
}
|
||||
.mouse_pointer{
|
||||
cursor: pointer;
|
||||
}
|
||||
.lb-md {
|
||||
font-size: 16px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<!-- Modal -->
|
||||
<div id="mymodal" class="modal fade" role="dialog">
|
||||
<div class="modal-dialog modal-lg">
|
||||
|
||||
<!-- Modal content-->
|
||||
<div id="mymodalcontent" class="modal-content">
|
||||
<div id="mymodalbody" class="modal-body" max-width="8500px">
|
||||
<p>Loading paste information...</p>
|
||||
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
|
||||
</div>
|
||||
<div class="modal-footer">
|
||||
<a id="button_show_plot" target="_blank" href=""><button type="button" class="btn btn-info">Plot term</button></a>
|
||||
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% include 'navbar.html' %}
|
||||
|
||||
<div id="page-wrapper">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<h1 class="page-header" data-page="page-termsfrequency" >Terms frequency: Management interface</h1>
|
||||
</div>
|
||||
<!-- /.col-lg-12 -->
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
<div class="row">
|
||||
|
||||
<!-- Panel OPTIONS -->
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div class="row">
|
||||
{% set uniq_id = namespace(modal_id=0)%}
|
||||
<div class="col-lg-12">
|
||||
<label class="switch">
|
||||
<input id="per_paste" class="switch-input" value="per_paste" type="checkbox" onclick="reload_per_paste()">
|
||||
<span class="switch-label" data-on="On" data-off="Off"></span>
|
||||
<span class="switch-handle"></span>
|
||||
</label>
|
||||
<strong style="top: 3px; position: relative;">1 term per paste</strong>
|
||||
|
||||
<div id="panel-today" class="panel panel-success">
|
||||
<div class="panel-heading">
|
||||
<strong>Manage tracked terms</strong>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
|
||||
<div style="margin-bottom: 10px;">
|
||||
<table>
|
||||
<tr><td><b>Regex</b>: surround the term by '<b>/</b>'. </td> <td><b style="margin-left: 20px;">/([a-z])\w+([a-z])\n/</b></td></tr>
|
||||
<tr><td><b>Set of terms</b>: surround the list by '<b>\</b>'. </td> <td><b style="margin-left: 20px;">\[term1, term2, ...]\</b></td></tr>
|
||||
<tr><td> - To set a custom matching <b>threshold</b> (defaut=50), append it at the end as a inner list '<b>[thresh]</b>'. </td> <td><b style="margin-left: 20px;">\[term1, term2, ..., [75]]\</b></td></tr>
|
||||
</table>
|
||||
</div>
|
||||
<div class="form-group input-group" style="margin-bottom: 30px;">
|
||||
<span class="input-group-addon"><span class="fa fa-eye"></span></span>
|
||||
<input id="followTermInput" class="form-control" placeholder="Term to track." type="text" style="max-width: 400px;">
|
||||
<input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (optional, space separated)" type="text" style="max-width: 400px;">
|
||||
<input id="followTermTag" class="form-control" placeholder="Tags (optional, space separated)" type="text" style="max-width: 400px;">
|
||||
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button>
|
||||
</div>
|
||||
|
||||
<table class="table table-striped table-bordered table-hover" id="myTable">
|
||||
<thead>
|
||||
<tr>
|
||||
<th style="max-width: 800px;">Term</th>
|
||||
<th>Added date</th>
|
||||
<th>Day occurence</th>
|
||||
<th>Week occurence</th>
|
||||
<th>Month occurence</th>
|
||||
<th># tracked paste</th>
|
||||
<th>Action</th>
|
||||
<th>Notification E-Mails</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<!-- SET -->
|
||||
{% for set in trackSet_list %}
|
||||
<tr style="background-color: #cdffca;">
|
||||
<td>
|
||||
<span class="term_name">{{ set }}</span>
|
||||
<div>
|
||||
{% for tag in notificationTagsTermMapping[set] %}
|
||||
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
|
||||
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
{% if notificationTagsTermMapping[set] %}
|
||||
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
|
||||
|
||||
<div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
|
||||
<div class="modal-dialog">
|
||||
|
||||
<!-- Modal content-->
|
||||
<div id="mymodalcontent" class="modal-content">
|
||||
<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
|
||||
<h2 class="text-center">Remove Custom Tag</h2>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="modal-body">
|
||||
<form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
|
||||
{% for tag in notificationTagsTermMapping[set] %}
|
||||
<div class="form-check">
|
||||
<input type="hidden" class="form-control" name="term" value="{{ set }}">
|
||||
<input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
|
||||
<label class="form-check-label">
|
||||
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
|
||||
</label>
|
||||
<br>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div class="modal-footer">
|
||||
<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
|
||||
<span class="glyphicon glyphicon-trash"></span>
|
||||
<span class="label-icon">Remove Tags</span>
|
||||
</button>
|
||||
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
|
||||
{% endif %}
|
||||
</div>
|
||||
</td>
|
||||
<td>{{ trackSet_list_values[loop.index0][3] }}</td>
|
||||
<td>{{ trackSet_list_values[loop.index0][0] }}</td>
|
||||
<td>{{ trackSet_list_values[loop.index0][1] }}</td>
|
||||
<td>{{ trackSet_list_values[loop.index0][2] }}</td>
|
||||
<td>{{ trackSet_list_num_of_paste[loop.index0] }}</td>
|
||||
<td><p style="margin: 0px; white-space: nowrap;">
|
||||
<span data-toggle="modal" data-target="#mymodal" data-term="{{ set }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
|
||||
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
|
||||
<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ set }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[set] %} checked {% endif %}>
|
||||
</p></td>
|
||||
<td>
|
||||
{% for email in notificationEMailTermMapping[set] %}
|
||||
<a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{set}}">
|
||||
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
|
||||
<span class="glyphicon glyphicon-trash" style="color:Red;" ></span>
|
||||
</div>
|
||||
</a>
|
||||
{{ email }}
|
||||
<br>
|
||||
{% endfor %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
<!-- REGEX -->
|
||||
{% for regex in trackReg_list %}
|
||||
<tr style="background-color: #fffdca;">
|
||||
<td>
|
||||
<span class="term_name">{{ regex }}</span>
|
||||
<div>
|
||||
{% for tag in notificationTagsTermMapping[regex] %}
|
||||
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
|
||||
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
{% if notificationTagsTermMapping[regex] %}
|
||||
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
|
||||
|
||||
<div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
|
||||
<div class="modal-dialog">
|
||||
|
||||
<!-- Modal content-->
|
||||
<div id="mymodalcontent" class="modal-content">
|
||||
<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
|
||||
<h2 class="text-center">Remove Custom Tag</h2>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="modal-body">
|
||||
<form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
|
||||
{% for tag in notificationTagsTermMapping[regex] %}
|
||||
<div class="form-check">
|
||||
<input type="hidden" class="form-control" name="term" value="{{ regex }}">
|
||||
<input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
|
||||
<label class="form-check-label">
|
||||
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
|
||||
</label>
|
||||
<br>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div class="modal-footer">
|
||||
<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
|
||||
<span class="glyphicon glyphicon-trash"></span>
|
||||
<span class="label-icon">Remove Tags</span>
|
||||
</button>
|
||||
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
|
||||
{% endif %}
|
||||
</div>
|
||||
</td>
|
||||
<td>{{ trackReg_list_values[loop.index0][3] }}</td>
|
||||
<td>{{ trackReg_list_values[loop.index0][0] }}</td>
|
||||
<td>{{ trackReg_list_values[loop.index0][1] }}</td>
|
||||
<td>{{ trackReg_list_values[loop.index0][2] }}</td>
|
||||
<td>{{ trackReg_list_num_of_paste[loop.index0] }}</td>
|
||||
<td><p style="margin: 0px; white-space: nowrap;">
|
||||
<span data-toggle="modal" data-target="#mymodal" data-term="{{ regex }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
|
||||
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ regex }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
|
||||
<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ regex }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[regex] %} checked {% endif %}>
|
||||
</p></td>
|
||||
<td>
|
||||
{% for email in notificationEMailTermMapping[regex] %}
|
||||
<a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{regex}}">
|
||||
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
|
||||
<span class="glyphicon glyphicon-trash" style="color:Red;"></span>
|
||||
</div>
|
||||
</a>
|
||||
{{ email }}
|
||||
<br>
|
||||
{% endfor %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
<!-- Normal term -->
|
||||
{% for term in track_list %}
|
||||
<tr>
|
||||
<td>
|
||||
<span class="term_name">{{ term }}</span>
|
||||
<div>
|
||||
{% for tag in notificationTagsTermMapping[term] %}
|
||||
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
|
||||
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
{% if notificationTagsTermMapping[term] %}
|
||||
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
|
||||
|
||||
<div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
|
||||
<div class="modal-dialog">
|
||||
|
||||
<!-- Modal content-->
|
||||
<div id="mymodalcontent" class="modal-content">
|
||||
<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
|
||||
<h2 class="text-center">Remove Custom Tag</h2>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="modal-body">
|
||||
<form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
|
||||
{% for tag in notificationTagsTermMapping[term] %}
|
||||
<div class="form-check">
|
||||
<input type="hidden" class="form-control" name="term" value="{{ term }}">
|
||||
<input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
|
||||
<label class="form-check-label">
|
||||
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
|
||||
</label>
|
||||
<br>
|
||||
</div>
|
||||
{% endfor %}
|
||||
</form>
|
||||
</div>
|
||||
|
||||
<div class="modal-footer">
|
||||
<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
|
||||
<span class="glyphicon glyphicon-trash"></span>
|
||||
<span class="label-icon">Remove Tags</span>
|
||||
</button>
|
||||
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
|
||||
{% endif %}
|
||||
</div>
|
||||
</td>
|
||||
<td>{{ track_list_values[loop.index0][3] }}</td>
|
||||
<td>{{ track_list_values[loop.index0][0] }}</td>
|
||||
<td>{{ track_list_values[loop.index0][1] }}</td>
|
||||
<td>{{ track_list_values[loop.index0][2] }}</td>
|
||||
<td>{{ track_list_num_of_paste[loop.index0] }}</td>
|
||||
<td><p style="margin: 0px; white-space: nowrap;">
|
||||
<span data-toggle="modal" data-target="#mymodal" data-term="{{ term }}" ><button class="btn-link" data-toggle="tooltip" data-placement="right" title="Show concerned paste(s)"><span class="glyphicon glyphicon-info-sign"></span></button></span>
|
||||
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ term }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
|
||||
<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ term }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[term] %} checked {% endif %}>
|
||||
</p></td>
|
||||
<td>
|
||||
{% for email in notificationEMailTermMapping[term] %}
|
||||
<a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{term}}">
|
||||
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
|
||||
<span class="glyphicon glyphicon-trash" style="color:Red;"></span>
|
||||
</div>
|
||||
</a>
|
||||
{{ email }}
|
||||
<br>
|
||||
{% endfor %}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
<!-- /.panel-body -->
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.panel -->
|
||||
</div>
|
||||
<!-- /.panel -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Panel OPTIONS -->
|
||||
<div class="col-lg-12">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div id="panel-today" class="panel panel-danger">
|
||||
<div class="panel-heading">
|
||||
<strong>Manage blacklisted terms</strong>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
|
||||
<div class="form-group input-group" style="margin-bottom: 30px;">
|
||||
<span class="input-group-addon"><span class="fa fa-eye-slash "></span></span>
|
||||
<input id="blacklistTermInput" class="form-control" placeholder="Term to track" type="text" style="max-width: 400px;">
|
||||
<button id="blacklistTermBtn" class="btn btn-danger btn-interaction" style="margin-left: 10px;" data-section="blacklistTerm" data-action="add"> Black list a term</button>
|
||||
</div>
|
||||
|
||||
<table class="table table-striped table-bordered table-hover" id="myTable2">
|
||||
<thead>
|
||||
<tr>
|
||||
<th style="max-width: 800px;">Term</th>
|
||||
<th>Added date</th>
|
||||
<th>Action</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for term, date in black_list %}
|
||||
<tr>
|
||||
<td>{{ term }}</td>
|
||||
<td>{{ date }}</td>
|
||||
<td><p style="margin: 0px;">
|
||||
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="right" title="Remove this term" data-content="{{ term }}" data-section="blacklistTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
|
||||
</p></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
<!-- /.panel-body -->
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.panel -->
|
||||
</div>
|
||||
<!-- /.panel -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- /.row -->
|
||||
</div>
|
||||
<!-- /#page-wrapper -->
|
||||
</div>
|
||||
|
||||
|
||||
<!-- import graph function -->
|
||||
<script>
|
||||
function reload_per_paste() {
|
||||
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
|
||||
window.location.href = {{ url_for('terms.terms_management') }}+"?per_paste="+checked;
|
||||
}
|
||||
|
||||
|
||||
var table_track;
|
||||
var table_black;
|
||||
|
||||
function bindEventsForCurrentPage() {
|
||||
// On click, get html content from url and update the corresponding modal
|
||||
$("[data-toggle='modal']").unbind().on("click.openmodal", function (event) {
|
||||
//console.log(data);
|
||||
event.preventDefault();
|
||||
var the_modal=$(this);
|
||||
var url = "{{ url_for('terms.terms_management_query_paste') }}?term=" + encodeURIComponent($(this).attr('data-term'));
|
||||
$.getJSON(url, function (data) {
|
||||
if (data.length != 0) {
|
||||
var html_to_add = "";
|
||||
html_to_add += "<table id=\"modal-table\" class=\"table table-striped\">";
|
||||
html_to_add += "<thead>";
|
||||
html_to_add += "<tr>";
|
||||
html_to_add += "<th>Source</th>";
|
||||
html_to_add += "<th>Date</th>";
|
||||
html_to_add += "<th>Mime</th>";
|
||||
html_to_add += "<th>Size (Kb)</th>";
|
||||
html_to_add += "<th># lines</th>";
|
||||
html_to_add += "<th>Max length</th>";
|
||||
html_to_add += "<th>Preview</th>";
|
||||
html_to_add += "</tr>";
|
||||
html_to_add += "</thead>";
|
||||
html_to_add += "<tbody>";
|
||||
for (i=0; i<data.length; i++) {
|
||||
curr_data = data[i];
|
||||
html_to_add += "<tr>";
|
||||
html_to_add += "<td>"+curr_data.source+"</td>";
|
||||
html_to_add += "<td>"+curr_data.date+"</td>";
|
||||
html_to_add += "<td>"+curr_data.mime+"</td>";
|
||||
html_to_add += "<td>"+curr_data.size+"</td>";
|
||||
html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
|
||||
html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";
|
||||
html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content.replace(/\"/g, "\'")+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('objects_item.showItem') }}?id="+curr_data.path+"&num=0\"> <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
|
||||
|
||||
html_to_add += "</tr>";
|
||||
}
|
||||
html_to_add += "</tbody>";
|
||||
html_to_add += "</table>";
|
||||
$("#mymodalbody").html(html_to_add);
|
||||
$("[data-toggle=popover]").popover();
|
||||
$("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
|
||||
$('#modal-table').DataTable();
|
||||
} else {
|
||||
$("#mymodalbody").html("No paste containing this term has been received yet.");
|
||||
$("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
|
||||
$(document).ready(function(){
|
||||
bindEventsForCurrentPage();
|
||||
activePage = $('h1.page-header').attr('data-page');
|
||||
$("#"+activePage).addClass("active");
|
||||
if({{ per_paste }} == 1) {
|
||||
$("#per_paste").attr('checked', true)
|
||||
}
|
||||
|
||||
$('[data-toggle="tooltip"]').tooltip();
|
||||
table_track = $('#myTable').DataTable();
|
||||
table_black = $('#myTable2').DataTable();
|
||||
|
||||
table_track.on( 'draw.dt', function () {
|
||||
perform_binding();
|
||||
});
|
||||
table_black.on( 'draw.dt', function () {
|
||||
perform_binding();
|
||||
});
|
||||
|
||||
|
||||
$("#followTermInput").keyup(function(event){
|
||||
if(event.keyCode == 13){
|
||||
$("#followTermBtn").click();
|
||||
$("#followTermInput").val("");
|
||||
}
|
||||
});
|
||||
|
||||
$("#blacklistTermInput").keyup(function(event){
|
||||
if(event.keyCode == 13){
|
||||
$("#blacklistTermBtn").click();
|
||||
$("#blacklistTermInput").val("");
|
||||
}
|
||||
});
|
||||
|
||||
perform_binding();
|
||||
|
||||
$("#mymodal").on('hidden.bs.modal', function () {
|
||||
$("#mymodalbody").html("<p>Loading paste information...</p>");
|
||||
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
|
||||
$("#mymodalbody").append(loading_gif); // Show the loading GIF
|
||||
});
|
||||
|
||||
|
||||
});
|
||||
</script>
|
||||
|
||||
<script>
|
||||
|
||||
|
||||
|
||||
function perform_binding() {
|
||||
$(".btn-interaction").unbind("click.interaction");
|
||||
$(".btn-interaction").bind("click.interaction", perform_operation);
|
||||
}
|
||||
|
||||
function perform_operation(){
|
||||
var curr_section = $(this).attr('data-section');
|
||||
var curr_action = $(this).attr('data-action');
|
||||
var row_tr = $(this).closest("tr");
|
||||
if (curr_action == "add") {
|
||||
var curr_term = $('#'+curr_section+'Input').val();
|
||||
var email_addresses = $('#followTermEMailNotificationReceiversInput').val();
|
||||
var tags = $('#followTermTag').val();
|
||||
} else {
|
||||
var curr_term = $(this).attr('data-content');
|
||||
var email_addresses = "";
|
||||
}
|
||||
var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses, tags: tags};
|
||||
|
||||
if (curr_term != "") {
|
||||
//console.log(data_to_send);
|
||||
$.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){
|
||||
if(status == "success") {
|
||||
var json = data;
|
||||
|
||||
if(json.section == "followTerm") {
|
||||
if(json.action == "add") {
|
||||
// query data
|
||||
$.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
|
||||
reload_per_paste();
|
||||
});
|
||||
} else if (json.action == "delete") {
|
||||
row_tr.remove()
|
||||
}
|
||||
} else if(json.section == "blacklistTerm"){
|
||||
if(json.action == "add") {
|
||||
$.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
|
||||
console.log(data2);
|
||||
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"blacklistTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
|
||||
table_black.row.add( [ json.term, data2[3], action_button ] ).draw( false );
|
||||
perform_binding();
|
||||
});
|
||||
} else if (json.action == "delete") {
|
||||
// Find indexes of row which have the term in the first column
|
||||
var index = table_black.rows().eq( 0 ).filter( function (rowIdx) {
|
||||
return table_black.cell( rowIdx, 0 ).data() === json.term;
|
||||
} );
|
||||
table_black.rows(index).remove().draw( false );
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
</script>
|
|
@ -1,275 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Terms Plot Tool - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
|
||||
<link href="{{ url_for('static', filename='css/jquery-ui.min.css') }}" rel="stylesheet" type="text/css" />
|
||||
<link href="{{ url_for('static', filename='css/switch_checkbox.css') }}" rel="stylesheet" type="text/css" />
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery-ui.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
|
||||
|
||||
<style>
|
||||
.sparkLineStats ul {
|
||||
padding-left:0;
|
||||
list-style:none
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'navbar.html' %}
|
||||
|
||||
<div id="page-wrapper">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<h1 class="page-header" data-page="page-termsfrequency" >Terms plot tool</h1>
|
||||
</div>
|
||||
<!-- /.col-lg-12 -->
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
<div class="row">
|
||||
|
||||
<!-- Panel OPTIONS -->
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div id="panel-today" class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<strong>Select options</strong>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div aria-disabled="false" class="slider sliderRange sliderBlue ui-slider ui-slider-horizontal ui-widget ui-widget-content ui-corner-all" style="margin-bottom: 5px;"></div>
|
||||
<strong>Date:</strong> <input type="text" id="amount" readonly style="border:0; color:#f6931f; font-weight:bold;"> <button id="plot-month" onclick="replot();" class="btn btn-info" style="float: right;">Replot</button>
|
||||
|
||||
<div class="form-group input-group" style="margin-top: 30px;">
|
||||
<span class="input-group-addon"><span class="glyphicon glyphicon-stats"></span></span>
|
||||
<input id="TermInput" class="form-control" placeholder="Term to plot" type="text" style="max-width: 400px;" data-init-plot="{{ term }}">
|
||||
<button id="plot-btn" class="btn btn-info" style="margin-left: 10px;"><span class="fa fa-caret-down"> Plot a term</button>
|
||||
<button id="plot-btn-add" class="btn btn-success" style="margin-left: 6px;"><span class="fa fa-plus"></span> Add the term to the chart</button>
|
||||
</div>
|
||||
|
||||
<div class='pull_right' style="float: right;">
|
||||
<label class="switch">
|
||||
<input id="per_paste" class="switch-input" value="per_paste" type="checkbox" checked>
|
||||
<span class="switch-label" data-on="On" data-off="Off"></span>
|
||||
<span class="switch-handle"></span>
|
||||
</label>
|
||||
<strong style="top: 3px; position: relative;">1 term per paste</strong>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<!-- /.panel-body -->
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.panel -->
|
||||
</div>
|
||||
<!-- /.panel -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Panel PLOT -->
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div id="panel-today" class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<strong>Graph</strong>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div id="graph" style="height: 300px;"></div>
|
||||
</div>
|
||||
<!-- /.panel-body -->
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.panel -->
|
||||
</div>
|
||||
<!-- /.panel -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<!-- /.row -->
|
||||
</div>
|
||||
<!-- /#page-wrapper -->
|
||||
</div>
|
||||
|
||||
|
||||
<div style="position: absolute; border: 1px solid rgb(255, 221, 221); padding: 2px; background-color: #333; color:white; opacity: 0.8; top: 423px; left: 616px; display: none;" id="tooltip"></div>
|
||||
|
||||
|
||||
<!-- import graph function -->
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
activePage = $('h1.page-header').attr('data-page');
|
||||
$("#"+activePage).addClass("active");
|
||||
|
||||
|
||||
/* Create the slider and button*/
|
||||
$('#plot-btn-add').hide();
|
||||
var today = Date.now();
|
||||
var old_day = today - (12*31*24*60*60)*1000;
|
||||
$( ".sliderRange" ).slider({
|
||||
range: true,
|
||||
min: old_day,
|
||||
max: today,
|
||||
values: [ today - (7*24*60*60)*1000, today ],
|
||||
step: 24*60*60*1000,
|
||||
slide: function( event, ui ) {
|
||||
$( "#amount" ).val( new Date(ui.values[ 0 ]).toLocaleDateString() + " - " + new Date(ui.values[ 1 ]).toLocaleDateString() );
|
||||
}
|
||||
});
|
||||
|
||||
$( "#amount" ).val( new Date($( ".sliderRange" ).slider( "values", 0 )).toLocaleDateString() +
|
||||
" - " + new Date($( ".sliderRange" ).slider( "values", 1 )).toLocaleDateString() );
|
||||
|
||||
$('#plot-btn').click(plotData);
|
||||
$('#plot-btn-add').click(addData);
|
||||
|
||||
$("#TermInput").val($("#TermInput").attr("data-init-plot"));
|
||||
if($("#TermInput").attr("data-init-plot") != "") {
|
||||
$("#plot-btn").click();
|
||||
}
|
||||
|
||||
$("#TermInput").keyup(function(event){
|
||||
if(event.keyCode == 13){
|
||||
$("#plot-btn").click();
|
||||
$("#TermInput").val("");
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
</script>
|
||||
|
||||
<script>
|
||||
var plot;
|
||||
var graph_data = [];
|
||||
var plotted_terms = [];
|
||||
var graph_options = {
|
||||
series: {
|
||||
lines: {
|
||||
show: true,
|
||||
lineWidth: 2
|
||||
},
|
||||
bars: {show: false, barWidth: 60*60*1000},
|
||||
shadowSize: 0
|
||||
},
|
||||
grid: {
|
||||
hoverable: true,
|
||||
clickable: true,
|
||||
tickColor: "#f9f9f9",
|
||||
borderWidth: 0
|
||||
},
|
||||
xaxis: {
|
||||
mode: "time",
|
||||
timeformat: "%m/%d",
|
||||
minTickSize: [1, "day"]
|
||||
},
|
||||
yaxis: {
|
||||
autoscaleMargin: 0.1,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
function plotData() {
|
||||
$('#plot-btn-add').show("fast");
|
||||
var curthis = $(this);
|
||||
var term = $('#TermInput').val();
|
||||
plotted_terms = [term]
|
||||
var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
|
||||
var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
|
||||
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
|
||||
|
||||
$.getJSON("{{ url_for('terms.terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
|
||||
graph_data = [];
|
||||
var to_plot = [];
|
||||
var curr_data = [];
|
||||
for(i=1; i<data.length; i++) {
|
||||
curr_data.push([data[i][0]*1000, data[i][1]]);
|
||||
}
|
||||
to_plot.push({ data: curr_data, label: term});
|
||||
graph_data.push({ data: curr_data, label: term});
|
||||
plot = $.plot($("#graph"), to_plot, graph_options);
|
||||
|
||||
$("#graph").bind("plothover", function (event, pos, item) {
|
||||
if (item) {
|
||||
var date = new Date(item.datapoint[0]);
|
||||
var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
|
||||
var y = item.datapoint[1];
|
||||
|
||||
$("#tooltip").html(item.series.label + " for "+x + " = " + y)
|
||||
.css({top: item.pageY-15, left: item.pageX+5})
|
||||
.fadeIn(200);
|
||||
} else {
|
||||
$("#tooltip").hide();
|
||||
}
|
||||
});
|
||||
|
||||
$("#TermInput").val("");
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
function addData() {
|
||||
var curthis = $(this);
|
||||
var term = $('#TermInput').val();
|
||||
plotted_terms.push(term)
|
||||
var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
|
||||
var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
|
||||
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
|
||||
|
||||
$.getJSON("{{ url_for('terms.terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
|
||||
var curr_data = [];
|
||||
for(i=1; i<data.length; i++) {
|
||||
curr_data.push([data[i][0]*1000, data[i][1]]);
|
||||
}
|
||||
graph_data.push({ data: curr_data, label: term});
|
||||
plot = $.plot($("#graph"), graph_data, graph_options);
|
||||
$("#TermInput").val("");
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
function replot() {
|
||||
graph_data = [];
|
||||
promises = [];
|
||||
for(i=0; i<plotted_terms.length; i++) {
|
||||
var term = plotted_terms[i];
|
||||
var range_start = new Date($( ".sliderRange" ).slider( "values", 0 )).getTime() / 1000;
|
||||
var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
|
||||
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
|
||||
|
||||
promises.push($.getJSON("{{ url_for('terms.terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
|
||||
var curr_data = [];
|
||||
for(i=1; i<data.length; i++) {
|
||||
curr_data.push([data[i][0]*1000, data[i][1]]);
|
||||
}
|
||||
graph_data.push({ data: curr_data, label: data[0]});
|
||||
$("#TermInput").val("");
|
||||
}))
|
||||
}
|
||||
|
||||
$.when.apply($, promises).done( function () {
|
||||
plot = $.plot($("#graph"), graph_data, graph_options);
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
</script>
|
|
@ -1,576 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Terms Plot Top - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
|
||||
<link href="{{ url_for('static', filename='css/switch_checkbox.css') }}" rel="stylesheet" type="text/css" />
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
|
||||
|
||||
<style>
|
||||
.sparkLineStats ul {
|
||||
padding-left:0;
|
||||
list-style:none
|
||||
}
|
||||
.table > tbody > tr > td {
|
||||
padding: 5px;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'navbar.html' %}
|
||||
|
||||
<div id="page-wrapper">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<h1 class="page-header" data-page="page-termsfrequency" >Terms frequency: Top set information</h1>
|
||||
</div>
|
||||
<!-- /.col-lg-12 -->
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
<div class="row">
|
||||
|
||||
<!-- Panel OPTIONS -->
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div class='pull_right'>
|
||||
<label class="switch">
|
||||
<input id="per_paste" class="switch-input" value="per_paste" type="checkbox" onclick="reload_per_paste()">
|
||||
<span class="switch-label" data-on="On" data-off="Off"></span>
|
||||
<span class="switch-handle"></span>
|
||||
</label>
|
||||
<strong style="top: 3px; position: relative;">1 term per paste</strong>
|
||||
<div>
|
||||
<div id="panel-today" class="panel panel-info">
|
||||
<div class="panel-heading">
|
||||
<strong>Today</strong>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div id="panel-today" class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<strong>Today top word</strong>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div class="col-lg-6">
|
||||
<table class="table table-striped">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Term</th>
|
||||
<th>Value</th>
|
||||
<th>Action</th>
|
||||
<th>Show</th>
|
||||
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the week and month set">Position</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="table-today">
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="col-lg-6">
|
||||
<table class="table table-striped">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Term</th>
|
||||
<th>Value</th>
|
||||
<th>Action</th>
|
||||
<th>Show</th>
|
||||
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the week and month set">Position</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="table-today2">
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.panel-body -->
|
||||
</div>
|
||||
<div id="panel-today" class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<strong>Graph</strong>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div id="graph-today" style="height: 400px;"></div>
|
||||
</div>
|
||||
<!-- /.panel-body -->
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<!-- /.panel-body -->
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.panel -->
|
||||
</div>
|
||||
<!-- /.panel -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Panel OPTIONS -->
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div id="panel-today" class="panel panel-info">
|
||||
<div class="panel-heading">
|
||||
<strong>Week</strong>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div id="panel-week" class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<strong>Week top word</strong>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div class="col-lg-6">
|
||||
<table class="table table-striped">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Term</th>
|
||||
<th>Value</th>
|
||||
<th>Action</th>
|
||||
<th>Show</th>
|
||||
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and month set">Position</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="table-week">
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="col-lg-6">
|
||||
<table class="table table-striped">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Term</th>
|
||||
<th>Value</th>
|
||||
<th>Action</th>
|
||||
<th>Show</th>
|
||||
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and month set">Position</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="table-week2">
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.panel-body -->
|
||||
</div>
|
||||
<div id="panel-week" class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<strong>Graph</strong>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div id="graph-week" style="height: 400px;"></div>
|
||||
</div>
|
||||
<!-- /.panel-body -->
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<!-- /.panel-body -->
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.panel -->
|
||||
</div>
|
||||
<!-- /.panel -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Panel OPTIONS -->
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div id="panel-today" class="panel panel-info">
|
||||
<div class="panel-heading">
|
||||
<strong>Month</strong>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div id="panel-month" class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<strong>Month top word</strong>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div class="col-lg-6">
|
||||
<table class="table table-striped">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Term</th>
|
||||
<th>Value</th>
|
||||
<th>Action</th>
|
||||
<th>Show</th>
|
||||
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and week set">Position</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="table-month">
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="col-lg-6">
|
||||
<table class="table table-striped">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Term</th>
|
||||
<th>Value</th>
|
||||
<th>Action</th>
|
||||
<th>Show</th>
|
||||
<th data-toggle="tooltip" data-placement="left" title="Position of the term in the day and week set">Position</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="table-month2">
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<!-- /.panel-body -->
|
||||
</div>
|
||||
<div id="panel-month" class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<strong>Graph</strong>
|
||||
</div>
|
||||
<div class="panel-body">
|
||||
<div id="graph-month" style="height: 400px;"></div>
|
||||
</div>
|
||||
<!-- /.panel-body -->
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<!-- /.panel-body -->
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.panel -->
|
||||
</div>
|
||||
<!-- /.panel -->
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<!-- /.row -->
|
||||
</div>
|
||||
<!-- /#page-wrapper -->
|
||||
</div>
|
||||
|
||||
<div style="position: absolute; border: 1px solid rgb(255, 221, 221); padding: 2px; background-color: #333; color:white; opacity: 0.8; top: 423px; left: 616px; display: none;" id="tooltip"></div>
|
||||
|
||||
<!-- import graph function -->
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
activePage = $('h1.page-header').attr('data-page');
|
||||
$("#"+activePage).addClass("active");
|
||||
if({{ per_paste }} == 1) {
|
||||
$("#per_paste").attr('checked', true)
|
||||
}
|
||||
});
|
||||
|
||||
function reload_per_paste() {
|
||||
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
|
||||
window.location.href = {{ url_for('terms.terms_plot_top') }}+"?per_paste="+checked;
|
||||
}
|
||||
</script>
|
||||
|
||||
|
||||
<script>
|
||||
var graph_options = {
|
||||
series: {
|
||||
lines: {
|
||||
show: true,
|
||||
lineWidth: 2
|
||||
},
|
||||
bars: {show: false, barWidth: 60*60*1000},
|
||||
shadowSize: 0
|
||||
},
|
||||
grid: {
|
||||
hoverable: true,
|
||||
clickable: true,
|
||||
tickColor: "#f9f9f9",
|
||||
borderWidth: 0
|
||||
},
|
||||
legend: { show: true,
|
||||
noColumns: 1,
|
||||
position: "nw",
|
||||
labelFormatter: function(label, series) {
|
||||
return "<a href=\"#\" onClick=\"hide_or_show2("+series.idx+", "+series.graphNum+"); return false;\" >" + label + "</a>";
|
||||
}
|
||||
},
|
||||
xaxis: {
|
||||
mode: "time",
|
||||
timeformat: "%m/%d",
|
||||
minTickSize: [1, "day"]
|
||||
},
|
||||
yaxis: {
|
||||
//transform: function (v) { return v < 1 ? v : Math.log(v); }
|
||||
autoscaleMargin: 0.1,
|
||||
},
|
||||
tooltip: true,
|
||||
tooltipOpts: {
|
||||
content: " %s (%x.1 is %y.4) ",
|
||||
shifts: {
|
||||
x: -60,
|
||||
y: 25
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
set_today = "TopTermFreq_set_day";
|
||||
set_week = "TopTermFreq_set_week";
|
||||
set_month = "TopTermFreq_set_month";
|
||||
default_num_curves = 8;
|
||||
per_paste = {{ per_paste }}
|
||||
|
||||
var plot_today;
|
||||
var plot_week;
|
||||
var plot_month;
|
||||
|
||||
var promises = []; // Used to know when everything has been received
|
||||
|
||||
promises.push($.getJSON("{{ url_for('terms.terms_plot_top_data') }}", { set: set_today, num_day: 5, per_paste: per_paste }, function(data, status){
|
||||
data.sort(function(a, b){return b[2]-a[2];});
|
||||
// Sort data
|
||||
var table_today = $("#table-today")
|
||||
var table_today2 = $("#table-today2")
|
||||
var to_plot = [];
|
||||
var unchecked_label = [];
|
||||
for(i=0; i<data.length; i++) {
|
||||
var highlight = data[i][3].week == "<20"? " style=\"background-color: lightgreen;\" " : "";
|
||||
var curr_data = [];
|
||||
for(j=0; j<data[i][1].length; j++) {
|
||||
curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
|
||||
}
|
||||
if (i>=default_num_curves) {
|
||||
unchecked_label.push(data[i][0]);
|
||||
}
|
||||
to_plot.push({ data: curr_data, label: data[i][0], idx: i});
|
||||
if ( i < (data.length/2))
|
||||
table_today.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("today", data[i][0], i<default_num_curves)+"</td><td>"+data[i][3].week+", "+data[i][3].month+"</td></tr>");
|
||||
else
|
||||
table_today2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("today", data[i][0], i<default_num_curves)+"</td><td>"+data[i][3].week+", "+data[i][3].month+"</td></tr>");
|
||||
}
|
||||
graph_options.series.graphNum=1;
|
||||
plot_today = $.plot($("#graph-today"), to_plot, graph_options);
|
||||
hide_unchecked_curves(plot_today, unchecked_label);
|
||||
$("#graph-today").bind("plothover", function (event, pos, item) {
|
||||
if (item) {
|
||||
var date = new Date(item.datapoint[0]);
|
||||
var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
|
||||
var y = item.datapoint[1];
|
||||
|
||||
$("#tooltip").html(item.series.label + " for "+x + " = " + y)
|
||||
.css({top: item.pageY+5, left: item.pageX+5})
|
||||
.fadeIn(200);
|
||||
} else {
|
||||
$("#tooltip").hide();
|
||||
}
|
||||
});
|
||||
|
||||
}));
|
||||
|
||||
promises.push($.getJSON("{{ url_for('terms.terms_plot_top_data') }}", { set: set_week, num_day: 7, per_paste: per_paste }, function(data, status){
|
||||
data.sort(function(a, b){return b[2]-a[2];});
|
||||
// Sort data
|
||||
var table = $("#table-week")
|
||||
var table2 = $("#table-week2")
|
||||
var to_plot = [];
|
||||
var unchecked_label = [];
|
||||
for(i=0; i<data.length; i++) {
|
||||
var highlight = parseInt(data[i][3].day) > 20? " style=\"background-color: orange;\" " : "";
|
||||
var curr_data = [];
|
||||
for(j=0; j<data[i][1].length; j++) {
|
||||
curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
|
||||
}
|
||||
if (i>=default_num_curves) {
|
||||
unchecked_label.push(data[i][0]);
|
||||
}
|
||||
to_plot.push({ data: curr_data, label: data[i][0], idx: i});
|
||||
if ( i < (data.length/2))
|
||||
table.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("week", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].month+"</td></tr>");
|
||||
else
|
||||
table2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("week", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].month+"</td></tr>");
|
||||
}
|
||||
graph_options.series.graphNum=2;
|
||||
plot_week = $.plot($("#graph-week"), to_plot, graph_options);
|
||||
hide_unchecked_curves(plot_week, unchecked_label);
|
||||
$("#graph-week").bind("plothover", function (event, pos, item) {
|
||||
if (item) {
|
||||
var date = new Date(item.datapoint[0]);
|
||||
var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
|
||||
var y = item.datapoint[1];
|
||||
|
||||
$("#tooltip").html(item.series.label + " for "+x + " = " + y)
|
||||
.css({top: item.pageY+5, left: item.pageX+5})
|
||||
.fadeIn(200);
|
||||
} else {
|
||||
$("#tooltip").hide();
|
||||
}
|
||||
});
|
||||
}));
|
||||
|
||||
promises.push($.getJSON("{{ url_for('terms.terms_plot_top_data') }}", { set: set_month, num_day: 31, per_paste: per_paste }, function(data, status){
|
||||
data.sort(function(a, b){return b[2]-a[2];});
|
||||
// Sort data
|
||||
var table = $("#table-month")
|
||||
var table2 = $("#table-month2")
|
||||
var to_plot = [];
|
||||
var unchecked_label = [];
|
||||
for(i=0; i<data.length; i++) {
|
||||
var highlight = parseInt(data[i][3].day) > 20? " style=\"background-color: orange;\" " : "";
|
||||
var curr_data = [];
|
||||
for(j=0; j<data[i][1].length; j++) {
|
||||
curr_data.push([data[i][1][j][0]*1000, data[i][1][j][1]]);
|
||||
}
|
||||
if (i>=default_num_curves) {
|
||||
unchecked_label.push(data[i][0]);
|
||||
}
|
||||
to_plot.push({ data: curr_data, label: data[i][0], idx: i});
|
||||
if ( i < (data.length/2))
|
||||
table.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("month", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].week+"</td></tr>");
|
||||
else
|
||||
table2.append("<tr"+highlight+"><td>"+data[i][0]+"</td><td>"+data[i][2]+"</td><td>"+addbuttons(data[i][0])+"</td><td>"+addcheckbox("month", data[i][0], i<default_num_curves)+"</td><td><strong>"+data[i][3].day+"</strong>, "+data[i][3].week+"</td></tr>");
|
||||
}
|
||||
graph_options.series.graphNum=3;
|
||||
plot_month = $.plot($("#graph-month"), to_plot, graph_options);
|
||||
hide_unchecked_curves(plot_month, unchecked_label);
|
||||
$("#graph-month").bind("plothover", function (event, pos, item) {
|
||||
if (item) {
|
||||
var date = new Date(item.datapoint[0]);
|
||||
var x = parseInt(date.getUTCMonth())+1 + "/" + date.getUTCDate();
|
||||
var y = item.datapoint[1];
|
||||
|
||||
$("#tooltip").html(item.series.label + " for "+x + " = " + y)
|
||||
.css({top: item.pageY+5, left: item.pageX+5})
|
||||
.fadeIn(200);
|
||||
} else {
|
||||
$("#tooltip").hide();
|
||||
}
|
||||
});
|
||||
}));
|
||||
|
||||
/* When everything has been received, start adding tooltip */
|
||||
$.when.apply($, promises).done( function (arg) {
|
||||
$('[data-toggle="tooltip"]').tooltip();
|
||||
|
||||
$(".btn-interaction").unbind("click.interaction");
|
||||
$(".btn-interaction").bind("click.interaction", perform_operation);
|
||||
$(".check-interaction").unbind("click.interaction");
|
||||
$(".check-interaction").bind("click.interaction", hide_or_show);
|
||||
});
|
||||
|
||||
|
||||
function addbuttons(term) {
|
||||
return "<button class=\"fa fa-eye btn-interaction btn btn-success\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Add to tracked list\" style=\"margin-right: 5px;\""+
|
||||
"data-section=\"followTerm\" data-term=\""+term+"\"></button>"+
|
||||
"<button class=\"fa fa-eye-slash btn-interaction btn btn-danger\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Add to black-list\""+
|
||||
"data-section=\"blacklistTerm\" data-term=\""+term+"\"></button>";
|
||||
}
|
||||
|
||||
function addcheckbox(graph, term, checked) {
|
||||
var checked_text = checked ? "checked" : "";
|
||||
return "<input type=checkbox "+checked_text+" class=\"check-interaction\" data-term=\""+term+"\" data-graph=\""+graph+"\"></input>";
|
||||
}
|
||||
|
||||
function perform_operation(){
|
||||
var curr_section = $(this).attr('data-section');
|
||||
var curr_term = $(this).attr('data-term');
|
||||
|
||||
var data_to_send = { section: curr_section, action:"add", term: curr_term};
|
||||
|
||||
$.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){
|
||||
if(status == "success") {
|
||||
location.reload();
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
function hide_unchecked_curves(plot, unchecked_label) {
|
||||
var graphData = plot.getData();
|
||||
var index;
|
||||
for(i=0; i<graphData.length; i++) {
|
||||
if($.inArray( graphData[i].label, unchecked_label ) != -1){
|
||||
graphData[i].lines.show = false;
|
||||
}
|
||||
}
|
||||
plot.setData(graphData);
|
||||
plot.draw();
|
||||
}
|
||||
|
||||
|
||||
function hide_or_show() {
|
||||
var curr_term = $(this).attr('data-term');
|
||||
var graph = $(this).attr('data-graph');
|
||||
var checked = $(this).prop('checked')
|
||||
|
||||
if(graph == "today") {
|
||||
var graphData = plot_today.getData();
|
||||
var index;
|
||||
for(i=0; i<graphData.length; i++){
|
||||
if(graphData[i].label == curr_term){
|
||||
index = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
graphData[index].lines.show = checked;
|
||||
plot_today.setData(graphData);
|
||||
plot_today.draw();
|
||||
} else if (graph == "week") {
|
||||
var graphData = plot_week.getData();
|
||||
var index;
|
||||
for(i=0; i<graphData.length; i++){
|
||||
if(graphData[i].label == curr_term){
|
||||
index = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
graphData[index].lines.show = checked;
|
||||
plot_week.setData(graphData);
|
||||
plot_week.draw();
|
||||
} else if (graph == "month") {
|
||||
var graphData = plot_month.getData();
|
||||
var index;
|
||||
for(i=0; i<graphData.length; i++){
|
||||
if(graphData[i].label == curr_term){
|
||||
index = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
graphData[index].lines.show = checked;
|
||||
plot_month.setData(graphData);
|
||||
plot_month.draw();
|
||||
}
|
||||
|
||||
// graph, hide curve
|
||||
}
|
||||
|
||||
function hide_or_show2(index, graphNum) {
|
||||
if (graphNum == 1)
|
||||
var plot = plot_today;
|
||||
else if (graphNum == 2)
|
||||
var plot = plot_week;
|
||||
else if (graphNum == 3)
|
||||
var plot = plot_month;
|
||||
|
||||
var graphData = plot.getData();
|
||||
graphData[index].lines.show = !graphData[index].lines.show;
|
||||
plot.setData(graphData);
|
||||
plot.draw();
|
||||
}
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
|
@ -6,9 +6,9 @@
|
|||
<img id="hive-logo" src="{{url_for('static', filename='image/thehive-logo.png') }}" width="500" >
|
||||
</div>
|
||||
|
||||
<form method="post" action="{{ url_for('PasteSubmit.create_hive_case') }}" target="_blank">
|
||||
<form method="post" action="{{ url_for('import_export.create_thehive_case') }}" target="_blank">
|
||||
|
||||
<div class="modal-body">
|
||||
<div class="modal-body">
|
||||
|
||||
<div class="input clear required">
|
||||
<label for="EventThreatLevelId">Threat Level</label>
|
||||
|
@ -35,8 +35,9 @@
|
|||
<label for="hive_description">Description</label>
|
||||
<input name="hive_description" class="form-control span6" placeholder="Quick Case Description" type="text" id="hive_description"/>
|
||||
</div>
|
||||
<input type="hidden" id="obj_type" name="obj_type" value="item">
|
||||
<input type="hidden" id="obj_id" name="obj_id" value="{{ dict_item['id'] }}">
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="modal-footer">
|
||||
<button class="btn btn-primary" type="submit">
|
||||
|
|
Loading…
Reference in a new issue