AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-12 17:48:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [objects + retro hunt] refactor retro hunt + objects retro hunts + get objects generator by filters (date_from, sources, mimetypes, ...)

Browse source
This commit is contained in:
Terrtia 2023-05-10 16:26:46 +02:00
parent ee828a04bc
commit 37c71b8438
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
14 changed files with 1026 additions and 882 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

824
bin/lib/Tracker.py
View file

File diff suppressed because it is too large Load diff

18
bin/lib/ail_core.py
View file

@ -43,6 +43,9 @@ def get_object_all_subtypes(obj_type):
def get_objects_tracked(): def get_objects_tracked():
return ['decoded', 'item', 'pgp'] return ['decoded', 'item', 'pgp']
def get_objects_retro_hunted():
return ['decoded', 'item']
def get_all_objects_with_subtypes_tuple(): def get_all_objects_with_subtypes_tuple():
str_objs = [] str_objs = []
for obj_type in get_all_objects(): for obj_type in get_all_objects():
@ -56,6 +59,21 @@ def get_all_objects_with_subtypes_tuple():
##-- AIL OBJECTS --## ##-- AIL OBJECTS --##
#### Redis ####
def _parse_zscan(response):
cursor, r = response
it = iter(r)
return str(cursor), list(it)
def zscan_iter(r_redis, name): # count ???
cursor = 0
while cursor != "0":
cursor, data = _parse_zscan(r_redis.zscan(name, cursor=cursor))
yield from data
## -- Redis -- ##
def paginate_iterator(iter_elems, nb_obj=50, page=1): def paginate_iterator(iter_elems, nb_obj=50, page=1):
dict_page = {'nb_all_elem': len(iter_elems)} dict_page = {'nb_all_elem': len(iter_elems)}
nb_pages = dict_page['nb_all_elem'] / nb_obj nb_pages = dict_page['nb_all_elem'] / nb_obj

71
bin/lib/objects/Decodeds.py
View file

@ -372,6 +372,67 @@ def search_decodeds_by_name(name_to_search, r_pos=False):
decodeds[decoded_name]['hl-end'] = res.end() decodeds[decoded_name]['hl-end'] = res.end()
return decodeds return decodeds
############################################################################
def get_decodeds_dir():
decodeds_dir = os.path.join(os.environ['AIL_HOME'], HASH_DIR)
if not decodeds_dir.endswith("/"):
decodeds_dir = f"{decodeds_dir}/"
return decodeds_dir
# Generator
def get_nb_decodeds_objects(filters={}):
nb = 0
if 'mimetypes' in filters:
mimetypes = filters['mimetypes']
else:
mimetypes = get_all_mimetypes()
d_dir = get_decodeds_dir()
for mimetype in mimetypes:
for root, dirs, files in os.walk(os.path.join(d_dir, mimetype)):
nb += len(files)
return nb
def get_all_decodeds_objects(filters={}):
if 'mimetypes' in filters:
# TODO sanityze mimetype
mimetypes = filters['mimetypes']
else:
mimetypes = get_all_mimetypes()
mimetypes = sorted(mimetypes)
if filters.get('start'):
_, start_id = filters['start'].split(':', 1)
decoded = Decoded(start_id)
# remove sources
start_mimetype = decoded.get_mimetype()
i = 0
while start_mimetype and len(mimetypes) > i:
if mimetypes[i] == start_mimetype:
mimetypes = mimetypes[i:]
start_mimetype = None
i += 1
else:
start_id = None
d_dir = get_decodeds_dir()
for mimetype in mimetypes:
for root, dirs, files in os.walk(os.path.join(d_dir, mimetype)):
if start_id:
i = 0
while start_id and len(files) > i:
if files[i] == start_id:
files = files[i:]
start_id = None
i += 1
if i >= len(files):
files = []
for file in files:
yield Decoded(file).id
############################################################################ ############################################################################
def sanityze_decoder_names(decoder_name): def sanityze_decoder_names(decoder_name):
@ -538,6 +599,12 @@ def get_all_decodeds_files():
return decodeds return decodeds
# if __name__ == '__main__': if __name__ == '__main__':
# name_to_search = '4d36' # name_to_search = '4d36'
# print(search_decodeds_by_name(name_to_search)) # print(search_decodeds_by_name(name_to_search))
# filters = {'mimetypes': ['text/html']}
filters = {'start': ':1a005f82a4ae0940205c8fd81fd14838845696be'}
# filters = {}
gen = get_all_decodeds_objects(filters=filters)
for f in gen:
print(f)

118
bin/lib/objects/Items.py
View file

@ -22,7 +22,8 @@ from lib.ail_core import get_ail_uuid
from lib.objects.abstract_object import AbstractObject from lib.objects.abstract_object import AbstractObject
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib import item_basic from lib import item_basic
from lib.data_retention_engine import update_obj_date from lib.data_retention_engine import update_obj_date, get_obj_date_first
from packages import Date
from flask import url_for from flask import url_for
@ -406,6 +407,115 @@ def _manual_set_items_date_first_last():
if last != 0: if last != 0:
update_obj_date(last, 'item') update_obj_date(last, 'item')
################################################################################
################################################################################
################################################################################
def get_nb_items_objects(filters={}):
nb = 0
date_from = filters.get('date_from')
date_to = filters.get('date_to')
if 'sources' in filters:
sources = filters['sources']
else:
sources = get_all_sources()
sources = sorted(sources)
# date
if date_from and date_to:
daterange = Date.get_daterange(date_from, date_to)
elif date_from:
daterange = Date.get_daterange(date_from, Date.get_today_date_str())
elif date_to:
date_from = get_obj_date_first('item')
daterange = Date.get_daterange(date_from, date_to)
else:
date_from = get_obj_date_first('item')
daterange = Date.get_daterange(date_from, Date.get_today_date_str())
for source in sources:
for date in daterange:
date = f'{date[0:4]}/{date[4:6]}/{date[6:8]}'
full_dir = os.path.join(ITEMS_FOLDER, source, date)
if not os.path.isdir(full_dir):
continue
nb += len(os.listdir(full_dir))
return nb
def get_all_items_objects(filters={}):
date_from = filters.get('date_from')
date_to = filters.get('date_to')
if 'sources' in filters:
sources = filters['sources']
else:
sources = get_all_sources()
sources = sorted(sources)
if filters.get('start'):
_, start_id = filters['start'].split(':', 1)
item = Item(start_id)
# remove sources
start_source = item.get_source()
i = 0
while start_source and len(sources) > i:
if sources[i] == start_source:
sources = sources[i:]
start_source = None
i += 1
start_date = item.get_date()
else:
start_id = None
start_date = None
# date
if date_from and date_to:
daterange = Date.get_daterange(date_from, date_to)
elif date_from:
daterange = Date.get_daterange(date_from, Date.get_today_date_str())
elif date_to:
date_from = get_obj_date_first('item')
daterange = Date.get_daterange(date_from, date_to)
else:
date_from = get_obj_date_first('item')
daterange = Date.get_daterange(date_from, Date.get_today_date_str())
if start_date:
if int(start_date) > int(date_from):
i = 0
while start_date and len(daterange) > i:
if daterange[i] == start_date:
daterange = daterange[i:]
start_date = None
i += 1
for source in sources:
for date in daterange:
date = f'{date[0:4]}/{date[4:6]}/{date[6:8]}'
full_dir = os.path.join(ITEMS_FOLDER, source, date)
s_dir = os.path.join(source, date)
if not os.path.isdir(full_dir):
continue
# TODO replace by os.scandir() ????
all_items = sorted([os.path.join(s_dir, f)
for f in os.listdir(full_dir)
if os.path.isfile(os.path.join(full_dir, f))])
# start obj id
if start_id:
i = 0
while start_id and len(all_items) > i:
if all_items[i] == start_id:
if i == len(all_items):
all_items = []
else:
all_items = all_items[i+1:]
start_id = None
i += 1
for obj_id in all_items:
yield Item(obj_id)
################################################################################
################################################################################
################################################################################
#### API #### #### API ####
def api_get_item(data): def api_get_item(data):
@ -810,9 +920,13 @@ def create_item(obj_id, obj_metadata, io_content):
# delete_item(child_id) # delete_item(child_id)
# if __name__ == '__main__': if __name__ == '__main__':
# content = 'test file content' # content = 'test file content'
# duplicates = {'tests/2020/01/02/test.gz': [{'algo':'ssdeep', 'similarity':75}, {'algo':'tlsh', 'similarity':45}]} # duplicates = {'tests/2020/01/02/test.gz': [{'algo':'ssdeep', 'similarity':75}, {'algo':'tlsh', 'similarity':45}]}
# #
# item = Item('tests/2020/01/02/test_save.gz') # item = Item('tests/2020/01/02/test_save.gz')
# item.create(content, _save=False) # item.create(content, _save=False)
filters = {'date_from': '20230101', 'date_to': '20230501', 'sources': ['crawled', 'submitted'], 'start': ':submitted/2023/04/28/submitted_2b3dd861-a75d-48e4-8cec-6108d41450da.gz'}
gen = get_all_items_objects(filters=filters)
for obj_id in gen:
print(obj_id.id)

18
bin/lib/objects/Pgps.py
View file

@ -13,7 +13,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id from lib.objects.abstract_subtype_object import AbstractSubtypeObject, get_all_id, get_all_id_iterator
config_loader = ConfigLoader() config_loader = ConfigLoader()
baseurl = config_loader.get_config_str("Notifications", "ail_domain") baseurl = config_loader.get_config_str("Notifications", "ail_domain")
@ -128,8 +128,22 @@ def search_pgps_by_name(name_to_search, subtype, r_pos=False):
pgps[pgp_name]['hl-end'] = res.end() pgps[pgp_name]['hl-end'] = res.end()
return pgps return pgps
def get_all_pgps_objects(filters={}):
if 'subtypes' in filters:
subtypes = filters['subtypes']
else:
subtypes = get_all_subtypes()
for subtype in subtypes:
for z_tuple in get_all_id_iterator('pgp', subtype):
obj_id, _ = z_tuple
yield Pgp(obj_id, subtype)
# if __name__ == '__main__':
if __name__ == '__main__':
# name_to_search = 'ex' # name_to_search = 'ex'
# subtype = 'name' # subtype = 'name'
# print(search_pgps_by_name(name_to_search, subtype)) # print(search_pgps_by_name(name_to_search, subtype))
gen = get_all_pgps_objects(filters={'subtypes': ['key']})
for f in gen:
print(f)

5
bin/lib/objects/abstract_subtype_object.py
View file

@ -17,7 +17,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from lib.objects.abstract_object import AbstractObject from lib.objects.abstract_object import AbstractObject
from lib.ail_core import get_object_all_subtypes from lib.ail_core import get_object_all_subtypes, zscan_iter
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib.item_basic import is_crawled, get_item_domain from lib.item_basic import is_crawled, get_item_domain
from lib.data_retention_engine import update_obj_date from lib.data_retention_engine import update_obj_date
@ -176,6 +176,9 @@ class AbstractSubtypeObject(AbstractObject, ABC):
def get_all_id(obj_type, subtype): def get_all_id(obj_type, subtype):
return r_object.zrange(f'{obj_type}_all:{subtype}', 0, -1) return r_object.zrange(f'{obj_type}_all:{subtype}', 0, -1)
def get_all_id_iterator(obj_type, subtype):
return zscan_iter(r_object, f'{obj_type}_all:{subtype}')
def get_subtypes_objs_by_date(obj_type, subtype, date): def get_subtypes_objs_by_date(obj_type, subtype, date):
return r_object.hkeys(f'{obj_type}:{subtype}:{date}') return r_object.hkeys(f'{obj_type}:{subtype}:{date}')

34
bin/lib/objects/ail_objects.py
View file

@ -9,16 +9,16 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib.ail_core import get_all_objects, get_object_all_subtypes, get_all_objects_with_subtypes_tuple from lib.ail_core import get_all_objects, get_object_all_subtypes
from lib import correlations_engine from lib import correlations_engine
from lib import btc_ail from lib import btc_ail
from lib import Tag from lib import Tag
from lib.objects import CryptoCurrencies from lib.objects import CryptoCurrencies
from lib.objects.Cves import Cve from lib.objects.Cves import Cve
from lib.objects.Decodeds import Decoded from lib.objects.Decodeds import Decoded, get_all_decodeds_objects, get_nb_decodeds_objects
from lib.objects.Domains import Domain from lib.objects.Domains import Domain
from lib.objects.Items import Item from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects
from lib.objects import Pgps from lib.objects import Pgps
from lib.objects.Screenshots import Screenshot from lib.objects.Screenshots import Screenshot
from lib.objects import Usernames from lib.objects import Usernames
@ -114,6 +114,9 @@ def get_obj_tags(obj_type, subtype, id):
obj = get_object(obj_type, subtype, id) obj = get_object(obj_type, subtype, id)
return obj.get_tags() return obj.get_tags()
def is_obj_tags_safe(obj_type, subtype, id):
obj = get_object(obj_type, subtype, id)
return obj.is_tags_safe()
def add_obj_tag(obj_type, subtype, id, tag): def add_obj_tag(obj_type, subtype, id, tag):
obj = get_object(obj_type, subtype, id) obj = get_object(obj_type, subtype, id)
@ -143,6 +146,10 @@ def get_objects_meta(objs, options=set(), flask_context=False):
obj_type = obj['type'] obj_type = obj['type']
subtype = obj['subtype'] subtype = obj['subtype']
obj_id = obj['id'] obj_id = obj['id']
elif isinstance(obj, tuple):
obj_type = obj[0]
subtype = obj[1]
obj_id = obj[2]
else: else:
obj_type, subtype, obj_id = obj.split(':', 2) obj_type, subtype, obj_id = obj.split(':', 2)
metas.append(get_object_meta(obj_type, subtype, obj_id, options=options, flask_context=flask_context)) metas.append(get_object_meta(obj_type, subtype, obj_id, options=options, flask_context=flask_context))
@ -184,6 +191,27 @@ def is_filtered(obj, filters):
return True return True
return False return False
def obj_iterator(obj_type, filters):
if obj_type == 'decoded':
return get_all_decodeds_objects(filters=filters)
elif obj_type == 'item':
return get_all_items_objects(filters=filters)
elif obj_type == 'pgp':
return Pgps.get_all_pgps_objects(filters=filters)
def card_objs_iterators(filters):
nb = 0
for obj_type in filters:
nb += int(card_obj_iterator(obj_type, filters.get(obj_type, {})))
return nb
def card_obj_iterator(obj_type, filters):
if obj_type == 'decoded':
return get_nb_decodeds_objects(filters=filters)
elif obj_type == 'item':
return get_nb_items_objects(filters=filters)
elif obj_type == 'pgp':
return Pgps.nb_all_pgps_objects(filters=filters)
def get_ui_obj_tag_table_keys(obj_type): # TODO REMOVE ME def get_ui_obj_tag_table_keys(obj_type): # TODO REMOVE ME
""" """

7
bin/packages/Date.py
View file

@ -79,8 +79,11 @@ class Date(object):
comp_day = str(computed_date.day).zfill(2) comp_day = str(computed_date.day).zfill(2)
return comp_year + comp_month + comp_day return comp_year + comp_month + comp_day
def get_today_date_str(): def get_today_date_str(separator=False):
return datetime.date.today().strftime("%Y%m%d") if separator:
datetime.date.today().strftime("%Y/%m/%d")
else:
return datetime.date.today().strftime("%Y%m%d")
def date_add_day(date, num_day=1): def date_add_day(date, num_day=1):
new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day) new_date = datetime.date(int(date[0:4]), int(date[4:6]), int(date[6:8])) + datetime.timedelta(num_day)

160
bin/trackers/Retro_Hunt.py
View file

@ -19,149 +19,122 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from modules.abstract_module import AbstractModule from modules.abstract_module import AbstractModule
from lib.ail_core import get_objects_retro_hunted
from lib.ConfigLoader import ConfigLoader from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item from lib.objects import ail_objects
from packages import Date
from lib import Tracker from lib import Tracker
import NotificationHelper # # TODO: refractor class Retro_Hunt_Module(AbstractModule):
class Retro_Hunt(AbstractModule):
# mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}"
""" """
Retro_Hunt module for AIL framework Retro_Hunt module for AIL framework
""" """
def __init__(self): def __init__(self):
super(Retro_Hunt, self).__init__() super(Retro_Hunt_Module, self).__init__()
config_loader = ConfigLoader() config_loader = ConfigLoader()
self.pending_seconds = 5 self.pending_seconds = 5
self.full_item_url = config_loader.get_config_str("Notifications", "ail_domain") + "/object/item?id="
# reset on each loop # reset on each loop
self.task_uuid = None self.retro_hunt = None
self.date_from = 0 self.nb_objs = 0
self.date_to = 0 self.nb_done = 0
self.nb_src_done = 0
self.progress = 0 self.progress = 0
self.item = None self.obj = None
self.tags = [] self.tags = []
self.redis_logger.info(f"Module: {self.module_name} Launched") self.redis_logger.info(f"Module: {self.module_name} Launched")
# # TODO: send mails # # TODO: # start_time
# # TODO: # start_time # end_time # # end_time
def compute(self, task_uuid): def compute(self, task_uuid):
self.redis_logger.warning(f'{self.module_name}, starting Retro hunt task {task_uuid}') self.redis_logger.warning(f'{self.module_name}, starting Retro hunt task {task_uuid}')
print(f'starting Retro hunt task {task_uuid}') print(f'starting Retro hunt task {task_uuid}')
self.task_uuid = task_uuid
self.progress = 0 self.progress = 0
# First launch # First launch
# restart # restart
retro_hunt = Tracker.RetroHunt(task_uuid) # TODO SELF self.retro_hunt = Tracker.RetroHunt(task_uuid)
rule = retro_hunt.get_rule(r_compile=True) rule = self.retro_hunt.get_rule(r_compile=True)
timeout = self.retro_hunt.get_timeout()
self.tags = self.retro_hunt.get_tags()
timeout = retro_hunt.get_timeout()
self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}') self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
sources = retro_hunt.get_sources(r_sort=True)
self.date_from = retro_hunt.get_date_from() # Filters
self.date_to = retro_hunt.get_date_to() filters = self.retro_hunt.get_filters()
self.tags = retro_hunt.get_tags() if not filters:
curr_date = Tracker.get_retro_hunt_task_current_date(task_uuid) filters = {}
self.nb_src_done = Tracker.get_retro_hunt_task_nb_src_done(task_uuid, sources=sources) for obj_type in get_objects_retro_hunted():
self.update_progress(sources, curr_date) filters[obj_type] = {}
# iterate on date
filter_last = True
while int(curr_date) <= int(self.date_to):
print(curr_date)
dirs_date = Tracker.get_retro_hunt_dir_day_to_analyze(task_uuid, curr_date, filter_last=filter_last, sources=sources)
filter_last = False
nb_id = 0
self.nb_src_done = 0
self.update_progress(sources, curr_date)
# # TODO: Filter previous item
for dir in dirs_date:
print(dir)
self.redis_logger.debug(f'{self.module_name}, Retro Hunt searching in directory {dir}')
l_obj = Tracker.get_items_to_analyze(dir)
for id in l_obj:
# print(f'{dir} / {id}')
self.item = Item(id)
# save current item in cache
Tracker.set_cache_retro_hunt_task_id(task_uuid, id)
self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid}, searching item {id}') self.nb_objs = ail_objects.card_objs_iterators(filters)
yara_match = rule.match(data=self.item.get_content(), callback=self.yara_rules_match, # Resume
which_callbacks=yara.CALLBACK_MATCHES, timeout=timeout) last_obj = self.retro_hunt.get_last_analyzed()
if last_obj:
last_obj_type, last_obj_subtype, last_obj_id = last_obj.split(':', 2)
else:
last_obj_type = None
last_obj_subtype = None
last_obj_id = None
# save last item self.nb_done = 0
if nb_id % 10 == 0: # # TODO: Add nb before save in DB self.update_progress()
Tracker.set_retro_hunt_last_analyzed(task_uuid, id)
nb_id += 1
self.update_progress(sources, curr_date)
# PAUSE for obj_type in filters:
self.update_progress(sources, curr_date) if last_obj_type:
if retro_hunt.to_pause(): filters['start'] = f'{last_obj_subtype}:{last_obj_id}'
Tracker.set_retro_hunt_last_analyzed(task_uuid, id) last_obj_type = None
# self.update_progress(sources, curr_date, save_db=True) for obj in ail_objects.obj_iterator(obj_type, filters):
retro_hunt.pause() self.obj = obj
return None content = obj.get_content(r_str=True)
rule.match(data=content, callback=self.yara_rules_match,
which_callbacks=yara.CALLBACK_MATCHES, timeout=timeout)
self.nb_src_done += 1 self.nb_done += 1
self.update_progress(sources, curr_date) if self.nb_done % 10 == 0:
curr_date = Date.date_add_day(curr_date) self.retro_hunt.set_last_analyzed(obj.get_type(), obj.get_subtype(r_str=True), obj.get_id())
print('-----') self.retro_hunt.set_last_analyzed_cache(obj.get_type(), obj.get_subtype(r_str=True), obj.get_id())
self.update_progress(sources, curr_date) # update progress
self.update_progress()
retro_hunt.complete() # PAUSE
if self.retro_hunt.to_pause():
self.retro_hunt.set_last_analyzed(obj.get_type(), obj.get_subtype(r_str=True), obj.get_id())
self.retro_hunt.pause()
return None
# Completed
self.retro_hunt.complete()
print(f'Retro Hunt {task_uuid} completed') print(f'Retro Hunt {task_uuid} completed')
self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed') self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed')
# # TODO: stop def update_progress(self):
new_progress = self.nb_done * 100 / self.nb_objs
def update_progress(self, sources, curr_date, save_db=False): if int(self.progress) != int(new_progress):
retro_hunt = Tracker.RetroHunt(retro_hubt) # TODO USE SELF print(new_progress)
progress = retro_hunt.compute_progress(date_from=self.date_from, date_to=self.date_to, self.retro_hunt.set_progress(new_progress)
sources=sources, curr_date=curr_date, nb_src_done=self.nb_src_done) self.progress = new_progress
if self.progress != progress:
retro_hunt.set_progress(progress)
self.progress = progress
# if save_db:
# Tracker.set_retro_hunt_task_progress(task_uuid, progress)
def yara_rules_match(self, data): def yara_rules_match(self, data):
id = self.item.get_id() obj_id = self.obj.get_id()
# print(data) # print(data)
task_uuid = data['namespace'] task_uuid = data['namespace']
self.redis_logger.info(f'{self.module_name}, Retro hunt {task_uuid} match found: {id}') self.redis_logger.info(f'{self.module_name}, Retro hunt {task_uuid} match found: {obj_id}')
print(f'Retro hunt {task_uuid} match found: {id}') print(f'Retro hunt {task_uuid} match found: {self.obj.get_type()} {obj_id}')
Tracker.save_retro_hunt_match(task_uuid, id) self.retro_hunt.add(self.obj.get_type(), self.obj.get_subtype(), obj_id)
# TODO FILTER Tags
# Tags # Tags
for tag in self.tags: for tag in self.tags:
msg = f'{tag};{id}' msg = f'{tag};{id}'
self.add_message_to_queue(msg, 'Tags') self.add_message_to_queue(msg, 'Tags')
# # Mails # # Mails
# mail_to_notify = Tracker.get_tracker_mails(tracker_uuid) # EXPORTER MAILS
# if mail_to_notify:
# mail_subject = Tracker.get_email_subject(tracker_uuid)
# mail_body = Tracker_Yara.mail_body_template.format(data['rule'], item_id, self.full_item_url, item_id)
# for mail in mail_to_notify:
# self.redis_logger.debug(f'Send Mail {mail_subject}')
# print(f'Send Mail {mail_subject}')
# NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
return yara.CALLBACK_CONTINUE return yara.CALLBACK_CONTINUE
def run(self): def run(self):
@ -188,6 +161,5 @@ class Retro_Hunt(AbstractModule):
if __name__ == '__main__': if __name__ == '__main__':
module = Retro_Hunt_Module()
module = Retro_Hunt()
module.run() module.run()

2
configs/modules.cfg
View file

@ -40,7 +40,7 @@ publish = D4_client
[D4Client] [D4Client]
subscribe = D4_client subscribe = D4_client
[Retro_Hunt] [Retro_Hunt_Module]
publish = Tags publish = Tags
[Tracker_Typo_Squatting] [Tracker_Typo_Squatting]

142
var/www/blueprints/hunters.py
View file

@ -9,7 +9,7 @@ import os
import sys import sys
import json import json
from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response, escape
from flask_login import login_required, current_user, login_user, logout_user from flask_login import login_required, current_user, login_user, logout_user
sys.path.append('modules') sys.path.append('modules')
@ -23,6 +23,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages # Import Project packages
################################## ##################################
from lib import ail_core from lib import ail_core
from lib.objects import ail_objects
from lib import item_basic from lib import item_basic
from lib import Tracker from lib import Tracker
from lib import Tag from lib import Tag
@ -149,6 +150,8 @@ def add_tracked_menu():
galaxies_tags = json.loads(galaxies_tags) galaxies_tags = json.loads(galaxies_tags)
except: except:
galaxies_tags = [] galaxies_tags = []
else:
galaxies_tags = []
# custom tags # custom tags
if tags: if tags:
tags = tags.split() tags = tags.split()
@ -242,7 +245,7 @@ def tracker_delete():
@login_required @login_required
@login_read_only @login_read_only
def retro_hunt_all_tasks(): def retro_hunt_all_tasks():
retro_hunts = Tracker.get_retro_hunt_tasks_metas() retro_hunts = Tracker.get_retro_hunt_metas()
return render_template("retro_hunt_tasks.html", retro_hunts=retro_hunts, bootstrap_label=bootstrap_label) return render_template("retro_hunt_tasks.html", retro_hunts=retro_hunts, bootstrap_label=bootstrap_label)
@hunters.route('/retro_hunt/task/show', methods=['GET']) @hunters.route('/retro_hunt/task/show', methods=['GET'])
@ -250,40 +253,35 @@ def retro_hunt_all_tasks():
@login_read_only @login_read_only
def retro_hunt_show_task(): def retro_hunt_show_task():
task_uuid = request.args.get('uuid', None) task_uuid = request.args.get('uuid', None)
objs = request.args.get('objs', False)
date_from = request.args.get('date_from') date_from_item = request.args.get('date_from')
date_to = request.args.get('date_to') date_to_item = request.args.get('date_to')
if date_from: if date_from_item:
date_from = date_from.replace('-', '') date_from_item = date_from_item.replace('-', '')
if date_to: if date_to_item:
date_to = date_to.replace('-', '') date_to_item = date_to_item.replace('-', '')
res = Tracker.api_check_retro_hunt_task_uuid(task_uuid) res = Tracker.api_check_retro_hunt_task_uuid(task_uuid)
if res: if res:
return create_json_response(res[0], res[1]) return create_json_response(res[0], res[1])
retro_hunt = Tracker.RetroHunt(task_uuid) retro_hunt = Tracker.RetroHunt(task_uuid)
dict_task = retro_hunt.get_meta(options={'creator', 'date', 'description', 'progress', 'sources', 'tags'}) dict_task = retro_hunt.get_meta(options={'creator', 'date', 'description', 'progress', 'filters', 'nb_objs', 'tags'})
rule_content = Tracker.get_yara_rule_content(dict_task['rule']) rule_content = Tracker.get_yara_rule_content(dict_task['rule'])
dict_task['filters'] = json.dumps(dict_task['filters'], indent=4)
if date_from: if objs:
res = Tracker.api_get_retro_hunt_items({'uuid': task_uuid, 'date_from': date_from, 'date_to': date_to}) dict_task['objs'] = ail_objects.get_objects_meta(retro_hunt.get_objs(), flask_context=True)
if res[1] != 200:
return create_json_response(res[0], res[1])
dict_task['items'] = res[0]['items']
dict_task['date_from_input'] = res[0]['date_from']
dict_task['date_to_input'] = res[0]['date_to']
else: else:
dict_task['items'] = [] dict_task['objs'] = []
dict_task['date_from_input'] = dict_task['date_from']
dict_task['date_to_input'] = dict_task['date_to']
return render_template("show_retro_hunt.html", dict_task=dict_task, return render_template("show_retro_hunt.html", dict_task=dict_task,
rule_content=rule_content, rule_content=rule_content,
bootstrap_label=bootstrap_label) bootstrap_label=bootstrap_label)
@hunters.route('/retro_hunt/task/add', methods=['GET', 'POST']) @hunters.route('/retro_hunt/add', methods=['GET', 'POST'])
@login_required @login_required
@login_analyst @login_analyst
def retro_hunt_add_task(): def retro_hunt_add_task():
@ -291,23 +289,69 @@ def retro_hunt_add_task():
name = request.form.get("name", '') name = request.form.get("name", '')
description = request.form.get("description", '') description = request.form.get("description", '')
timeout = request.form.get("timeout", 30) timeout = request.form.get("timeout", 30)
# TAGS
tags = request.form.get("tags", []) tags = request.form.get("tags", [])
taxonomies_tags = request.form.get('taxonomies_tags')
if taxonomies_tags:
try:
taxonomies_tags = json.loads(taxonomies_tags)
except:
taxonomies_tags = []
else:
taxonomies_tags = []
galaxies_tags = request.form.get('galaxies_tags')
if galaxies_tags:
try:
galaxies_tags = json.loads(galaxies_tags)
except:
galaxies_tags = []
else:
galaxies_tags = []
# custom tags
if tags: if tags:
tags = tags.split() tags = tags.split()
escaped_tags = []
for tag in tags:
escaped_tags.append(escape(tag))
tags = escaped_tags
else:
tags = []
tags = tags + taxonomies_tags + galaxies_tags
# mails = request.form.get("mails", []) # mails = request.form.get("mails", [])
# if mails: # if mails:
# mails = mails.split() # mails = mails.split()
sources = request.form.get("sources", []) # FILTERS
if sources: filters = {}
sources = json.loads(sources) for obj_type in Tracker.get_objects_tracked():
new_filter = request.form.get(f'{obj_type}_obj')
date_from = request.form.get('date_from') if new_filter == 'on':
date_to = request.form.get('date_to') filters[obj_type] = {}
if date_from: # Date From
date_from = date_from.replace('-', '') date_from = request.form.get(f'date_from_{obj_type}', '').replace('-', '')
if date_to: if date_from:
date_to = date_to.replace('-', '') filters[obj_type]['date_from'] = date_from
# Date to
date_to = request.form.get(f'date_to_{obj_type}', '').replace('-', '')
if date_to:
filters[obj_type]['date_to'] = date_to
# Mimetypes
mimetypes = request.form.get(f'mimetypes_{obj_type}', [])
if mimetypes:
mimetypes = json.loads(mimetypes)
filters[obj_type]['mimetypes'] = mimetypes
# Sources
sources = request.form.get(f'sources_{obj_type}', [])
if sources:
sources = json.loads(sources)
filters[obj_type]['sources'] = sources
# Subtypes
for obj_subtype in ail_core.get_object_all_subtypes(obj_type):
subtype = request.form.get(f'filter_{obj_type}_{obj_subtype}')
if subtype == 'on':
if 'subtypes' not in filters[obj_type]:
filters[obj_type]['subtypes'] = []
filters[obj_type]['subtypes'].append(obj_subtype)
# YARA # # YARA #
yara_default_rule = request.form.get("yara_default_rule") yara_default_rule = request.form.get("yara_default_rule")
@ -322,9 +366,9 @@ def retro_hunt_add_task():
user_id = current_user.get_id() user_id = current_user.get_id()
input_dict = {"name": name, "description": description, "creator": user_id, input_dict = {"name": name, "description": description, "creator": user_id,
"rule": rule, "type": rule_type, "rule": rule, "type": rule_type,
"tags": tags, "sources": sources, "timeout": timeout, #"mails": mails, "tags": tags, "filters": filters, "timeout": timeout, # "mails": mails
"date_from": date_from, "date_to": date_to} }
res = Tracker.api_create_retro_hunt_task(input_dict, user_id) res = Tracker.api_create_retro_hunt_task(input_dict, user_id)
if res[1] == 200: if res[1] == 200:
@ -334,8 +378,9 @@ def retro_hunt_add_task():
return create_json_response(res[0], res[1]) return create_json_response(res[0], res[1])
else: else:
return render_template("add_retro_hunt_task.html", return render_template("add_retro_hunt_task.html",
all_yara_files=Tracker.get_all_default_yara_files(), all_yara_files=Tracker.get_all_default_yara_files(),
all_sources=item_basic.get_all_items_sources(r_list=True)) tags_selector_data=Tag.get_tags_selector_data(),
items_sources=item_basic.get_all_items_sources(r_list=True))
@hunters.route('/retro_hunt/task/pause', methods=['GET']) @hunters.route('/retro_hunt/task/pause', methods=['GET'])
@login_required @login_required
@ -368,27 +413,4 @@ def retro_hunt_delete_task():
return redirect(url_for('hunters.retro_hunt_all_tasks')) return redirect(url_for('hunters.retro_hunt_all_tasks'))
#### JSON ####
@hunters.route("/retro_hunt/nb_items/date/json", methods=['GET'])
@login_required
@login_read_only
def get_json_retro_hunt_nb_items_by_date():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
if date_from:
date_from = date_from.replace('-', '')
if date_to:
date_to = date_to.replace('-', '')
task_uuid = request.args.get('uuid')
if date_from and date_to:
res = Tracker.get_retro_hunt_nb_item_by_day([task_uuid], date_from=date_from, date_to=date_to)
else:
res = Tracker.get_retro_hunt_nb_item_by_day([task_uuid])
return jsonify(res)
## - - ## ## - - ##

1
var/www/modules/hunter/Flask_hunter.py
View file

@ -127,6 +127,7 @@ def show_tracker():
meta['date_to'] = date_to meta['date_to'] = date_to
print(meta['filters']) print(meta['filters'])
meta['item_sources'] = sorted(meta['filters'].get('item', {}).get('sources', [])) meta['item_sources'] = sorted(meta['filters'].get('item', {}).get('sources', []))
# meta['filters'] = json.dumps(meta['filters'], indent=4)
return render_template("showTracker.html", tracker_metadata=meta, return render_template("showTracker.html", tracker_metadata=meta,
yara_rule_content=yara_rule_content, yara_rule_content=yara_rule_content,

268
var/www/templates/hunter/add_retro_hunt_task.html
View file

@ -31,28 +31,22 @@
<div class="col-12 col-lg-10" id="core_content"> <div class="col-12 col-lg-10" id="core_content">
<div class="card my-3"> <div class="card my-3">
<div class="card-header bg-dark text-white"> <div class="card-header bg-dark text-white">
<h5 class="card-title">Create a new Retro Hunt task</h5> <h5 class="card-title">Create a new Retro Hunt task</h5>
</div> </div>
<div class="card-body"> <div class="card-body">
<form action="{{ url_for('hunters.retro_hunt_add_task') }}" method='post'> <form action="{{ url_for('hunters.retro_hunt_add_task') }}" method='post'>
<div class="row"> <div class="row">
<div class="col-12 col-xl-9"> <div class="col-12 col-xl-9">
<div class="input-group mb-2 mr-sm-2"> <div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend"> <div class="input-group-prepend">
<div class="input-group-text bg-secondary text-white"><i class="fas fa-quote-right"></i></div> <div class="input-group-text bg-secondary text-white"><i class="fas fa-quote-right"></i></div>
</div> </div>
<input id="name" name="name" class="form-control" placeholder="Retro Hunt Name" type="text" required> <input id="name" name="name" class="form-control" placeholder="Retro Hunt Name" type="text" required>
</div> </div>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
</div>
<input id="tags" name="tags" class="form-control" placeholder="Tags to add on match (optional, space separated)" type="text">
</div>
<!-- <div class="input-group mb-2 mr-sm-2"> <!-- <div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend"> <div class="input-group-prepend">
@ -60,85 +54,168 @@
</div> </div>
<input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text"}> <input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text"}>
</div> --> </div> -->
<div class="input-group mb-2 mr-sm-2"> <div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend"> <div class="input-group-prepend">
<div class="input-group-text bg-info text-white"><i class="fas fa-pencil-alt"></i></div> <div class="input-group-text bg-info text-white"><i class="fas fa-pencil-alt"></i></div>
</div> </div>
<textarea id="description" name="description" class="form-control" placeholder="Retro Hunt Description (optional)" rows="3"></textarea> <textarea id="description" name="description" class="form-control" placeholder="Retro Hunt Description (optional)" rows="3"></textarea>
</div> </div>
<div class="input-group mb-2 mr-sm-2"> <div class="card my-4">
<div class="input-group-prepend"> <div class="card-header bg-info text-white">
<div class="input-group-text bg-dark text-white"><i class="fas fa-folder"></i></div> <b>Objects to Track:</b>
</div> </div>
<input id="sources" class="form-control" type="text" name="sources" placeholder="Sources to track (ALL IF EMPTY)" autocomplete="off"> <div class="card-body">
</div> {# <div class="custom-control custom-switch mt-1">#}
{# <input class="custom-control-input" type="checkbox" name="cve_obj" id="cve_obj" checked="">#}
{# <label class="custom-control-label" for="cve_obj"><i class="fas fa-bug"></i>&nbsp;CVE</label>#}
{# </div>#}
{# <div class="custom-control custom-switch mt-1">#}
{# <input class="custom-control-input" type="checkbox" name="crypto_obj" id="crypto_obj" checked="">#}
{# <label class="custom-control-label" for="crypto_obj"><i class="fas fa-coins"></i>&nbsp;Cryptocurrency</label>#}
{# </div>#}
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="decoded_obj" id="decoded_obj">
<label class="custom-control-label" for="decoded_obj"><i class="fas fa-lock-open"></i>&nbsp;Decoded</label>
</div>
{# <div class="custom-control custom-switch mt-1">#}
{# <input class="custom-control-input" type="checkbox" name="domain_obj" id="domain_obj" checked="">#}
{# <label class="custom-control-label" for="domain_obj"><i class="fas fa-spider"></i>&nbsp;Domain</label>#}
{# </div>#}
<div class="custom-control custom-switch mt-1">
<input class="custom-control-input" type="checkbox" name="item_obj" id="item_obj" checked="">
<label class="custom-control-label" for="item_obj"><i class="fas fa-file"></i>&nbsp;Item</label>
</div>
<div class="card border-dark mb-4" id="sources_item_div">
<div class="card-body">
<h5>Filter Item by sources</h5>
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-dark text-white"><i class="fas fa-folder"></i></div>
</div>
<input id="sources_item" class="form-control" type="text" name="sources_item" placeholder="Item Sources to track (ALL IF EMPTY)" autocomplete="off">
</div>
<h5>Date range:</h5>
<div class="row mb-2">
<div class="col-lg-6">
<div class="input-group" id="date-range-from">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from_item" autocomplete="off">
</div>
</div>
<div class="col-lg-6">
<div class="input-group" id="date-range-to">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to_item" autocomplete="off">
</div>
</div>
</div>
</div>
</div>
<h6>Date range:</h6>
<div class="row mb-2">
<div class="col-lg-6">
<div class="input-group" id="date-range-from">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from" autocomplete="off" required>
</div>
</div>
<div class="col-lg-6">
<div class="input-group" id="date-range-to">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to" autocomplete="off" required>
</div>
</div>
</div>
</div> {# <div class="custom-control custom-switch mt-1">#}
<div class="col-12 col-xl-3"> {# <input class="custom-control-input" type="checkbox" name="pgp_obj" id="pgp_obj" checked="">#}
{# <label class="custom-control-label" for="pgp_obj"><i class="fas fa-key"></i>&nbsp;PGP</label>#}
{# </div>#}
{# <div class="card border-dark mb-4" id="sources_pgp_div">#}
{# <div class="card-body">#}
{# <h6>Filter PGP by subtype:</h6>#}
{# <div class="custom-control custom-switch mt-1">#}
{# <input class="custom-control-input" type="checkbox" name="filter_pgp_name" id="filter_pgp_name" checked="">#}
{# <label class="custom-control-label" for="filter_pgp_name">#}
{# <svg height="26" width="26">#}
{# <g class="nodes">#}
{# <circle cx="13" cy="13" r="13" fill="#44AA99"></circle>#}
{# <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf507;</text>#}
{# </g>#}
{# </svg>#}
{# name#}
{# </label>#}
{# </div>#}
{# <div class="custom-control custom-switch mt-1">#}
{# <input class="custom-control-input" type="checkbox" name="filter_pgp_mail" id="filter_pgp_mail" checked="">#}
{# <label class="custom-control-label" for="filter_pgp_mail">#}
{# <svg height="26" width="26">#}
{# <g class="nodes">#}
{# <circle cx="13" cy="13" r="13" fill="#44AA99"></circle>#}
{# <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="fas" font-size="16px">&#xf1fa;</text>#}
{# </g>#}
{# </svg>#}
{# mail#}
{# </label>#}
{# </div>#}
{# </div>#}
{# </div>#}
</div>
</div> </div>
</div>
<div class="card my-4">
<div class="card-header bg-secondary text-white">
<b>Tags</b>
</div>
<div class="card-body">
<div class="input-group mb-2 mr-sm-2">
<div class="input-group-prepend">
<div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
</div>
<input id="tags" name="tags" class="form-control" placeholder="Custom Tags (optional, space separated)" type="text">
</div>
{% include 'tags/block_tags_selector.html' %}
</div>
</div>
</div>
<div class="col-12 col-xl-3">
</div>
</div>
<div class="mb-2" id="yara_rule"> <div class="mb-2" id="yara_rule">
<div class="" id="yara_default_rules"> <div class="" id="yara_default_rules">
<h6>Default YARA rules:</h6> <h6>Default YARA rules:</h6>
<select class="custom-select w-100 mb-3" id="yara_default_rule" name="yara_default_rule" onchange="get_default_rule_content(this);"> <select class="custom-select w-100 mb-3" id="yara_default_rule" name="yara_default_rule" onchange="get_default_rule_content(this);">
<option selected>Select a default rule</option> <option selected>Select a default rule</option>
{% for yara_types in all_yara_files %} {% for yara_types in all_yara_files %}
{% for yara_file_name in all_yara_files[yara_types] %} {% for yara_file_name in all_yara_files[yara_types] %}
<option value="{{yara_types}}/{{yara_file_name}}">{{yara_types}} - {{yara_file_name}}</option> <option value="{{yara_types}}/{{yara_file_name}}">{{yara_types}} - {{yara_file_name}}</option>
{% endfor %} {% endfor %}
{% endfor %} {% endfor %}
</select> </select>
<pre class="border bg-light" id="default_yara_rule_content"></pre> <pre class="border bg-light" id="default_yara_rule_content"></pre>
</div> </div>
<hr> <hr>
<h6>Custom YARA rules:</h6> <h6>Custom YARA rules:</h6>
<div class="row" id="textarea"> <div class="row" id="textarea">
<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5">{%if dict_tracker%}{%if dict_tracker['type']=='yara' and dict_tracker['content']%}{{dict_tracker['content']}}{%endif%}{%endif%}</textarea> <textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5"></textarea>
</div> </div>
</div> </div>
<br> <br>
<button class="btn btn-primary mt-2"> <button class="btn btn-primary mt-2">
<i class="fas fa-plus"></i> Create Retro Hunt Task <i class="fas fa-plus"></i> Create Retro Hunt Task
</button> </button>
</form> </form>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
@ -177,22 +254,21 @@ $(document).ready(function(){
} }
}); });
sources = $('#sources').tagSuggest({ sources_item = $('#sources_item').tagSuggest({
data: {{all_sources|safe}}, data: {{items_sources|safe}},
{%if dict_tracker%}{%if dict_tracker['sources']%}value: {{dict_tracker['sources']|safe}},{%endif%}{%endif%}
sortOrder: 'name', sortOrder: 'name',
maxDropHeight: 200, maxDropHeight: 200,
name: 'sources', name: 'sources_item',
emptyText: 'Sources to track (ALL IF EMPTY)', emptyText: 'Item Sources to track (ALL IF EMPTY)',
}); });
{%if dict_tracker%} $('#item_obj').on("change", function () {
$('#tracker_type').val('{{dict_tracker['type']}}').change(); item_source_input_controller();
});
/*$('#pgp_obj').on("change", function () {
pgp_source_input_controller();
});*/
{%if dict_tracker['type']=='yara' and dict_tracker['yara_file']%}
$('#yara_default_rule').val('{{dict_tracker['yara_file']}}').change();
{%endif%}
{%endif%}
}); });
@ -210,7 +286,21 @@ function toggle_sidebar(){
} }
} }
function item_source_input_controller() {
if($('#item_obj').is(':checked')){
$("#sources_item_div").show();
}else{
$("#sources_item_div").hide();
}
}
function pgp_source_input_controller() {
if($('#pgp_obj').is(':checked')){
$("#sources_pgp_div").show();
}else{
$("#sources_pgp_div").hide();
}
}
function get_default_rule_content(selector){ function get_default_rule_content(selector){
var yara_name = selector.value var yara_name = selector.value

240
var/www/templates/hunter/show_retro_hunt.html
View file

@ -20,7 +20,6 @@
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script> <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script> <script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script> <script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/d3/graphlinesgroup.js')}}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script> <script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script> <script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
@ -48,9 +47,9 @@
<div class="col-12 col-lg-10" id="core_content"> <div class="col-12 col-lg-10" id="core_content">
<div class="row"> <div class="row">
<div class="col-lg-4"> <div class="col-lg-4">
<div class="card my-2"> <div class="card my-2">
<div class="card-header bg-dark text-white"> <div class="card-header bg-dark text-white">
<div class=""> <div class="">
<span class="badge badge-pill badge-light lex-row-reverse float-right"> <span class="badge badge-pill badge-light lex-row-reverse float-right">
@ -68,13 +67,13 @@
<h3 class="card-title"> <h3 class="card-title">
{{dict_task['name']}} {{dict_task['name']}}
{%if dict_task['state']!='completed'%} {%if dict_task['state']!='completed'%}
<span class="text-info">[{{ dict_task['progress']}}%]<span> <span class="text-info">[{{ dict_task['progress']}}%]</span>
{%endif%} {%endif%}
</h3> </h3>
</div> </div>
</div> </div>
<div class="card-body bg-light pt-2"> <div class="card-body bg-light pt-2">
<div class="d-flex justify-content-end"> <div class="d-flex justify-content-end">
{%if dict_task['state']=='paused'%} {%if dict_task['state']=='paused'%}
<a href="{{ url_for('hunters.retro_hunt_resume_task') }}?uuid={{dict_task['uuid']}}" class="mx-1"> <a href="{{ url_for('hunters.retro_hunt_resume_task') }}?uuid={{dict_task['uuid']}}" class="mx-1">
@ -94,83 +93,85 @@
</a> </a>
</div> </div>
<hr> <hr>
<table class="table table-borderless"> <table class="table table-borderless">
<tbody> <tbody>
<tr> <tr>
<td class="text-right"><b>Date</b></td> <td class="text-right"><b>Date</b></td>
<td> <td>
{{dict_task['date'][0:4]}}/{{dict_task['date'][4:6]}}/{{dict_task['date'][6:8]}} {{dict_task['date'][0:4]}}/{{dict_task['date'][4:6]}}/{{dict_task['date'][6:8]}}
</td> </td>
</tr> </tr>
<tr> <tr>
<td class="text-right"><b>Search date</b></td> <td class="text-right"><b>Description</b></td>
<td> <td>{{dict_task['description']}}</td>
{{dict_task['date_from'][0:4]}}/{{dict_task['date_from'][4:6]}}/{{dict_task['date_from'][6:8]}} - </tr>
{{dict_task['date_to'][0:4]}}/{{dict_task['date_to'][4:6]}}/{{dict_task['date_to'][6:8]}} <tr>
</td> <td class="text-right"><b>Tags</b></td>
</tr> <td>
<tr> {%for tag in dict_task['tags']%}
<td class="text-right"><b>Description</b></td> <a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item&ltags={{ tag }}">
<td>{{dict_task['description']}}</td> <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
</tr> </a>
<tr> {%endfor%}
<td class="text-right"><b>Tags</b></td> </td>
<td> </tr>
{%for tag in dict_task['tags']%} <tr>
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item&ltags={{ tag }}"> <td class="text-right"><b>Creator</b></td>
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span> <td>{{dict_task['creator']}}</td>
</a> </tr>
{%endfor%} <tr>
</td> <td class="text-right"><b>Filters</b></td>
</tr> <td>
<tr> <div class="">
<td class="text-right"><b>Creator</b></td> {% if dict_task['filters'] %}
<td>{{dict_task['creator']}}</td> <pre>{{ dict_task['filters'] }}</pre>
</tr> {% else %}
<tr> <span class="badge badge-secondary">No Filters</span><br>
<td class="text-right"><b>Sources</b></td> {% endif %}
<td> </div>
<div class=""> </td>
{%if not dict_task['sources']%} </tr>
<span class="badge badge-secondary">All Souces</span> <tr>
{%else%} <td class="text-right"><b>Objects Match</b></td>
{%for source in dict_task['sources']%} <td>
<span class="badge badge-secondary">{{source}}</span> {%for obj_type in dict_task['nb_objs']%}
{%endfor%} <h4><span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">
{%endif%} {{ obj_type }}
</div> <span class="badge badge-light">{{ dict_task['nb_objs'][obj_type] }}</span>
</td> </span></h4>
</tr> {%endfor%}
</tbody> </td>
</table> </tr>
</div> </tbody>
</div> </table>
</div> </div>
</div>
</div>
<div class="col-lg-8"> <div class="col-lg-8">
<div class="card border-dark my-2"> <div class="card border-dark my-2">
<div class="card-body"> <div class="card-body">
<div class="row mb-2"> {# <div class="row mb-2">#}
<div class="col-lg-6"> {# <div class="col-lg-6">#}
<div class="input-group" id="date-range-from"> {# <div class="input-group" id="date-range-from">#}
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div> {# <div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>#}
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from" autocomplete="off" {# <input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from" autocomplete="off"> #}
value="{{ dict_task['date_from_input'][0:4] }}-{{ dict_task['date_from_input'][4:6] }}-{{ dict_task['date_from_input'][6:8] }}"> {# value="{{ dict_task['date_from_input'][0:4] }}-{{ dict_task['date_from_input'][4:6] }}-{{ dict_task['date_from_input'][6:8] }}"#}
</div> {# </div>#}
</div> {# </div>#}
<div class="col-lg-6"> {# <div class="col-lg-6">#}
<div class="input-group" id="date-range-to"> {# <div class="input-group" id="date-range-to">#}
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div> {# <div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>#}
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to" autocomplete="off" {# <input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to" autocomplete="off"> #}
value="{{ dict_task['date_to_input'][0:4] }}-{{ dict_task['date_to_input'][4:6] }}-{{ dict_task['date_to_input'][6:8] }}"> {# value="{{ dict_task['date_to_input'][0:4] }}-{{ dict_task['date_to_input'][4:6] }}-{{ dict_task['date_to_input'][6:8] }}"#}
</div> {# </div>#}
</div> {# </div>#}
</div> {# </div>#}
<button class="btn btn-info" type="button" id="button-search-tags" onclick="getItems();"> <button class="btn btn-info" type="button" id="button-search-tags" onclick="getItems();">
<i class="fas fa-search"></i> Show Items <i class="fas fa-search"></i> Show Objects
</button> </button>
</div> </div>
@ -178,40 +179,52 @@
<p class="my-0"><pre class="border bg-light">{{ rule_content }}</pre></p> <p class="my-0"><pre class="border bg-light">{{ rule_content }}</pre></p>
</div> </div>
</div> </div>
{%if dict_task['items']%} {% if dict_task['objs'] %}
<div class="mt-4"> <div class="mt-4">
<table class="table table-bordered table-hover" id="items_table"> <table class="table table-bordered table-hover" id="objs_table">
<thead class="thead-dark"> <thead class="thead-dark">
<tr> <tr>
<th>Date</th> <th>Type</th>
<th>Item Id</th> <th></th>
<th>Id</th>
<th>Tags</th>
<th></th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody>
{% for object in dict_task['objs'] %}
{% for item in dict_task['items'] %} <tr class="border-color: blue;">
<tr> <td>
<td> {% with style=object['icon']['style'], icon=object['icon']['icon'] , color=object['icon']['color'] %}
{{item['date'][0:4]}}/{{item['date'][4:6]}}/{{item['date'][6:8]}} {% include 'objects/obj_svg_block.html' %}
</td> {% endwith %}
<td> {{ object['type']}}
<a class="text-secondary" target="_blank" href="{{ url_for('objects_item.showItem') }}?id={{item['id']}}"> </td>
<div style="line-height:0.9;">{{ item['id'] }}</div> <td>
</a> {% if object['subtype'] %}
<div class="mb-2"> {{ object['subtype']}}
{% for tag in item['tags'] %} {% endif %}
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item&ltags={{ tag }}"> </td>
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span> <td>
</a> <a href="{{ object['link'] }}">
{% endfor %} {{ object['id']}}
</div> </a>
</td> </td>
</tr> <td>
{% for tag in object['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
{% endfor %}
</td>
<td class="text-right">
{# <a href="{{ url_for('investigations_b.unregister_investigation') }}?uuid={{ tracker_metadata['uuid']}}&type={{ object['type'] }}&subtype={{ object['subtype']}}&id={{ object['id']}}">#}
{# <button type="button" class="btn btn-danger"><i class="fas fa-trash-alt"></i></button>#}
{# </a>#}
</td>
</tr>
{% endfor %} {% endfor %}
</tbody> </tbody>
</table> </table>
</div> </div>
@ -219,13 +232,6 @@
<hr> <hr>
{% endif %} {% endif %}
<div id="graphline" class="text-center"></div>
<br>
<br>
<br>
</div> </div>
</div> </div>
</div> </div>
@ -234,7 +240,7 @@
$(document).ready(function(){ $(document).ready(function(){
$('#nav_title_retro_hunt').removeClass("text-muted"); $('#nav_title_retro_hunt').removeClass("text-muted");
$('#date-range-from').dateRangePicker({ /*$('#date-range-from').dateRangePicker({
separator : ' to ', separator : ' to ',
getValue: function(){ getValue: function(){
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() ) if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
@ -259,18 +265,15 @@ $(document).ready(function(){
$('#date-range-from-input').val(s1); $('#date-range-from-input').val(s1);
$('#date-range-to-input').val(s2); $('#date-range-to-input').val(s2);
} }
}); });*/
$('#items_table').DataTable({ {% if dict_task['objs'] %}
$('#objs_table').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]], "aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 10, "iDisplayLength": 10,
"order": [[ 0, "asc" ]] "order": [[ 0, "asc" ]]
}); });
{% endif%}
let div_width = $("#graphline").width();
$.getJSON( "{{ url_for('hunters.get_json_retro_hunt_nb_items_by_date') }}?uuid={{ dict_task['uuid'] }}&date_from={{ dict_task['date_from_input'] }}&date_to={{ dict_task['date_to_input'] }}",
function( data ) {multilines_group("graphline", data, {"width": div_width});}
);
}); });
@ -291,7 +294,8 @@ function toggle_sidebar(){
function getItems() { function getItems() {
var date_from = $('#date-range-from-input').val(); var date_from = $('#date-range-from-input').val();
var date_to =$('#date-range-to-input').val(); var date_to =$('#date-range-to-input').val();
window.location.replace("{{ url_for('hunters.retro_hunt_show_task') }}?uuid={{ dict_task['uuid'] }}&date_from="+date_from+"&date_to="+date_to); {#window.location.replace("{{ url_for('hunters.retro_hunt_show_task') }}?uuid={{ dict_task['uuid'] }}&date_from="+date_from+"&date_to="+date_to);#}
window.location.replace("{{ url_for('hunters.retro_hunt_show_task') }}?uuid={{ dict_task['uuid'] }}&objs=True");
} }
</script> </script>