mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 00:28:22 +00:00
chg: [trackers] refactor trackers: track objects + filter by object types/sources/subtypes
This commit is contained in:
Terrtia
parent
e363dcda62
commit
0daf5bad44
41 changed files with 2549 additions and 2124 deletions
|
|
@ -233,17 +233,61 @@ def ail_2_ail_migration():
|
|||
# item in queue
|
||||
ail_2_ail.set_last_updated_sync_config()
|
||||
|
||||
###############################
|
||||
# #
|
||||
# TRACKER MIGRATION #
|
||||
# #
|
||||
###############################
|
||||
|
||||
def get_tracker_level(tracker_uuid):
|
||||
level = r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'level')
|
||||
if not level:
|
||||
level = 0
|
||||
return int(level)
|
||||
|
||||
def get_tracker_metadata(tracker_uuid):
|
||||
meta = {'uuid': tracker_uuid,
|
||||
'tracked': r_serv_tracker.hget('tracker:{tracker_uuid}', 'tracked'),
|
||||
'type': r_serv_tracker.hget('tracker:{tracker_uuid}', 'type'),
|
||||
'date': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'date'),
|
||||
'first_seen': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'first_seen'),
|
||||
'last_seen': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'last_seen'),
|
||||
'user_id': r_serv_tracker.hget('tracker:{tracker_uuid}', 'user_id'),
|
||||
'level': get_tracker_level(tracker_uuid),
|
||||
'mails': list(r_serv_tracker.smembers('tracker:mail:{tracker_uuid}')),
|
||||
'sources': list(r_serv_tracker.smembers(f'tracker:sources:{tracker_uuid}')),
|
||||
'tags': list(r_serv_tracker.smembers(f'tracker:tags:{tracker_uuid}')),
|
||||
'description': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'description'),
|
||||
'webhook': r_serv_tracker.hget(f'tracker:{tracker_uuid}', 'webhook')}
|
||||
return meta
|
||||
|
||||
def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to):
|
||||
all_item_id = set()
|
||||
if date_from and date_to:
|
||||
l_date_match = r_serv_tracker.zrange(f'tracker:stat:{tracker_uuid}', 0, -1, withscores=True)
|
||||
if l_date_match:
|
||||
dict_date_match = dict(l_date_match)
|
||||
for date_day in Date.substract_date(date_from, date_to):
|
||||
if date_day in dict_date_match:
|
||||
all_item_id |= r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}')
|
||||
return all_item_id
|
||||
|
||||
# trackers + retro_hunts
|
||||
def trackers_migration():
|
||||
print('TRACKERS MIGRATION...')
|
||||
for tracker_uuid in old_Tracker.get_all_tracker_uuid():
|
||||
meta = old_Tracker.get_tracker_metadata(tracker_uuid, user_id=True, description=True, level=True, tags=True, mails=True, sources=True, sparkline=False, webhook=True)
|
||||
Tracker._re_create_tracker(meta['tracker'], meta['type'], meta['user_id'], meta['level'], meta['tags'], meta['mails'], meta['description'], meta['webhook'], 0, meta['uuid'], meta['sources'], meta['first_seen'], meta['last_seen'])
|
||||
meta = get_tracker_metadata(tracker_uuid)
|
||||
Tracker._re_create_tracker(meta['type'], meta['uuid'], meta['tracked'], meta['user_id'], meta['level'],
|
||||
tags=meta['tags'], mails=meta['mails'], description=meta['description'],
|
||||
webhook=meta['webhook'], sources=meta['sources'],
|
||||
first_seen=meta['first_seen'], last_seen=meta['last_seen'])
|
||||
|
||||
tracker = Tracker.Tracker(tracker_uuid)
|
||||
# object migration # # TODO: in background
|
||||
for item_id in old_Tracker.get_tracker_items_by_daterange(tracker_uuid, meta['first_seen'], meta['last_seen']):
|
||||
print(item_id)
|
||||
Tracker.add_tracked_item(tracker_uuid, item_id)
|
||||
item_date = get_item_date(item_id)
|
||||
tracker.add('item', '', item_id, date=item_date)
|
||||
|
||||
print('RETRO HUNT MIGRATION...')
|
||||
|
||||
|
|
@ -929,13 +973,13 @@ if __name__ == '__main__':
|
|||
# user_migration()
|
||||
#tags_migration()
|
||||
# items_migration()
|
||||
crawler_migration()
|
||||
# crawler_migration()
|
||||
# domain_migration() # TO TEST ###########################
|
||||
# decodeds_migration()
|
||||
# screenshots_migration()
|
||||
# subtypes_obj_migration()
|
||||
# ail_2_ail_migration()
|
||||
# trackers_migration()
|
||||
trackers_migration()
|
||||
# investigations_migration()
|
||||
## statistics_migration()
|
||||
|
||||
|
|
|
|||
|
|
@ -14,23 +14,9 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from packages import Date
|
||||
from packages import Term
|
||||
|
||||
from pubsublogger import publisher
|
||||
|
||||
def clean_term_db_stat_token():
|
||||
all_stat_date = Term.get_all_token_stat_history()
|
||||
|
||||
list_date_to_keep = Date.get_date_range(31)
|
||||
for date in all_stat_date:
|
||||
if date not in list_date_to_keep:
|
||||
# remove history
|
||||
Term.delete_token_statistics_by_date(date)
|
||||
|
||||
print('Term Stats Cleaned')
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
publisher.port = 6380
|
||||
|
|
@ -46,7 +32,7 @@ if __name__ == "__main__":
|
|||
while True:
|
||||
|
||||
if daily_cleaner:
|
||||
clean_term_db_stat_token()
|
||||
|
||||
daily_cleaner = False
|
||||
else:
|
||||
sys.exit(0)
|
||||
|
|
|
|||
|
|
@ -105,7 +105,7 @@ class MailExporterTracker(MailExporter):
|
|||
|
||||
def export(self, tracker, obj): # TODO match
|
||||
tracker_type = tracker.get_type()
|
||||
tracker_name = tracker.get_tracker()
|
||||
tracker_name = tracker.get_tracked()
|
||||
subject = f'AIL Framework Tracker: {tracker_name}' # TODO custom subject
|
||||
body = f"AIL Framework, New occurrence for {tracker_type} tracker: {tracker_name}\n"
|
||||
body += f'Item: {obj.id}\nurl:{obj.get_link()}'
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ class WebHookExporterTracker(WebHookExporter):
|
|||
data = {'trackerId': tracker.get_uuid(),
|
||||
'trackerType': tracker.get_type(),
|
||||
'tags': tracker.get_tags(),
|
||||
'tracker': tracker.get_tracker(),
|
||||
'tracker': tracker.get_tracked(),
|
||||
# object
|
||||
'itemId': obj.get_id(),
|
||||
'itemURL': obj.get_link()}
|
||||
|
|
|
|||
|
|
@ -186,9 +186,9 @@ class Investigation(object):
|
|||
def set_threat_level(self, threat_level):
|
||||
try:
|
||||
threat_level = int(threat_level)
|
||||
except:
|
||||
except TypeError:
|
||||
raise UpdateInvestigationError('threat_level Not an integer')
|
||||
if threat_level >= 1 and threat_level <= 4:
|
||||
if 1 <= threat_level <= 4:
|
||||
r_tracking.hset(f'investigations:data:{self.uuid}', 'threat_level', threat_level)
|
||||
else:
|
||||
raise UpdateInvestigationError(f'Invalid threat_level: {threat_level}')
|
||||
|
|
@ -196,9 +196,9 @@ class Investigation(object):
|
|||
def set_analysis(self, analysis):
|
||||
try:
|
||||
analysis = int(analysis)
|
||||
except:
|
||||
except TypeError:
|
||||
raise UpdateInvestigationError('analysis Not an integer')
|
||||
if analysis >= 0 and analysis <= 2:
|
||||
if 0 <= analysis <= 2:
|
||||
r_tracking.hset(f'investigations:data:{self.uuid}', 'analysis', analysis)
|
||||
else:
|
||||
raise UpdateInvestigationError(f'Invalid analysis: {analysis}')
|
||||
|
|
|
|||
1718
bin/lib/Tracker.py
1718
bin/lib/Tracker.py
File diff suppressed because it is too large
Load diff
|
|
@ -40,6 +40,9 @@ def get_object_all_subtypes(obj_type):
|
|||
return ['telegram', 'twitter', 'jabber']
|
||||
return []
|
||||
|
||||
def get_objects_tracked():
|
||||
return ['decoded', 'item', 'pgp']
|
||||
|
||||
def get_all_objects_with_subtypes_tuple():
|
||||
str_objs = []
|
||||
for obj_type in get_all_objects():
|
||||
|
|
|
|||
|
|
@ -95,17 +95,18 @@ def convert_byte_offset_to_string(b_content, offset):
|
|||
def get_tracker_match(obj_id, content):
|
||||
extracted = []
|
||||
extracted_yara = []
|
||||
trackers = Tracker.get_obj_all_trackers('item', '', obj_id)
|
||||
trackers = Tracker.get_obj_trackers('item', '', obj_id)
|
||||
for tracker_uuid in trackers:
|
||||
tracker_type = Tracker.get_tracker_type(tracker_uuid)
|
||||
tracker = Tracker.Tracker(tracker_uuid)
|
||||
tracker_type = tracker.get_type()
|
||||
# print(tracker_type)
|
||||
tracker = Tracker.get_tracker_by_uuid(tracker_uuid)
|
||||
tracked = tracker.get_tracked()
|
||||
if tracker_type == 'regex': # TODO Improve word detection -> word delimiter
|
||||
regex_match = regex_helper.regex_finditer(r_key, tracker, obj_id, content)
|
||||
regex_match = regex_helper.regex_finditer(r_key, tracked, obj_id, content)
|
||||
for match in regex_match:
|
||||
extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker_uuid}'])
|
||||
extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker.uuid}'])
|
||||
elif tracker_type == 'yara':
|
||||
rule = Tracker.get_yara_rule_by_uuid(tracker_uuid)
|
||||
rule = tracker.get_rule()
|
||||
rule.match(data=content.encode(), callback=_get_yara_match,
|
||||
which_callbacks=yara.CALLBACK_MATCHES, timeout=30)
|
||||
yara_match = r_cache.smembers(f'extractor:yara:match:{r_key}')
|
||||
|
|
@ -113,20 +114,20 @@ def get_tracker_match(obj_id, content):
|
|||
extracted = []
|
||||
for match in yara_match:
|
||||
start, end, value = match.split(':', 2)
|
||||
extracted_yara.append([int(start), int(end), value, f'tracker:{tracker_uuid}'])
|
||||
extracted_yara.append([int(start), int(end), value, f'tracker:{tracker.uuid}'])
|
||||
|
||||
elif tracker_type == 'word' or tracker_type == 'set':
|
||||
if tracker_type == 'set':
|
||||
tracker = tracker.rsplit(';', 1)[0]
|
||||
words = tracker.split(',')
|
||||
tracked = tracked.rsplit(';', 1)[0]
|
||||
words = tracked.split(',')
|
||||
else:
|
||||
words = [tracker]
|
||||
words = [tracked]
|
||||
for word in words:
|
||||
regex = _get_word_regex(word)
|
||||
regex_match = regex_helper.regex_finditer(r_key, regex, obj_id, content)
|
||||
# print(regex_match)
|
||||
for match in regex_match:
|
||||
extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker_uuid}'])
|
||||
extracted.append([int(match[0]), int(match[1]), match[2], f'tracker:{tracker.uuid}'])
|
||||
|
||||
# Convert byte offset to string offset
|
||||
if extracted_yara:
|
||||
|
|
|
|||
|
|
@ -2,12 +2,13 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from flask import url_for
|
||||
from hashlib import sha256
|
||||
|
||||
from pymisp import MISPObject, MISPAttribute
|
||||
from pymisp import MISPObject
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
|
|
@ -175,31 +176,57 @@ def get_all_cryptocurrencies():
|
|||
cryptos[subtype] = get_all_cryptocurrencies_by_subtype(subtype)
|
||||
return cryptos
|
||||
|
||||
|
||||
def get_all_cryptocurrencies_by_subtype(subtype):
|
||||
return get_all_id('cryptocurrency', subtype)
|
||||
|
||||
def sanitize_cryptocurrency_name_to_search(name_to_search, subtype): # TODO FILTER NAME + Key + mail
|
||||
if subtype == '':
|
||||
pass
|
||||
elif subtype == 'name':
|
||||
pass
|
||||
elif subtype == 'mail':
|
||||
pass
|
||||
return name_to_search
|
||||
|
||||
# TODO save object
|
||||
def import_misp_object(misp_obj):
|
||||
"""
|
||||
:type misp_obj: MISPObject
|
||||
"""
|
||||
obj_id = None
|
||||
obj_subtype = None
|
||||
for attribute in misp_obj.attributes:
|
||||
if attribute.object_relation == 'address': # TODO: handle xmr address field
|
||||
obj_id = attribute.value
|
||||
elif attribute.object_relation == 'symbol':
|
||||
obj_subtype = get_subtype_by_symbol(attribute.value)
|
||||
if obj_id and obj_subtype:
|
||||
obj = CryptoCurrency(obj_id, obj_subtype)
|
||||
first_seen, last_seen = obj.get_misp_object_first_last_seen(misp_obj)
|
||||
tags = obj.get_misp_object_tags(misp_obj)
|
||||
# for tag in tags:
|
||||
# obj.add_tag()
|
||||
def search_cryptocurrency_by_name(name_to_search, subtype, r_pos=False):
|
||||
cryptocurrencies = {}
|
||||
# for subtype in subtypes:
|
||||
r_name = sanitize_cryptocurrency_name_to_search(name_to_search, subtype)
|
||||
if not name_to_search or isinstance(r_name, dict):
|
||||
# break
|
||||
return cryptocurrencies
|
||||
r_name = re.compile(r_name)
|
||||
for crypto_name in get_all_cryptocurrencies_by_subtype(subtype):
|
||||
res = re.search(r_name, crypto_name)
|
||||
if res:
|
||||
cryptocurrencies[crypto_name] = {}
|
||||
if r_pos:
|
||||
cryptocurrencies[crypto_name]['hl-start'] = res.start()
|
||||
cryptocurrencies[crypto_name]['hl-end'] = res.end()
|
||||
return cryptocurrencies
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
res = get_all_cryptocurrencies()
|
||||
print(res)
|
||||
# # TODO save object
|
||||
# def import_misp_object(misp_obj):
|
||||
# """
|
||||
# :type misp_obj: MISPObject
|
||||
# """
|
||||
# obj_id = None
|
||||
# obj_subtype = None
|
||||
# for attribute in misp_obj.attributes:
|
||||
# if attribute.object_relation == 'address': # TODO: handle xmr address field
|
||||
# obj_id = attribute.value
|
||||
# elif attribute.object_relation == 'symbol':
|
||||
# obj_subtype = get_subtype_by_symbol(attribute.value)
|
||||
# if obj_id and obj_subtype:
|
||||
# obj = CryptoCurrency(obj_id, obj_subtype)
|
||||
# first_seen, last_seen = obj.get_misp_object_first_last_seen(misp_obj)
|
||||
# tags = obj.get_misp_object_tags(misp_obj)
|
||||
# # for tag in tags:
|
||||
# # obj.add_tag()
|
||||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# name_to_search = '3c'
|
||||
# subtype = 'bitcoin'
|
||||
# print(search_cryptocurrency_by_name(name_to_search, subtype))
|
||||
|
|
|
|||
|
|
@ -2,9 +2,11 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from flask import url_for
|
||||
|
||||
from pymisp import MISPObject
|
||||
|
||||
import requests
|
||||
|
|
@ -110,6 +112,25 @@ def get_cves_meta(cves_id, options=set()):
|
|||
dict_cve[cve_id] = cve.get_meta(options=options)
|
||||
return dict_cve
|
||||
|
||||
def sanitize_cve_name_to_search(name_to_search): # TODO FILTER NAME
|
||||
return name_to_search
|
||||
|
||||
def search_cves_by_name(name_to_search, r_pos=False):
|
||||
cves = {}
|
||||
# for subtype in subtypes:
|
||||
r_name = sanitize_cve_name_to_search(name_to_search)
|
||||
if not name_to_search or isinstance(r_name, dict):
|
||||
return cves
|
||||
r_name = re.compile(r_name)
|
||||
for cve_name in get_all_cves():
|
||||
res = re.search(r_name, cve_name)
|
||||
if res:
|
||||
cves[cve_name] = {}
|
||||
if r_pos:
|
||||
cves[cve_name]['hl-start'] = res.start()
|
||||
cves[cve_name]['hl-end'] = res.end()
|
||||
return cves
|
||||
|
||||
def api_get_cves_range_by_daterange(date_from, date_to):
|
||||
cves = []
|
||||
for date in Date.substract_date(date_from, date_to):
|
||||
|
|
@ -133,3 +154,5 @@ def get_cve_graphline(cve_id):
|
|||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# name_to_search = '98'
|
||||
# print(search_cves_by_name(name_to_search))
|
||||
|
|
@ -2,6 +2,7 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import magic
|
||||
import requests
|
||||
|
|
@ -9,6 +10,7 @@ import zipfile
|
|||
|
||||
from flask import url_for
|
||||
from io import BytesIO
|
||||
|
||||
from pymisp import MISPObject
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
|
|
@ -114,11 +116,15 @@ class Decoded(AbstractDaterangeObject):
|
|||
def get_filepath(self, mimetype=None):
|
||||
return os.path.join(os.environ['AIL_HOME'], self.get_rel_path(mimetype=mimetype))
|
||||
|
||||
def get_content(self, mimetype=None):
|
||||
def get_content(self, mimetype=None, r_str=False):
|
||||
filepath = self.get_filepath(mimetype=mimetype)
|
||||
if r_str:
|
||||
with open(filepath, 'r') as f:
|
||||
content = f.read()
|
||||
else:
|
||||
with open(filepath, 'rb') as f:
|
||||
file_content = BytesIO(f.read())
|
||||
return file_content
|
||||
content = BytesIO(f.read())
|
||||
return content
|
||||
|
||||
def get_zip_content(self):
|
||||
# mimetype = self.get_estimated_type()
|
||||
|
|
@ -347,6 +353,25 @@ def sanitise_mimetype(mimetype):
|
|||
else:
|
||||
return None
|
||||
|
||||
def sanitize_decoded_name_to_search(name_to_search): # TODO FILTER NAME
|
||||
return name_to_search
|
||||
|
||||
def search_decodeds_by_name(name_to_search, r_pos=False):
|
||||
decodeds = {}
|
||||
# for subtype in subtypes:
|
||||
r_name = sanitize_decoded_name_to_search(name_to_search)
|
||||
if not name_to_search or isinstance(r_name, dict):
|
||||
return decodeds
|
||||
r_name = re.compile(r_name)
|
||||
for decoded_name in get_all_decodeds():
|
||||
res = re.search(r_name, decoded_name)
|
||||
if res:
|
||||
decodeds[decoded_name] = {}
|
||||
if r_pos:
|
||||
decodeds[decoded_name]['hl-start'] = res.start()
|
||||
decodeds[decoded_name]['hl-end'] = res.end()
|
||||
return decodeds
|
||||
|
||||
############################################################################
|
||||
|
||||
def sanityze_decoder_names(decoder_name):
|
||||
|
|
@ -512,4 +537,7 @@ def get_all_decodeds_files():
|
|||
decodeds.append(file)
|
||||
return decodeds
|
||||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# name_to_search = '4d36'
|
||||
# print(search_decodeds_by_name(name_to_search))
|
||||
|
|
@ -96,6 +96,9 @@ class Domain(AbstractObject):
|
|||
elif int(last_check) < date:
|
||||
self._set_last_check(date)
|
||||
|
||||
def get_content(self):
|
||||
return self.id
|
||||
|
||||
def get_last_origin(self, obj=False):
|
||||
origin = {'item': r_crawler.hget(f'domain:meta:{self.id}', 'last_origin')}
|
||||
if obj and origin['item']:
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@ class Item(AbstractObject):
|
|||
else:
|
||||
return filename
|
||||
|
||||
def get_content(self, binary=False):
|
||||
def get_content(self, r_str=True, binary=False):
|
||||
"""
|
||||
Returns Item content
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -2,6 +2,7 @@
|
|||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from flask import url_for
|
||||
|
|
@ -100,5 +101,35 @@ def get_all_pgps():
|
|||
def get_all_pgps_by_subtype(subtype):
|
||||
return get_all_id('pgp', subtype)
|
||||
|
||||
# TODO FILTER NAME + Key + mail
|
||||
def sanitize_pgp_name_to_search(name_to_search, subtype): # TODO FILTER NAME + Key + mail
|
||||
if subtype == 'key':
|
||||
pass
|
||||
elif subtype == 'name':
|
||||
pass
|
||||
elif subtype == 'mail':
|
||||
pass
|
||||
return name_to_search
|
||||
|
||||
def search_pgps_by_name(name_to_search, subtype, r_pos=False):
|
||||
pgps = {}
|
||||
# for subtype in subtypes:
|
||||
r_name = sanitize_pgp_name_to_search(name_to_search, subtype)
|
||||
if not name_to_search or isinstance(r_name, dict):
|
||||
# break
|
||||
return pgps
|
||||
r_name = re.compile(r_name)
|
||||
for pgp_name in get_all_pgps_by_subtype(subtype):
|
||||
res = re.search(r_name, pgp_name)
|
||||
if res:
|
||||
pgps[pgp_name] = {}
|
||||
if r_pos:
|
||||
pgps[pgp_name]['hl-start'] = res.start()
|
||||
pgps[pgp_name]['hl-end'] = res.end()
|
||||
return pgps
|
||||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# name_to_search = 'ex'
|
||||
# subtype = 'name'
|
||||
# print(search_pgps_by_name(name_to_search, subtype))
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@
|
|||
|
||||
import base64
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
from hashlib import sha256
|
||||
|
|
@ -71,6 +72,9 @@ class Screenshot(AbstractObject):
|
|||
file_content = BytesIO(f.read())
|
||||
return file_content
|
||||
|
||||
def get_content(self):
|
||||
return self.get_file_content()
|
||||
|
||||
def get_misp_object(self):
|
||||
obj_attrs = []
|
||||
obj = MISPObject('file')
|
||||
|
|
@ -129,4 +133,26 @@ def create_screenshot(content, size_limit=5000000, b64=True, force=False):
|
|||
return screenshot
|
||||
return None
|
||||
|
||||
#if __name__ == '__main__':
|
||||
def sanitize_screenshot_name_to_search(name_to_search): # TODO FILTER NAME
|
||||
return name_to_search
|
||||
|
||||
def search_screenshots_by_name(name_to_search, r_pos=False):
|
||||
screenshots = {}
|
||||
# for subtype in subtypes:
|
||||
r_name = sanitize_screenshot_name_to_search(name_to_search)
|
||||
if not name_to_search or isinstance(r_name, dict):
|
||||
return screenshots
|
||||
r_name = re.compile(r_name)
|
||||
for screenshot_name in get_all_screenshots():
|
||||
res = re.search(r_name, screenshot_name)
|
||||
if res:
|
||||
screenshots[screenshot_name] = {}
|
||||
if r_pos:
|
||||
screenshots[screenshot_name]['hl-start'] = res.start()
|
||||
screenshots[screenshot_name]['hl-end'] = res.end()
|
||||
return screenshots
|
||||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# name_to_search = '29ba'
|
||||
# print(search_screenshots_by_name(name_to_search))
|
||||
|
|
|
|||
|
|
@ -3,13 +3,11 @@
|
|||
|
||||
import os
|
||||
import sys
|
||||
import redis
|
||||
import re
|
||||
|
||||
from flask import url_for
|
||||
from pymisp import MISPObject
|
||||
|
||||
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
|
|
@ -61,7 +59,7 @@ class Username(AbstractSubtypeObject):
|
|||
else:
|
||||
style = 'fas'
|
||||
icon = '\uf007'
|
||||
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius':5}
|
||||
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius': 5}
|
||||
|
||||
def get_meta(self, options=set()):
|
||||
meta = self._get_meta(options=options)
|
||||
|
|
@ -74,15 +72,15 @@ class Username(AbstractSubtypeObject):
|
|||
obj_attrs = []
|
||||
if self.subtype == 'telegram':
|
||||
obj = MISPObject('telegram-account', standalone=True)
|
||||
obj_attrs.append( obj.add_attribute('username', value=self.id) )
|
||||
obj_attrs.append(obj.add_attribute('username', value=self.id))
|
||||
|
||||
elif self.subtype == 'twitter':
|
||||
obj = MISPObject('twitter-account', standalone=True)
|
||||
obj_attrs.append( obj.add_attribute('name', value=self.id) )
|
||||
obj_attrs.append(obj.add_attribute('name', value=self.id))
|
||||
|
||||
else:
|
||||
obj = MISPObject('user-account', standalone=True)
|
||||
obj_attrs.append( obj.add_attribute('username', value=self.id) )
|
||||
obj_attrs.append(obj.add_attribute('username', value=self.id))
|
||||
|
||||
obj.first_seen = self.get_first_seen()
|
||||
obj.last_seen = self.get_last_seen()
|
||||
|
|
@ -107,9 +105,30 @@ def get_all_usernames():
|
|||
def get_all_usernames_by_subtype(subtype):
|
||||
return get_all_id('username', subtype)
|
||||
|
||||
# TODO FILTER NAME + Key + mail
|
||||
def sanitize_username_name_to_search(name_to_search, subtype): # TODO FILTER NAME
|
||||
|
||||
return name_to_search
|
||||
|
||||
def search_usernames_by_name(name_to_search, subtype, r_pos=False):
|
||||
usernames = {}
|
||||
# for subtype in subtypes:
|
||||
r_name = sanitize_username_name_to_search(name_to_search, subtype)
|
||||
if not name_to_search or isinstance(r_name, dict):
|
||||
# break
|
||||
return usernames
|
||||
r_name = re.compile(r_name)
|
||||
for user_name in get_all_usernames_by_subtype(subtype):
|
||||
res = re.search(r_name, user_name)
|
||||
if res:
|
||||
usernames[user_name] = {}
|
||||
if r_pos:
|
||||
usernames[user_name]['hl-start'] = res.start()
|
||||
usernames[user_name]['hl-end'] = res.end()
|
||||
return usernames
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
obj = Username('ninechantw', 'telegram')
|
||||
print(obj.get_misp_object().to_json())
|
||||
# if __name__ == '__main__':
|
||||
# name_to_search = 'co'
|
||||
# subtype = 'telegram'
|
||||
# print(search_usernames_by_name(name_to_search, subtype))
|
||||
|
|
|
|||
|
|
@ -108,6 +108,9 @@ class AbstractDaterangeObject(AbstractObject, ABC):
|
|||
sparkline.append(self.get_nb_seen_by_date(date))
|
||||
return sparkline
|
||||
|
||||
def get_content(self):
|
||||
return self.id
|
||||
|
||||
def _add_create(self):
|
||||
r_object.sadd(f'{self.type}:all', self.id)
|
||||
|
||||
|
|
|
|||
|
|
@ -19,9 +19,9 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
##################################
|
||||
from lib import Tag
|
||||
from lib import Duplicate
|
||||
from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type
|
||||
from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, delete_obj_correlations, exists_obj_correlation, is_obj_correlated, get_nb_correlation_by_correl_type
|
||||
from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
|
||||
from lib.Tracker import is_obj_tracked, get_obj_all_trackers, delete_obj_trackers
|
||||
from lib.Tracker import is_obj_tracked, get_obj_trackers, delete_obj_trackers
|
||||
|
||||
|
||||
class AbstractObject(ABC):
|
||||
|
|
@ -85,6 +85,13 @@ class AbstractObject(ABC):
|
|||
|
||||
#- Tags -#
|
||||
|
||||
@abstractmethod
|
||||
def get_content(self):
|
||||
"""
|
||||
Get Object Content
|
||||
"""
|
||||
pass
|
||||
|
||||
## Duplicates ##
|
||||
def get_duplicates(self):
|
||||
return Duplicate.get_obj_duplicates(self.type, self.get_subtype(r_str=True), self.id)
|
||||
|
|
@ -125,7 +132,7 @@ class AbstractObject(ABC):
|
|||
return is_obj_tracked(self.type, self.subtype, self.id)
|
||||
|
||||
def get_trackers(self):
|
||||
return get_obj_all_trackers(self.type, self.subtype, self.id)
|
||||
return get_obj_trackers(self.type, self.subtype, self.id)
|
||||
|
||||
def delete_trackers(self):
|
||||
return delete_obj_trackers(self.type, self.subtype, self.id)
|
||||
|
|
@ -134,13 +141,14 @@ class AbstractObject(ABC):
|
|||
|
||||
def _delete(self):
|
||||
# DELETE TAGS
|
||||
Tag.delete_obj_all_tags(self.id, self.type) # ########### # TODO: # TODO: # FIXME:
|
||||
Tag.delete_object_tags(self.type, self.get_subtype(r_str=True), self.id)
|
||||
# remove from tracker
|
||||
self.delete_trackers()
|
||||
# remove from retro hunt currently item only TODO
|
||||
# remove from investigations
|
||||
self.delete_investigations()
|
||||
|
||||
# # TODO: remove from correlation
|
||||
# Delete Correlations
|
||||
delete_obj_correlations(self.type, self.get_subtype(r_str=True), self.id)
|
||||
|
||||
@abstractmethod
|
||||
def delete(self):
|
||||
|
|
|
|||
|
|
@ -116,6 +116,9 @@ class AbstractSubtypeObject(AbstractObject, ABC):
|
|||
if date > last_seen:
|
||||
self.set_last_seen(date)
|
||||
|
||||
def get_content(self):
|
||||
return self.id
|
||||
|
||||
def get_sparkline(self):
|
||||
sparkline = []
|
||||
for date in Date.get_previous_date_list(6):
|
||||
|
|
|
|||
|
|
@ -138,9 +138,14 @@ def get_object_meta(obj_type, subtype, id, options=set(), flask_context=False):
|
|||
|
||||
def get_objects_meta(objs, options=set(), flask_context=False):
|
||||
metas = []
|
||||
for obj_dict in objs:
|
||||
metas.append(get_object_meta(obj_dict['type'], obj_dict['subtype'], obj_dict['id'], options=options,
|
||||
flask_context=flask_context))
|
||||
for obj in objs:
|
||||
if isinstance(obj, dict):
|
||||
obj_type = obj['type']
|
||||
subtype = obj['subtype']
|
||||
obj_id = obj['id']
|
||||
else:
|
||||
obj_type, subtype, obj_id = obj.split(':', 2)
|
||||
metas.append(get_object_meta(obj_type, subtype, obj_id, options=options, flask_context=flask_context))
|
||||
return metas
|
||||
|
||||
|
||||
|
|
@ -162,7 +167,25 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False):
|
|||
return meta
|
||||
|
||||
|
||||
def get_ui_obj_tag_table_keys(obj_type):
|
||||
#### OBJ FILTERS ####
|
||||
|
||||
def is_filtered(obj, filters):
|
||||
if 'mimetypes' in filters:
|
||||
mimetype = obj.get_mimetype()
|
||||
if mimetype not in filters['mimetypes']:
|
||||
return True
|
||||
if 'sources' in filters:
|
||||
obj_source = obj.get_source()
|
||||
if obj_source not in filters['sources']:
|
||||
return True
|
||||
if 'subtypes' in filters:
|
||||
subtype = obj.get_subtype(r_str=True)
|
||||
if subtype not in filters['subtypes']:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def get_ui_obj_tag_table_keys(obj_type): # TODO REMOVE ME
|
||||
"""
|
||||
Warning: use only in flask (dynamic templates)
|
||||
"""
|
||||
|
|
@ -170,6 +193,7 @@ def get_ui_obj_tag_table_keys(obj_type):
|
|||
return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot
|
||||
|
||||
|
||||
|
||||
# # # # MISP OBJECTS # # # #
|
||||
|
||||
# # TODO: CHECK IF object already have an UUID
|
||||
|
|
@ -265,6 +289,7 @@ def api_sanitize_object_type(obj_type):
|
|||
if not is_valid_object_type(obj_type):
|
||||
return {'status': 'error', 'reason': 'Incorrect object type'}, 400
|
||||
|
||||
#### CORRELATION ####
|
||||
|
||||
def get_obj_correlations(obj_type, subtype, obj_id):
|
||||
obj = get_object(obj_type, subtype, obj_id)
|
||||
|
|
@ -292,13 +317,14 @@ def get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=[], lvl=0,
|
|||
|
||||
def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types=[], lvl=0, nb_max=300):
|
||||
objs = get_obj_correlations_objs(obj_type, subtype, obj_id, filter_types=filter_types, lvl=lvl, nb_max=nb_max)
|
||||
# print(objs)
|
||||
for obj_tuple in objs:
|
||||
obj1_type, subtype1, id1 = obj_tuple
|
||||
add_obj_tags(obj1_type, subtype1, id1, tags)
|
||||
return objs
|
||||
|
||||
################################################################################
|
||||
################################################################################
|
||||
################################################################################ TODO
|
||||
################################################################################
|
||||
|
||||
def delete_obj_correlations(obj_type, subtype, obj_id):
|
||||
|
|
@ -357,7 +383,7 @@ def get_correlations_graph_node(obj_type, subtype, obj_id, filter_types=[], max_
|
|||
"links": create_correlation_graph_links(links)}
|
||||
|
||||
|
||||
###############
|
||||
# --- CORRELATION --- #
|
||||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
|
|
|
|||
|
|
@ -23,6 +23,9 @@ from modules.abstract_module import AbstractModule
|
|||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.Items import Item
|
||||
from lib.objects.Decodeds import Decoded
|
||||
from trackers.Tracker_Term import Tracker_Term
|
||||
from trackers.Tracker_Regex import Tracker_Regex
|
||||
from trackers.Tracker_Yara import Tracker_Yara
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
hex_max_execution_time = config_loader.get_config_int("Decoder", "max_execution_time_hexadecimal")
|
||||
|
|
@ -76,6 +79,10 @@ class Decoder(AbstractModule):
|
|||
# Waiting time in seconds between to message processed
|
||||
self.pending_seconds = 1
|
||||
|
||||
self.tracker_term = Tracker_Term(queue=False)
|
||||
self.tracker_regex = Tracker_Regex(queue=False)
|
||||
self.tracker_yara = Tracker_Yara(queue=False)
|
||||
|
||||
# Send module state to logs
|
||||
self.redis_logger.info(f'Module {self.module_name} initialized')
|
||||
|
||||
|
|
@ -84,6 +91,7 @@ class Decoder(AbstractModule):
|
|||
item = Item(message)
|
||||
content = item.get_content()
|
||||
date = item.get_date()
|
||||
new_decodeds = []
|
||||
|
||||
for decoder in self.decoder_order:
|
||||
find = False
|
||||
|
|
@ -96,8 +104,8 @@ class Decoder(AbstractModule):
|
|||
encodeds = set(encodeds)
|
||||
|
||||
for encoded in encodeds:
|
||||
find = False
|
||||
if len(encoded) >= decoder['encoded_min_size']:
|
||||
find = True
|
||||
decoded_file = self.decoder_function[dname](encoded)
|
||||
|
||||
sha1_string = sha1(decoded_file).hexdigest()
|
||||
|
|
@ -109,6 +117,7 @@ class Decoder(AbstractModule):
|
|||
print(sha1_string, item.id)
|
||||
raise Exception(f'Invalid mimetype: {decoded.id} {item.id}')
|
||||
decoded.save_file(decoded_file, mimetype)
|
||||
new_decodeds.append(decoded.id)
|
||||
else:
|
||||
mimetype = decoded.get_mimetype()
|
||||
decoded.add(dname, date, item.id, mimetype=mimetype)
|
||||
|
|
@ -125,6 +134,13 @@ class Decoder(AbstractModule):
|
|||
msg = f'infoleak:automatic-detection="{dname}";{item.id}'
|
||||
self.add_message_to_queue(msg, 'Tags')
|
||||
|
||||
####################
|
||||
# TRACKERS DECODED
|
||||
for decoded_id in new_decodeds:
|
||||
self.tracker_term.compute(decoded_id, obj_type='decoded')
|
||||
self.tracker_regex.compute(decoded_id, obj_type='decoded')
|
||||
self.tracker_yara.compute(decoded_id, obj_type='decoded')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
module = Decoder()
|
||||
|
|
|
|||
|
|
@ -216,16 +216,16 @@ class PgpDump(AbstractModule):
|
|||
pgp = Pgps.Pgp(name, 'name')
|
||||
pgp.add(date, self.item_id)
|
||||
print(f' name: {name}')
|
||||
self.tracker_term.compute(self.item_id, item_content=name)
|
||||
self.tracker_regex.compute(self.item_id, content=name)
|
||||
self.tracker_yara.compute(self.item_id, item_content=name)
|
||||
self.tracker_term.compute(name, obj_type='pgp', subtype='name')
|
||||
self.tracker_regex.compute(name, obj_type='pgp', subtype='name')
|
||||
self.tracker_yara.compute(name, obj_type='pgp', subtype='name')
|
||||
for mail in self.mails:
|
||||
pgp = Pgps.Pgp(mail, 'mail')
|
||||
pgp.add(date, self.item_id)
|
||||
print(f' mail: {mail}')
|
||||
self.tracker_term.compute(self.item_id, item_content=mail)
|
||||
self.tracker_regex.compute(self.item_id, content=mail)
|
||||
self.tracker_yara.compute(self.item_id, item_content=mail)
|
||||
self.tracker_term.compute(mail, obj_type='pgp', subtype='mail')
|
||||
self.tracker_regex.compute(mail, obj_type='pgp', subtype='mail')
|
||||
self.tracker_yara.compute(mail, obj_type='pgp', subtype='mail')
|
||||
|
||||
# Keys extracted from PGP PRIVATE KEY BLOCK
|
||||
for key in self.private_keys:
|
||||
|
|
|
|||
|
|
@ -1,532 +0,0 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import uuid
|
||||
import datetime
|
||||
|
||||
from collections import defaultdict
|
||||
from flask import escape
|
||||
|
||||
from nltk.tokenize import RegexpTokenizer
|
||||
from textblob import TextBlob
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib import ConfigLoader
|
||||
from lib import Tracker
|
||||
from packages import Date
|
||||
from lib.objects import Items
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_term = config_loader.get_db_conn("Kvrocks_Trackers")
|
||||
config_loader = None
|
||||
|
||||
email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
|
||||
email_regex = re.compile(email_regex)
|
||||
|
||||
special_characters = set('[<>~!?@#$%^&*|()_-+={}":;,.\'\n\r\t]/\\')
|
||||
special_characters.add('\\s')
|
||||
|
||||
# NLTK tokenizer
|
||||
tokenizer = RegexpTokenizer('[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]+',
|
||||
gaps=True, discard_empty=True)
|
||||
|
||||
def is_valid_uuid_v4(UUID):
|
||||
if not UUID:
|
||||
return False
|
||||
UUID = UUID.replace('-', '')
|
||||
try:
|
||||
uuid_test = uuid.UUID(hex=UUID, version=4)
|
||||
return uuid_test.hex == UUID
|
||||
except:
|
||||
return False
|
||||
|
||||
# # TODO: use new package => duplicate fct
|
||||
def is_in_role(user_id, role):
|
||||
if r_serv_term.sismember('user_role:{}'.format(role), user_id):
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def check_term_uuid_valid_access(term_uuid, user_id):
|
||||
if not is_valid_uuid_v4(term_uuid):
|
||||
return {"status": "error", "reason": "Invalid uuid"}, 400
|
||||
level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
|
||||
if not level:
|
||||
return {"status": "error", "reason": "Unknown uuid"}, 404
|
||||
if level == 0:
|
||||
if r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id') != user_id:
|
||||
if not is_in_role(user_id, 'admin'):
|
||||
return {"status": "error", "reason": "Unknown uuid"}, 404
|
||||
return None
|
||||
|
||||
|
||||
def is_valid_mail(email):
|
||||
result = email_regex.match(email)
|
||||
if result:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def verify_mail_list(mail_list):
|
||||
for mail in mail_list:
|
||||
if not is_valid_mail(mail):
|
||||
return {'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400
|
||||
return None
|
||||
|
||||
def is_valid_regex(term_regex):
|
||||
try:
|
||||
re.compile(term_regex)
|
||||
return True
|
||||
except:
|
||||
return False
|
||||
|
||||
def get_text_word_frequency(item_content, filtering=True):
|
||||
item_content = item_content.lower()
|
||||
words_dict = defaultdict(int)
|
||||
|
||||
if filtering:
|
||||
blob = TextBlob(item_content, tokenizer=tokenizer)
|
||||
else:
|
||||
blob = TextBlob(item_content)
|
||||
for word in blob.tokens:
|
||||
words_dict[word] += 1
|
||||
return words_dict
|
||||
|
||||
# # TODO: create all tracked words
|
||||
def get_tracked_words_list():
|
||||
return list(r_serv_term.smembers('all:tracker:word'))
|
||||
|
||||
def get_set_tracked_words_list():
|
||||
set_list = r_serv_term.smembers('all:tracker:set')
|
||||
all_set_list = []
|
||||
for elem in set_list:
|
||||
res = elem.split(';')
|
||||
num_words = int(res[1])
|
||||
ter_set = res[0].split(',')
|
||||
all_set_list.append((ter_set, num_words, elem))
|
||||
return all_set_list
|
||||
|
||||
def get_regex_tracked_words_dict():
|
||||
regex_list = r_serv_term.smembers('all:tracker:regex')
|
||||
dict_tracked_regex = {}
|
||||
for regex in regex_list:
|
||||
dict_tracked_regex[regex] = re.compile(regex)
|
||||
return dict_tracked_regex
|
||||
|
||||
def get_tracked_term_list_item(term_uuid, date_from, date_to):
|
||||
all_item_id = []
|
||||
if date_from and date_to:
|
||||
for date in r_serv_term.zrangebyscore('tracker:stat:{}'.format(term_uuid), int(date_from), int(date_to)):
|
||||
all_item_id = all_item_id + list(r_serv_term.smembers('tracker:item:{}:{}'.format(term_uuid, date)))
|
||||
return all_item_id
|
||||
|
||||
def is_term_tracked_in_global_level(term, term_type):
|
||||
res = r_serv_term.smembers('all:tracker_uuid:{}:{}'.format(term_type, term))
|
||||
if res:
|
||||
for elem_uuid in res:
|
||||
if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'level') == '1':
|
||||
return True
|
||||
return False
|
||||
|
||||
def is_term_tracked_in_user_level(term, term_type, user_id):
|
||||
res = r_serv_term.smembers('user:tracker:{}'.format(user_id))
|
||||
if res:
|
||||
for elem_uuid in res:
|
||||
if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'tracked') == term:
|
||||
if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'type') == term_type:
|
||||
return True
|
||||
return False
|
||||
|
||||
def parse_json_term_to_add(dict_input, user_id):
|
||||
term = dict_input.get('term', None)
|
||||
if not term:
|
||||
return {"status": "error", "reason": "Term not provided"}, 400
|
||||
term_type = dict_input.get('type', None)
|
||||
if not term_type:
|
||||
return {"status": "error", "reason": "Term type not provided"}, 400
|
||||
|
||||
nb_words = dict_input.get('nb_words', 1)
|
||||
|
||||
description = dict_input.get('description', '')
|
||||
description = escape(description)
|
||||
|
||||
webhook = dict_input.get('webhook', '')
|
||||
webhook = escape(webhook)
|
||||
|
||||
res = parse_tracked_term_to_add(term, term_type, nb_words=nb_words)
|
||||
if res[1] != 200:
|
||||
return res
|
||||
term = res[0]['term']
|
||||
term_type = res[0]['type']
|
||||
|
||||
tags = dict_input.get('tags', [])
|
||||
mails = dict_input.get('mails', [])
|
||||
res = verify_mail_list(mails)
|
||||
if res:
|
||||
return res
|
||||
|
||||
level = dict_input.get('level', 1)
|
||||
try:
|
||||
level = int(level)
|
||||
if level not in range(0, 1):
|
||||
level = 1
|
||||
except:
|
||||
level = 1
|
||||
|
||||
# check if term already tracked in global
|
||||
if level == 1:
|
||||
if is_term_tracked_in_global_level(term, term_type):
|
||||
return {"status": "error", "reason": "Term already tracked"}, 409
|
||||
else:
|
||||
if is_term_tracked_in_user_level(term, term_type, user_id):
|
||||
return {"status": "error", "reason": "Term already tracked"}, 409
|
||||
|
||||
term_uuid = add_tracked_term(term, term_type, user_id, level, tags, mails, description, webhook)
|
||||
|
||||
return {'term': term, 'type': term_type, 'uuid': term_uuid}, 200
|
||||
|
||||
|
||||
def parse_tracked_term_to_add(term, term_type, nb_words=1):
|
||||
if term_type == 'regex':
|
||||
if not is_valid_regex(term):
|
||||
return {"status": "error", "reason": "Invalid regex"}, 400
|
||||
elif term_type == 'word' or term_type == 'set':
|
||||
# force lowercase
|
||||
term = term.lower()
|
||||
word_set = set(term)
|
||||
set_inter = word_set.intersection(special_characters)
|
||||
if set_inter:
|
||||
return {"status": "error", "reason": "special character not allowed", "message": "Please use a regex or remove all special characters"}, 400
|
||||
words = term.split()
|
||||
# not a word
|
||||
if term_type == 'word' and len(words) > 1:
|
||||
term_type = 'set'
|
||||
|
||||
# output format: term1,term2,term3;2
|
||||
if term_type == 'set':
|
||||
try:
|
||||
nb_words = int(nb_words)
|
||||
except:
|
||||
nb_words = 1
|
||||
if nb_words == 0:
|
||||
nb_words = 1
|
||||
|
||||
words_set = set(words)
|
||||
words_set = sorted(words_set)
|
||||
|
||||
if nb_words > len(words_set):
|
||||
nb_words = len(words_set)
|
||||
|
||||
term = ",".join(words_set)
|
||||
term = "{};{}".format(term, nb_words)
|
||||
|
||||
elif term_type == 'yara_custom':
|
||||
if not Tracker.is_valid_yara_rule(term):
|
||||
return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400
|
||||
elif term_type == 'yara_default':
|
||||
if not Tracker.is_valid_default_yara_rule(term):
|
||||
return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400
|
||||
else:
|
||||
return {"status": "error", "reason": "Incorrect type"}, 400
|
||||
return {"status": "success", "term": term, "type": term_type}, 200
|
||||
|
||||
def add_tracked_term(term, term_type, user_id, level, tags, mails, description, webhook, dashboard=0):
|
||||
|
||||
term_uuid = str(uuid.uuid4())
|
||||
|
||||
# YARA
|
||||
if term_type == 'yara_custom' or term_type == 'yara_default':
|
||||
term = Tracker.save_yara_rule(term_type, term, tracker_uuid=term_uuid)
|
||||
term_type = 'yara'
|
||||
|
||||
# create metadata
|
||||
r_serv_term.hset('tracker:{}'.format(term_uuid), 'tracked', term) # # TODO: use hash
|
||||
r_serv_term.hset('tracker:{}'.format(term_uuid), 'type', term_type)
|
||||
r_serv_term.hset('tracker:{}'.format(term_uuid), 'date', datetime.date.today().strftime("%Y%m%d"))
|
||||
r_serv_term.hset('tracker:{}'.format(term_uuid), 'user_id', user_id)
|
||||
r_serv_term.hset('tracker:{}'.format(term_uuid), 'level', level)
|
||||
r_serv_term.hset('tracker:{}'.format(term_uuid), 'dashboard', dashboard)
|
||||
|
||||
if description:
|
||||
r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description)
|
||||
|
||||
if webhook:
|
||||
r_serv_term.hset('tracker:{}'.format(term_uuid), 'webhook', webhook)
|
||||
|
||||
# create all term set
|
||||
r_serv_term.sadd('all:tracker:{}'.format(term_type), term)
|
||||
|
||||
# create term - uuid map
|
||||
r_serv_term.sadd('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid)
|
||||
|
||||
# add display level set
|
||||
if level == 0: # user only
|
||||
r_serv_term.sadd('user:tracker:{}'.format(user_id), term_uuid)
|
||||
r_serv_term.sadd('user:tracker:{}:{}'.format(user_id, term_type), term_uuid)
|
||||
elif level == 1: # global
|
||||
r_serv_term.sadd('global:tracker', term_uuid)
|
||||
r_serv_term.sadd('global:tracker:{}'.format(term_type), term_uuid)
|
||||
|
||||
# create term tags list
|
||||
for tag in tags:
|
||||
r_serv_term.sadd('tracker:tags:{}'.format(term_uuid), escape(tag))
|
||||
|
||||
# create term tags mail notification list
|
||||
for mail in mails:
|
||||
r_serv_term.sadd('tracker:mail:{}'.format(term_uuid), escape(mail))
|
||||
|
||||
# toggle refresh module tracker list/set
|
||||
r_serv_term.set('tracker:refresh:{}'.format(term_type), time.time())
|
||||
|
||||
return term_uuid
|
||||
|
||||
|
||||
def parse_tracked_term_to_delete(dict_input, user_id):
|
||||
term_uuid = dict_input.get("uuid", None)
|
||||
res = check_term_uuid_valid_access(term_uuid, user_id)
|
||||
if res:
|
||||
return res
|
||||
delete_term(term_uuid)
|
||||
return {"uuid": term_uuid}, 200
|
||||
|
||||
|
||||
# # TODO: MOVE IN TRACKER
|
||||
def delete_term(term_uuid):
|
||||
term = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked')
|
||||
term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type')
|
||||
level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
|
||||
r_serv_term.srem('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid)
|
||||
|
||||
r_serv_term.srem(f'trackers:all', term_uuid)
|
||||
r_serv_term.srem(f'trackers:all:{term_type}', term_uuid)
|
||||
|
||||
# Term not tracked by other users
|
||||
if not r_serv_term.exists('all:tracker_uuid:{}:{}'.format(term_type, term)):
|
||||
r_serv_term.srem('all:tracker:{}'.format(term_type), term)
|
||||
|
||||
# toggle refresh module tracker list/set
|
||||
r_serv_term.set('tracker:refresh:{}'.format(term_type), time.time())
|
||||
|
||||
if level == '0': # user only
|
||||
user_id = term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id')
|
||||
r_serv_term.srem('user:tracker:{}'.format(user_id), term_uuid)
|
||||
r_serv_term.srem('user:tracker:{}:{}'.format(user_id, term_type), term_uuid)
|
||||
elif level == '1': # global
|
||||
r_serv_term.srem('global:tracker', term_uuid)
|
||||
r_serv_term.srem('global:tracker:{}'.format(term_type), term_uuid)
|
||||
|
||||
# delete metatadata
|
||||
r_serv_term.delete('tracker:{}'.format(term_uuid))
|
||||
|
||||
# remove tags
|
||||
r_serv_term.delete('tracker:tags:{}'.format(term_uuid))
|
||||
|
||||
# remove mails
|
||||
r_serv_term.delete('tracker:mail:{}'.format(term_uuid))
|
||||
|
||||
# remove sources
|
||||
r_serv_term.delete('tracker:sources:{}'.format(term_uuid))
|
||||
|
||||
# remove item set
|
||||
all_item_date = r_serv_term.zrange(f'tracker:stat:{term_uuid}', 0, -1, withscores=True)
|
||||
if all_item_date:
|
||||
all_item_date = dict(all_item_date)
|
||||
for date in all_item_date:
|
||||
for item_id in r_serv_term.smembers(f'tracker:item:{term_uuid}:{date}'):
|
||||
r_serv_term.srem(f'obj:trackers:item:{item_id}', term_uuid)
|
||||
r_serv_term.delete(f'tracker:item:{term_uuid}:{date}')
|
||||
r_serv_term.delete('tracker:stat:{}'.format(term_uuid))
|
||||
|
||||
if term_type == 'yara':
|
||||
# delete custom rule
|
||||
if not Tracker.is_default_yara_rule(term):
|
||||
filepath = Tracker.get_yara_rule_file_by_tracker_name(term)
|
||||
if filepath:
|
||||
os.remove(filepath)
|
||||
|
||||
def replace_tracker_description(term_uuid, description):
|
||||
description = escape(description)
|
||||
r_serv_term.hset('tracker:{}'.format(term_uuid), 'description', description)
|
||||
|
||||
def replace_tracked_term_tags(term_uuid, tags):
|
||||
r_serv_term.delete('tracker:tags:{}'.format(term_uuid))
|
||||
for tag in tags:
|
||||
tag = escape(tag)
|
||||
r_serv_term.sadd('tracker:tags:{}'.format(term_uuid), tag)
|
||||
|
||||
def replace_tracked_term_mails(term_uuid, mails):
|
||||
res = verify_mail_list(mails)
|
||||
if res:
|
||||
return res
|
||||
else:
|
||||
r_serv_term.delete('tracker:mail:{}'.format(term_uuid))
|
||||
for mail in mails:
|
||||
mail = escape(mail)
|
||||
r_serv_term.sadd('tracker:mail:{}'.format(term_uuid), mail)
|
||||
|
||||
def get_term_uuid_list(term, term_type):
|
||||
return list(r_serv_term.smembers('all:tracker_uuid:{}:{}'.format(term_type, term)))
|
||||
|
||||
def get_term_tags(term_uuid):
|
||||
return list(r_serv_term.smembers('tracker:tags:{}'.format(term_uuid)))
|
||||
|
||||
def get_term_mails(term_uuid):
|
||||
return list(r_serv_term.smembers('tracker:mail:{}'.format(term_uuid)))
|
||||
|
||||
def get_term_webhook(term_uuid):
|
||||
return r_serv_term.hget('tracker:{}'.format(term_uuid), "webhook")
|
||||
|
||||
def add_tracked_item(term_uuid, item_id, item_date):
|
||||
# track item
|
||||
r_serv_term.sadd('tracker:item:{}:{}'.format(term_uuid, item_date), item_id)
|
||||
# track nb item by date
|
||||
r_serv_term.zadd('tracker:stat:{}'.format(term_uuid), {item_date: item_date})
|
||||
|
||||
def create_token_statistics(item_date, word, nb):
|
||||
r_serv_term.zincrby('stat_token_per_item_by_day:{}'.format(item_date), 1, word)
|
||||
r_serv_term.zincrby('stat_token_total_by_day:{}'.format(item_date), nb, word)
|
||||
r_serv_term.sadd('stat_token_history', item_date)
|
||||
|
||||
def delete_token_statistics_by_date(item_date):
|
||||
r_serv_term.delete('stat_token_per_item_by_day:{}'.format(item_date))
|
||||
r_serv_term.delete('stat_token_total_by_day:{}'.format(item_date))
|
||||
r_serv_term.srem('stat_token_history', item_date)
|
||||
|
||||
def get_all_token_stat_history():
|
||||
return r_serv_term.smembers('stat_token_history')
|
||||
|
||||
def get_tracked_term_last_updated_by_type(term_type):
|
||||
epoch_update = r_serv_term.get('tracker:refresh:{}'.format(term_type))
|
||||
if not epoch_update:
|
||||
epoch_update = 0
|
||||
return float(epoch_update)
|
||||
|
||||
def parse_get_tracker_term_item(dict_input, user_id):
|
||||
term_uuid = dict_input.get('uuid', None)
|
||||
res = check_term_uuid_valid_access(term_uuid, user_id)
|
||||
if res:
|
||||
return res
|
||||
|
||||
date_from = dict_input.get('date_from', None)
|
||||
date_to = dict_input.get('date_to', None)
|
||||
|
||||
if date_from is None:
|
||||
date_from = get_tracked_term_first_seen(term_uuid)
|
||||
if date_from:
|
||||
date_from = date_from[0]
|
||||
|
||||
if date_to is None:
|
||||
date_to = date_from
|
||||
|
||||
if date_from > date_to:
|
||||
date_from = date_to
|
||||
|
||||
all_item_id = Tracker.get_tracker_items_by_daterange(term_uuid, date_from, date_to)
|
||||
all_item_id = Items.get_item_list_desc(all_item_id)
|
||||
|
||||
res_dict = {'uuid': term_uuid, 'date_from': date_from, 'date_to': date_to, 'items': all_item_id}
|
||||
return res_dict, 200
|
||||
|
||||
def get_tracked_term_first_seen(term_uuid):
|
||||
return Tracker.get_tracker_first_seen(term_uuid)
|
||||
|
||||
def get_tracked_term_last_seen(term_uuid):
|
||||
return Tracker.get_tracker_last_seen(term_uuid)
|
||||
|
||||
def get_term_metedata(term_uuid, user_id=False, description=False, level=False, tags=False, mails=False, sparkline=False):
|
||||
dict_uuid = {}
|
||||
dict_uuid['term'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked')
|
||||
dict_uuid['type'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type')
|
||||
dict_uuid['date'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'date')
|
||||
dict_uuid['first_seen'] = get_tracked_term_first_seen(term_uuid)
|
||||
dict_uuid['last_seen'] = get_tracked_term_last_seen(term_uuid)
|
||||
if user_id:
|
||||
dict_uuid['user_id'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id')
|
||||
if level:
|
||||
dict_uuid['level'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level')
|
||||
if mails:
|
||||
dict_uuid['mails'] = get_list_trackeed_term_mails(term_uuid)
|
||||
if tags:
|
||||
dict_uuid['tags'] = get_list_trackeed_term_tags(term_uuid)
|
||||
if sparkline:
|
||||
dict_uuid['sparkline'] = get_tracked_term_sparkline(term_uuid)
|
||||
if description:
|
||||
dict_uuid['description'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'description')
|
||||
dict_uuid['uuid'] = term_uuid
|
||||
return dict_uuid
|
||||
|
||||
def get_tracked_term_sparkline(tracker_uuid, num_day=6):
|
||||
date_range_sparkline = Date.get_date_range(num_day)
|
||||
sparklines_value = []
|
||||
for date_day in date_range_sparkline:
|
||||
nb_seen_this_day = r_serv_term.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day))
|
||||
if nb_seen_this_day is None:
|
||||
nb_seen_this_day = 0
|
||||
sparklines_value.append(int(nb_seen_this_day))
|
||||
return sparklines_value
|
||||
|
||||
def get_list_tracked_term_stats_by_day(list_tracker_uuid, num_day=31, date_from=None, date_to=None):
|
||||
if date_from and date_to:
|
||||
date_range = Date.substract_date(date_from, date_to)
|
||||
else:
|
||||
date_range = Date.get_date_range(num_day)
|
||||
list_tracker_stats = []
|
||||
for tracker_uuid in list_tracker_uuid:
|
||||
dict_tracker_data = []
|
||||
tracker = r_serv_term.hget('tracker:{}'.format(tracker_uuid), 'tracked')
|
||||
for date_day in date_range:
|
||||
nb_seen_this_day = r_serv_term.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day))
|
||||
if nb_seen_this_day is None:
|
||||
nb_seen_this_day = 0
|
||||
dict_tracker_data.append({"date": date_day, "value": int(nb_seen_this_day)})
|
||||
list_tracker_stats.append({"name": tracker, "Data": dict_tracker_data})
|
||||
return list_tracker_stats
|
||||
|
||||
def get_list_trackeed_term_tags(term_uuid):
|
||||
res = r_serv_term.smembers('tracker:tags:{}'.format(term_uuid))
|
||||
if res:
|
||||
return list(res)
|
||||
else:
|
||||
return []
|
||||
|
||||
def get_list_trackeed_term_mails(term_uuid):
|
||||
res = r_serv_term.smembers('tracker:mail:{}'.format(term_uuid))
|
||||
if res:
|
||||
return list(res)
|
||||
else:
|
||||
return []
|
||||
|
||||
def get_user_tracked_term_uuid(user_id, filter_type=None):
|
||||
if filter_type:
|
||||
return list(r_serv_term.smembers('user:tracker:{}:{}'.format(user_id, filter_type)))
|
||||
else:
|
||||
return list(r_serv_term.smembers('user:tracker:{}'.format(user_id)))
|
||||
|
||||
def get_global_tracked_term_uuid(filter_type=None):
|
||||
if filter_type:
|
||||
return list(r_serv_term.smembers('global:tracker:{}'.format(filter_type)))
|
||||
else:
|
||||
return list(r_serv_term.smembers('global:tracker'))
|
||||
|
||||
def get_all_user_tracked_terms(user_id, filter_type=None):
|
||||
all_user_term = []
|
||||
all_user_term_uuid = get_user_tracked_term_uuid(user_id, filter_type=filter_type)
|
||||
|
||||
for term_uuid in all_user_term_uuid:
|
||||
all_user_term.append(get_term_metedata(term_uuid, tags=True, mails=True, sparkline=True))
|
||||
return all_user_term
|
||||
|
||||
def get_all_global_tracked_terms(filter_type=None):
|
||||
all_user_term = []
|
||||
all_user_term_uuid = get_global_tracked_term_uuid(filter_type=filter_type)
|
||||
|
||||
for term_uuid in all_user_term_uuid:
|
||||
all_user_term.append(get_term_metedata(term_uuid, user_id=True, tags=True, mails=True, sparkline=True))
|
||||
return all_user_term
|
||||
|
|
@ -3,8 +3,6 @@
|
|||
import os
|
||||
import string
|
||||
|
||||
from pubsublogger import publisher
|
||||
|
||||
import calendar
|
||||
from datetime import date
|
||||
from dateutil.rrule import rrule, DAILY
|
||||
|
|
|
|||
|
|
@ -61,15 +61,17 @@ class Retro_Hunt(AbstractModule):
|
|||
self.progress = 0
|
||||
# First launch
|
||||
# restart
|
||||
rule = Tracker.get_retro_hunt_task_rule(task_uuid, r_compile=True)
|
||||
retro_hunt = Tracker.RetroHunt(task_uuid) # TODO SELF
|
||||
|
||||
timeout = Tracker.get_retro_hunt_task_timeout(task_uuid)
|
||||
rule = retro_hunt.get_rule(r_compile=True)
|
||||
|
||||
timeout = retro_hunt.get_timeout()
|
||||
self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
|
||||
sources = Tracker.get_retro_hunt_task_sources(task_uuid, r_sort=True)
|
||||
sources = retro_hunt.get_sources(r_sort=True)
|
||||
|
||||
self.date_from = Tracker.get_retro_hunt_task_date_from(task_uuid)
|
||||
self.date_to = Tracker.get_retro_hunt_task_date_to(task_uuid)
|
||||
self.tags = Tracker.get_retro_hunt_task_tags(task_uuid)
|
||||
self.date_from = retro_hunt.get_date_from()
|
||||
self.date_to = retro_hunt.get_date_to()
|
||||
self.tags = retro_hunt.get_tags()
|
||||
curr_date = Tracker.get_retro_hunt_task_current_date(task_uuid)
|
||||
self.nb_src_done = Tracker.get_retro_hunt_task_nb_src_done(task_uuid, sources=sources)
|
||||
self.update_progress(sources, curr_date)
|
||||
|
|
@ -106,11 +108,10 @@ class Retro_Hunt(AbstractModule):
|
|||
|
||||
# PAUSE
|
||||
self.update_progress(sources, curr_date)
|
||||
if Tracker.check_retro_hunt_pause(task_uuid):
|
||||
if retro_hunt.to_pause():
|
||||
Tracker.set_retro_hunt_last_analyzed(task_uuid, id)
|
||||
# self.update_progress(sources, curr_date, save_db=True)
|
||||
Tracker.pause_retro_hunt_task(task_uuid)
|
||||
Tracker.clear_retro_hunt_task_cache(task_uuid)
|
||||
retro_hunt.pause()
|
||||
return None
|
||||
|
||||
self.nb_src_done += 1
|
||||
|
|
@ -120,9 +121,7 @@ class Retro_Hunt(AbstractModule):
|
|||
|
||||
self.update_progress(sources, curr_date)
|
||||
|
||||
Tracker.set_retro_hunt_task_state(task_uuid, 'completed')
|
||||
Tracker.set_retro_hunt_nb_match(task_uuid)
|
||||
Tracker.clear_retro_hunt_task_cache(task_uuid)
|
||||
retro_hunt.complete()
|
||||
|
||||
print(f'Retro Hunt {task_uuid} completed')
|
||||
self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed')
|
||||
|
|
@ -130,10 +129,11 @@ class Retro_Hunt(AbstractModule):
|
|||
# # TODO: stop
|
||||
|
||||
def update_progress(self, sources, curr_date, save_db=False):
|
||||
progress = Tracker.compute_retro_hunt_task_progress(self.task_uuid, date_from=self.date_from, date_to=self.date_to,
|
||||
retro_hunt = Tracker.RetroHunt(retro_hubt) # TODO USE SELF
|
||||
progress = retro_hunt.compute_progress(date_from=self.date_from, date_to=self.date_to,
|
||||
sources=sources, curr_date=curr_date, nb_src_done=self.nb_src_done)
|
||||
if self.progress != progress:
|
||||
Tracker.set_cache_retro_hunt_task_progress(self.task_uuid, progress)
|
||||
retro_hunt.set_progress(progress)
|
||||
self.progress = progress
|
||||
# if save_db:
|
||||
# Tracker.set_retro_hunt_task_progress(task_uuid, progress)
|
||||
|
|
|
|||
|
|
@ -17,9 +17,8 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from lib.objects.Items import Item
|
||||
from lib.objects import ail_objects
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from packages import Term
|
||||
from lib import Tracker
|
||||
|
||||
from exporter.MailExporter import MailExporterTracker
|
||||
|
|
@ -39,7 +38,7 @@ class Tracker_Regex(AbstractModule):
|
|||
self.max_execution_time = config_loader.get_config_int(self.module_name, "max_execution_time")
|
||||
|
||||
# refresh Tracked Regex
|
||||
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
|
||||
self.tracked_regexs = Tracker.get_tracked_regexs()
|
||||
self.last_refresh = time.time()
|
||||
|
||||
# Exporter
|
||||
|
|
@ -48,56 +47,65 @@ class Tracker_Regex(AbstractModule):
|
|||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, item_id, content=None):
|
||||
def compute(self, obj_id, obj_type='item', subtype=''):
|
||||
# refresh Tracked regex
|
||||
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('regex'):
|
||||
self.dict_regex_tracked = Term.get_regex_tracked_words_dict()
|
||||
self.tracked_regexs = Tracker.get_tracked_regexs()
|
||||
self.last_refresh = time.time()
|
||||
self.redis_logger.debug('Tracked regex refreshed')
|
||||
print('Tracked regex refreshed')
|
||||
|
||||
item = Item(item_id)
|
||||
item_id = item.get_id()
|
||||
if not content:
|
||||
content = item.get_content()
|
||||
obj = ail_objects.get_object(obj_type, subtype, obj_id)
|
||||
obj_id = obj.get_id()
|
||||
obj_type = obj.get_type()
|
||||
|
||||
for regex in self.dict_regex_tracked:
|
||||
matched = self.regex_findall(self.dict_regex_tracked[regex], item_id, content)
|
||||
# Object Filter
|
||||
if obj_type not in self.tracked_regexs:
|
||||
return None
|
||||
|
||||
content = obj.get_content(r_str=True)
|
||||
|
||||
for dict_regex in self.tracked_regexs[obj_type]:
|
||||
matched = self.regex_findall(dict_regex['regex'], obj_id, content)
|
||||
if matched:
|
||||
self.new_tracker_found(regex, 'regex', item)
|
||||
self.new_tracker_found(dict_regex['tracked'], 'regex', obj)
|
||||
|
||||
def new_tracker_found(self, tracker_name, tracker_type, item):
|
||||
uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type)
|
||||
|
||||
item_id = item.get_id()
|
||||
# date = item.get_date()
|
||||
item_source = item.get_source()
|
||||
|
||||
for tracker_uuid in uuid_list:
|
||||
def new_tracker_found(self, tracker_name, tracker_type, obj):
|
||||
obj_id = obj.get_id()
|
||||
for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type(tracker_type, obj.get_type(), tracker_name):
|
||||
tracker = Tracker.Tracker(tracker_uuid)
|
||||
|
||||
# Source Filtering
|
||||
tracker_sources = tracker.get_sources()
|
||||
if tracker_sources and item_source not in tracker_sources:
|
||||
# Filter Object
|
||||
filters = tracker.get_filters()
|
||||
if ail_objects.is_filtered(obj, filters):
|
||||
continue
|
||||
|
||||
print(f'new tracked regex found: {tracker_name} in {item_id}')
|
||||
self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {item_id}')
|
||||
# TODO
|
||||
Tracker.add_tracked_item(tracker_uuid, item_id)
|
||||
print(f'new tracked regex found: {tracker_name} in {obj_id}')
|
||||
self.redis_logger.warning(f'new tracked regex found: {tracker_name} in {obj_id}')
|
||||
|
||||
if obj.get_type() == 'item':
|
||||
date = obj.get_date()
|
||||
else:
|
||||
date = None
|
||||
|
||||
tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id, date=date)
|
||||
|
||||
for tag in tracker.get_tags():
|
||||
msg = f'{tag};{item_id}'
|
||||
if obj.get_type() == 'item':
|
||||
msg = f'{tag};{obj_id}'
|
||||
self.add_message_to_queue(msg, 'Tags')
|
||||
else:
|
||||
obj.add_tag(tag)
|
||||
|
||||
if tracker.mail_export():
|
||||
# TODO add matches + custom subjects
|
||||
self.exporters['mail'].export(tracker, item)
|
||||
self.exporters['mail'].export(tracker, obj)
|
||||
|
||||
if tracker.webhook_export():
|
||||
self.exporters['webhook'].export(tracker, item)
|
||||
self.exporters['webhook'].export(tracker, obj)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
module = Tracker_Regex()
|
||||
module.run()
|
||||
# module.compute('submitted/2023/05/02/submitted_b1e518f1-703b-40f6-8238-d1c22888197e.gz')
|
||||
|
|
|
|||
|
|
@ -21,8 +21,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.Items import Item
|
||||
from packages import Term
|
||||
from lib.objects import ail_objects
|
||||
from lib import Tracker
|
||||
|
||||
from exporter.MailExporter import MailExporterTracker
|
||||
|
|
@ -54,9 +53,9 @@ class Tracker_Term(AbstractModule):
|
|||
self.max_execution_time = config_loader.get_config_int('Tracker_Term', "max_execution_time")
|
||||
|
||||
# loads tracked words
|
||||
self.list_tracked_words = Term.get_tracked_words_list()
|
||||
self.tracked_words = Tracker.get_tracked_words()
|
||||
self.last_refresh_word = time.time()
|
||||
self.set_tracked_words_list = Term.get_set_tracked_words_list()
|
||||
self.tracked_sets = Tracker.get_tracked_sets()
|
||||
self.last_refresh_set = time.time()
|
||||
|
||||
# Exporter
|
||||
|
|
@ -65,93 +64,94 @@ class Tracker_Term(AbstractModule):
|
|||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, item_id, item_content=None):
|
||||
def compute(self, obj_id, obj_type='item', subtype=''):
|
||||
# refresh Tracked term
|
||||
if self.last_refresh_word < Term.get_tracked_term_last_updated_by_type('word'):
|
||||
self.list_tracked_words = Term.get_tracked_words_list()
|
||||
if self.last_refresh_word < Tracker.get_tracker_last_updated_by_type('word'):
|
||||
self.tracked_words = Tracker.get_tracked_words()
|
||||
self.last_refresh_word = time.time()
|
||||
self.redis_logger.debug('Tracked word refreshed')
|
||||
print('Tracked word refreshed')
|
||||
|
||||
if self.last_refresh_set < Term.get_tracked_term_last_updated_by_type('set'):
|
||||
self.set_tracked_words_list = Term.get_set_tracked_words_list()
|
||||
if self.last_refresh_set < Tracker.get_tracker_last_updated_by_type('set'):
|
||||
self.tracked_sets = Tracker.get_tracked_sets()
|
||||
self.last_refresh_set = time.time()
|
||||
self.redis_logger.debug('Tracked set refreshed')
|
||||
print('Tracked set refreshed')
|
||||
|
||||
# Cast message as Item
|
||||
item = Item(item_id)
|
||||
if not item_content:
|
||||
item_content = item.get_content()
|
||||
obj = ail_objects.get_object(obj_type, subtype, obj_id)
|
||||
obj_type = obj.get_type()
|
||||
|
||||
# Object Filter
|
||||
if obj_type not in self.tracked_words and obj_type not in self.tracked_sets:
|
||||
return None
|
||||
|
||||
content = obj.get_content(r_str=True)
|
||||
|
||||
signal.alarm(self.max_execution_time)
|
||||
|
||||
dict_words_freq = None
|
||||
try:
|
||||
dict_words_freq = Term.get_text_word_frequency(item_content)
|
||||
dict_words_freq = Tracker.get_text_word_frequency(content)
|
||||
except TimeoutException:
|
||||
self.redis_logger.warning(f"{item.get_id()} processing timeout")
|
||||
self.redis_logger.warning(f"{obj.get_id()} processing timeout")
|
||||
else:
|
||||
signal.alarm(0)
|
||||
|
||||
if dict_words_freq:
|
||||
# create token statistics
|
||||
# for word in dict_words_freq:
|
||||
# Term.create_token_statistics(item_date, word, dict_words_freq[word])
|
||||
|
||||
# check solo words
|
||||
# ###### # TODO: check if source needed #######
|
||||
for word in self.list_tracked_words:
|
||||
for word in self.tracked_words[obj_type]:
|
||||
if word in dict_words_freq:
|
||||
self.new_term_found(word, 'word', item)
|
||||
self.new_tracker_found(word, 'word', obj)
|
||||
|
||||
# check words set
|
||||
for elem in self.set_tracked_words_list:
|
||||
list_words = elem[0]
|
||||
nb_words_threshold = elem[1]
|
||||
word_set = elem[2]
|
||||
for tracked_set in self.tracked_sets[obj_type]:
|
||||
nb_uniq_word = 0
|
||||
|
||||
for word in list_words:
|
||||
for word in tracked_set['words']:
|
||||
if word in dict_words_freq:
|
||||
nb_uniq_word += 1
|
||||
if nb_uniq_word >= nb_words_threshold:
|
||||
self.new_term_found(word_set, 'set', item)
|
||||
if nb_uniq_word >= tracked_set['nb']:
|
||||
self.new_tracker_found(tracked_set['tracked'], 'set', obj)
|
||||
|
||||
def new_term_found(self, tracker_name, tracker_type, item):
|
||||
uuid_list = Tracker.get_tracker_uuid_list(tracker_name, tracker_type)
|
||||
def new_tracker_found(self, tracker_name, tracker_type, obj): # TODO FILTER
|
||||
obj_id = obj.get_id()
|
||||
|
||||
item_id = item.get_id()
|
||||
item_source = item.get_source()
|
||||
|
||||
for tracker_uuid in uuid_list:
|
||||
for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type(tracker_type, obj.get_type(), tracker_name):
|
||||
tracker = Tracker.Tracker(tracker_uuid)
|
||||
|
||||
# Source Filtering
|
||||
tracker_sources = tracker.get_sources()
|
||||
if tracker_sources and item_source not in tracker_sources:
|
||||
# Filter Object
|
||||
filters = tracker.get_filters()
|
||||
if ail_objects.is_filtered(obj, filters):
|
||||
continue
|
||||
|
||||
print(f'new tracked term found: {tracker_name} in {item_id}')
|
||||
self.redis_logger.warning(f'new tracked term found: {tracker_name} in {item_id}')
|
||||
# TODO
|
||||
Tracker.add_tracked_item(tracker_uuid, item_id)
|
||||
print(f'new tracked term found: {tracker_name} in {obj_id}')
|
||||
self.redis_logger.warning(f'new tracked term found: {tracker_name} in {obj_id}')
|
||||
|
||||
if obj.get_type() == 'item':
|
||||
date = obj.get_date()
|
||||
else:
|
||||
date = None
|
||||
tracker.add(obj.get_type(), obj.get_subtype(), obj_id, date=date)
|
||||
|
||||
# Tags
|
||||
for tag in tracker.get_tags():
|
||||
msg = f'{tag};{item_id}'
|
||||
if obj.get_type() == 'item':
|
||||
msg = f'{tag};{obj_id}'
|
||||
self.add_message_to_queue(msg, 'Tags')
|
||||
else:
|
||||
obj.add_tag(tag)
|
||||
|
||||
# Mail
|
||||
if tracker.mail_export():
|
||||
# TODO add matches + custom subjects
|
||||
self.exporters['mail'].export(tracker, item)
|
||||
self.exporters['mail'].export(tracker, obj)
|
||||
|
||||
# Webhook
|
||||
if tracker.webhook_export():
|
||||
self.exporters['webhook'].export(tracker, item)
|
||||
self.exporters['webhook'].export(tracker, obj)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
module = Tracker_Term()
|
||||
module.run()
|
||||
# module.compute('submitted/2023/05/02/submitted_b1e518f1-703b-40f6-8238-d1c22888197e.gz')
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from lib.objects.Items import Item
|
||||
from lib.objects import ail_objects
|
||||
from lib import Tracker
|
||||
|
||||
from exporter.MailExporter import MailExporterTracker
|
||||
|
|
@ -36,8 +36,8 @@ class Tracker_Typo_Squatting(AbstractModule):
|
|||
self.pending_seconds = 5
|
||||
|
||||
# Refresh typo squatting
|
||||
self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
|
||||
self.last_refresh_typosquat = time.time()
|
||||
self.tracked_typosquattings = Tracker.get_tracked_typosquatting()
|
||||
self.last_refresh_typosquatting = time.time()
|
||||
|
||||
# Exporter
|
||||
self.exporters = {'mail': MailExporterTracker(),
|
||||
|
|
@ -45,50 +45,62 @@ class Tracker_Typo_Squatting(AbstractModule):
|
|||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, message):
|
||||
def compute(self, message, obj_type='item', subtype=''):
|
||||
# refresh Tracked typo
|
||||
if self.last_refresh_typosquat < Tracker.get_tracker_last_updated_by_type('typosquatting'):
|
||||
self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
|
||||
self.last_refresh_typosquat = time.time()
|
||||
if self.last_refresh_typosquatting < Tracker.get_tracker_last_updated_by_type('typosquatting'):
|
||||
self.tracked_typosquattings = Tracker.get_tracked_typosquatting()
|
||||
self.last_refresh_typosquatting = time.time()
|
||||
self.redis_logger.debug('Tracked typosquatting refreshed')
|
||||
print('Tracked typosquatting refreshed')
|
||||
|
||||
host, item_id = message.split()
|
||||
host, obj_id = message.split()
|
||||
obj = ail_objects.get_object(obj_type, subtype, obj_id)
|
||||
obj_type = obj.get_type()
|
||||
|
||||
# Cast message as Item
|
||||
for tracker in self.typosquat_tracked_words_list:
|
||||
if host in self.typosquat_tracked_words_list[tracker]:
|
||||
item = Item(item_id)
|
||||
self.new_tracker_found(tracker, 'typosquatting', item)
|
||||
# Object Filter
|
||||
if obj_type not in self.tracked_typosquattings:
|
||||
return None
|
||||
|
||||
def new_tracker_found(self, tracker, tracker_type, item):
|
||||
item_id = item.get_id()
|
||||
item_source = item.get_source()
|
||||
print(f'new tracked typosquatting found: {tracker} in {item_id}')
|
||||
self.redis_logger.warning(f'tracker typosquatting: {tracker} in {item_id}')
|
||||
for typo in self.tracked_typosquattings[obj_type]:
|
||||
if host in typo['domains']:
|
||||
self.new_tracker_found(typo['tracked'], 'typosquatting', obj)
|
||||
|
||||
for tracker_uuid in Tracker.get_tracker_uuid_list(tracker, tracker_type):
|
||||
def new_tracker_found(self, tracked, tracker_type, obj):
|
||||
obj_id = obj.get_id()
|
||||
for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type(tracker_type, obj.get_type(), tracked):
|
||||
tracker = Tracker.Tracker(tracker_uuid)
|
||||
|
||||
# Source Filtering
|
||||
tracker_sources = tracker.get_sources()
|
||||
if tracker_sources and item_source not in tracker_sources:
|
||||
# Filter Object
|
||||
filters = tracker.get_filters()
|
||||
if ail_objects.is_filtered(obj, filters):
|
||||
continue
|
||||
|
||||
Tracker.add_tracked_item(tracker_uuid, item_id)
|
||||
print(f'new tracked typosquatting found: {tracked} in {obj_id}')
|
||||
self.redis_logger.warning(f'tracker typosquatting: {tracked} in {obj_id}')
|
||||
|
||||
if obj.get_type() == 'item':
|
||||
date = obj.get_date()
|
||||
else:
|
||||
date = None
|
||||
|
||||
tracker.add(obj.get_type(), obj.get_subtype(r_str=True), obj_id, date=date)
|
||||
|
||||
# Tags
|
||||
for tag in tracker.get_tags():
|
||||
msg = f'{tag};{item_id}'
|
||||
if obj.get_type() == 'item':
|
||||
msg = f'{tag};{obj_id}'
|
||||
self.add_message_to_queue(msg, 'Tags')
|
||||
else:
|
||||
obj.add_tag(tag)
|
||||
|
||||
if tracker.mail_export():
|
||||
self.exporters['mail'].export(tracker, item)
|
||||
self.exporters['mail'].export(tracker, obj)
|
||||
|
||||
if tracker.webhook_export():
|
||||
self.exporters['webhook'].export(tracker, item)
|
||||
self.exporters['webhook'].export(tracker, obj)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
module = Tracker_Typo_Squatting()
|
||||
module.run()
|
||||
#module.compute('g00gle.com tests/2020/01/01/test.gz')
|
||||
# module.compute('g00gle.com tests/2020/01/01/test.gz')
|
||||
|
|
|
|||
|
|
@ -19,7 +19,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from lib.objects.Items import Item
|
||||
from lib.objects import ail_objects
|
||||
from lib import Tracker
|
||||
|
||||
from exporter.MailExporter import MailExporterTracker
|
||||
|
|
@ -35,10 +35,10 @@ class Tracker_Yara(AbstractModule):
|
|||
self.pending_seconds = 5
|
||||
|
||||
# Load Yara rules
|
||||
self.rules = Tracker.reload_yara_rules()
|
||||
self.rules = Tracker.get_tracked_yara_rules()
|
||||
self.last_refresh = time.time()
|
||||
|
||||
self.item = None
|
||||
self.obj = None
|
||||
|
||||
# Exporter
|
||||
self.exporters = {'mail': MailExporterTracker(),
|
||||
|
|
@ -46,58 +46,67 @@ class Tracker_Yara(AbstractModule):
|
|||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
def compute(self, item_id, item_content=None):
|
||||
def compute(self, obj_id, obj_type='item', subtype=''):
|
||||
# refresh YARA list
|
||||
if self.last_refresh < Tracker.get_tracker_last_updated_by_type('yara'):
|
||||
self.rules = Tracker.reload_yara_rules()
|
||||
self.rules = Tracker.get_tracked_yara_rules()
|
||||
self.last_refresh = time.time()
|
||||
self.redis_logger.debug('Tracked set refreshed')
|
||||
print('Tracked set refreshed')
|
||||
|
||||
self.item = Item(item_id)
|
||||
if not item_content:
|
||||
item_content = self.item.get_content()
|
||||
self.obj = ail_objects.get_object(obj_type, subtype, obj_id)
|
||||
obj_type = self.obj.get_type()
|
||||
|
||||
# Object Filter
|
||||
if obj_type not in self.rules:
|
||||
return None
|
||||
|
||||
content = self.obj.get_content(r_str=True)
|
||||
|
||||
try:
|
||||
yara_match = self.rules.match(data=item_content, callback=self.yara_rules_match,
|
||||
yara_match = self.rules[obj_type].match(data=content, callback=self.yara_rules_match,
|
||||
which_callbacks=yara.CALLBACK_MATCHES, timeout=60)
|
||||
if yara_match:
|
||||
self.redis_logger.warning(f'tracker yara: new match {self.item.get_id()}: {yara_match}')
|
||||
print(f'{self.item.get_id()}: {yara_match}')
|
||||
self.redis_logger.warning(f'tracker yara: new match {self.obj.get_id()}: {yara_match}')
|
||||
print(f'{self.obj.get_id()}: {yara_match}')
|
||||
except yara.TimeoutError:
|
||||
print(f'{self.item.get_id()}: yara scanning timed out')
|
||||
self.redis_logger.info(f'{self.item.get_id()}: yara scanning timed out')
|
||||
print(f'{self.obj.get_id()}: yara scanning timed out')
|
||||
self.redis_logger.info(f'{self.obj.get_id()}: yara scanning timed out')
|
||||
|
||||
def yara_rules_match(self, data):
|
||||
tracker_uuid = data['namespace']
|
||||
item_id = self.item.get_id()
|
||||
|
||||
item = Item(item_id)
|
||||
item_source = self.item.get_source()
|
||||
|
||||
tracker_name = data['namespace']
|
||||
obj_id = self.obj.get_id()
|
||||
for tracker_uuid in Tracker.get_trackers_by_tracked_obj_type('yara', self.obj.get_type(), tracker_name):
|
||||
tracker = Tracker.Tracker(tracker_uuid)
|
||||
|
||||
# Source Filtering
|
||||
tracker_sources = tracker.get_sources()
|
||||
if tracker_sources and item_source not in tracker_sources:
|
||||
print(f'Source Filtering: {data["rule"]}')
|
||||
return yara.CALLBACK_CONTINUE
|
||||
# Filter Object
|
||||
filters = tracker.get_filters()
|
||||
if ail_objects.is_filtered(self.obj, filters):
|
||||
continue
|
||||
|
||||
Tracker.add_tracked_item(tracker_uuid, item_id) # TODO
|
||||
if self.obj.get_type() == 'item':
|
||||
date = self.obj.get_date()
|
||||
else:
|
||||
date = None
|
||||
|
||||
tracker.add(self.obj.get_type(), self.obj.get_subtype(r_str=True), obj_id, date=date)
|
||||
|
||||
# Tags
|
||||
for tag in tracker.get_tags():
|
||||
msg = f'{tag};{item_id}'
|
||||
if self.obj.get_type() == 'item':
|
||||
msg = f'{tag};{obj_id}'
|
||||
self.add_message_to_queue(msg, 'Tags')
|
||||
else:
|
||||
self.obj.add_tag(tag)
|
||||
|
||||
# Mails
|
||||
if tracker.mail_export():
|
||||
# TODO add matches + custom subjects
|
||||
self.exporters['mail'].export(tracker, item)
|
||||
self.exporters['mail'].export(tracker, self.obj)
|
||||
|
||||
# Webhook
|
||||
if tracker.webhook_export():
|
||||
self.exporters['webhook'].export(tracker, item)
|
||||
self.exporters['webhook'].export(tracker, self.obj)
|
||||
|
||||
return yara.CALLBACK_CONTINUE
|
||||
|
||||
|
|
@ -105,3 +114,4 @@ class Tracker_Yara(AbstractModule):
|
|||
if __name__ == '__main__':
|
||||
module = Tracker_Yara()
|
||||
module.run()
|
||||
# module.compute('archive/gist.github.com/2023/04/13/chipzoller_d8d6d2d737d02ad4fe9d30a897170761.gz')
|
||||
|
|
|
|||
|
|
@ -39,74 +39,75 @@ if __name__ == '__main__':
|
|||
|
||||
r_serv_term_stats.flushdb()
|
||||
|
||||
# convert all regex:
|
||||
all_regex = r_serv_termfreq.smembers('TrackedRegexSet')
|
||||
for regex in all_regex:
|
||||
tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)))
|
||||
mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)))
|
||||
|
||||
new_term = regex[1:-1]
|
||||
res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1},
|
||||
'admin@admin.test')
|
||||
if res[1] == 200:
|
||||
term_uuid = res[0]['uuid']
|
||||
list_items = r_serv_termfreq.smembers('regex_{}'.format(regex))
|
||||
for paste_item in list_items:
|
||||
item_id = get_item_id(paste_item)
|
||||
item_date = get_item_date(item_id)
|
||||
Term.add_tracked_item(term_uuid, item_id, item_date)
|
||||
|
||||
# Invalid Tracker => remove it
|
||||
else:
|
||||
print('Invalid Regex Removed: {}'.format(regex))
|
||||
print(res[0])
|
||||
# allow reprocess
|
||||
r_serv_termfreq.srem('TrackedRegexSet', regex)
|
||||
|
||||
all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet')
|
||||
for token in all_tokens:
|
||||
tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)))
|
||||
mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)))
|
||||
|
||||
res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
|
||||
if res[1] == 200:
|
||||
term_uuid = res[0]['uuid']
|
||||
list_items = r_serv_termfreq.smembers('tracked_{}'.format(token))
|
||||
for paste_item in list_items:
|
||||
item_id = get_item_id(paste_item)
|
||||
item_date = get_item_date(item_id)
|
||||
Term.add_tracked_item(term_uuid, item_id, item_date)
|
||||
# Invalid Tracker => remove it
|
||||
else:
|
||||
print('Invalid Token Removed: {}'.format(token))
|
||||
print(res[0])
|
||||
# allow reprocess
|
||||
r_serv_termfreq.srem('TrackedSetTermSet', token)
|
||||
|
||||
all_set = r_serv_termfreq.smembers('TrackedSetSet')
|
||||
for curr_set in all_set:
|
||||
tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)))
|
||||
mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)))
|
||||
|
||||
to_remove = ',{}'.format(curr_set.split(',')[-1])
|
||||
new_set = rreplace(curr_set, to_remove, '', 1)
|
||||
new_set = new_set[2:]
|
||||
new_set = new_set.replace(',', '')
|
||||
|
||||
res = Term.parse_json_term_to_add({"term": new_set, "type": 'set', "nb_words": 1, "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
|
||||
if res[1] == 200:
|
||||
term_uuid = res[0]['uuid']
|
||||
list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set))
|
||||
for paste_item in list_items:
|
||||
item_id = get_item_id(paste_item)
|
||||
item_date = get_item_date(item_id)
|
||||
Term.add_tracked_item(term_uuid, item_id, item_date)
|
||||
# Invalid Tracker => remove it
|
||||
else:
|
||||
print('Invalid Set Removed: {}'.format(curr_set))
|
||||
print(res[0])
|
||||
# allow reprocess
|
||||
r_serv_termfreq.srem('TrackedSetSet', curr_set)
|
||||
# Disabled. Checkout the v2.2 branch if you need it
|
||||
# # convert all regex:
|
||||
# all_regex = r_serv_termfreq.smembers('TrackedRegexSet')
|
||||
# for regex in all_regex:
|
||||
# tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(regex)))
|
||||
# mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(regex)))
|
||||
#
|
||||
# new_term = regex[1:-1]
|
||||
# res = Term.parse_json_term_to_add({"term": new_term, "type": 'regex', "tags": tags, "mails": mails, "level": 1},
|
||||
# 'admin@admin.test')
|
||||
# if res[1] == 200:
|
||||
# term_uuid = res[0]['uuid']
|
||||
# list_items = r_serv_termfreq.smembers('regex_{}'.format(regex))
|
||||
# for paste_item in list_items:
|
||||
# item_id = get_item_id(paste_item)
|
||||
# item_date = get_item_date(item_id)
|
||||
# Term.add_tracked_item(term_uuid, item_id, item_date)
|
||||
#
|
||||
# # Invalid Tracker => remove it
|
||||
# else:
|
||||
# print('Invalid Regex Removed: {}'.format(regex))
|
||||
# print(res[0])
|
||||
# # allow reprocess
|
||||
# r_serv_termfreq.srem('TrackedRegexSet', regex)
|
||||
#
|
||||
# all_tokens = r_serv_termfreq.smembers('TrackedSetTermSet')
|
||||
# for token in all_tokens:
|
||||
# tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(token)))
|
||||
# mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(token)))
|
||||
#
|
||||
# res = Term.parse_json_term_to_add({"term": token, "type": 'word', "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
|
||||
# if res[1] == 200:
|
||||
# term_uuid = res[0]['uuid']
|
||||
# list_items = r_serv_termfreq.smembers('tracked_{}'.format(token))
|
||||
# for paste_item in list_items:
|
||||
# item_id = get_item_id(paste_item)
|
||||
# item_date = get_item_date(item_id)
|
||||
# Term.add_tracked_item(term_uuid, item_id, item_date)
|
||||
# # Invalid Tracker => remove it
|
||||
# else:
|
||||
# print('Invalid Token Removed: {}'.format(token))
|
||||
# print(res[0])
|
||||
# # allow reprocess
|
||||
# r_serv_termfreq.srem('TrackedSetTermSet', token)
|
||||
#
|
||||
# all_set = r_serv_termfreq.smembers('TrackedSetSet')
|
||||
# for curr_set in all_set:
|
||||
# tags = list(r_serv_termfreq.smembers('TrackedNotificationTags_{}'.format(curr_set)))
|
||||
# mails = list(r_serv_termfreq.smembers('TrackedNotificationEmails_{}'.format(curr_set)))
|
||||
#
|
||||
# to_remove = ',{}'.format(curr_set.split(',')[-1])
|
||||
# new_set = rreplace(curr_set, to_remove, '', 1)
|
||||
# new_set = new_set[2:]
|
||||
# new_set = new_set.replace(',', '')
|
||||
#
|
||||
# res = Term.parse_json_term_to_add({"term": new_set, "type": 'set', "nb_words": 1, "tags": tags, "mails": mails, "level": 1}, 'admin@admin.test')
|
||||
# if res[1] == 200:
|
||||
# term_uuid = res[0]['uuid']
|
||||
# list_items = r_serv_termfreq.smembers('tracked_{}'.format(curr_set))
|
||||
# for paste_item in list_items:
|
||||
# item_id = get_item_id(paste_item)
|
||||
# item_date = get_item_date(item_id)
|
||||
# Term.add_tracked_item(term_uuid, item_id, item_date)
|
||||
# # Invalid Tracker => remove it
|
||||
# else:
|
||||
# print('Invalid Set Removed: {}'.format(curr_set))
|
||||
# print(res[0])
|
||||
# # allow reprocess
|
||||
# r_serv_termfreq.srem('TrackedSetSet', curr_set)
|
||||
|
||||
r_serv_termfreq.flushdb()
|
||||
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ class Updater(AIL_Updater):
|
|||
print('Fixing Tracker_uuid list ...')
|
||||
Tracker.fix_all_tracker_uuid_list()
|
||||
nb = 0
|
||||
for tracker_uuid in Tracker.get_all_tracker_uuid():
|
||||
for tracker_uuid in Tracker.get_trackers():
|
||||
self.r_serv.sadd('trackers_update_v3.7', tracker_uuid)
|
||||
nb += 1
|
||||
|
||||
|
|
|
|||
|
|
@ -1,15 +1,15 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
"""
|
||||
Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
|
||||
'''
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, make_response
|
||||
from flask import render_template, jsonify, request, Blueprint, redirect, url_for, Response
|
||||
from flask_login import login_required, current_user, login_user, logout_user
|
||||
|
||||
sys.path.append('modules')
|
||||
|
|
@ -22,8 +22,10 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib import ail_core
|
||||
from lib import item_basic
|
||||
from lib import Tracker
|
||||
from lib import Tag
|
||||
|
||||
|
||||
bootstrap_label = Flask_config.bootstrap_label
|
||||
|
|
@ -34,7 +36,6 @@ hunters = Blueprint('hunters', __name__, template_folder=os.path.join(os.environ
|
|||
# ============ VARIABLES ============
|
||||
|
||||
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
def api_validator(api_response):
|
||||
if api_response:
|
||||
|
|
@ -45,11 +46,203 @@ def create_json_response(data, status_code):
|
|||
|
||||
# ============= ROUTES ==============
|
||||
|
||||
##################
|
||||
# TRACKERS #
|
||||
##################
|
||||
|
||||
@hunters.route('/trackers', methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def trackers_dashboard():
|
||||
user_id = current_user.get_id() # TODO
|
||||
trackers = Tracker.get_trackers_dashboard()
|
||||
stats = Tracker.get_trackers_stats(user_id)
|
||||
return render_template("trackers_dashboard.html", trackers=trackers, stats=stats, bootstrap_label=bootstrap_label)
|
||||
|
||||
@hunters.route("/trackers/all")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu():
|
||||
user_id = current_user.get_id()
|
||||
user_trackers = Tracker.get_user_trackers_meta(user_id)
|
||||
global_trackers = Tracker.get_global_trackers_meta()
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label)
|
||||
|
||||
@hunters.route("/trackers/word")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu_word():
|
||||
tracker_type = 'word'
|
||||
user_id = current_user.get_id()
|
||||
user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type='word')
|
||||
global_trackers = Tracker.get_global_trackers_meta(tracker_type='word')
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
|
||||
|
||||
@hunters.route("/trackers/set")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu_set():
|
||||
tracker_type = 'set'
|
||||
user_id = current_user.get_id()
|
||||
user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type)
|
||||
global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type)
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
|
||||
|
||||
@hunters.route("/trackers/regex")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu_regex():
|
||||
tracker_type = 'regex'
|
||||
user_id = current_user.get_id()
|
||||
user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type)
|
||||
global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type)
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
|
||||
|
||||
@hunters.route("/trackers/yara")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu_yara():
|
||||
tracker_type = 'yara'
|
||||
user_id = current_user.get_id()
|
||||
user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type)
|
||||
global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type)
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
|
||||
|
||||
@hunters.route("/trackers/typosquatting")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu_typosquatting():
|
||||
tracker_type = 'typosquatting'
|
||||
user_id = current_user.get_id()
|
||||
user_trackers = Tracker.get_user_trackers_meta(user_id, tracker_type=tracker_type)
|
||||
global_trackers = Tracker.get_global_trackers_meta(tracker_type=tracker_type)
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers,
|
||||
bootstrap_label=bootstrap_label, tracker_type=tracker_type)
|
||||
|
||||
@hunters.route("/tracker/add", methods=['GET', 'POST'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def add_tracked_menu():
|
||||
if request.method == 'POST':
|
||||
to_track = request.form.get("tracker")
|
||||
tracker_uuid = request.form.get("tracker_uuid")
|
||||
tracker_type = request.form.get("tracker_type")
|
||||
nb_words = request.form.get("nb_word", 1)
|
||||
description = request.form.get("description", '')
|
||||
webhook = request.form.get("webhook", '')
|
||||
level = request.form.get("level", 0)
|
||||
mails = request.form.get("mails", [])
|
||||
|
||||
# TAGS
|
||||
tags = request.form.get("tags", [])
|
||||
taxonomies_tags = request.form.get('taxonomies_tags')
|
||||
if taxonomies_tags:
|
||||
try:
|
||||
taxonomies_tags = json.loads(taxonomies_tags)
|
||||
except:
|
||||
taxonomies_tags = []
|
||||
else:
|
||||
taxonomies_tags = []
|
||||
galaxies_tags = request.form.get('galaxies_tags')
|
||||
if galaxies_tags:
|
||||
try:
|
||||
galaxies_tags = json.loads(galaxies_tags)
|
||||
except:
|
||||
galaxies_tags = []
|
||||
# custom tags
|
||||
if tags:
|
||||
tags = tags.split()
|
||||
else:
|
||||
tags = []
|
||||
tags = tags + taxonomies_tags + galaxies_tags
|
||||
|
||||
# YARA #
|
||||
if tracker_type == 'yara':
|
||||
yara_default_rule = request.form.get("yara_default_rule")
|
||||
yara_custom_rule = request.form.get("yara_custom_rule")
|
||||
if yara_custom_rule:
|
||||
to_track = yara_custom_rule
|
||||
tracker_type = 'yara_custom'
|
||||
else:
|
||||
to_track = yara_default_rule
|
||||
tracker_type = 'yara_default'
|
||||
|
||||
if level == 'on':
|
||||
level = 1
|
||||
else:
|
||||
level = 0
|
||||
if mails:
|
||||
mails = mails.split()
|
||||
else:
|
||||
tags = []
|
||||
|
||||
# FILTERS
|
||||
filters = {}
|
||||
for obj_type in Tracker.get_objects_tracked():
|
||||
new_filter = request.form.get(f'{obj_type}_obj')
|
||||
if new_filter == 'on':
|
||||
filters[obj_type] = {}
|
||||
# Mimetypes
|
||||
mimetypes = request.form.get(f'mimetypes_{obj_type}', [])
|
||||
if mimetypes:
|
||||
mimetypes = json.loads(mimetypes)
|
||||
filters[obj_type]['mimetypes'] = mimetypes
|
||||
# Sources
|
||||
sources = request.form.get(f'sources_{obj_type}', [])
|
||||
if sources:
|
||||
sources = json.loads(sources)
|
||||
filters[obj_type]['sources'] = sources
|
||||
# Subtypes
|
||||
for obj_subtype in ail_core.get_object_all_subtypes(obj_type):
|
||||
subtype = request.form.get(f'filter_{obj_type}_{obj_subtype}')
|
||||
if subtype == 'on':
|
||||
if 'subtypes' not in filters[obj_type]:
|
||||
filters[obj_type]['subtypes'] = []
|
||||
filters[obj_type]['subtypes'].append(obj_subtype)
|
||||
|
||||
input_dict = {"tracked": to_track, "type": tracker_type,
|
||||
"tags": tags, "mails": mails, "filters": filters,
|
||||
"level": level, "description": description, "webhook": webhook}
|
||||
if tracker_type == 'set':
|
||||
try:
|
||||
input_dict['nb_words'] = int(nb_words)
|
||||
except TypeError:
|
||||
input_dict['nb_words'] = 1
|
||||
|
||||
user_id = current_user.get_id()
|
||||
res = Tracker.api_add_tracker(input_dict, user_id)
|
||||
if res[1] == 200:
|
||||
return redirect(url_for('hunters.trackers_dashboard'))
|
||||
else:
|
||||
return create_json_response(res[0], res[1])
|
||||
else:
|
||||
return render_template("tracker_add.html",
|
||||
all_sources=item_basic.get_all_items_sources(r_list=True),
|
||||
tags_selector_data=Tag.get_tags_selector_data(),
|
||||
all_yara_files=Tracker.get_all_default_yara_files())
|
||||
|
||||
@hunters.route('/tracker/delete', methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def tracker_delete():
|
||||
user_id = current_user.get_id()
|
||||
tracker_uuid = request.args.get('uuid')
|
||||
res = Tracker.api_delete_tracker({'uuid': tracker_uuid}, user_id)
|
||||
if res[1] != 200:
|
||||
return create_json_response(res[0], res[1])
|
||||
else:
|
||||
return redirect(url_for('hunter.tracked_menu'))
|
||||
|
||||
|
||||
####################
|
||||
# RETRO HUNT #
|
||||
####################
|
||||
|
||||
@hunters.route('/retro_hunt/tasks', methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def retro_hunt_all_tasks():
|
||||
retro_hunts = Tracker.get_all_retro_hunt_tasks_with_metadata()
|
||||
retro_hunts = Tracker.get_retro_hunt_tasks_metas()
|
||||
return render_template("retro_hunt_tasks.html", retro_hunts=retro_hunts, bootstrap_label=bootstrap_label)
|
||||
|
||||
@hunters.route('/retro_hunt/task/show', methods=['GET'])
|
||||
|
|
@ -69,8 +262,8 @@ def retro_hunt_show_task():
|
|||
if res:
|
||||
return create_json_response(res[0], res[1])
|
||||
|
||||
dict_task = Tracker.get_retro_hunt_task_metadata(task_uuid, date=True, progress=True, creator=True,
|
||||
sources=True, tags=True, description=True)
|
||||
retro_hunt = Tracker.RetroHunt(task_uuid)
|
||||
dict_task = retro_hunt.get_meta(options={'creator', 'date', 'description', 'progress', 'sources', 'tags'})
|
||||
rule_content = Tracker.get_yara_rule_content(dict_task['rule'])
|
||||
|
||||
if date_from:
|
||||
|
|
@ -177,7 +370,7 @@ def retro_hunt_delete_task():
|
|||
|
||||
#### JSON ####
|
||||
|
||||
@hunters.route("/tracker/get_json_retro_hunt_nb_items_by_date", methods=['GET'])
|
||||
@hunters.route("/retro_hunt/nb_items/date/json", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def get_json_retro_hunt_nb_items_by_date():
|
||||
|
|
|
|||
|
|
@ -17,10 +17,11 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib.objects import ail_objects
|
||||
from lib import item_basic
|
||||
from lib import Tracker
|
||||
from lib import Tag
|
||||
from packages import Term
|
||||
from packages import Date
|
||||
|
||||
|
||||
# ============ VARIABLES ============
|
||||
|
|
@ -34,142 +35,11 @@ hunter = Blueprint('hunter', __name__, template_folder='templates')
|
|||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
def create_json_response(data, status_code):
|
||||
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
|
||||
|
||||
# ============ ROUTES ============
|
||||
|
||||
@hunter.route("/trackers")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu():
|
||||
user_id = current_user.get_id()
|
||||
user_trackers = Tracker.get_user_trackers_metadata(user_id)
|
||||
global_trackers = Tracker.get_global_trackers_metadata()
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label)
|
||||
|
||||
@hunter.route("/trackers/word")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu_word():
|
||||
tracker_type = 'word'
|
||||
user_id = current_user.get_id()
|
||||
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type='word')
|
||||
global_trackers = Tracker.get_global_trackers_metadata(tracker_type='word')
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
|
||||
|
||||
@hunter.route("/trackers/set")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu_set():
|
||||
tracker_type = 'set'
|
||||
user_id = current_user.get_id()
|
||||
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
|
||||
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
|
||||
|
||||
@hunter.route("/trackers/regex")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu_regex():
|
||||
tracker_type = 'regex'
|
||||
user_id = current_user.get_id()
|
||||
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
|
||||
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
|
||||
|
||||
@hunter.route("/trackers/yara")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu_yara():
|
||||
tracker_type = 'yara'
|
||||
user_id = current_user.get_id()
|
||||
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
|
||||
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers, bootstrap_label=bootstrap_label, tracker_type=tracker_type)
|
||||
|
||||
@hunter.route("/trackers/typosquatting")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def tracked_menu_typosquatting():
|
||||
tracker_type = 'typosquatting'
|
||||
user_id = current_user.get_id()
|
||||
user_trackers = Tracker.get_user_trackers_metadata(user_id, tracker_type=tracker_type)
|
||||
global_trackers = Tracker.get_global_trackers_metadata(tracker_type=tracker_type)
|
||||
return render_template("trackersManagement.html", user_trackers=user_trackers, global_trackers=global_trackers,
|
||||
bootstrap_label=bootstrap_label, tracker_type=tracker_type)
|
||||
|
||||
@hunter.route("/tracker/add", methods=['GET', 'POST'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def add_tracked_menu():
|
||||
if request.method == 'POST':
|
||||
tracker = request.form.get("tracker")
|
||||
tracker_uuid = request.form.get("tracker_uuid")
|
||||
tracker_type = request.form.get("tracker_type")
|
||||
nb_words = request.form.get("nb_word", 1)
|
||||
description = request.form.get("description", '')
|
||||
webhook = request.form.get("webhook", '')
|
||||
level = request.form.get("level", 0)
|
||||
mails = request.form.get("mails", [])
|
||||
sources = request.form.get("sources", [])
|
||||
|
||||
tags = request.form.get("tags", [])
|
||||
taxonomies_tags = request.form.get('taxonomies_tags')
|
||||
if taxonomies_tags:
|
||||
try:
|
||||
taxonomies_tags = json.loads(taxonomies_tags)
|
||||
except Exception:
|
||||
taxonomies_tags = []
|
||||
else:
|
||||
taxonomies_tags = []
|
||||
galaxies_tags = request.form.get('galaxies_tags')
|
||||
if galaxies_tags:
|
||||
try:
|
||||
galaxies_tags = json.loads(galaxies_tags)
|
||||
except Exception:
|
||||
galaxies_tags = []
|
||||
|
||||
|
||||
# YARA #
|
||||
if tracker_type == 'yara':
|
||||
yara_default_rule = request.form.get("yara_default_rule")
|
||||
yara_custom_rule = request.form.get("yara_custom_rule")
|
||||
if yara_custom_rule:
|
||||
tracker = yara_custom_rule
|
||||
tracker_type='yara_custom'
|
||||
else:
|
||||
tracker = yara_default_rule
|
||||
tracker_type='yara_default'
|
||||
|
||||
if level == 'on':
|
||||
level = 1
|
||||
if mails:
|
||||
mails = mails.split()
|
||||
if tags:
|
||||
tags = tags.split()
|
||||
if sources:
|
||||
sources = json.loads(sources)
|
||||
|
||||
input_dict = {"tracker": tracker, "type": tracker_type, "nb_words": nb_words,
|
||||
"tags": tags, "mails": mails, "sources": sources,
|
||||
"level": level, "description": description, "webhook": webhook}
|
||||
user_id = current_user.get_id()
|
||||
# edit tracker
|
||||
if tracker_uuid:
|
||||
input_dict['uuid'] = tracker_uuid
|
||||
res = Tracker.api_add_tracker(input_dict, user_id)
|
||||
if res[1] == 200:
|
||||
if 'uuid' in res[0]:
|
||||
return redirect(url_for('hunter.show_tracker', uuid=res[0]['uuid']))
|
||||
else:
|
||||
return redirect(url_for('hunter.tracked_menu'))
|
||||
else:
|
||||
## TODO: use modal
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
else:
|
||||
return render_template("edit_tracker.html",
|
||||
all_sources=item_basic.get_all_items_sources(r_list=True),
|
||||
tags_selector_data=Tag.get_tags_selector_data(),
|
||||
all_yara_files=Tracker.get_all_default_yara_files())
|
||||
|
||||
@hunter.route("/tracker/edit", methods=['GET', 'POST'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
|
|
@ -181,7 +51,8 @@ def edit_tracked_menu():
|
|||
if res[1] != 200: # invalid access
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
dict_tracker = Tracker.get_tracker_metadata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, webhook=True)
|
||||
tracker = Tracker.Tracker(tracker_uuid)
|
||||
dict_tracker = tracker.get_meta(options={'description', 'level', 'mails', 'sources', 'tags', 'user', 'webhook'})
|
||||
dict_tracker['tags'] = ' '.join(dict_tracker['tags'])
|
||||
dict_tracker['mails'] = ' '.join(dict_tracker['mails'])
|
||||
|
||||
|
|
@ -205,7 +76,7 @@ def edit_tracked_menu():
|
|||
# word
|
||||
# set of word + nb words
|
||||
# regex
|
||||
# yara custum
|
||||
# yara custom
|
||||
# yara default ???? => allow edit ?
|
||||
|
||||
#### EDIT SHow Trackers ??????????????????????????????????????????????????
|
||||
|
|
@ -228,34 +99,36 @@ def show_tracker():
|
|||
if date_to:
|
||||
date_to = date_to.replace('-', '')
|
||||
|
||||
tracker_metadata = Tracker.get_tracker_metadata(tracker_uuid, user_id=True, level=True, description=True, tags=True, mails=True, sources=True, sparkline=True, webhook=True)
|
||||
tracker = Tracker.Tracker(tracker_uuid)
|
||||
meta = tracker.get_meta(options={'description', 'level', 'mails', 'filters', 'sparkline', 'tags',
|
||||
'user', 'webhook'})
|
||||
|
||||
if tracker_metadata['type'] == 'yara':
|
||||
yara_rule_content = Tracker.get_yara_rule_content(tracker_metadata['tracker'])
|
||||
if meta['type'] == 'yara':
|
||||
yara_rule_content = Tracker.get_yara_rule_content(meta['tracked'])
|
||||
else:
|
||||
yara_rule_content = None
|
||||
|
||||
if tracker_metadata['type'] == 'typosquatting':
|
||||
typo_squatting = list(Tracker.get_tracker_typosquatting_domains(tracker_uuid))
|
||||
typo_squatting.sort()
|
||||
if meta['type'] == 'typosquatting':
|
||||
typo_squatting = Tracker.get_tracked_typosquatting_domains(meta['tracked'])
|
||||
sorted(typo_squatting)
|
||||
else:
|
||||
typo_squatting = None
|
||||
typo_squatting = set()
|
||||
|
||||
if date_from:
|
||||
res = Term.parse_get_tracker_term_item({'uuid': tracker_uuid, 'date_from': date_from, 'date_to': date_to}, user_id)
|
||||
if res[1] != 200:
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
tracker_metadata['items'] = res[0]['items']
|
||||
tracker_metadata['date_from'] = res[0]['date_from']
|
||||
tracker_metadata['date_to'] = res[0]['date_to']
|
||||
date_from, date_to = Date.sanitise_daterange(meta['first_seen'], meta['last_seen'])
|
||||
objs = tracker.get_objs_by_daterange(date_from, date_to)
|
||||
meta['objs'] = ail_objects.get_objects_meta(objs, flask_context=True)
|
||||
else:
|
||||
tracker_metadata['items'] = []
|
||||
tracker_metadata['date_from'] = ''
|
||||
tracker_metadata['date_to'] = ''
|
||||
date_from = ''
|
||||
date_to = ''
|
||||
meta['objs'] = []
|
||||
|
||||
tracker_metadata['sources'] = sorted(tracker_metadata['sources'])
|
||||
meta['date_from'] = date_from
|
||||
meta['date_to'] = date_to
|
||||
print(meta['filters'])
|
||||
meta['item_sources'] = sorted(meta['filters'].get('item', {}).get('sources', []))
|
||||
|
||||
return render_template("showTracker.html", tracker_metadata=tracker_metadata,
|
||||
return render_template("showTracker.html", tracker_metadata=meta,
|
||||
yara_rule_content=yara_rule_content,
|
||||
typo_squatting=typo_squatting,
|
||||
bootstrap_label=bootstrap_label)
|
||||
|
|
@ -309,21 +182,16 @@ def update_tracker_mails():
|
|||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
return redirect(url_for('hunter.show_tracker', uuid=term_uuid))
|
||||
|
||||
@hunter.route("/tracker/delete", methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def delete_tracker():
|
||||
user_id = current_user.get_id()
|
||||
term_uuid = request.args.get('uuid')
|
||||
res = Term.parse_tracked_term_to_delete({'uuid': term_uuid}, user_id)
|
||||
if res[1] !=200:
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
return redirect(url_for('hunter.tracked_menu'))
|
||||
|
||||
@hunter.route("/tracker/get_json_tracker_stats", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def get_json_tracker_stats():
|
||||
user_id = current_user.get_id()
|
||||
tracker_uuid = request.args.get('uuid')
|
||||
res = Tracker.api_check_tracker_acl(tracker_uuid, user_id)
|
||||
if res:
|
||||
return create_json_response(res[0], res[1])
|
||||
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
|
||||
|
|
@ -331,13 +199,10 @@ def get_json_tracker_stats():
|
|||
date_from = date_from.replace('-', '')
|
||||
if date_to:
|
||||
date_to = date_to.replace('-', '')
|
||||
|
||||
tracker_uuid = request.args.get('uuid')
|
||||
|
||||
if date_from and date_to:
|
||||
res = Term.get_list_tracked_term_stats_by_day([tracker_uuid], date_from=date_from, date_to=date_to)
|
||||
res = Tracker.get_trackers_graph_by_day([tracker_uuid], date_from=date_from, date_to=date_to)
|
||||
else:
|
||||
res = Term.get_list_tracked_term_stats_by_day([tracker_uuid])
|
||||
res = Tracker.get_trackers_graph_by_day([tracker_uuid])
|
||||
return jsonify(res)
|
||||
|
||||
@hunter.route("/tracker/yara/default_rule/content", methods=['GET'])
|
||||
|
|
@ -348,5 +213,6 @@ def get_default_yara_rule_content():
|
|||
res = Tracker.api_get_default_rule_content(default_yara_rule)
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
|
||||
# ========= REGISTRATION =========
|
||||
app.register_blueprint(hunter, url_prefix=baseUrl)
|
||||
|
|
|
|||
|
|
@ -1,260 +0,0 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<title>AIL-Framework</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
|
||||
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'hunter/menu_sidebar.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="card my-3">
|
||||
<div class="card-header bg-dark text-white">
|
||||
<h5 class="card-title">{%if dict_tracker%}Edit a{%else%}Create a new{%endif%} Tracker</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
|
||||
<form action="{{ url_for('hunter.add_tracked_menu') }}" method='post'>
|
||||
{%if dict_tracker%}
|
||||
<input id="tracker_uuid" name="tracker_uuid" class="form-control" type="text" value="{{dict_tracker['uuid']}}" hidden>
|
||||
{%endif%}
|
||||
|
||||
<div class="row">
|
||||
<div class="col-12 col-xl-9">
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-secondary text-white"><i class="fas fa-at"></i></div>
|
||||
</div>
|
||||
<input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['mails']%}value="{{dict_tracker['mails']}}"{%endif%}{%endif%}>
|
||||
</div>
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-info text-white"><i class="fas fa-anchor"></i></div>
|
||||
</div>
|
||||
<input id="webhook" name="webhook" class="form-control" placeholder="Webhook URL" type="text" {%if dict_tracker%}{%if dict_tracker['webhook']%}value="{{dict_tracker['webhook']}}"{%endif%}{%endif%}>
|
||||
</div>
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-info text-white"><i class="fas fa-pencil-alt"></i></div>
|
||||
</div>
|
||||
<input id="description" name="description" class="form-control" placeholder="Tracker Description (optional)" type="text" {%if dict_tracker%}{%if dict_tracker['description']%}value="{{dict_tracker['description']}}"{%endif%}{%endif%}>
|
||||
</div>
|
||||
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-dark text-white"><i class="fas fa-folder"></i></div>
|
||||
</div>
|
||||
<input id="sources" class="form-control" type="text" name="sources" placeholder="Sources to track (ALL IF EMPTY)" autocomplete="off">
|
||||
</div>
|
||||
|
||||
|
||||
<div class="card my-4">
|
||||
<div class="card-header bg-secondary text-white">
|
||||
<b>Tags</b>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
|
||||
</div>
|
||||
<input id="tags" name="tags" class="form-control" placeholder="Custom Tags (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tags']%}value="{{dict_tracker['tags']}}"{%endif%}{%endif%}>
|
||||
</div>
|
||||
{% include 'tags/block_tags_selector.html' %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class="col-12 col-xl-3">
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="level" id="id_level" {%if dict_tracker%}{%if dict_tracker['level']==1%}checked{%endif%}{%else%}checked{%endif%}>
|
||||
<label class="custom-control-label" for="id_level">
|
||||
<i class="fas fa-users"></i> Show tracker to all Users
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<hr>
|
||||
<h4>Tracker Type:</h4>
|
||||
|
||||
<select id="tracker_type" name="tracker_type" class="custom-select w-25 mb-3">
|
||||
<option disabled selected value> -- Select a tracker type -- </option>
|
||||
<option value="word">Word</option>
|
||||
<option value="set">Set</option>
|
||||
<option value="regex">Regex</option>
|
||||
<option value="yara">YARA rule</option>
|
||||
<option value="typosquatting">Typo-squatting</option>
|
||||
</select>
|
||||
|
||||
<p id="tracker_desc">Terms to track (space separated)</p>
|
||||
|
||||
<div class="row" id="simple_input">
|
||||
<div class="col-12 col-lg-10">
|
||||
<input id="tracker" name="tracker" class="form-control" placeholder="Terms to track (space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tracker']!='yara'%}value="{{dict_tracker['tracker']}}"{%endif%}{%endif%}>
|
||||
</div>
|
||||
<div class="col-12 col-lg-2">
|
||||
<input type="number" id="nb_word" name="nb_word" name="quantity" min="1" placeholder="Nb of keywords" {%if dict_tracker%}{%if dict_tracker['nb_words']%}value="{{dict_tracker['nb_words']}}"{%endif%}{%endif%}>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="mb-2" id="yara_rule">
|
||||
<div class="" id="yara_default_rules">
|
||||
|
||||
<h6>Default YARA rules:</h6>
|
||||
<select class="custom-select w-100 mb-3" id="yara_default_rule" name="yara_default_rule" onchange="get_default_rule_content(this);">
|
||||
<option selected>Select a default rule</option>
|
||||
{% for yara_types in all_yara_files %}
|
||||
{% for yara_file_name in all_yara_files[yara_types] %}
|
||||
<option value="{{yara_types}}/{{yara_file_name}}">{{yara_types}} - {{yara_file_name}}</option>
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
</select>
|
||||
|
||||
<pre class="border bg-light" id="default_yara_rule_content"></pre>
|
||||
|
||||
</div>
|
||||
|
||||
<hr>
|
||||
|
||||
<h6>Custom YARA rules:</h6>
|
||||
<div class="row" id="textarea">
|
||||
<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5">{%if dict_tracker%}{%if dict_tracker['type']=='yara' and dict_tracker['content']%}{{dict_tracker['content']}}{%endif%}{%endif%}</textarea>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<br>
|
||||
<button class="btn btn-success mt-2">
|
||||
<i class="fas fa-plus"></i> {%if dict_tracker%}Edit{%else%}Create{%endif%} Tracker
|
||||
</button>
|
||||
|
||||
</form>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</body>
|
||||
|
||||
<script>
|
||||
var ltags;
|
||||
var chart = {};
|
||||
$(document).ready(function(){
|
||||
$("#page-Tracker").addClass("active");
|
||||
$("#nav_manual_crawler").addClass("active");
|
||||
$("#tracker_desc").hide();
|
||||
$("#tracker").hide();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").hide();
|
||||
|
||||
sources = $('#sources').tagSuggest({
|
||||
data: {{all_sources|safe}},
|
||||
{%if dict_tracker%}{%if dict_tracker['sources']%}value: {{dict_tracker['sources']|safe}},{%endif%}{%endif%}
|
||||
sortOrder: 'name',
|
||||
maxDropHeight: 200,
|
||||
name: 'sources',
|
||||
emptyText: 'Sources to track (ALL IF EMPTY)',
|
||||
});
|
||||
|
||||
$('#tracker_type').on('change', function() {
|
||||
var tracker_type = this.value;
|
||||
if (tracker_type=="word") {
|
||||
$("#tracker_desc").text("Token to track. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
|
||||
$("#tracker_desc").show();
|
||||
$("#tracker").show();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").hide();
|
||||
} else if (tracker_type=="set") {
|
||||
$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
|
||||
$("#tracker_desc").show();
|
||||
$("#tracker").show();
|
||||
$("#nb_word").show();
|
||||
$("#yara_rule").hide();
|
||||
} else if (tracker_type=="regex") {
|
||||
$("#tracker_desc").text("Enter a valid Python regex");
|
||||
$("#tracker_desc").show();
|
||||
$("#tracker").show();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").hide();
|
||||
} else if (tracker_type=="yara") {
|
||||
$("#tracker_desc").text("Select a default yara rule or create your own rule:");
|
||||
$("#tracker_desc").show();
|
||||
$("#tracker").hide();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").show();
|
||||
} else if (tracker_type=="typosquatting") {
|
||||
$("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time.");
|
||||
$("#tracker_desc").show();
|
||||
$("#tracker").show();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").hide();
|
||||
}
|
||||
});
|
||||
|
||||
{%if dict_tracker%}
|
||||
$('#tracker_type').val('{{dict_tracker['type']}}').change();
|
||||
|
||||
{%if dict_tracker['type']=='yara' and dict_tracker['yara_file']%}
|
||||
$('#yara_default_rule').val('{{dict_tracker['yara_file']}}').change();
|
||||
{%endif%}
|
||||
{%endif%}
|
||||
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
function get_default_rule_content(selector){
|
||||
var yara_name = selector.value
|
||||
if (yara_name === "Select a default rule") {
|
||||
jQuery("#default_yara_rule_content").text("")
|
||||
} else {
|
||||
$.getJSON("{{ url_for('hunter.get_default_yara_rule_content') }}?rule_name=" + yara_name,
|
||||
function(data) {
|
||||
jQuery("#default_yara_rule_content").text(data['content'])
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
</script>
|
||||
|
|
@ -72,7 +72,7 @@
|
|||
<tr>
|
||||
<th>Type</th>
|
||||
<th>Tracker</th>
|
||||
<th>Date added</th>
|
||||
<th>Created</th>
|
||||
<th>Access Level</th>
|
||||
<th>Created by</th>
|
||||
<th>First seen</th>
|
||||
|
|
@ -94,7 +94,7 @@
|
|||
{% if tracker_metadata['type'] == 'typosquatting' %}
|
||||
<td>
|
||||
<a class="btn btn-primary" data-toggle="collapse" href="#collapseTypo" role="button" aria-expanded="false" aria-controls="collapseTypo">
|
||||
{{ tracker_metadata['tracker'].split(",")[0] }}
|
||||
{{ tracker_metadata['tracked'].split(",")[0] }}
|
||||
</a>
|
||||
<div class="collapse" id="collapseTypo">
|
||||
<div class="card card-body">
|
||||
|
|
@ -108,7 +108,7 @@
|
|||
</div>
|
||||
</td>
|
||||
{% else %}
|
||||
<td>{{ tracker_metadata['tracker'] }}</td>
|
||||
<td>{{ tracker_metadata['tracked'] }}</td>
|
||||
{% endif %}
|
||||
<td>{{ tracker_metadata['date'][0:4] }}/{{ tracker_metadata['date'][4:6] }}/{{ tracker_metadata['date'][6:8] }}</td>
|
||||
<td>
|
||||
|
|
@ -118,19 +118,15 @@
|
|||
Global
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>{{ tracker_metadata['user_id'] }}</td>
|
||||
<td>{{ tracker_metadata['user'] }}</td>
|
||||
<td>
|
||||
{% if tracker_metadata['first_seen'] %}
|
||||
{{ tracker_metadata['first_seen'][0:4] }}/
|
||||
{{ tracker_metadata['first_seen'][4:6] }}/
|
||||
{{ tracker_metadata['first_seen'][6:8] }}
|
||||
{{ tracker_metadata['first_seen'][0:4] }}/{{ tracker_metadata['first_seen'][4:6] }}/{{ tracker_metadata['first_seen'][6:8] }}
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{% if tracker_metadata['last_seen'] %}
|
||||
{{ tracker_metadata['last_seen'][0:4] }}/
|
||||
{{ tracker_metadata['last_seen'][4:6] }}/
|
||||
{{ tracker_metadata['last_seen'][6:8] }}
|
||||
{{ tracker_metadata['last_seen'][0:4] }}/{{ tracker_metadata['last_seen'][4:6] }}/{{ tracker_metadata['last_seen'][6:8] }}
|
||||
{% endif %}
|
||||
</td>
|
||||
{% if tracker_metadata['webhook'] %}
|
||||
|
|
@ -159,14 +155,22 @@
|
|||
<div id="sparkline"></div>
|
||||
</div>
|
||||
</div>
|
||||
<h6>Sources:</h6>
|
||||
{% if tracker_metadata['sources'] %}
|
||||
{% for sources in tracker_metadata['sources'] %}
|
||||
<span class="badge badge-secondary">{{ sources }}</span><br>
|
||||
{% endfor %}
|
||||
|
||||
<h6>Filters:</h6>
|
||||
{% if tracker_metadata['filters'] %}
|
||||
<pre>{{ tracker_metadata['filters'] }}</pre>
|
||||
{% else %}
|
||||
<span class="badge badge-secondary">All Souces</span><br>
|
||||
<span class="badge badge-secondary">No Filters</span><br>
|
||||
{% endif %}
|
||||
|
||||
{# <h6>Sources:</h6>#}
|
||||
{# {% if tracker_metadata['sources'] %}#}
|
||||
{# {% for sources in tracker_metadata['sources'] %}#}
|
||||
{# <span class="badge badge-secondary">{{ sources }}</span><br>#}
|
||||
{# {% endfor %}#}
|
||||
{# {% else %}#}
|
||||
{# <span class="badge badge-secondary">All Sources</span><br>#}
|
||||
{# {% endif %}#}
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
|
|
@ -232,7 +236,7 @@
|
|||
</div>
|
||||
|
||||
<div class="d-flex flex-row-reverse">
|
||||
<a href="{{ url_for('hunter.delete_tracker') }}?uuid={{ tracker_metadata['uuid'] }}"
|
||||
<a href="{{ url_for('hunters.tracker_delete') }}?uuid={{ tracker_metadata['uuid'] }}"
|
||||
style="font-size: 15px">
|
||||
<button class='btn btn-danger'><i class="fas fa-trash-alt"></i></button>
|
||||
</a>
|
||||
|
|
@ -262,8 +266,9 @@
|
|||
class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd"
|
||||
name="date_from" autocomplete="off"
|
||||
{% if tracker_metadata['date_from'] %}value="{{ tracker_metadata['date_from'] }}"
|
||||
{% else %}value="{{ tracker_metadata['first_seen'] }}"{% endif %}>
|
||||
{% if tracker_metadata['date_from'] %}value="{{ tracker_metadata['date_from'][0:4] }}-{{ tracker_metadata['date_from'][4:6] }}-{{ tracker_metadata['date_from'][6:8] }}"
|
||||
{% elif tracker_metadata['first_seen'] %}value="{{ tracker_metadata['first_seen'][0:4] }}-{{ tracker_metadata['first_seen'][4:6] }}-{{ tracker_metadata['first_seen'][6:8] }}"
|
||||
{% endif %}>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
|
|
@ -272,8 +277,9 @@
|
|||
class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd"
|
||||
name="date_to" autocomplete="off"
|
||||
{% if tracker_metadata['date_to'] %}value="{{ tracker_metadata['date_to'] }}"
|
||||
{% else %}value="{{ tracker_metadata['last_seen'] }}"{% endif %}>
|
||||
{% if tracker_metadata['date_to'] %}value="{{ tracker_metadata['date_to'][0:4] }}-{{ tracker_metadata['date_to'][4:6] }}-{{ tracker_metadata['date_to'][6:8] }}"
|
||||
{% elif tracker_metadata['last_seen'] %}value="{{ tracker_metadata['last_seen'][0:4] }}-{{ tracker_metadata['last_seen'][4:6] }}-{{ tracker_metadata['last_seen'][6:8] }}"
|
||||
{% endif %}>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -285,40 +291,52 @@
|
|||
</div>
|
||||
</div>
|
||||
|
||||
{% if tracker_metadata['items'] %}
|
||||
{% if tracker_metadata['objs'] %}
|
||||
<div class="mt-4">
|
||||
<table class="table table-bordered table-hover" id="myTable_">
|
||||
<thead class="thead-dark">
|
||||
<table id="myTable_" class="table table-striped border-primary">
|
||||
<thead class="bg-dark text-white">
|
||||
<tr>
|
||||
<th>Date</th>
|
||||
<th>Item Id</th>
|
||||
<th>Type</th>
|
||||
<th></th>
|
||||
<th>Id</th>
|
||||
<th>Tags</th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
|
||||
{% for item in tracker_metadata['items'] %}
|
||||
<tr>
|
||||
<tbody style="font-size: 15px;">
|
||||
{% for object in tracker_metadata['objs'] %}
|
||||
<tr class="border-color: blue;">
|
||||
<td>
|
||||
{{ item['date'][0:4] }}/{{ item['date'][4:6] }}/{{ item['date'][6:8] }}
|
||||
{% with style=object['icon']['style'], icon=object['icon']['icon'] , color=object['icon']['color'] %}
|
||||
{% include 'objects/obj_svg_block.html' %}
|
||||
{% endwith %}
|
||||
{{ object['type']}}
|
||||
</td>
|
||||
<td>
|
||||
<a class="text-secondary" target="_blank"
|
||||
href="{{ url_for('objects_item.showItem') }}?id={{ item['id'] }}">
|
||||
<div style="line-height:0.9;">{{ item['id'] }}</div>
|
||||
{% if object['subtype'] %}
|
||||
{{ object['subtype']}}
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
<a href="{{ object['link'] }}">
|
||||
{{ object['id']}}
|
||||
</a>
|
||||
<div class="mb-2">
|
||||
{% for tag in item['tags'] %}
|
||||
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item<ags={{ tag }}">
|
||||
</td>
|
||||
<td>
|
||||
{% for tag in object['tags'] %}
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</td>
|
||||
<td class="text-right">
|
||||
{# <a href="{{ url_for('investigations_b.unregister_investigation') }}?uuid={{ tracker_metadata['uuid']}}&type={{ object['type'] }}&subtype={{ object['subtype']}}&id={{ object['id']}}">#}
|
||||
{# <button type="button" class="btn btn-danger"><i class="fas fa-trash-alt"></i></button>#}
|
||||
{# </a>#}
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
|
|
@ -331,7 +349,7 @@
|
|||
$('#div_edit_mails').hide();
|
||||
$('#div_edit_tags').hide();
|
||||
$('#div_edit_description').hide();
|
||||
$("#page-Decoded").addClass("active");
|
||||
$("#page-Tracker").addClass("active");
|
||||
|
||||
$('#date-range-from').dateRangePicker({
|
||||
separator: ' to ',
|
||||
|
|
|
|||
|
|
@ -22,8 +22,6 @@ from lib.objects import Items
|
|||
from lib import Tag
|
||||
from lib import Tracker
|
||||
|
||||
from packages import Term # TODO REMOVE ME
|
||||
|
||||
from packages import Import_helper
|
||||
|
||||
from importer.FeederImporter import api_add_json_feeder_to_queue
|
||||
|
|
@ -324,9 +322,10 @@ def get_all_tags():
|
|||
res = {'tags': Tag.get_all_tags()}
|
||||
return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # # # TRACKER # # # # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TODO
|
||||
# # # # # # # # # # # # # # TRACKER # # # # # # # # # # # # # # # # # TODO
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # TODO
|
||||
'''
|
||||
@restApi.route("api/v1/add/tracker", methods=['POST'])
|
||||
@token_required('analyst')
|
||||
def add_tracker_term():
|
||||
|
|
@ -371,11 +370,24 @@ def get_tracker_metadata_api():
|
|||
data = request.get_json()
|
||||
tracker_uuid = data.get('tracker_uuid', None)
|
||||
req_data = {'tracker_uuid': tracker_uuid}
|
||||
|
||||
tracker_uuid = request_dict.get('tracker_uuid', None)
|
||||
if not request_dict:
|
||||
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
|
||||
if not tracker_uuid:
|
||||
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
|
||||
if not is_valid_uuid_v4(tracker_uuid):
|
||||
return {"status": "error", "reason": "Invalid Tracker UUID"}, 400
|
||||
if not r_serv_tracker.exists(f'tracker:{tracker_uuid}'):
|
||||
return {'status': 'error', 'reason': 'Tracker not found'}, 404
|
||||
|
||||
res = Tracker.get_tracker_metadata_api(req_data)
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=False), mimetype='application/json'), res[1]
|
||||
|
||||
'''
|
||||
|
||||
'''
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
|
|
@ -530,27 +542,27 @@ def get_domain_status_minimal():
|
|||
res[0]['domain'] = domain
|
||||
return create_json_response(res[0], res[1])
|
||||
|
||||
@restApi.route("api/v1/get/crawled/domain/list", methods=['POST'])
|
||||
@token_required('analyst')
|
||||
def get_crawled_domain_list():
|
||||
data = request.get_json()
|
||||
res = get_mandatory_fields(data, ['date_from', 'date_to'])
|
||||
if res:
|
||||
return create_json_response(res[0], res[1])
|
||||
|
||||
date_from = data.get('date_from', None)
|
||||
date_to = data.get('date_to', None)
|
||||
domain_type = data.get('domain_type', None)
|
||||
domain_status = 'UP'
|
||||
# TODO TO MIGRATE
|
||||
raise Exception('TO MIGRATE')
|
||||
# res = Domain.api_get_domains_by_status_daterange(date_from, date_to, domain_type)
|
||||
dict_res = res[0]
|
||||
dict_res['date_from'] = date_from
|
||||
dict_res['date_to'] = date_to
|
||||
dict_res['domain_status'] = domain_status
|
||||
dict_res['domain_type'] = domain_type
|
||||
return create_json_response(dict_res, res[1])
|
||||
# @restApi.route("api/v1/get/crawled/domain/list", methods=['POST'])
|
||||
# @token_required('analyst')
|
||||
# def get_crawled_domain_list():
|
||||
# data = request.get_json()
|
||||
# res = get_mandatory_fields(data, ['date_from', 'date_to'])
|
||||
# if res:
|
||||
# return create_json_response(res[0], res[1])
|
||||
#
|
||||
# date_from = data.get('date_from', None)
|
||||
# date_to = data.get('date_to', None)
|
||||
# domain_type = data.get('domain_type', None)
|
||||
# domain_status = 'UP'
|
||||
# # TODO TO MIGRATE
|
||||
# raise Exception('TO MIGRATE')
|
||||
# # res = Domain.api_get_domains_by_status_daterange(date_from, date_to, domain_type)
|
||||
# dict_res = res[0]
|
||||
# dict_res['date_from'] = date_from
|
||||
# dict_res['date_to'] = date_to
|
||||
# dict_res['domain_status'] = domain_status
|
||||
# dict_res['domain_type'] = domain_type
|
||||
# return create_json_response(dict_res, res[1])
|
||||
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # # IMPORT # # # # # # # # # # # # # # # # # #
|
||||
|
|
|
|||
|
|
@ -8,43 +8,43 @@
|
|||
<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
|
||||
<h5 class="d-flex text-muted w-100" id="nav_title_trackers">
|
||||
<span>Trackers </span>
|
||||
<a class="ml-auto" href="{{url_for('hunter.add_tracked_menu')}}">
|
||||
<a class="ml-auto" href="{{url_for('hunters.add_tracked_menu')}}">
|
||||
<i class="fas fa-plus-circle ml-auto"></i>
|
||||
</a>
|
||||
</h5>
|
||||
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu')}}" id="nav_tracker_">
|
||||
<a class="nav-link" href="{{url_for('hunters.tracked_menu')}}" id="nav_tracker_">
|
||||
<i class="fas fa-ruler-combined"></i>
|
||||
<span>All Trackers</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu_word')}}" id="nav_tracker_word">
|
||||
<a class="nav-link" href="{{url_for('hunters.tracked_menu_word')}}" id="nav_tracker_word">
|
||||
<i class="fas fa-font"></i>
|
||||
<span>Words</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu_set')}}" id="nav_tracker_set">
|
||||
<a class="nav-link" href="{{url_for('hunters.tracked_menu_set')}}" id="nav_tracker_set">
|
||||
<i class="fas fa-layer-group"></i>
|
||||
<span>Set</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu_regex')}}" id="nav_tracker_regex">
|
||||
<a class="nav-link" href="{{url_for('hunters.tracked_menu_regex')}}" id="nav_tracker_regex">
|
||||
<i class="fas fa-drafting-compass"></i>
|
||||
<span>Regex</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu_yara')}}" id="nav_tracker_yara">
|
||||
<a class="nav-link" href="{{url_for('hunters.tracked_menu_yara')}}" id="nav_tracker_yara">
|
||||
<span class="bg-danger text-white font-weight-bold" style="font-size: 120%"> { </span>
|
||||
<span> YARA</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hunter.tracked_menu_typosquatting')}}" id="nav_tracker_typosquatting">
|
||||
<a class="nav-link" href="{{url_for('hunters.tracked_menu_typosquatting')}}" id="nav_tracker_typosquatting">
|
||||
<i class="fa fa-clone"></i>
|
||||
<span>Typo-squatting</span>
|
||||
</a>
|
||||
|
|
|
|||
396
var/www/templates/hunter/tracker_add.html
Normal file
396
var/www/templates/hunter/tracker_add.html
Normal file
|
|
@ -0,0 +1,396 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<title>AIL-Framework</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
|
||||
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'hunter/menu_sidebar.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="card my-3">
|
||||
<div class="card-header bg-dark text-white">
|
||||
<h5 class="card-title">{%if dict_tracker%}Edit a{%else%}Create a new{%endif%} Tracker</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
|
||||
<form action="{{ url_for('hunters.add_tracked_menu') }}" method='post'>
|
||||
{%if dict_tracker%}
|
||||
<input id="tracker_uuid" name="tracker_uuid" class="form-control" type="text" value="{{dict_tracker['uuid']}}" hidden>
|
||||
{%endif%}
|
||||
|
||||
<div class="row">
|
||||
<div class="col-12 col-xl-9">
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-secondary text-white"><i class="fas fa-at"></i></div>
|
||||
</div>
|
||||
<input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['mails']%}value="{{dict_tracker['mails']}}"{%endif%}{%endif%}>
|
||||
</div>
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-info text-white"><i class="fas fa-anchor"></i></div>
|
||||
</div>
|
||||
<input id="webhook" name="webhook" class="form-control" placeholder="Webhook URL" type="text" {%if dict_tracker%}{%if dict_tracker['webhook']%}value="{{dict_tracker['webhook']}}"{%endif%}{%endif%}>
|
||||
</div>
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-info text-white"><i class="fas fa-pencil-alt"></i></div>
|
||||
</div>
|
||||
<input id="description" name="description" class="form-control" placeholder="Tracker Description (optional)" type="text" {%if dict_tracker%}{%if dict_tracker['description']%}value="{{dict_tracker['description']}}"{%endif%}{%endif%}>
|
||||
</div>
|
||||
|
||||
<div class="card my-4">
|
||||
<div class="card-header bg-info text-white">
|
||||
<b>Objects to Track:</b>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
{# <div class="custom-control custom-switch mt-1">#}
|
||||
{# <input class="custom-control-input" type="checkbox" name="cve_obj" id="cve_obj" checked="">#}
|
||||
{# <label class="custom-control-label" for="cve_obj"><i class="fas fa-bug"></i> CVE</label>#}
|
||||
{# </div>#}
|
||||
{# <div class="custom-control custom-switch mt-1">#}
|
||||
{# <input class="custom-control-input" type="checkbox" name="crypto_obj" id="crypto_obj" checked="">#}
|
||||
{# <label class="custom-control-label" for="crypto_obj"><i class="fas fa-coins"></i> Cryptocurrency</label>#}
|
||||
{# </div>#}
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="decoded_obj" id="decoded_obj" checked="">
|
||||
<label class="custom-control-label" for="decoded_obj"><i class="fas fa-lock-open"></i> Decoded</label>
|
||||
</div>
|
||||
{# <div class="custom-control custom-switch mt-1">#}
|
||||
{# <input class="custom-control-input" type="checkbox" name="domain_obj" id="domain_obj" checked="">#}
|
||||
{# <label class="custom-control-label" for="domain_obj"><i class="fas fa-spider"></i> Domain</label>#}
|
||||
{# </div>#}
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="item_obj" id="item_obj" checked="">
|
||||
<label class="custom-control-label" for="item_obj"><i class="fas fa-file"></i> Item</label>
|
||||
</div>
|
||||
<div class="card border-dark mb-4" id="sources_item_div">
|
||||
<div class="card-body">
|
||||
<h5>Filter Item by sources</h5>
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-dark text-white"><i class="fas fa-folder"></i></div>
|
||||
</div>
|
||||
<input id="sources_item" class="form-control" type="text" name="sources_item" placeholder="Item Sources to track (ALL IF EMPTY)" autocomplete="off">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="pgp_obj" id="pgp_obj" checked="">
|
||||
<label class="custom-control-label" for="pgp_obj"><i class="fas fa-key"></i> PGP</label>
|
||||
</div>
|
||||
<div class="card border-dark mb-4" id="sources_pgp_div">
|
||||
<div class="card-body">
|
||||
<h6>Filter PGP by subtype:</h6>
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="filter_pgp_name" id="filter_pgp_name" checked="">
|
||||
<label class="custom-control-label" for="filter_pgp_name">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
name
|
||||
</label>
|
||||
</div>
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="filter_pgp_mail" id="filter_pgp_mail" checked="">
|
||||
<label class="custom-control-label" for="filter_pgp_mail">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
mail
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{# <div class="custom-control custom-switch mt-1">#}
|
||||
{# <input class="custom-control-input" type="checkbox" name="level" id="screenshot_obj" checked="">#}
|
||||
{# <label class="custom-control-label" for="screenshot_obj"><i class="fas fa-image"></i> Screenshot</label>#}
|
||||
{# </div>#}
|
||||
{# <div class="custom-control custom-switch mt-1">#}
|
||||
{# <input class="custom-control-input" type="checkbox" name="level" id="username_obj" checked="">#}
|
||||
{# <label class="custom-control-label" for="username_obj"><i class="fas fa-user"></i> Username</label>#}
|
||||
{# </div>#}
|
||||
|
||||
{# <li class="list-group-item text-left">#}
|
||||
{# <div class="form-check">#}
|
||||
{# <input class="form-check-input" type="checkbox" value="True" id="CveCheck" name="CveCheck" checked="">#}
|
||||
{# <label class="form-check-label" for="CveCheck"><i class="fas fa-bug"></i> Cve</label>#}
|
||||
{# </div>#}
|
||||
{# <div class="form-check">#}
|
||||
{# <input class="form-check-input" type="checkbox" value="True" id="CryptocurrencyCheck" name="CryptocurrencyCheck" checked="">#}
|
||||
{# <label class="form-check-label" for="CryptocurrencyCheck"><i class="fas fa-coins"></i> Cryptocurrency</label>#}
|
||||
{# </div>#}
|
||||
{# <div class="form-check">#}
|
||||
{# <input class="form-check-input" type="checkbox" value="True" id="DecodedCheck" name="DecodedCheck" checked="">#}
|
||||
{# <label class="form-check-label" for="DecodedCheck"><i class="fas fa-lock-open"></i> Decoded</label>#}
|
||||
{# </div>#}
|
||||
{# <div class="form-check">#}
|
||||
{# <input class="form-check-input" type="checkbox" value="True" id="ScreenshotCheck" name="ScreenshotCheck" checked="">#}
|
||||
{# <label class="form-check-label" for="ScreenshotCheck"><i class="fas fa-image"></i> Screenshot</label>#}
|
||||
{# </div>#}
|
||||
{# <div class="form-check">#}
|
||||
{# <input class="form-check-input" type="checkbox" value="True" id="PgpCheck" name="PgpCheck" checked="">#}
|
||||
{# <label class="form-check-label" for="PgpCheck"><i class="fas fa-key"></i> PGP</label>#}
|
||||
{# </div>#}
|
||||
{# <div class="form-check">#}
|
||||
{# <input class="form-check-input" type="checkbox" value="True" id="UsernameCheck" name="UsernameCheck" checked="">#}
|
||||
{# <label class="form-check-label" for="UsernameCheck"><i class="fas fa-user"></i> Username</label>#}
|
||||
{# </div>#}
|
||||
{# <div class="form-check">#}
|
||||
{# <input class="form-check-input" type="checkbox" value="True" id="DomainCheck" name="DomainCheck" checked="">#}
|
||||
{# <label class="form-check-label" for="DomainCheck"><i class="fas fa-spider"></i> Domain</label>#}
|
||||
{# </div>#}
|
||||
{# <div class="form-check">#}
|
||||
{# <input class="form-check-input" type="checkbox" value="True" id="ItemCheck" name="ItemCheck" checked="">#}
|
||||
{# <label class="form-check-label" for="ItemCheck"><i class="fas fa-user"></i> Item</label>#}
|
||||
{# </div>#}
|
||||
{##}
|
||||
{# </li>#}
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="card my-4">
|
||||
<div class="card-header bg-secondary text-white">
|
||||
<b>Tags</b>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
|
||||
</div>
|
||||
<input id="tags" name="tags" class="form-control" placeholder="Custom Tags (optional, space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tags']%}value="{{dict_tracker['tags']}}"{%endif%}{%endif%}>
|
||||
</div>
|
||||
{% include 'tags/block_tags_selector.html' %}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class="col-12 col-xl-3">
|
||||
<div class="custom-control custom-switch mt-1">
|
||||
<input class="custom-control-input" type="checkbox" name="level" id="id_level" {%if dict_tracker%}{%if dict_tracker['level']==1%}checked{%endif%}{%else%}checked{%endif%}>
|
||||
<label class="custom-control-label" for="id_level">
|
||||
<i class="fas fa-users"></i> Show tracker to all Users
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<hr>
|
||||
<h4>Tracker Type:</h4>
|
||||
|
||||
<select id="tracker_type" name="tracker_type" class="custom-select w-25 mb-3">
|
||||
<option disabled selected value> -- Select a tracker type -- </option>
|
||||
<option value="word">Word</option>
|
||||
<option value="set">Set</option>
|
||||
<option value="regex">Regex</option>
|
||||
<option value="yara">YARA rule</option>
|
||||
<option value="typosquatting">Typo-squatting</option>
|
||||
</select>
|
||||
|
||||
<p id="tracker_desc">Terms to track (space separated)</p>
|
||||
|
||||
<div class="row" id="simple_input">
|
||||
<div class="col-12 col-lg-10">
|
||||
<input id="tracker" name="tracker" class="form-control" placeholder="Terms to track (space separated)" type="text" {%if dict_tracker%}{%if dict_tracker['tracker']!='yara'%}value="{{dict_tracker['tracker']}}"{%endif%}{%endif%}>
|
||||
</div>
|
||||
<div class="col-12 col-lg-2">
|
||||
<input type="number" id="nb_word" name="nb_word" name="quantity" min="1" placeholder="Nb of keywords" {%if dict_tracker%}{%if dict_tracker['nb_words']%}value="{{dict_tracker['nb_words']}}"{%endif%}{%endif%}>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="mb-2" id="yara_rule">
|
||||
<div class="" id="yara_default_rules">
|
||||
|
||||
<h6>Default YARA rules:</h6>
|
||||
<select class="custom-select w-100 mb-3" id="yara_default_rule" name="yara_default_rule" onchange="get_default_rule_content(this);">
|
||||
<option selected>Select a default rule</option>
|
||||
{% for yara_types in all_yara_files %}
|
||||
{% for yara_file_name in all_yara_files[yara_types] %}
|
||||
<option value="{{yara_types}}/{{yara_file_name}}">{{yara_types}} - {{yara_file_name}}</option>
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
</select>
|
||||
|
||||
<pre class="border bg-light" id="default_yara_rule_content"></pre>
|
||||
|
||||
</div>
|
||||
|
||||
<hr>
|
||||
|
||||
<h6>Custom YARA rules:</h6>
|
||||
<div class="row" id="textarea">
|
||||
<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5">{%if dict_tracker%}{%if dict_tracker['type']=='yara' and dict_tracker['content']%}{{dict_tracker['content']}}{%endif%}{%endif%}</textarea>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<br>
|
||||
<button class="btn btn-success mt-2">
|
||||
<i class="fas fa-plus"></i> {%if dict_tracker%}Edit{%else%}Create{%endif%} Tracker
|
||||
</button>
|
||||
|
||||
</form>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</body>
|
||||
|
||||
<script>
|
||||
var ltags;
|
||||
var chart = {};
|
||||
$(document).ready(function(){
|
||||
$("#page-Tracker").addClass("active");
|
||||
$("#nav_manual_crawler").addClass("active");
|
||||
$("#tracker_desc").hide();
|
||||
$("#tracker").hide();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").hide();
|
||||
|
||||
|
||||
sources_item = $('#sources_item').tagSuggest({
|
||||
data: {{all_sources|safe}},
|
||||
{%if dict_tracker%}{%if dict_tracker['item_sources']%}value: {{dict_tracker['item_sources']|safe}},{%endif%}{%endif%}
|
||||
sortOrder: 'name',
|
||||
maxDropHeight: 200,
|
||||
name: 'sources_item',
|
||||
emptyText: 'Item Sources to track (ALL IF EMPTY)',
|
||||
});
|
||||
|
||||
$('#tracker_type').on('change', function() {
|
||||
var tracker_type = this.value;
|
||||
if (tracker_type=="word") {
|
||||
$("#tracker_desc").text("Token to track. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
|
||||
$("#tracker_desc").show();
|
||||
$("#tracker").show();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").hide();
|
||||
} else if (tracker_type=="set") {
|
||||
$("#tracker_desc").text("Set of Terms to track (space separated). This tracker is used to check if an item contain one or more terms specified in a set. If an item contain NB unique terms (by default NB of unique keywords = 1), this tracker is triggered. You need to use a regex if you want to use one of the following special characters [<>~!?@#$%^&*|()_-+={}\":;,.\'\n\r\t]/\\ ");
|
||||
$("#tracker_desc").show();
|
||||
$("#tracker").show();
|
||||
$("#nb_word").show();
|
||||
$("#yara_rule").hide();
|
||||
} else if (tracker_type=="regex") {
|
||||
$("#tracker_desc").text("Enter a valid Python regex");
|
||||
$("#tracker_desc").show();
|
||||
$("#tracker").show();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").hide();
|
||||
} else if (tracker_type=="yara") {
|
||||
$("#tracker_desc").text("Select a default yara rule or create your own rule:");
|
||||
$("#tracker_desc").show();
|
||||
$("#tracker").hide();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").show();
|
||||
} else if (tracker_type=="typosquatting") {
|
||||
$("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time.");
|
||||
$("#tracker_desc").show();
|
||||
$("#tracker").show();
|
||||
$("#nb_word").hide();
|
||||
$("#yara_rule").hide();
|
||||
}
|
||||
});
|
||||
|
||||
{%if dict_tracker%}
|
||||
$('#tracker_type').val('{{dict_tracker['type']}}').change();
|
||||
|
||||
{%if dict_tracker['type']=='yara' and dict_tracker['yara_file']%}
|
||||
$('#yara_default_rule').val('{{dict_tracker['yara_file']}}').change();
|
||||
{%endif%}
|
||||
{%endif%}
|
||||
|
||||
$('#item_obj').on("change", function () {
|
||||
item_source_input_controller();
|
||||
});
|
||||
$('#pgp_obj').on("change", function () {
|
||||
pgp_source_input_controller();
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
|
||||
function item_source_input_controller() {
|
||||
if($('#item_obj').is(':checked')){
|
||||
$("#sources_item_div").show();
|
||||
}else{
|
||||
$("#sources_item_div").hide();
|
||||
}
|
||||
}
|
||||
|
||||
function pgp_source_input_controller() {
|
||||
if($('#pgp_obj').is(':checked')){
|
||||
$("#sources_pgp_div").show();
|
||||
}else{
|
||||
$("#sources_pgp_div").hide();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
function get_default_rule_content(selector){
|
||||
var yara_name = selector.value
|
||||
if (yara_name === "Select a default rule") {
|
||||
jQuery("#default_yara_rule_content").text("")
|
||||
} else {
|
||||
$.getJSON("{{ url_for('hunter.get_default_yara_rule_content') }}?rule_name=" + yara_name,
|
||||
function(data) {
|
||||
jQuery("#default_yara_rule_content").text(data['content'])
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
</script>
|
||||
|
|
@ -71,11 +71,11 @@
|
|||
<td>
|
||||
<span>
|
||||
<a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}">
|
||||
{% if dict_uuid['tracker'] %}
|
||||
{% if dict_uuid['tracker']|length > 256 %}
|
||||
{{ dict_uuid['tracker'][0:256] }}...
|
||||
{% if dict_uuid['tracked'] %}
|
||||
{% if dict_uuid['tracked']|length > 256 %}
|
||||
{{ dict_uuid['tracked'][0:256] }}...
|
||||
{% else %}
|
||||
{{ dict_uuid['tracker'] }}
|
||||
{{ dict_uuid['tracked'] }}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</a>
|
||||
|
|
@ -136,11 +136,11 @@
|
|||
<td>
|
||||
<span>
|
||||
<a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}">
|
||||
{% if dict_uuid['tracker'] %}
|
||||
{% if dict_uuid['tracker']|length > 256 %}
|
||||
{{ dict_uuid['tracker'][0:256] }}...
|
||||
{% if dict_uuid['tracked'] %}
|
||||
{% if dict_uuid['tracked']|length > 256 %}
|
||||
{{ dict_uuid['tracked'][0:256] }}...
|
||||
{% else %}
|
||||
{{ dict_uuid['tracker'] }}
|
||||
{{ dict_uuid['tracked'] }}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</a>
|
||||
|
|
@ -178,7 +178,7 @@
|
|||
</div>
|
||||
|
||||
|
||||
<a class="btn btn-info my-4" href="{{ url_for('hunter.add_tracked_menu') }}">
|
||||
<a class="btn btn-info my-4" href="{{ url_for('hunters.add_tracked_menu') }}">
|
||||
<i class="fas fa-plus-circle ml-auto"></i>
|
||||
Create New Tracker
|
||||
</a>
|
||||
174
var/www/templates/hunter/trackers_dashboard.html
Normal file
174
var/www/templates/hunter/trackers_dashboard.html
Normal file
|
|
@ -0,0 +1,174 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Trackers - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
|
||||
<script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3/sparklines.js') }}"></script>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'hunter/menu_sidebar.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="d-flex justify-content-around my-2">
|
||||
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu') }}">
|
||||
<div class="card-body">
|
||||
<h5 class="card-title"><i class="fas fa-ruler-combined"></i>
|
||||
All Trackers <span class="badge badge-light">{{ stats['all'] }}</span>
|
||||
</h5>
|
||||
</div>
|
||||
</a>
|
||||
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_word') }}">
|
||||
<div class="card-body">
|
||||
<h5 class="card-title"><i class="fas fa-font"></i>
|
||||
Words <span class="badge badge-light">{{ stats['word'] }}</span>
|
||||
</h5>
|
||||
</div>
|
||||
</a>
|
||||
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_set') }}">
|
||||
<div class="card-body">
|
||||
<h5 class="card-title"><i class="fas fa-layer-group"></i>
|
||||
Set <span class="badge badge-light">{{ stats['set'] }}</span>
|
||||
</h5>
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div class="d-flex justify-content-around my-2">
|
||||
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_regex') }}">
|
||||
<div class="card-body">
|
||||
<h5 class="card-title">
|
||||
<i class="fas fa-drafting-compass"></i>
|
||||
Regex
|
||||
<span class="badge badge-light">{{ stats['regex'] }}</span>
|
||||
</h5>
|
||||
</div>
|
||||
</a>
|
||||
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_typosquatting') }}">
|
||||
<div class="card-body">
|
||||
<h5 class="card-title">
|
||||
<i class="fa fa-clone"></i>
|
||||
Typo Squatting
|
||||
<span class="badge badge-light">{{ stats['typosquatting'] }}</span>
|
||||
</h5>
|
||||
</div>
|
||||
</a>
|
||||
<a class="btn btn-info border-secondary" style="width: 20rem;" href="{{ url_for('hunters.tracked_menu_yara') }}">
|
||||
<div class="card-body">
|
||||
<h5 class="card-title">
|
||||
<span class="bg-danger text-white font-weight-bold" style="font-size: 120%"> { </span>
|
||||
Yara
|
||||
<span class="badge badge-light">{{ stats['yara'] }}</span>
|
||||
</h5>
|
||||
</div>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
{# TODO Buttons#}
|
||||
|
||||
<div class="card my-3">
|
||||
<div class="card-header bg-dark text-white">
|
||||
<h5 class="card-title"><b>Trackers Last Matches</b></h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
<table id="table_global_trackers" class="table table-striped table-borderless table-condensed">
|
||||
<tbody style="font-size: 15px;">
|
||||
{% for meta in trackers %}
|
||||
<tr>
|
||||
<td>{{ meta['type'] }}</td>
|
||||
<td>
|
||||
<span>
|
||||
<a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ meta['uuid'] }}">
|
||||
{% if meta['tracked'] %}
|
||||
{% if meta['tracked']|length > 256 %}
|
||||
{{ meta['tracked'][0:256] }}...
|
||||
{% else %}
|
||||
{{ meta['tracked'] }}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</a>
|
||||
</span>
|
||||
</td>
|
||||
<td>
|
||||
{% for tag in meta['tags'] %}
|
||||
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item<ags={{ tag }}">
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</td>
|
||||
<td>
|
||||
{% if meta['first_seen'] %}
|
||||
{{ meta['first_seen'][0:4] }}/{{ meta['first_seen'][4:6] }}/{{ meta['first_seen'][6:8] }}
|
||||
{% endif %}
|
||||
</td>
|
||||
<td>
|
||||
{{ meta['timestamp'] }}
|
||||
</td>
|
||||
{# <td id="sparklines_{{ dict_uuid['uuid'] }}" style="text-align:center;"></td>#}
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<a class="btn btn-primary my-4" href="{{ url_for('hunters.add_tracked_menu') }}">
|
||||
<i class="fas fa-plus-circle ml-auto"></i>
|
||||
Create New Tracker
|
||||
</a>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
$(document).ready(function () {
|
||||
$("#page-Tracker").addClass("active");
|
||||
$("#nav_tracker_{{tracker_type}}").addClass("active");
|
||||
|
||||
{# {% for dict_uuid in global_trackers %}#}
|
||||
{# sparkline("sparklines_{{ dict_uuid['uuid'] }}", {{ dict_uuid['sparkline'] }}, {height: 40});#}
|
||||
{# {% endfor %}#}
|
||||
|
||||
});
|
||||
|
||||
function toggle_sidebar() {
|
||||
if ($('#nav_menu').is(':visible')) {
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
} else {
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -19,7 +19,7 @@
|
|||
<a class="nav-link" id="navbar-tags" href="{{ url_for('tags_ui.tags_search_items') }}" aria-disabled="true"><i class="fas fa-tag"></i> Tags</a>
|
||||
</li>
|
||||
<li class="nav-item mr-3">
|
||||
<a class="nav-link" id="page-Tracker" href="{{ url_for('hunter.tracked_menu') }}" aria-disabled="true"><i class="fas fa-crosshairs"></i> Leaks Hunter</a>
|
||||
<a class="nav-link" id="page-Tracker" href="{{ url_for('hunters.trackers_dashboard') }}" aria-disabled="true"><i class="fas fa-crosshairs"></i> Leaks Hunter</a>
|
||||
</li>
|
||||
<li class="nav-item mr-3">
|
||||
<a class="nav-link" id="page-Crawler" href="{{ url_for('crawler_splash.crawlers_dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>
|
||||
|
|
|
|||
Loading…
Reference in a new issue