ail-framework/bin/lib/Tracker.py

2042 lines
73 KiB
Python
Executable file

#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import json
import os
import logging
import logging.config
import re
import sys
import time
import uuid
import yara
import datetime
import base64
import math
from collections import defaultdict
from markupsafe import escape
from textblob import TextBlob
from nltk.tokenize import RegexpTokenizer
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from packages import Date
from lib.ail_core import get_objects_tracked, get_object_all_subtypes, get_objects_retro_hunted
from lib import ail_logger
from lib import ail_orgs
from lib import ConfigLoader
from lib import item_basic
from lib import Tag
# LOGS
logging.config.dictConfig(ail_logger.get_config(name='modules'))
logger = logging.getLogger()
config_loader = ConfigLoader.ConfigLoader()
r_cache = config_loader.get_redis_conn("Redis_Cache")
r_tracker = config_loader.get_db_conn("Kvrocks_Trackers")
config_loader = None
# NLTK tokenizer
TOKENIZER = None
def init_tokenizer():
global TOKENIZER
TOKENIZER = RegexpTokenizer('[\&\~\:\;\,\.\(\)\{\}\|\[\]\\\\/\-/\=\'\"\%\$\?\@\+\#\_\^\<\>\!\*\n\r\t\s]+',
gaps=True, discard_empty=True)
def get_special_characters():
special_characters = set('[<>~!?@#$%^&*|()_-+={}":;,.\'\n\r\t]/\\')
special_characters.add('\\s')
return special_characters
###############
#### UTILS ####
def is_valid_uuid_v4(curr_uuid):
if not curr_uuid:
return False
curr_uuid = curr_uuid.replace('-', '')
try:
uuid_test = uuid.UUID(hex=curr_uuid, version=4)
return uuid_test.hex == curr_uuid
except:
return False
def is_valid_regex(tracker_regex):
try:
re.compile(tracker_regex)
return True
except:
return False
def is_valid_mail(email):
email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
email_regex = re.compile(email_regex)
result = email_regex.match(email)
if result:
return True
else:
return False
def verify_mail_list(mail_list):
for mail in mail_list:
if not is_valid_mail(mail):
return {'status': 'error', 'reason': 'Invalid email', 'value': mail}, 400
return None
## -- UTILS -- ##
#################
###################
#### TRACKER ####
class Tracker:
def __init__(self, tracker_uuid):
self.uuid = tracker_uuid
def get_uuid(self):
return self.uuid
def exists(self):
return r_tracker.exists(f'tracker:{self.uuid}')
def _get_field(self, field):
return r_tracker.hget(f'tracker:{self.uuid}', field)
def _set_field(self, field, value):
r_tracker.hset(f'tracker:{self.uuid}', field, value)
def get_date(self):
return self._get_field('date')
def get_last_change(self, r_str=False):
last_change = self._get_field('last_change')
if r_str and last_change:
last_change = datetime.datetime.fromtimestamp(float(last_change)).strftime('%Y-%m-%d %H:%M:%S')
return last_change
def get_first_seen(self):
return self._get_field('first_seen')
def get_last_seen(self):
return self._get_field('last_seen')
def _set_first_seen(self, date):
self._set_field('first_seen', date)
def _set_last_seen(self, date):
self._set_field('last_seen', date)
def _exist_date(self, date):
return r_tracker.exists(f'tracker:objs:{self.uuid}:{date}')
# TODO: ADD CACHE ???
def update_daterange(self, date=None):
first_seen = self.get_first_seen()
# Added Object
if date:
date = int(date)
first_seen = self.get_first_seen()
# if op == 'add':
if not first_seen:
self._set_first_seen(date)
self._set_last_seen(date)
else:
first_seen = int(first_seen)
last_seen = int(self.get_last_seen())
if date < first_seen:
self._set_first_seen(date)
if date > last_seen:
self._set_last_seen(date)
else:
last_seen = self.get_last_seen()
if first_seen and last_seen:
valid_first_seen = self._exist_date(first_seen)
valid_last_seen = self._exist_date(last_seen)
# update first seen
if not valid_first_seen:
for date in Date.get_daterange(first_seen, last_seen):
if self._exist_date(date):
self._set_first_seen(date)
valid_first_seen = True
break
# update last seen
if not valid_last_seen:
for date in reversed(Date.get_daterange(first_seen, last_seen)):
if self._exist_date(date):
self._set_first_seen(date)
valid_last_seen = True
break
if not valid_first_seen or not valid_last_seen:
r_tracker.hdel(f'tracker:{self.uuid}', 'first_seen')
r_tracker.hdel(f'tracker:{self.uuid}', 'last_seen')
def get_description(self):
return self._get_field('description')
## LEVEL ##
def get_level(self):
level = int(self._get_field('level'))
if not level:
level = 0
return int(level)
def set_level(self, level, org_uuid):
tracker_type = self.get_type()
if level == 0: # user only
user_id = self.get_user()
r_tracker.sadd(f'user:tracker:{user_id}', self.uuid)
r_tracker.sadd(f'user:tracker:{user_id}:{tracker_type}', self.uuid)
elif level == 1: # global
r_tracker.sadd('global:tracker', self.uuid)
r_tracker.sadd(f'global:tracker:{tracker_type}', self.uuid)
elif level == 2: # org only
r_tracker.sadd(f'org:tracker:{org_uuid}', self.uuid)
r_tracker.sadd(f'org:tracker:{org_uuid}:{tracker_type}', self.uuid)
self.add_to_org(org_uuid)
self._set_field('level', level)
def reset_level(self, old_level, new_level, new_org_uuid):
if old_level == 0:
user_id = self.get_user()
r_tracker.srem(f'user:tracker:{user_id}', self.uuid)
r_tracker.srem(f'user:tracker:{user_id}:{self.get_type()}', self.uuid)
elif old_level == 1:
r_tracker.srem('global:tracker', self.uuid)
r_tracker.srem(f'global:tracker:{self.get_type()}', self.uuid)
# Org
elif old_level == 2:
old_org = self.get_org()
r_tracker.srem(f'org:tracker:{old_org}', self.uuid)
r_tracker.srem(f'org:tracker:{old_org}:{self.get_type()}', self.uuid)
ail_orgs.remove_obj_to_org(old_org, 'tracker', self.uuid)
self.set_level(new_level, new_org_uuid)
def check_level(self, user_org, user_id):
level = self.get_level()
if level == 1:
return True
elif level == 0:
return self.get_user() == user_id
elif level == 2:
return self.get_org() == user_org
def is_level_user(self):
return self.get_level() == 0
def is_level_org(self):
return self.get_level() == 2
def is_level_global(self):
return self.get_level() == 1
## ORG ##
def get_creator_org(self):
return self._get_field('creator_org')
def get_org(self):
return self._get_field('org')
def add_to_org(self, org_uuid):
self._set_field('org', org_uuid)
ail_orgs.add_obj_to_org(org_uuid, 'tracker', self.uuid)
## -ORG- ##
def get_filters(self):
filters = self._get_field('filters')
if not filters:
return {}
else:
return json.loads(filters)
def set_filters(self, filters):
if filters:
self._set_field('filters', json.dumps(filters))
def del_filters(self, tracker_type, to_track):
filters = self.get_filters()
if not filters:
filters = get_objects_tracked()
for obj_type in filters:
r_tracker.srem(f'trackers:objs:{tracker_type}:{obj_type}', to_track)
r_tracker.srem(f'trackers:uuid:{tracker_type}:{to_track}', f'{self.uuid}:{obj_type}')
r_tracker.hdel(f'tracker:{self.uuid}', 'filters')
def get_tracked(self):
return self._get_field('tracked')
def get_type(self):
return self._get_field('type')
def get_tags(self):
return r_tracker.smembers(f'tracker:tags:{self.uuid}')
def _set_tags(self, tags):
for tag in tags:
r_tracker.sadd(f'tracker:tags:{self.uuid}', tag)
Tag.create_custom_tag(tag) # TODO CUSTOM TAGS
def _del_tags(self):
return r_tracker.delete(f'tracker:tags:{self.uuid}')
def mail_export(self):
return r_tracker.exists(f'tracker:mail:{self.uuid}')
def get_mails(self):
return r_tracker.smembers(f'tracker:mail:{self.uuid}')
def _set_mails(self, mails):
for mail in mails:
r_tracker.sadd(f'tracker:mail:{self.uuid}', escape(mail))
def _del_mails(self):
r_tracker.delete(f'tracker:mail:{self.uuid}')
def get_user(self):
return self._get_field('user_id')
def webhook_export(self):
webhook = self.get_webhook()
return webhook is not None and webhook
def get_webhook(self):
return r_tracker.hget(f'tracker:{self.uuid}', 'webhook')
def get_sparkline(self, nb_day=6):
date_range_sparkline = Date.get_date_range(nb_day)
sparkline = []
for date in date_range_sparkline:
nb_seen_this_day = self.get_nb_objs_by_date(date)
if nb_seen_this_day is None:
nb_seen_this_day = 0
sparkline.append(int(nb_seen_this_day))
return sparkline
def get_rule(self):
yar_path = self.get_tracked()
return yara.compile(filepath=os.path.join(get_yara_rules_dir(), yar_path))
def get_meta(self, options):
if not options:
options = set()
meta = {'uuid': self.uuid,
'tracked': self.get_tracked(), # TODO TO CHECK
'type': self.get_type(),
'date': self.get_date(),
'first_seen': self.get_first_seen(),
'last_seen': self.get_last_seen()}
if 'org' in options:
meta['org'] = self.get_org()
if 'org_name' in options:
meta['org_name'] = ail_orgs.Organisation(self.get_org()).get_name()
if 'user' in options:
meta['user'] = self.get_user()
if 'level' in options:
meta['level'] = self.get_level()
if 'description' in options:
meta['description'] = self.get_description()
if 'nb_objs' in options:
meta['nb_objs'] = self.get_nb_objs()
if 'tags' in options:
meta['tags'] = self.get_tags()
if 'filters' in options:
meta['filters'] = self.get_filters()
if 'mails' in options:
meta['mails'] = self.get_mails()
if 'webhooks' in options:
meta['webhook'] = self.get_webhook()
if 'sparkline' in options:
meta['sparkline'] = self.get_sparkline(6)
return meta
def _add_to_dashboard(self, obj_type, subtype, obj_id):
mess = f'{self.uuid}:{int(time.time())}:{obj_type}:{subtype}:{obj_id}'
if self.is_level_user():
user = self.get_user()
r_tracker.lpush(f'trackers:user:{user}', mess)
r_tracker.ltrim(f'trackers:user:{user}', 0, 9)
else:
r_tracker.lpush('trackers:dashboard', mess)
r_tracker.ltrim(f'trackers:dashboard', 0, 9)
def get_nb_objs_by_type(self, obj_type):
return r_tracker.scard(f'tracker:objs:{self.uuid}:{obj_type}')
def get_objs_by_type(self, obj_type):
return r_tracker.smembers(f'tracker:objs:{self.uuid}:{obj_type}')
def get_nb_objs(self):
objs = {}
for obj_type in get_objects_tracked():
nb = self.get_nb_objs_by_type(obj_type)
if nb:
objs[obj_type] = nb
return objs
def get_objs(self):
objs = []
for obj_type in get_objects_tracked():
for obj in self.get_objs_by_type(obj_type):
subtype, obj_id = obj.split(':', 1)
objs.append((obj_type, subtype, obj_id))
return objs
def get_nb_objs_by_date(self, date):
return r_tracker.scard(f'tracker:objs:{self.uuid}:{date}')
def get_objs_by_date(self, date):
return r_tracker.smembers(f'tracker:objs:{self.uuid}:{date}')
def get_objs_by_daterange(self, date_from, date_to):
objs = set()
for date in Date.get_daterange(date_from, date_to):
objs |= self.get_objs_by_date(date)
return objs
def get_obj_dates(self, obj_type, subtype, obj_id):
return r_tracker.smembers(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}')
# - TODO Data Retention TO Implement - #
# Or Daily/Monthly Global DB Cleanup:
# Iterate on each tracker:
# Iterate on each Obj:
# Iterate on each date:
# Delete from tracker range if date limit exceeded
# - TODO
def add(self, obj_type, subtype, obj_id, date=None):
if not subtype:
subtype = ''
if not date:
date = Date.get_today_date_str()
new_obj_date = r_tracker.sadd(f'tracker:objs:{self.uuid}:{date}', f'{obj_type}:{subtype}:{obj_id}')
r_tracker.sadd(f'obj:trackers:{obj_type}:{subtype}:{obj_id}', self.uuid)
# Only save object match date - Needed for the DB Cleaner
r_tracker.sadd(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}', date)
r_tracker.sadd(f'tracker:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}')
if new_obj_date:
self.update_daterange(date)
self._add_to_dashboard(obj_type, subtype, obj_id)
def remove(self, obj_type, subtype, obj_id):
if not subtype:
subtype = ''
for date in self.get_obj_dates(obj_type, subtype, obj_id):
r_tracker.srem(f'tracker:objs:{self.uuid}:{date}', f'{obj_type}:{subtype}:{obj_id}')
r_tracker.srem(f'obj:tracker:{obj_type}:{subtype}:{obj_id}:{self.uuid}', date)
r_tracker.srem(f'obj:trackers:{obj_type}:{subtype}:{obj_id}', self.uuid)
r_tracker.srem(f'tracker:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}')
self.update_daterange()
# TODO escape custom tags
# TODO escape mails ????
def create(self, tracker_type, to_track, org, user_id, level, description=None, filters={}, tags=[], mails=[], webhook=None):
if self.exists():
raise Exception('Error: Tracker already exists')
# YARA
if tracker_type == 'yara_custom' or tracker_type == 'yara_default':
to_track = save_yara_rule(tracker_type, to_track, tracker_uuid=self.uuid)
tracker_type = 'yara'
elif tracker_type == 'typosquatting':
from ail_typo_squatting import runAll
domain = to_track.split(" ")[0]
typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False) # TODO REPLACE LIMIT BY -1
for typo in typo_generation:
r_tracker.sadd(f'tracker:typosquatting:{to_track}', typo)
# create metadata
self._set_field('tracked', to_track)
self._set_field('type', tracker_type)
self._set_field('date', datetime.date.today().strftime("%Y%m%d"))
self._set_field('creator_org', org)
self._set_field('user_id', user_id)
if description:
self._set_field('description', escape(description))
if webhook:
self._set_field('webhook', webhook)
# create all tracker set
r_tracker.sadd(f'all:tracker:{tracker_type}', to_track)
# create tracker - uuid map
r_tracker.sadd(f'all:tracker_uuid:{tracker_type}:{to_track}', self.uuid)
r_tracker.sadd('trackers:all', self.uuid)
r_tracker.sadd(f'trackers:all:{tracker_type}', self.uuid)
# TRACKER LEVEL
self.set_level(level, org)
# create tracker tags list
if tags:
self._set_tags(tags)
# create tracker mail notification list
if mails:
self._set_mails(mails)
# Filters
if not filters:
filters = {}
for obj_type in get_objects_tracked():
filters[obj_type] = {}
else:
self.set_filters(filters)
for obj_type in filters:
r_tracker.sadd(f'trackers:objs:{tracker_type}:{obj_type}', to_track)
r_tracker.sadd(f'trackers:uuid:{tracker_type}:{to_track}', f'{self.uuid}:{obj_type}')
self._set_field('last_change', time.time())
# toggle refresh module tracker list/set
trigger_trackers_refresh(tracker_type)
return self.uuid
def edit(self, tracker_type, to_track, level, org, description=None, filters={}, tags=[], mails=[], webhook=None):
# edit tracker
old_type = self.get_type()
old_to_track = self.get_tracked()
old_level = self.get_level()
user_id = self.get_user()
# YARA
if tracker_type == 'yara_custom' or tracker_type == 'yara_default':
# create yara rule
if tracker_type == 'yara_default' and old_type == 'yara':
if not is_default_yara_rule(old_to_track):
filepath = get_yara_rule_file_by_tracker_name(old_to_track)
if filepath:
os.remove(filepath)
to_track = save_yara_rule(tracker_type, to_track, tracker_uuid=self.uuid)
tracker_type = 'yara'
# TODO TYPO EDIT
elif tracker_type == 'typosquatting':
pass
if tracker_type != old_type:
# LEVEL
self.reset_level(old_level, level, org)
# Delete OLD YARA Rule File
if old_type == 'yara':
if not is_default_yara_rule(old_to_track):
filepath = get_yara_rule_file_by_tracker_name(old_to_track)
if filepath:
os.remove(filepath)
if old_type == 'typosquatting':
r_tracker.delete(f'tracker:typosquatting:{old_to_track}')
self._set_field('type', tracker_type)
# create all tracker set
r_tracker.srem(f'all:tracker:{old_type}', old_to_track)
r_tracker.sadd(f'all:tracker:{tracker_type}', to_track)
# create tracker - uuid map
r_tracker.srem(f'all:tracker_uuid:{old_type}:{old_to_track}', self.uuid)
r_tracker.sadd(f'all:tracker_uuid:{tracker_type}:{to_track}', self.uuid)
# create all tracker set by type
r_tracker.srem(f'trackers:all:{old_type}', self.uuid)
r_tracker.sadd(f'trackers:all:{tracker_type}', self.uuid)
# Same Type
# LEVEL
self.reset_level(old_level, level, org)
# To Track Edited
if to_track != old_to_track:
self._set_field('tracked', to_track)
self._set_field('description', description)
self._set_field('webhook', webhook)
# Tags
nb_old_tags = r_tracker.scard(f'tracker:tags:{self.uuid}')
if nb_old_tags > 0 or tags:
self._del_tags()
self._set_tags(tags)
# Mails
nb_old_mails = r_tracker.scard(f'tracker:mail:{self.uuid}')
if nb_old_mails > 0 or mails:
self._del_mails()
self._set_mails(mails)
# Filters
self.del_filters(old_type, old_to_track)
if not filters:
filters = {}
for obj_type in get_objects_tracked():
filters[obj_type] = {}
else:
self.set_filters(filters)
for obj_type in filters:
r_tracker.sadd(f'trackers:objs:{tracker_type}:{obj_type}', to_track)
r_tracker.sadd(f'trackers:uuid:{tracker_type}:{to_track}', f'{self.uuid}:{obj_type}')
self._set_field('last_change', time.time())
# Refresh Trackers
trigger_trackers_refresh(tracker_type)
if tracker_type != old_type:
trigger_trackers_refresh(old_type)
return self.uuid
def delete(self):
for obj in self.get_objs():
self.remove(obj[0], obj[1], obj[2])
tracker_type = self.get_type()
tracked = self.get_tracked()
if tracker_type == 'typosquatting':
r_tracker.delete(f'tracker:typosquatting:{tracked}')
elif tracker_type == 'yara':
if not is_default_yara_rule(tracked):
filepath = get_yara_rule_file_by_tracker_name(tracked)
if filepath:
os.remove(filepath)
# Filters
filters = get_objects_tracked()
for obj_type in filters:
r_tracker.srem(f'trackers:objs:{tracker_type}:{obj_type}', tracked)
r_tracker.srem(f'trackers:uuid:{tracker_type}:{tracked}', f'{self.uuid}:{obj_type}')
self._del_mails()
self._del_tags()
level = self.get_level()
if level == 0: # user only
user = self.get_user()
r_tracker.srem(f'user:tracker:{user}', self.uuid)
r_tracker.srem(f'user:tracker:{user}:{tracker_type}', self.uuid)
elif level == 1: # global
r_tracker.srem('global:tracker', self.uuid)
r_tracker.srem(f'global:tracker:{tracker_type}', self.uuid)
elif level == 2:
org = self.get_org()
r_tracker.srem(f'org:tracker:{org}', self.uuid)
r_tracker.srem(f'org:tracker:{org}:{tracker_type}', self.uuid)
r_tracker.srem(f'all:tracker:{tracker_type}', tracked)
# tracker - uuid map
r_tracker.srem(f'all:tracker_uuid:{tracker_type}:{tracked}', self.uuid)
r_tracker.srem('trackers:all', self.uuid)
r_tracker.srem(f'trackers:all:{tracker_type}', self.uuid)
ail_orgs.remove_obj_to_org(self.get_org(), 'tracker', self.uuid)
# meta
r_tracker.delete(f'tracker:{self.uuid}')
trigger_trackers_refresh(tracker_type)
def create_tracker(tracker_type, to_track, org, user_id, level, description=None, filters={}, tags=[], mails=[], webhook=None, tracker_uuid=None):
if not tracker_uuid:
tracker_uuid = str(uuid.uuid4())
tracker = Tracker(tracker_uuid)
return tracker.create(tracker_type, to_track, org, user_id, level, description=description, filters=filters, tags=tags,
mails=mails, webhook=webhook)
def _re_create_tracker(tracker_type, tracker_uuid, to_track, org, user_id, level, description=None, filters={}, tags=[], mails=[], webhook=None, first_seen=None, last_seen=None):
create_tracker(tracker_type, to_track, org, user_id, level, description=description, filters=filters,
tags=tags, mails=mails, webhook=webhook, tracker_uuid=tracker_uuid)
def get_trackers_types():
return ['word', 'set', 'regex', 'typosquatting', 'yara']
def get_trackers():
return r_tracker.smembers(f'trackers:all')
def get_trackers_by_type(tracker_type):
return r_tracker.smembers(f'trackers:all:{tracker_type}')
def _get_tracked_by_obj_type(tracker_type, obj_type):
return r_tracker.smembers(f'trackers:objs:{tracker_type}:{obj_type}')
def get_trackers_by_tracked_obj_type(tracker_type, obj_type, tracked):
trackers_uuid = set()
for res in r_tracker.smembers(f'trackers:uuid:{tracker_type}:{tracked}'):
tracker_uuid, tracker_obj_type = res.split(':', 1)
if tracker_obj_type == obj_type:
trackers_uuid.add(tracker_uuid)
return trackers_uuid
def get_trackers_by_tracked(tracker_type, tracked):
return r_tracker.smembers(f'all:tracker_uuid:{tracker_type}:{tracked}')
def get_user_trackers_by_tracked(tracker_type, tracked, user_id):
user_trackers = get_user_trackers(user_id, tracker_type=tracker_type)
trackers_uuid = get_trackers_by_tracked(tracker_type, tracked)
return trackers_uuid.intersection(user_trackers)
def get_trackers_tracked_by_type(tracker_type):
return r_tracker.smembers(f'all:tracker:{tracker_type}')
def get_global_trackers(tracker_type=None):
if tracker_type:
return r_tracker.smembers(f'global:tracker:{tracker_type}')
else:
return r_tracker.smembers('global:tracker')
def get_user_trackers(user_id, tracker_type=None):
if tracker_type:
return r_tracker.smembers(f'user:tracker:{user_id}:{tracker_type}')
else:
return r_tracker.smembers(f'user:tracker:{user_id}')
def get_org_trackers(org, tracker_type=None):
if tracker_type:
return r_tracker.smembers(f'org:tracker:{org}:{tracker_type}')
else:
return r_tracker.smembers(f'org:tracker:{org}')
def get_nb_global_trackers(tracker_type=None):
if tracker_type:
return r_tracker.scard(f'global:tracker:{tracker_type}')
else:
return r_tracker.scard('global:tracker')
def get_nb_user_trackers(user_id, tracker_type=None):
if tracker_type:
return r_tracker.scard(f'user:tracker:{user_id}:{tracker_type}')
else:
return r_tracker.scard(f'user:tracker:{user_id}')
def get_nb_org_trackers(org, tracker_type=None):
if tracker_type:
return r_tracker.scard(f'org:tracker:{org}:{tracker_type}')
else:
return r_tracker.scard(f'org:tracker:{org}')
def get_user_trackers_meta(user_id, tracker_type=None):
metas = []
for tracker_uuid in get_user_trackers(user_id, tracker_type=tracker_type):
tracker = Tracker(tracker_uuid)
metas.append(tracker.get_meta(options={'description', 'mails', 'org', 'org_name', 'sparkline', 'tags'}))
return metas
def get_global_trackers_meta(tracker_type=None):
metas = []
for tracker_uuid in get_global_trackers(tracker_type=tracker_type):
tracker = Tracker(tracker_uuid)
metas.append(tracker.get_meta(options={'description', 'mails', 'org', 'org_name', 'sparkline', 'tags'}))
return metas
def get_org_trackers_meta(user_org, tracker_type=None):
metas = []
for tracker_uuid in get_org_trackers(user_org, tracker_type=tracker_type):
tracker = Tracker(tracker_uuid)
metas.append(tracker.get_meta(options={'description', 'mails', 'org', 'org_name', 'sparkline', 'tags'}))
return metas
def get_users_trackers_meta():
trackers = []
for tracker_uuid in get_trackers():
tracker = Tracker(tracker_uuid)
if tracker.is_level_user():
trackers.append(tracker.get_meta(options={'mails', 'sparkline', 'tags'}))
return trackers
def get_orgs_trackers_meta():
trackers = []
for tracker_uuid in get_trackers():
tracker = Tracker(tracker_uuid)
if tracker.is_level_org():
trackers.append(tracker.get_meta(options={'mails', 'sparkline', 'tags'}))
return trackers
def get_trackers_graph_by_day(l_trackers, num_day=31, date_from=None, date_to=None):
if date_from and date_to:
date_range = Date.substract_date(date_from, date_to)
else:
date_range = Date.get_date_range(num_day)
list_tracker_stats = []
for tracker_uuid in l_trackers:
dict_tracker_data = []
tracker = Tracker(tracker_uuid)
for date_day in date_range:
nb_seen_this_day = tracker.get_nb_objs_by_date(date_day)
if nb_seen_this_day is None:
nb_seen_this_day = 0
dict_tracker_data.append({"date": date_day, "value": int(nb_seen_this_day)})
list_tracker_stats.append({"name": tracker.get_tracked(), "Data": dict_tracker_data})
return list_tracker_stats
def get_trackers_dashboard():
trackers = []
for raw in r_tracker.lrange('trackers:dashboard', 0, -1):
tracker_uuid, timestamp, obj_type, subtype, obj_id = raw.split(':', 4)
tracker = Tracker(tracker_uuid)
meta = tracker.get_meta(options={'description', 'tags'})
if not meta.get('type'):
meta['type'] = 'Tracker DELETED'
timestamp = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y-%m-%d %H:%M:%S')
meta['timestamp'] = timestamp
trackers.append(meta)
return trackers
def get_user_dashboard(user_id): # TODO SORT + REMOVE OLDER ROWS (trim)
trackers = []
for raw in r_tracker.lrange(f'trackers:user:{user_id}', 0, -1):
tracker_uuid, timestamp, obj_type, subtype, obj_id = raw.split(':', 4)
tracker = Tracker(tracker_uuid)
meta = tracker.get_meta(options={'tags'})
timestamp = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y-%m-%d %H:%M:%S')
meta['timestamp'] = timestamp
trackers.append(meta)
return trackers
def get_trackers_stats(user_org, user_id):
stats = {'all': 0}
for tracker_type in get_trackers_types():
nb_global = get_nb_global_trackers(tracker_type=tracker_type)
nb_user = get_nb_user_trackers(user_id, tracker_type=tracker_type)
nb_org = get_nb_org_trackers(user_org, tracker_type=tracker_type)
stats[tracker_type] = nb_global + nb_user + nb_org
stats['all'] += nb_global + nb_user + nb_org
return stats
## Cache ##
# TODO API: Check Tracker type
def trigger_trackers_refresh(tracker_type):
r_cache.set(f'tracker:refresh:{tracker_type}', time.time())
def get_tracker_last_updated_by_type(tracker_type):
epoch_update = r_cache.get(f'tracker:refresh:{tracker_type}')
if not epoch_update:
epoch_update = 0
return float(epoch_update)
# - Cache - #
## Objects ##
def is_obj_tracked(obj_type, subtype, obj_id):
return r_tracker.exists(f'obj:trackers:{obj_type}:{subtype}:{obj_id}')
def get_obj_trackers(obj_type, subtype, obj_id):
return r_tracker.smembers(f'obj:trackers:{obj_type}:{subtype}:{obj_id}')
def delete_obj_trackers(obj_type, subtype, obj_id):
for tracker_uuid in get_obj_trackers(obj_type, subtype, obj_id):
tracker = Tracker(tracker_uuid)
tracker.remove(obj_type, subtype, obj_id)
######################
#### TRACKERS ACL ####
## LEVEL ##
def is_tracker_global_level(tracker_uuid):
return int(r_tracker.hget(f'tracker:{tracker_uuid}', 'level')) == 1
def is_tracked_in_global_level(tracked, tracker_type):
for tracker_uuid in get_trackers_by_tracked(tracker_type, tracked):
tracker = Tracker(tracker_uuid)
if tracker.is_level_global():
return True
return False
def is_tracked_in_user_level(tracked, tracker_type, user_id):
trackers_uuid = get_user_trackers_by_tracked(tracker_type, tracked, user_id)
if trackers_uuid:
return True
else:
return False
## API ##
def api_check_tracker_uuid(tracker_uuid):
if not is_valid_uuid_v4(tracker_uuid):
return {"status": "error", "reason": "Invalid uuid"}, 400
if not r_tracker.exists(f'tracker:{tracker_uuid}'):
return {"status": "error", "reason": "Unknown uuid"}, 404
return None
def api_check_tracker_acl(tracker_uuid, user_org, user_id, user_role, action):
res = api_check_tracker_uuid(tracker_uuid)
if res:
return res
tracker = Tracker(tracker_uuid)
if not ail_orgs.check_obj_access_acl(tracker, user_org, user_id, user_role, action):
return {"status": "error", "reason": "Access Denied"}, 403
def api_is_allowed_to_edit_tracker_level(tracker_uuid, user_org, user_id, user_role, new_level):
tracker = Tracker(tracker_uuid)
if not ail_orgs.check_acl_edit_level(tracker, user_org, user_id, user_role, new_level):
return {"status": "error", "reason": "Access Denied - Tracker level"}, 403
## --ACL-- ##
#### FIX DB #### TODO ###################################################################
def fix_tracker_stats_per_day(tracker_uuid):
tracker = Tracker(tracker_uuid)
date_from = tracker.get_date()
date_to = Date.get_today_date_str()
# delete stats
r_tracker.delete(f'tracker:stat:{tracker_uuid}')
r_tracker.hdel(f'tracker:{tracker_uuid}', 'first_seen')
r_tracker.hdel(f'tracker:{tracker_uuid}', 'last_seen')
# create new stats
for date_day in Date.substract_date(date_from, date_to):
date_day = int(date_day)
nb_items = r_tracker.scard(f'tracker:item:{tracker_uuid}:{date_day}')
if nb_items:
r_tracker.zincrby(f'tracker:stat:{tracker_uuid}', nb_items, int(date_day))
# update first_seen/last_seen
tracker.update_daterange(date_day)
def fix_tracker_item_link(tracker_uuid):
tracker = Tracker(tracker_uuid)
date_from = tracker.get_first_seen()
date_to = tracker.get_last_seen()
if date_from and date_to:
for date_day in Date.substract_date(date_from, date_to):
l_items = r_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}')
for item_id in l_items:
r_tracker.sadd(f'obj:trackers:item:{item_id}', tracker_uuid)
def fix_all_tracker_uuid_list():
r_tracker.delete(f'trackers:all')
for tracker_type in get_trackers_types():
r_tracker.delete(f'trackers:all:{tracker_type}')
for tracked in get_trackers_tracked_by_type(tracker_type):
l_tracker_uuid = get_trackers_by_tracked(tracker_type, tracked)
for tracker_uuid in l_tracker_uuid:
r_tracker.sadd(f'trackers:all', tracker_uuid)
r_tracker.sadd(f'trackers:all:{tracker_type}', tracker_uuid)
## --FIX DB-- ##
#### CREATE TRACKER ####
def api_validate_tracker_to_add(to_track, tracker_type, nb_words=1):
if tracker_type == 'regex':
if not is_valid_regex(to_track):
return {"status": "error", "reason": "Invalid regex"}, 400
elif tracker_type == 'word' or tracker_type == 'set':
# force lowercase
to_track = to_track.lower()
word_set = set(to_track)
set_inter = word_set.intersection(get_special_characters())
if set_inter:
return {"status": "error",
"reason": f'special character(s) not allowed: {set_inter}',
"message": "Please use a python regex or remove all special characters"}, 400
words = to_track.split()
# not a word
if tracker_type == 'word' and len(words) > 1:
tracker_type = 'set'
# output format: tracker1,tracker2,tracker3;2
if tracker_type == 'set':
try:
nb_words = int(nb_words)
except TypeError:
nb_words = 1
if nb_words == 0:
nb_words = 1
words_set = set(words)
words_set = sorted(words_set)
if nb_words > len(words_set):
nb_words = len(words_set)
to_track = ",".join(words_set)
to_track = f"{to_track};{nb_words}"
elif tracker_type == 'typosquatting':
to_track = to_track.lower()
# Take only the first term
domain = to_track.split(" ")
if len(domain) > 1:
return {"status": "error", "reason": "Only one domain is accepted at a time"}, 400
if "." not in to_track:
return {"status": "error", "reason": "Invalid domain name"}, 400
elif tracker_type == 'yara_custom':
if not is_valid_yara_rule(to_track):
return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400
elif tracker_type == 'yara_default':
if not is_valid_default_yara_rule(to_track):
return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400
else:
return {"status": "error", "reason": "Incorrect type"}, 400
return {"status": "success", "tracked": to_track, "type": tracker_type}, 200
def api_add_tracker(dict_input, org, user_id):
to_track = dict_input.get('tracked', None)
if not to_track:
return {"status": "error", "reason": "Tracker not provided"}, 400
tracker_type = dict_input.get('type', None)
if not tracker_type:
return {"status": "error", "reason": "Tracker type not provided"}, 400
nb_words = dict_input.get('nb_words', 1)
description = dict_input.get('description', '')
description = escape(description)
webhook = dict_input.get('webhook', '')
webhook = escape(webhook)
res = api_validate_tracker_to_add(to_track, tracker_type, nb_words=nb_words)
if res[1] != 200:
return res
to_track = res[0]['tracked']
tracker_type = res[0]['type']
tags = dict_input.get('tags', [])
mails = dict_input.get('mails', [])
res = verify_mail_list(mails)
if res:
return res
# Filters # TODO MOVE ME
filters = dict_input.get('filters', {})
if filters:
if filters.keys() == set(get_objects_tracked()) and set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}:
filters = {}
for obj_type in filters:
if obj_type not in get_objects_tracked():
return {"status": "error", "reason": "Invalid Tracker Object type"}, 400
if obj_type == 'pgp':
if set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}:
filters['pgp'].pop('subtypes')
for filter_name in filters[obj_type]:
if filter_name not in {'mimetypes', 'sources', 'subtypes'}:
return {"status": "error", "reason": "Invalid Filter"}, 400
elif filter_name == 'mimetypes': # TODO
pass
elif filter_name == 'sources':
if obj_type == 'item':
res = item_basic.verify_sources_list(filters['item']['sources'])
if res:
return res
else:
return {"status": "error", "reason": "Invalid Filter sources"}, 400
elif filter_name == 'subtypes':
obj_subtypes = set(get_object_all_subtypes(obj_type))
for subtype in filters[obj_type]['subtypes']:
if subtype not in obj_subtypes:
return {"status": "error", "reason": "Invalid Tracker Object subtype"}, 400
level = dict_input.get('level', 1)
try:
level = int(level)
except TypeError:
level = 1
if level not in range(0, 3):
level = 1
tracker_uuid = create_tracker(tracker_type, to_track, org, user_id, level, description=description, filters=filters,
tags=tags, mails=mails, webhook=webhook)
return {'tracked': to_track, 'type': tracker_type, 'uuid': tracker_uuid}, 200
def api_edit_tracker(dict_input, user_org, user_id, user_role):
tracker_uuid = dict_input.get('uuid')
res = api_check_tracker_acl(tracker_uuid, user_org, user_id, user_role, 'edit')
if res:
return res
tracker = Tracker(tracker_uuid)
to_track = dict_input.get('tracked', None)
if not to_track:
return {"status": "error", "reason": "Tracker not provided"}, 400
tracker_type = dict_input.get('type', None)
if not tracker_type:
return {"status": "error", "reason": "Tracker type not provided"}, 400
level = dict_input.get('level', 1)
try:
level = int(level)
except TypeError:
level = 1
if level not in range(0, 3):
level = 1
res = api_is_allowed_to_edit_tracker_level(tracker_uuid, user_org, user_id, user_role, level)
if res:
return res
nb_words = dict_input.get('nb_words', 1)
description = dict_input.get('description', '')
description = escape(description)
webhook = dict_input.get('webhook', '')
webhook = escape(webhook)
res = api_validate_tracker_to_add(to_track, tracker_type, nb_words=nb_words)
if res[1] != 200:
return res
to_track = res[0]['tracked']
tracker_type = res[0]['type']
tags = dict_input.get('tags', [])
mails = dict_input.get('mails', [])
res = verify_mail_list(mails)
if res:
return res
# Filters # TODO MOVE ME
filters = dict_input.get('filters', {})
if filters:
if filters.keys() == set(get_objects_tracked()) and set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}:
if not filters['decoded'] and not filters['item']:
filters = {}
for obj_type in filters:
if obj_type not in get_objects_tracked():
return {"status": "error", "reason": "Invalid Tracker Object type"}, 400
if obj_type == 'pgp':
if set(filters['pgp'].get('subtypes', [])) == {'mail', 'name'}:
filters['pgp'].pop('subtypes')
for filter_name in filters[obj_type]:
if filter_name not in {'mimetypes', 'sources', 'subtypes'}:
return {"status": "error", "reason": "Invalid Filter"}, 400
elif filter_name == 'mimetypes': # TODO
pass
elif filter_name == 'sources':
if obj_type == 'item':
res = item_basic.verify_sources_list(filters['item']['sources'])
if res:
return res
else:
return {"status": "error", "reason": "Invalid Filter sources"}, 400
elif filter_name == 'subtypes':
obj_subtypes = set(get_object_all_subtypes(obj_type))
for subtype in filters[obj_type]['subtypes']:
if subtype not in obj_subtypes:
return {"status": "error", "reason": "Invalid Tracker Object subtype"}, 400
tracker.edit(tracker_type, to_track, level, user_org, description=description, filters=filters,
tags=tags, mails=mails, webhook=webhook)
return {'tracked': to_track, 'type': tracker_type, 'uuid': tracker_uuid}, 200
def api_delete_tracker(data, user_org, user_id, user_role):
tracker_uuid = data.get('uuid')
res = api_check_tracker_acl(tracker_uuid, user_org, user_id, user_role, 'delete')
if res:
return res
tracker = Tracker(tracker_uuid)
return tracker.delete(), 200
def api_tracker_add_object(data, user_org, user_id, user_role):
tracker_uuid = data.get('uuid')
res = api_check_tracker_acl(tracker_uuid, user_org, user_id, user_role, 'edit')
if res:
return res
tracker = Tracker(tracker_uuid)
object_gid = data.get('gid')
date = data.get('date')
if date:
if not Date.validate_str_date(date):
date = None
try:
obj_type, subtype, obj_id = object_gid.split(':', 2)
except (AttributeError, IndexError):
return {"status": "error", "reason": "Invalid Object"}, 400
return tracker.add(obj_type, subtype, obj_id, date=date), 200
def api_tracker_remove_object(data, user_org, user_id, user_role):
tracker_uuid = data.get('uuid')
res = api_check_tracker_acl(tracker_uuid, user_org, user_id, user_role, 'edit')
if res:
return res
tracker = Tracker(tracker_uuid)
object_gid = data.get('gid')
try:
obj_type, subtype, obj_id = object_gid.split(':', 2)
except (AttributeError, IndexError):
return {"status": "error", "reason": "Invalid Object"}, 400
return tracker.remove(obj_type, subtype, obj_id), 200
## -- CREATE TRACKER -- ##
####################
#### WORD - SET ####
def get_tracked_words():
to_track = {}
for obj_type in get_objects_tracked():
to_track[obj_type] = _get_tracked_by_obj_type('word', obj_type)
return to_track
def get_tracked_sets():
to_track = {}
for obj_type in get_objects_tracked():
to_track[obj_type] = []
for tracked in _get_tracked_by_obj_type('set', obj_type):
res = tracked.split(';')
nb_words = int(res[1])
words_set = res[0].split(',')
to_track[obj_type].append({'words': words_set, 'nb': nb_words, 'tracked': tracked})
return to_track
def get_text_word_frequency(content, filtering=True):
content = content.lower()
words_dict = defaultdict(int)
if filtering:
if TOKENIZER is None:
init_tokenizer()
blob = TextBlob(content, tokenizer=TOKENIZER)
else:
blob = TextBlob(content)
for word in blob.tokens:
words_dict[word] += 1
return words_dict
###############
#### REGEX ####
def get_tracked_regexs():
to_track = {}
for obj_type in get_objects_tracked():
to_track[obj_type] = []
for tracked in _get_tracked_by_obj_type('regex', obj_type):
to_track[obj_type].append({'regex': re.compile(tracked), 'tracked': tracked})
return to_track
########################
#### TYPO SQUATTING ####
def get_tracked_typosquatting_domains(tracked):
return r_tracker.smembers(f'tracker:typosquatting:{tracked}')
def get_tracked_typosquatting():
to_track = {}
for obj_type in get_objects_tracked():
to_track[obj_type] = []
for tracked in _get_tracked_by_obj_type('typosquatting', obj_type):
to_track[obj_type].append({'domains': get_tracked_typosquatting_domains(tracked), 'tracked': tracked})
return to_track
##############
#### YARA ####
def get_yara_rules_dir():
return os.path.join(os.environ['AIL_BIN'], 'trackers', 'yara')
def get_yara_rules_default_dir():
return os.path.join(os.environ['AIL_BIN'], 'trackers', 'yara', 'ail-yara-rules', 'rules')
# # TODO: cache + update
def get_all_default_yara_rules_types():
yara_dir = get_yara_rules_default_dir()
all_yara_types = next(os.walk(yara_dir))[1]
# save in cache ?
return all_yara_types
# # TODO: cache + update
def get_all_default_yara_files():
yara_dir = get_yara_rules_default_dir()
all_default_yara_files = {}
for rules_type in get_all_default_yara_rules_types():
all_default_yara_files[rules_type] = os.listdir(os.path.join(yara_dir, rules_type))
return all_default_yara_files
def get_all_default_yara_rules_by_type(yara_types):
all_default_yara_files = get_all_default_yara_files()
if yara_types in all_default_yara_files:
return all_default_yara_files[yara_types]
else:
return []
def get_all_tracked_yara_files(filter_disabled=False):
yara_files = r_tracker.smembers('all:tracker:yara')
if not yara_files:
yara_files = []
if filter_disabled:
pass
return yara_files
def get_tracked_yara_rules():
to_track = {}
for obj_type in get_objects_tracked():
rules = {}
for tracked in _get_tracked_by_obj_type('yara', obj_type):
rule = os.path.join(get_yara_rules_dir(), tracked)
if not os.path.exists(rule):
logger.critical(f"Yara rule don't exists {tracked} : {obj_type}")
else:
rules[tracked] = rule
to_track[obj_type] = yara.compile(filepaths=rules)
return to_track
def reload_yara_rules():
yara_files = get_all_tracked_yara_files()
# {uuid: filename}
rule_dict = {}
for yar_path in yara_files:
for tracker_uuid in get_trackers_by_tracked('yara', yar_path):
rule_dict[tracker_uuid] = os.path.join(get_yara_rules_dir(), yar_path)
for tracker_uuid in rule_dict:
if not os.path.isfile(rule_dict[tracker_uuid]):
# TODO IGNORE + LOGS
raise Exception(f"Error: {rule_dict[tracker_uuid]} doesn't exists")
rules = yara.compile(filepaths=rule_dict)
return rules
def is_valid_yara_rule(yara_rule):
try:
yara.compile(source=yara_rule)
return True
except:
return False
def is_default_yara_rule(tracked_yara_name):
yara_dir = get_yara_rules_dir()
filename = os.path.join(yara_dir, tracked_yara_name)
filename = os.path.realpath(filename)
try:
if tracked_yara_name.split('/')[0] == 'custom-rules':
return False
except:
return False
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
return False
else:
if os.path.isfile(filename):
return True
return False
def is_valid_default_yara_rule(yara_rule, verbose=True):
yara_dir = get_yara_rules_default_dir()
filename = os.path.join(yara_dir, yara_rule)
filename = os.path.realpath(filename)
# incorrect filename
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
if verbose:
print('error: file transversal')
print(yara_dir)
print(filename)
return False
else:
if os.path.isfile(filename):
return True
else:
return False
def save_yara_rule(yara_rule_type, yara_rule, tracker_uuid=None):
if yara_rule_type == 'yara_custom':
if not tracker_uuid:
tracker_uuid = str(uuid.uuid4())
filename = os.path.join('custom-rules', tracker_uuid + '.yar')
with open(os.path.join(get_yara_rules_dir(), filename), 'w') as f:
f.write(str(yara_rule))
elif yara_rule_type == 'yara_default':
filename = os.path.join('ail-yara-rules', 'rules', yara_rule)
return filename
def get_yara_rule_file_by_tracker_name(tracked_yara_name):
yara_dir = get_yara_rules_dir()
filename = os.path.join(yara_dir, tracked_yara_name)
filename = os.path.realpath(filename)
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
print('error: file transversal')
print(yara_dir)
print(filename)
return None
return filename
def get_yara_rule_content(yara_rule):
yara_dir = get_yara_rules_dir()
filename = os.path.join(yara_dir, yara_rule)
filename = os.path.realpath(filename)
# incorrect filename
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
return '' # # TODO: throw exception
with open(filename, 'r') as f:
rule_content = f.read()
return rule_content
def api_get_default_rule_content(default_yara_rule):
yara_dir = get_yara_rules_default_dir()
filename = os.path.join(yara_dir, default_yara_rule)
filename = os.path.realpath(filename)
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
return {'status': 'error', 'reason': 'file traversal detected'}, 400
if not os.path.isfile(filename):
return {'status': 'error', 'reason': 'yara rule not found'}, 400
with open(filename, 'r') as f:
rule_content = f.read()
return {'rule_name': default_yara_rule, 'content': rule_content}, 200
def get_yara_rule_content_restapi(request_dict):
rule_name = request_dict.get('rule_name', None)
if not request_dict:
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
if not rule_name:
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
yara_dir = get_yara_rules_dir()
filename = os.path.join(yara_dir, rule_name)
filename = os.path.realpath(filename)
if not os.path.commonprefix([filename, yara_dir]) == yara_dir:
return {'status': 'error', 'reason': 'File Path Traversal'}, 400
if not os.path.isfile(filename):
return {'status': 'error', 'reason': 'yara rule not found'}, 400
with open(filename, 'r') as f:
rule_content = f.read()
rule_content = base64.b64encode((rule_content.encode('utf-8'))).decode('UTF-8')
return {'status': 'success', 'content': rule_content}, 200
## -- YARA -- ##
######################
#### RETRO - HUNT ####
# state: pending/running/completed/paused
class RetroHunt:
def __init__(self, task_uuid):
self.uuid = task_uuid
def exists(self):
return r_tracker.exists(f'retro_hunt:{self.uuid}')
def _get_field(self, field):
return r_tracker.hget(f'retro_hunt:{self.uuid}', field)
def _set_field(self, field, value):
return r_tracker.hset(f'retro_hunt:{self.uuid}', field, value)
## LEVEL ##
def get_level(self):
level = int(self._get_field('level'))
if not level:
level = 1
return int(level)
def set_level(self, level, org_uuid):
if level == 1: # global
r_tracker.sadd('retro_hunts', self.uuid)
elif level == 2: # org only
self.add_to_org(org_uuid)
self._set_field('level', level)
def delete_level(self, level=None):
if not level:
level = self.get_level()
if level == 1:
r_tracker.srem('retro_hunts', self.uuid)
# Org
elif level == 2:
ail_orgs.remove_obj_to_org(self.get_org(), 'retro_hunt', self.uuid)
def reset_level(self, old_level, new_level, new_org_uuid):
self.delete_level(old_level)
self.set_level(new_level, new_org_uuid)
def check_level(self, user_org):
level = self.get_level()
if level == 1:
return True
elif level == 2:
return self.get_org() == user_org
## ORG ##
def get_creator_org(self):
return self._get_field('creator_org')
def get_org(self):
return self._get_field('org')
def add_to_org(self, org_uuid):
self._set_field('org', org_uuid)
ail_orgs.add_obj_to_org(org_uuid, 'retro_hunt', self.uuid)
## -ORG- ##
def get_creator(self):
return self._get_field('creator')
def get_date(self):
return self._get_field('date')
def get_last_analyzed(self):
return self._get_field('last')
def set_last_analyzed(self, obj_type, subtype, obj_id):
return self._set_field('last', f'{obj_type}:{subtype}:{obj_id}')
def get_last_analyzed_cache(self):
r_cache.hget(f'retro_hunt:task:{self.uuid}', 'obj')
def set_last_analyzed_cache(self, obj_type, subtype, obj_id):
r_cache.hset(f'retro_hunt:task:{self.uuid}', 'obj', f'{obj_type}:{subtype}:{obj_id}')
def get_name(self):
return self._get_field('name')
def get_description(self):
return self._get_field('description')
def get_timeout(self):
res = self._get_field('timeout')
if res:
return int(res)
else:
return 30 # # TODO: FIXME use instance limit
def get_filters(self):
filters = self._get_field('filters')
if not filters:
return {}
else:
return json.loads(filters)
def set_filters(self, filters):
if filters:
self._set_field('filters', json.dumps(filters))
def get_tags(self):
return r_tracker.smembers(f'retro_hunt:tags:{self.uuid}')
def get_mails(self):
return r_tracker.smembers(f'retro_hunt:mails:{self.uuid}')
def get_state(self):
return self._get_field('state')
def _set_state(self, new_state):
curr_state = self.get_state()
if curr_state:
r_tracker.srem(f'retro_hunt:task:{curr_state}', self.uuid)
r_tracker.sadd(f'retro_hunts:{new_state}', self.uuid)
self._set_field('state', new_state)
def get_rule(self, r_compile=False):
rule = self._get_field('rule')
if r_compile:
rule = os.path.join(get_yara_rules_dir(), rule)
rule_dict = {self.uuid: os.path.join(get_yara_rules_dir(), rule)}
rule = yara.compile(filepaths=rule_dict)
return rule
# add timeout ?
def get_meta(self, options=set()):
meta = {'uuid': self.uuid,
'name': self.get_name(),
'state': self.get_state(),
'rule': self.get_rule(),
}
if 'creator' in options:
meta['creator'] = self.get_creator()
if 'date' in options:
meta['date'] = self.get_date()
if 'description' in options:
meta['description'] = self.get_description()
if 'level' in options:
meta['level'] = self.get_level()
if 'mails' in options:
meta['mails'] = self.get_mails()
if 'nb_match' in options:
meta['nb_match'] = self.get_nb_match()
if 'nb_objs' in options:
meta['nb_objs'] = self.get_nb_objs()
if 'org' in options:
meta['org'] = self.get_org()
if 'org_name' in options:
meta['org_name'] = ail_orgs.Organisation(self.get_org()).get_name()
if 'progress' in options:
meta['progress'] = self.get_progress()
if 'filters' in options:
meta['filters'] = self.get_filters()
if 'tags' in options:
meta['tags'] = self.get_tags()
return meta
def is_paused(self):
return r_tracker.sismember('retro_hunts:paused', self.uuid)
def to_pause(self):
to_pause = r_cache.hget(f'retro_hunt:{self.uuid}', 'pause')
if to_pause:
return True
else:
return False
def pause(self):
self._set_state('paused')
r_cache.hset(f'retro_hunt:{self.uuid}', 'pause', time.time())
self.clear_cache()
def resume(self):
r_cache.hdel(f'retro_hunt:{self.uuid}', 'pause')
self._set_state('pending')
def is_running(self):
return r_tracker.sismember('retro_hunts:running', self.uuid)
def run(self): # TODO ADD MORE CHECK
self._set_state('running')
def complete(self):
self._set_state('completed')
self.clear_cache()
r_tracker.hdel(f'retro_hunt:{self.uuid}', 'last')
def get_progress(self):
if self.get_state() == 'completed':
progress = 100
else:
progress = r_cache.hget(f'retro_hunt:{self.uuid}', 'progress')
if not progress:
progress = self._get_field('progress')
return progress
def set_progress(self, progress):
res = r_cache.hset(f'retro_hunt:{self.uuid}', 'progress', progress)
if res:
self._set_field('progress', progress)
def get_nb_match(self):
return self._get_field('nb_match')
def _incr_nb_match(self):
r_tracker.hincrby(f'retro_hunt:{self.uuid}', 'nb_match', 1)
def _decr_nb_match(self):
r_tracker.hincrby(f'retro_hunt:{self.uuid}', 'nb_match', -1)
def _set_nb_match(self, nb_match):
self._set_field('nb_match', nb_match)
def clear_cache(self):
r_cache.delete(f'retro_hunt:{self.uuid}')
def get_nb_objs_by_type(self, obj_type):
return r_tracker.scard(f'retro_hunt:objs:{self.uuid}:{obj_type}')
def get_objs_by_type(self, obj_type):
return r_tracker.smembers(f'retro_hunt:objs:{self.uuid}:{obj_type}')
def get_nb_objs(self):
objs = {}
for obj_type in get_objects_retro_hunted():
nb = self.get_nb_objs_by_type(obj_type)
if nb:
objs[obj_type] = nb
return objs
def get_objs(self):
objs = []
for obj_type in get_objects_retro_hunted():
for obj in self.get_objs_by_type(obj_type):
subtype, obj_id = obj.split(':', 1)
objs.append((obj_type, subtype, obj_id))
return objs
def add(self, obj_type, subtype, obj_id):
# match by object type:
r_tracker.sadd(f'retro_hunt:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}')
# MAP object -> retro hunt
r_tracker.sadd(f'obj:retro_hunts:{obj_type}:{subtype}:{obj_id}', self.uuid)
self._incr_nb_match()
def remove(self, obj_type, subtype, obj_id):
# match by object type:
r_tracker.srem(f'retro_hunt:objs:{self.uuid}:{obj_type}', f'{subtype}:{obj_id}')
# MAP object -> retro hunt
r_tracker.srem(f'obj:retro_hunts:{obj_type}:{subtype}:{obj_id}', self.uuid)
self._decr_nb_match()
def create(self, org_uuid, user_id, level, name, rule, description=None, filters=[], mails=[], tags=[], timeout=30, state='pending'):
if self.exists():
raise Exception('Error: Retro Hunt Task already exists')
self._set_field('name', escape(name))
self._set_field('rule', rule)
self._set_field('date', datetime.date.today().strftime("%Y%m%d"))
self._set_field('name', escape(name))
self._set_field('creator_org', org_uuid)
self._set_field('creator', user_id)
if description:
self._set_field('description', description)
if timeout:
self._set_field('timeout', int(timeout))
for tag in tags:
# tag = escape(tag)
r_tracker.sadd(f'retro_hunt:tags:{self.uuid}', tag)
Tag.create_custom_tag(tag)
for mail in mails:
r_tracker.sadd(f'retro_hunt:mails:{self.uuid}', escape(mail))
if filters:
self.set_filters(filters)
self.set_level(level, org_uuid)
r_tracker.sadd('retro_hunts:all', self.uuid)
# add to pending tasks
if state not in ('pending', 'completed', 'paused'):
state = 'pending'
self._set_state(state)
def delete(self):
if self.is_running() and self.get_state() not in ['completed', 'paused']:
return None
# Delete custom rule
rule = self.get_rule()
if not is_default_yara_rule(rule):
filepath = get_yara_rule_file_by_tracker_name(rule)
if filepath:
os.remove(filepath)
self.delete_level()
r_tracker.srem('retro_hunts:pending', self.uuid)
r_tracker.delete(f'retro_hunts:{self.uuid}')
r_tracker.delete(f'retro_hunt:tags:{self.uuid}')
r_tracker.delete(f'retro_hunt:mails:{self.uuid}')
for obj in self.get_objs():
self.remove(obj[0], obj[1], obj[2])
r_tracker.srem('retro_hunts:all', self.uuid)
r_tracker.srem('retro_hunts:pending', self.uuid)
r_tracker.srem('retro_hunts:paused', self.uuid)
r_tracker.srem('retro_hunts:completed', self.uuid)
self.clear_cache()
return self.uuid
def create_retro_hunt(user_org, user_id, level, name, rule_type, rule, description=None, filters=[], mails=[], tags=[], timeout=30, state='pending', task_uuid=None):
if not task_uuid:
task_uuid = str(uuid.uuid4())
retro_hunt = RetroHunt(task_uuid)
# rule_type: yara_default - yara custom
rule = save_yara_rule(rule_type, rule, tracker_uuid=retro_hunt.uuid)
retro_hunt.create(user_org, user_id, level, name, rule, description=description, mails=mails, tags=tags,
timeout=timeout, filters=filters, state=state)
return retro_hunt.uuid
# TODO
# def _re_create_retro_hunt_task(name, rule, date, date_from, date_to, creator, sources, tags, mails, timeout, description, task_uuid, state='pending', nb_match=0, last_id=None):
# retro_hunt = RetroHunt(task_uuid)
# retro_hunt.create(name, rule, date_from, date_to, creator, description=description, mails=mails, tags=tags,
# timeout=timeout, sources=sources, state=state)
# if last_id:
# set_retro_hunt_last_analyzed(task_uuid, last_id)
# retro_hunt._set_nb_match(nb_match)
# retro_hunt._set_field('date', date)
## ? ? ?
# set tags
# set mails
# limit mail
# SET Retro Hunts
def get_all_retro_hunt_tasks():
return r_tracker.smembers('retro_hunts:all')
def get_retro_hunts_global():
return r_tracker.smembers('retro_hunts')
def get_retro_hunts_org(org_uuid):
return ail_orgs.get_org_objs_by_type(org_uuid, 'retro_hunt')
def get_retro_hunts_orgs():
retros = []
for retro_uuid in get_all_retro_hunt_tasks():
retro = RetroHunt(retro_uuid)
if retro.get_level() == 2:
retros.append(retro_uuid)
return retros
def get_retro_hunt_pending_tasks():
return r_tracker.smembers('retro_hunts:pending')
def get_retro_hunt_running_tasks():
return r_tracker.smembers('retro_hunts:running')
def get_retro_hunt_paused_tasks():
return r_tracker.smembers('retro_hunts:paused')
def get_retro_hunt_completed_tasks():
return r_tracker.smembers('retro_hunts:completed')
## Change STATES ##
def get_retro_hunt_task_to_start():
task_uuid = r_tracker.spop('retro_hunts:pending')
if task_uuid:
retro_hunt = RetroHunt(task_uuid)
retro_hunt.run()
return task_uuid
## Metadata ##
def get_retro_hunt_metas(trackers_uuid):
tasks = []
for task_uuid in trackers_uuid:
retro_hunt = RetroHunt(task_uuid)
tasks.append(retro_hunt.get_meta(options={'date', 'progress', 'org', 'org_name', 'nb_match', 'tags'}))
return tasks
## Objects ##
def is_obj_retro_hunted(obj_type, subtype, obj_id):
return r_tracker.exists(f'obj:retro_hunts:{obj_type}:{subtype}:{obj_id}')
def get_obj_retro_hunts(obj_type, subtype, obj_id):
return r_tracker.smembers(f'obj:retro_hunts:{obj_type}:{subtype}:{obj_id}')
def delete_obj_retro_hunts(obj_type, subtype, obj_id):
for retro_uuid in get_obj_retro_hunts(obj_type, subtype, obj_id):
retro_hunt = RetroHunt(retro_uuid)
retro_hunt.remove(obj_type, subtype, obj_id)
#### ACL ####
def api_check_retro_hunt_acl(retro_hunt, user_org, user_id, user_role, action):
if not ail_orgs.check_obj_access_acl(retro_hunt, user_org, user_id, user_role, action):
return {"status": "error", "reason": "Access Denied"}, 403
# TODO
def api_is_allowed_to_edit_retro_hunt_level(retro_hunt, user_org, user_id, user_role, new_level):
if not ail_orgs.check_acl_edit_level(retro_hunt, user_org, user_id, user_role, new_level):
return {"status": "error", "reason": "Access Denied - Tracker level"}, 403
#### API ####
def api_check_retro_hunt_task_uuid(task_uuid):
if not is_valid_uuid_v4(task_uuid):
return {"status": "error", "reason": "Invalid uuid"}, 400
retro_hunt = RetroHunt(task_uuid)
if not retro_hunt.exists():
return {"status": "error", "reason": "Unknown uuid"}, 404
return None
def api_pause_retro_hunt_task(user_org, user_id, user_role, task_uuid):
res = api_check_retro_hunt_task_uuid(task_uuid)
if res:
return res
retro_hunt = RetroHunt(task_uuid)
res = api_check_retro_hunt_acl(retro_hunt, user_org, user_id, user_role, 'edit')
if res:
return res
task_state = retro_hunt.get_state()
if task_state not in ['pending', 'running']:
return {"status": "error", "reason": f"Task {task_uuid} not paused, current state: {task_state}"}, 400
retro_hunt.pause()
return task_uuid, 200
def api_resume_retro_hunt_task(user_org, user_id, user_role, task_uuid):
res = api_check_retro_hunt_task_uuid(task_uuid)
if res:
return res
retro_hunt = RetroHunt(task_uuid)
res = api_check_retro_hunt_acl(retro_hunt, user_org, user_id, user_role, 'edit')
if res:
return res
if not retro_hunt.is_paused():
return {"status": "error",
"reason": f"Task {task_uuid} not paused, current state: {retro_hunt.get_state()}"}, 400
retro_hunt.resume()
return task_uuid, 200
def api_validate_rule_to_add(rule, rule_type):
if rule_type == 'yara_custom':
if not is_valid_yara_rule(rule):
return {"status": "error", "reason": "Invalid custom Yara Rule"}, 400
elif rule_type == 'yara_default':
if not is_valid_default_yara_rule(rule):
return {"status": "error", "reason": "The Yara Rule doesn't exist"}, 400
else:
return {"status": "error", "reason": "Incorrect type"}, 400
return {"status": "success", "rule": rule, "type": rule_type}, 200
def api_create_retro_hunt_task(dict_input, user_org, user_id):
# # TODO: API: check mandatory arg
# # TODO: TIMEOUT
# timeout=30
rule = dict_input.get('rule', None)
if not rule:
return {"status": "error", "reason": "Retro Hunt Rule not provided"}, 400
task_type = dict_input.get('type', None)
if not task_type:
return {"status": "error", "reason": "type not provided"}, 400
# Level
level = dict_input.get('level', 1)
try:
level = int(level)
except TypeError:
level = 1
if level not in range(1, 3):
level = 1
# # TODO: limit
name = dict_input.get('name', '')
name = escape(name)
name = name[:60]
# # TODO: limit
description = dict_input.get('description', '')
description = escape(description)
description = description[:1000]
res = api_validate_rule_to_add(rule, task_type)
if res[1] != 200:
return res
tags = dict_input.get('tags', []) # TODO escape custom tags
mails = dict_input.get('mails', []) # TODO escape mails
res = verify_mail_list(mails)
if res:
return res
# Filters # TODO MOVE ME
filters = dict_input.get('filters', {})
if filters:
if filters.keys() == get_objects_retro_hunted():
filters = {}
for obj_type in filters:
if obj_type not in get_objects_retro_hunted():
return {"status": "error", "reason": "Invalid Tracker Object type"}, 400
for filter_name in filters[obj_type]:
if filter_name not in {'date_from', 'date_to', 'mimetypes', 'sources', 'subtypes'}:
return {"status": "error", "reason": "Invalid Filter"}, 400
elif filter_name == 'date_from':
if not Date.validate_str_date(filters[obj_type]['date_from']):
return {"status": "error", "reason": "Invalid date_from"}, 400
elif filter_name == 'date_to':
if not Date.validate_str_date(filters[obj_type]['date_from']):
return {"status": "error", "reason": "Invalid date_to"}, 400
elif filter_name == 'mimetypes': # TODO sanityze mimetypes
pass
elif filter_name == 'sources':
if obj_type == 'item':
res = item_basic.verify_sources_list(filters['item']['sources'])
if res:
return res
else:
return {"status": "error", "reason": "Invalid Filter sources"}, 400
elif filter_name == 'subtypes':
obj_subtypes = set(get_object_all_subtypes(obj_type))
for subtype in filters[obj_type]['subtypes']:
if subtype not in obj_subtypes:
return {"status": "error", "reason": "Invalid Tracker Object subtype"}, 400
if 'date_from' and 'date_to' in filters:
res = Date.api_validate_str_date_range(filters[obj_type]['date_from'], filters[obj_type]['date_to'])
if res:
return res
task_uuid = create_retro_hunt(user_org, user_id, level, name, task_type, rule, description=description,
mails=mails, tags=tags, timeout=30, filters=filters)
return {'name': name, 'rule': rule, 'type': task_type, 'uuid': task_uuid}, 200
def api_delete_retro_hunt_task(user_org, user_id, user_role, task_uuid):
res = api_check_retro_hunt_task_uuid(task_uuid)
if res:
return res
retro_hunt = RetroHunt(task_uuid)
res = api_check_retro_hunt_acl(retro_hunt, user_org, user_id, user_role, 'delete')
if res:
return res
if retro_hunt.is_running() and retro_hunt.get_state() not in ['completed', 'paused']:
return {"status": "error", "reason": "You can't delete a running task"}, 400
else:
return retro_hunt.delete(), 200
################################################################################
################################################################################
################################################################################
################################################################################
#### DB FIX ####
# def _fix_db_custom_tags():
# for tag in get_trackers_tags():
# if not Tag.is_taxonomie_tag(tag) and not Tag.is_galaxy_tag(tag):
# Tag.create_custom_tag(tag)
#### -- ####
# if __name__ == '__main__':
# _fix_db_custom_tags()
# fix_all_tracker_uuid_list()
# res = get_all_tracker_uuid()
# print(len(res))
# import Term
# Term.delete_term('5262ab6c-8784-4a55-b0ff-a471018414b4')
# fix_tracker_stats_per_day('5262ab6c-8784-4a55-b0ff-a471018414b4')
# tracker_uuid = '5262ab6c-8784-4a55-b0ff-a471018414b4'
# fix_tracker_item_link(tracker_uuid)
# res = get_item_all_trackers_uuid('archive/')
# print(res)
# res = is_valid_yara_rule('rule dummy { }')
# res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None, sources=['crawled', 'pastebin.com', 'rt/pastebin.com'])
# res = create_tracker('circl\.lu', 'regex', 'admin@admin.test', 1, [], [], None, sources=['crawled','pastebin.com'])
# print(res)
# t_uuid = '1c2d35b0-9330-4feb-b454-da13007aa9f7'
# res = get_tracker_sources('ail-yara-rules/rules/crypto/certificate.yar', 'yara')
# sys.path.append(os.environ['AIL_BIN'])
# from packages import Term
# Term.delete_term('074ab4be-6049-45b5-a20e-8125a4e4f500')
# res = get_items_to_analyze('archive/pastebin.com_pro/2020/05/15', last='archive/pastebin.com_pro/2020/05/15/zkHEgqjQ.gz')
# get_retro_hunt_task_progress('0', nb_src_done=2)
# res = set_cache_retro_hunt_task_progress('0', 100)
# res = get_retro_hunt_task_nb_src_done('0', sources=['pastebin.com_pro', 'alerts/pastebin.com_pro', 'crawled'])
# print(res)
# sources = ['pastebin.com_pro', 'alerts/pastebin.com_pro', 'crawled']
# rule = 'custom-rules/4a8a3d04-f0b6-43ce-8e00-bdf47a8df241.yar'
# name = 'retro_hunt_test_1'
# description = 'circl retro hunt first test'
# tags = ['retro_circl', 'circl']
# creator = 'admin@admin.test'
# date_from = '20200610'
# date_to = '20210630'
# res = create_retro_hunt_task(name, rule, date_from, date_to, creator, sources=sources, tags=tags, description=description)
# get_retro_hunt_nb_item_by_day(['80b402ef-a8a9-4e97-adb6-e090edcfd571'], date_from=None, date_to=None, num_day=31)
# res = get_retro_hunt_nb_item_by_day(['c625f971-16e6-4331-82a7-b1e1b9efdec1'], date_from='20200610', date_to='20210630')
# res = delete_retro_hunt_task('598687b6-f765-4f8b-861a-09ad76d0ab34')
# print(res)