mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-09-19 23:48:23 +00:00
chg: add Retro Hunt
This commit is contained in:
Terrtia
parent
b81d70172c
commit
b1793f18e4
16 changed files with 1655 additions and 18 deletions
|
|
@ -210,6 +210,10 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tracker_Yara" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Yara.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Retro_Hunt" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Retro_Hunt.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Retro_Hunt" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Retro_Hunt.py; read x"
|
||||
sleep 0.1
|
||||
|
||||
##################################
|
||||
# DISABLED MODULES #
|
||||
|
|
|
|||
|
|
@ -20,6 +20,8 @@ import ConfigLoader
|
|||
import item_basic
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker")
|
||||
config_loader = None
|
||||
|
|
@ -139,12 +141,14 @@ def get_tracker_sparkline(tracker_uuid, num_day=6):
|
|||
sparklines_value.append(int(nb_seen_this_day))
|
||||
return sparklines_value
|
||||
|
||||
def add_tracked_item(tracker_uuid, item_id, item_date):
|
||||
def add_tracked_item(tracker_uuid, item_id):
|
||||
item_date = item_basic.get_item_date(id)
|
||||
# track item
|
||||
res = r_serv_tracker.sadd(f'tracker:item:{tracker_uuid}:{item_date}', item_id)
|
||||
# track nb item by date
|
||||
if res == 1:
|
||||
r_serv_tracker.zadd('tracker:stat:{}'.format(tracker_uuid), item_date, int(item_date))
|
||||
r_serv_tracker.zincrby('tracker:stat:{}'.format(tracker_uuid), int(item_date), 1)
|
||||
|
||||
|
||||
def get_email_subject(tracker_uuid):
|
||||
tracker_description = get_tracker_description(tracker_uuid)
|
||||
|
|
@ -563,12 +567,526 @@ def api_get_default_rule_content(default_yara_rule):
|
|||
|
||||
##-- YARA --##
|
||||
|
||||
if __name__ == '__main__':
|
||||
######################
|
||||
#### RETRO - HUNT ####
|
||||
|
||||
# state: pending/running/completed/paused
|
||||
|
||||
# task keys:
|
||||
## tracker:retro_hunt:task:{task_uuid} state
|
||||
# start_time
|
||||
# end_time
|
||||
# date_from
|
||||
# date_to
|
||||
# creator
|
||||
# timeout
|
||||
# date
|
||||
# type
|
||||
|
||||
## ? ? ?
|
||||
# set tags
|
||||
# set mails
|
||||
# limit mail
|
||||
|
||||
# SET Retro Hunts
|
||||
|
||||
def get_all_retro_hunt_tasks():
|
||||
return r_serv_tracker.smembers('tracker:retro_hunt:task:all')
|
||||
|
||||
def get_all_pending_retro_hunt_tasks():
|
||||
return r_serv_tracker.smembers('tracker:retro_hunt:task:pending')
|
||||
|
||||
def get_all_running_retro_hunt_tasks():
|
||||
return r_serv_tracker.smembers('tracker:retro_hunt:task:running')
|
||||
|
||||
def get_all_paused_retro_hunt_tasks():
|
||||
return r_serv_tracker.smembers('tracker:retro_hunt:task:paused')
|
||||
|
||||
## Change STATES ##
|
||||
|
||||
def get_all_completed_retro_hunt_tasks():
|
||||
return r_serv_tracker.smembers('tracker:retro_hunt:task:completed')
|
||||
|
||||
def get_retro_hunt_task_to_start():
|
||||
task_uuid = r_serv_tracker.spop('tracker:retro_hunt:task:pending')
|
||||
if task_uuid:
|
||||
set_retro_hunt_task_state(task_uuid, 'running')
|
||||
return task_uuid
|
||||
|
||||
def pause_retro_hunt_task(task_uuid):
|
||||
set_retro_hunt_task_state(task_uuid, 'paused')
|
||||
r_cache.hset(f'tracker:retro_hunt:task:{task_uuid}', 'pause', time.time())
|
||||
|
||||
def check_retro_hunt_pause(task_uuid):
|
||||
is_paused = r_cache.hget(f'tracker:retro_hunt:task:{task_uuid}', 'pause')
|
||||
if is_paused:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
def resume_retro_hunt_task(task_uuid):
|
||||
r_cache.hdel(f'tracker:retro_hunt:task:{task_uuid}', 'pause')
|
||||
set_retro_hunt_task_state(task_uuid, 'pending')
|
||||
|
||||
## Metadata ##
|
||||
|
||||
def get_retro_hunt_task_name(task_uuid):
|
||||
return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'name')
|
||||
|
||||
def get_retro_hunt_task_state(task_uuid):
|
||||
return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'state')
|
||||
|
||||
def set_retro_hunt_task_state(task_uuid, new_state):
|
||||
curr_state = get_retro_hunt_task_state(task_uuid)
|
||||
if curr_state:
|
||||
r_serv_tracker.srem(f'tracker:retro_hunt:task:{curr_state}', task_uuid)
|
||||
r_serv_tracker.sadd(f'tracker:retro_hunt:task:{new_state}', task_uuid)
|
||||
r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'state', new_state)
|
||||
|
||||
def get_retro_hunt_task_type(task_uuid):
|
||||
return r_serv_tracker(f'tracker:retro_hunt:task:{task_uuid}', 'type')
|
||||
|
||||
# # TODO: yararule
|
||||
def get_retro_hunt_task_rule(task_uuid, r_compile=False):
|
||||
#rule_type = 'yara'
|
||||
rule = r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'rule')
|
||||
if r_compile:
|
||||
#if rule_type == 'yara'
|
||||
rule = os.path.join(get_yara_rules_dir(), rule)
|
||||
rule_dict = {task_uuid : os.path.join(get_yara_rules_dir(), rule)}
|
||||
rule = yara.compile(filepaths=rule_dict)
|
||||
return rule
|
||||
|
||||
def get_retro_hunt_task_timeout(task_uuid):
|
||||
res = r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'timeout')
|
||||
if res:
|
||||
return int(res)
|
||||
else:
|
||||
return 30 # # TODO: FIXME use instance limit
|
||||
|
||||
def get_retro_hunt_task_date_from(task_uuid):
|
||||
return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'date_from')
|
||||
|
||||
def get_retro_hunt_task_date_to(task_uuid):
|
||||
return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'date_to')
|
||||
|
||||
def get_retro_hunt_task_creator(task_uuid):
|
||||
return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'creator')
|
||||
|
||||
def get_retro_hunt_last_analyzed(task_uuid):
|
||||
return r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'last')
|
||||
|
||||
# Keep history to relaunch on error/pause
|
||||
def set_retro_hunt_last_analyzed(task_uuid, last_id):
|
||||
r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'last', last_id)
|
||||
|
||||
def get_retro_hunt_task_sources(task_uuid, r_sort=False):
|
||||
sources = r_serv_tracker.smembers(f'tracker:retro_hunt:task:sources:{task_uuid}')
|
||||
if not sources:
|
||||
sources = set(item_basic.get_all_items_sources(filter_dir=False))
|
||||
if r_sort:
|
||||
sources = sorted(sources)
|
||||
return sources
|
||||
|
||||
def get_retro_hunt_task_tags(task_uuid):
|
||||
return r_serv_tracker.smembers(f'tracker:retro_hunt:task:tags:{task_uuid}')
|
||||
|
||||
def get_retro_hunt_task_mails(task_uuid):
|
||||
return r_serv_tracker.smembers(f'tracker:retro_hunt:task:mails:{task_uuid}')
|
||||
|
||||
# # TODO: ADD TYPE + TIMEOUT
|
||||
def get_retro_hunt_task_metadata(task_uuid, date=False, progress=False, creator=False, sources=None, tags=None, description=False, nb_match=False):
|
||||
task_metadata = {'uuid': task_uuid}
|
||||
task_metadata['state'] = get_retro_hunt_task_state(task_uuid)
|
||||
task_metadata['name'] = get_retro_hunt_task_name(task_uuid)
|
||||
task_metadata['rule'] = get_retro_hunt_task_rule(task_uuid)
|
||||
if creator:
|
||||
task_metadata['creator'] = get_retro_hunt_task_creator(task_uuid)
|
||||
if date:
|
||||
task_metadata['date'] = r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'date')
|
||||
task_metadata['date_from'] = get_retro_hunt_task_date_from(task_uuid)
|
||||
task_metadata['date_to'] = get_retro_hunt_task_date_to(task_uuid)
|
||||
if description:
|
||||
task_metadata['description'] = r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'description')
|
||||
if nb_match:
|
||||
task_metadata['nb_match'] = get_retro_hunt_nb_match(task_uuid)
|
||||
if progress:
|
||||
task_metadata['progress'] = get_retro_hunt_task_progress(task_uuid)
|
||||
if sources:
|
||||
task_metadata['sources'] = get_retro_hunt_task_sources(task_uuid, r_sort=True)
|
||||
if tags:
|
||||
task_metadata['tags'] = get_retro_hunt_task_tags(task_uuid)
|
||||
return task_metadata
|
||||
|
||||
def get_all_retro_hunt_tasks_with_metadata():
|
||||
l_retro_hunt = []
|
||||
for task_uuid in get_all_retro_hunt_tasks():
|
||||
l_retro_hunt.append(get_retro_hunt_task_metadata(task_uuid, date=True, progress=True, tags=True, nb_match=True))
|
||||
return l_retro_hunt
|
||||
|
||||
def get_retro_hunt_task_progress(task_uuid):
|
||||
if get_retro_hunt_task_state(task_uuid) == 'completed':
|
||||
progress = 100
|
||||
else:
|
||||
progress = r_cache.hget(f'tracker:retro_hunt:task:{task_uuid}', 'progress')
|
||||
if not progress:
|
||||
progress = compute_retro_hunt_task_progress(task_uuid)
|
||||
return progress
|
||||
|
||||
def set_cache_retro_hunt_task_progress(task_uuid, progress):
|
||||
r_cache.hset(f'tracker:retro_hunt:task:{task_uuid}', 'progress', progress)
|
||||
|
||||
def set_cache_retro_hunt_task_id(task_uuid, id):
|
||||
r_cache.hset(f'tracker:retro_hunt:task:{task_uuid}', 'id', id)
|
||||
|
||||
def clear_retro_hunt_task_cache(task_uuid):
|
||||
r_cache.delete(f'tracker:retro_hunt:task:{task_uuid}')
|
||||
|
||||
# Others
|
||||
|
||||
# date
|
||||
# type
|
||||
# tags
|
||||
# mails
|
||||
# name
|
||||
# description
|
||||
|
||||
# # # TODO: TYPE
|
||||
def create_retro_hunt_task(name, rule, date_from, date_to, creator, sources=[], tags=[], mails=[], timeout=30, description=None, task_uuid=None):
|
||||
if not task_uuid:
|
||||
task_uuid = str(uuid.uuid4())
|
||||
|
||||
r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'name', escape(name))
|
||||
|
||||
r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'rule', rule)
|
||||
|
||||
r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'date', datetime.date.today().strftime("%Y%m%d"))
|
||||
r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'date_from', date_from)
|
||||
r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'date_to', date_to)
|
||||
|
||||
r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'creator', creator)
|
||||
if description:
|
||||
r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'description', description)
|
||||
if timeout:
|
||||
r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'timeout', int(timeout))
|
||||
for source in sources:
|
||||
r_serv_tracker.sadd(f'tracker:retro_hunt:task:sources:{task_uuid}', escape(source))
|
||||
for tag in tags:
|
||||
r_serv_tracker.sadd(f'tracker:retro_hunt:task:tags:{task_uuid}', escape(tag))
|
||||
for mail in mails:
|
||||
r_serv_tracker.sadd(f'tracker:retro_hunt:task:mails:{task_uuid}', escape(mail))
|
||||
|
||||
r_serv_tracker.sadd('tracker:retro_hunt:task:all', task_uuid)
|
||||
# add to pending tasks
|
||||
r_serv_tracker.sadd('tracker:retro_hunt:task:pending', task_uuid)
|
||||
r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'state', 'pending')
|
||||
return task_uuid
|
||||
|
||||
# # TODO: delete rule
|
||||
def delete_retro_hunt_task(task_uuid):
|
||||
if r_serv_tracker.sismember('tracker:retro_hunt:task:running', task_uuid):
|
||||
return None
|
||||
|
||||
r_serv_tracker.srem('tracker:retro_hunt:task:pending', task_uuid)
|
||||
r_serv_tracker.delete(f'tracker:retro_hunt:task:{task_uuid}')
|
||||
r_serv_tracker.delete(f'tracker:retro_hunt:task:sources:{task_uuid}')
|
||||
r_serv_tracker.delete(f'tracker:retro_hunt:task:tags:{task_uuid}')
|
||||
r_serv_tracker.delete(f'tracker:retro_hunt:task:mails:{task_uuid}')
|
||||
|
||||
for item_date in get_retro_hunt_all_item_dates(task_uuid):
|
||||
r_serv_tracker.delete(f'tracker:retro_hunt:task:item:{task_uuid}:{item_date}')
|
||||
|
||||
r_serv_tracker.srem('tracker:retro_hunt:task:all', task_uuid)
|
||||
r_serv_tracker.srem('tracker:retro_hunt:task:pending', task_uuid)
|
||||
r_serv_tracker.srem('tracker:retro_hunt:task:paused', task_uuid)
|
||||
r_serv_tracker.srem('tracker:retro_hunt:task:completed', task_uuid)
|
||||
|
||||
clear_retro_hunt_task_cache(task_uuid)
|
||||
return task_uuid
|
||||
|
||||
def get_retro_hunt_task_current_date(task_uuid):
|
||||
last = get_retro_hunt_last_analyzed(task_uuid)
|
||||
if last:
|
||||
curr_date = item_basic.get_item_date(last)
|
||||
else:
|
||||
curr_date = get_retro_hunt_task_date_from(task_uuid)
|
||||
return curr_date
|
||||
|
||||
def get_retro_hunt_task_nb_src_done(task_uuid, sources=[]):
|
||||
if not sources:
|
||||
sources = list(get_retro_hunt_task_sources(task_uuid, r_sort=True))
|
||||
else:
|
||||
sources = list(sources)
|
||||
last_id = get_retro_hunt_last_analyzed(task_uuid)
|
||||
if last_id:
|
||||
last_source = item_basic.get_source(last_id)
|
||||
try:
|
||||
nb_src_done = sources.index(last_source)
|
||||
except ValueError:
|
||||
nb_src_done = 0
|
||||
else:
|
||||
nb_src_done = 0
|
||||
return nb_src_done
|
||||
|
||||
def get_retro_hunt_dir_day_to_analyze(task_uuid, date, filter_last=False, sources=[]):
|
||||
if not sources:
|
||||
sources = get_retro_hunt_task_sources(task_uuid, r_sort=True)
|
||||
|
||||
# filter last
|
||||
if filter_last:
|
||||
last = get_retro_hunt_last_analyzed(task_uuid)
|
||||
if last:
|
||||
curr_source = item_basic.get_source(last)
|
||||
# remove processed sources
|
||||
set_sources = sources.copy()
|
||||
for source in sources:
|
||||
if source != curr_source:
|
||||
set_sources.remove(source)
|
||||
else:
|
||||
break
|
||||
sources = set_sources
|
||||
|
||||
# return all dirs by day
|
||||
date = f'{date[0:4]}/{date[4:6]}/{date[6:8]}'
|
||||
dirs = set()
|
||||
for source in sources:
|
||||
dirs.add(os.path.join(source, date))
|
||||
return dirs
|
||||
|
||||
# # TODO: move me
|
||||
def get_items_to_analyze(dir, last=None):
|
||||
full_dir = os.path.join(os.environ['AIL_HOME'], 'PASTES', dir) # # TODO: # FIXME: use item config dir
|
||||
if os.path.isdir(full_dir):
|
||||
all_items = sorted([os.path.join(dir, f) for f in os.listdir(full_dir) if os.path.isfile(os.path.join(full_dir, f))])
|
||||
# remove processed items
|
||||
if last:
|
||||
items_set = all_items.copy()
|
||||
for item in all_items:
|
||||
if item != last:
|
||||
items_set.remove(item)
|
||||
else:
|
||||
break
|
||||
all_items = items_set
|
||||
return all_items
|
||||
else:
|
||||
return []
|
||||
|
||||
def compute_retro_hunt_task_progress(task_uuid, date_from=None, date_to=None, sources=[], curr_date=None, nb_src_done=0):
|
||||
# get nb days
|
||||
if not date_from:
|
||||
date_from = get_retro_hunt_task_date_from(task_uuid)
|
||||
if not date_to:
|
||||
date_to = get_retro_hunt_task_date_to(task_uuid)
|
||||
nb_days = Date.get_nb_days_by_daterange(date_from, date_to)
|
||||
|
||||
# nb days completed
|
||||
if not curr_date:
|
||||
curr_date = get_retro_hunt_task_current_date(task_uuid)
|
||||
nb_days_done = Date.get_nb_days_by_daterange(date_from, curr_date) - 1
|
||||
|
||||
# sources
|
||||
if not sources:
|
||||
nb_sources = len(get_retro_hunt_task_sources(task_uuid))
|
||||
else:
|
||||
nb_sources = len(sources)
|
||||
|
||||
# get progress
|
||||
progress = ((nb_days_done * nb_sources) + nb_src_done) * 100 / (nb_days * nb_sources)
|
||||
return int(progress)
|
||||
|
||||
# # TODO: # FIXME: # Cache
|
||||
|
||||
def save_retro_hunt_match(task_uuid, id, object_type='item'):
|
||||
item_date = item_basic.get_item_date(id)
|
||||
res = r_serv_tracker.sadd(f'tracker:retro_hunt:task:item:{task_uuid}:{item_date}', id)
|
||||
# track nb item by date
|
||||
if res == 1:
|
||||
r_serv_tracker.zincrby(f'tracker:retro_hunt:task:stat:{task_uuid}', int(item_date), 1)
|
||||
|
||||
def get_retro_hunt_all_item_dates(task_uuid):
|
||||
return r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1)
|
||||
|
||||
def get_retro_hunt_nb_match(task_uuid):
|
||||
nb_match = r_serv_tracker.hget(f'tracker:retro_hunt:task:{task_uuid}', 'nb_match')
|
||||
if not nb_match:
|
||||
l_date_value = r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1, withscores=True)
|
||||
nb_match = 0
|
||||
for tuple in l_date_value:
|
||||
nb_match += int(tuple[1])
|
||||
return int(nb_match)
|
||||
|
||||
def set_retro_hunt_nb_match(task_uuid):
|
||||
l_date_value = r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1, withscores=True)
|
||||
nb_match = 0
|
||||
for tuple in l_date_value:
|
||||
nb_match += int(tuple[1])
|
||||
r_serv_tracker.hset(f'tracker:retro_hunt:task:{task_uuid}', 'nb_match', nb_match)
|
||||
|
||||
def get_retro_hunt_items_by_daterange(task_uuid, date_from, date_to):
|
||||
all_item_id = set()
|
||||
if date_from and date_to:
|
||||
l_date_match = r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1, withscores=True)
|
||||
if l_date_match:
|
||||
dict_date_match = dict(l_date_match)
|
||||
for date_day in Date.substract_date(date_from, date_to):
|
||||
if date_day in dict_date_match:
|
||||
all_item_id |= r_serv_tracker.smembers(f'tracker:retro_hunt:task:item:{task_uuid}:{date_day}')
|
||||
return all_item_id
|
||||
|
||||
def get_retro_hunt_nb_item_by_day(l_task_uuid, date_from=None, date_to=None):
|
||||
list_stats = []
|
||||
for task_uuid in l_task_uuid:
|
||||
dict_task_data = []
|
||||
retro_name = get_retro_hunt_task_name(task_uuid)
|
||||
l_date_match = r_serv_tracker.zrange(f'tracker:retro_hunt:task:stat:{task_uuid}', 0, -1, withscores=True)
|
||||
if l_date_match:
|
||||
dict_date_match = dict(l_date_match)
|
||||
if not date_from:
|
||||
date_from = min(dict_date_match)
|
||||
if not date_to:
|
||||
date_to = max(dict_date_match)
|
||||
|
||||
date_range = Date.substract_date(date_from, date_to)
|
||||
for date_day in date_range:
|
||||
nb_seen_this_day = int(dict_date_match.get(date_day, 0))
|
||||
dict_task_data.append({"date": date_day,"value": int(nb_seen_this_day)})
|
||||
list_stats.append({"name": retro_name,"Data": dict_task_data})
|
||||
return list_stats
|
||||
|
||||
## API ##
|
||||
def api_check_retro_hunt_task_uuid(task_uuid):
|
||||
if not is_valid_uuid_v4(task_uuid):
|
||||
return ({"status": "error", "reason": "Invalid uuid"}, 400)
|
||||
if not r_serv_tracker.exists(f'tracker:retro_hunt:task:{task_uuid}'):
|
||||
return ({"status": "error", "reason": "Unknown uuid"}, 404)
|
||||
return None
|
||||
|
||||
def api_get_retro_hunt_items(dict_input):
|
||||
task_uuid = dict_input.get('uuid', None)
|
||||
res = api_check_retro_hunt_task_uuid(task_uuid)
|
||||
if res:
|
||||
return res
|
||||
|
||||
date_from = dict_input.get('date_from', None)
|
||||
date_to = dict_input.get('date_to', None)
|
||||
if date_from is None:
|
||||
date_from = get_retro_hunt_task_date_from(task_uuid)
|
||||
if date_from:
|
||||
date_from = date_from[0]
|
||||
if date_to is None:
|
||||
date_to = date_from
|
||||
if date_from > date_to:
|
||||
date_from = date_to
|
||||
|
||||
all_items_id = get_retro_hunt_items_by_daterange(task_uuid, date_from, date_to)
|
||||
all_items_id = item_basic.get_all_items_metadata_dict(all_items_id)
|
||||
|
||||
res_dict = {}
|
||||
res_dict['uuid'] = task_uuid
|
||||
res_dict['date_from'] = date_from
|
||||
res_dict['date_to'] = date_to
|
||||
res_dict['items'] = all_items_id
|
||||
return (res_dict, 200)
|
||||
|
||||
def api_pause_retro_hunt_task(task_uuid):
|
||||
res = api_check_retro_hunt_task_uuid(task_uuid)
|
||||
if res:
|
||||
return res
|
||||
task_state = get_retro_hunt_task_state(task_uuid)
|
||||
if task_state not in ['pending', 'running']:
|
||||
return ({"status": "error", "reason": f"Task {task_uuid} not paused, current state: {task_state}"}, 400)
|
||||
pause_retro_hunt_task(task_uuid)
|
||||
return (task_uuid, 200)
|
||||
|
||||
def api_resume_retro_hunt_task(task_uuid):
|
||||
res = api_check_retro_hunt_task_uuid(task_uuid)
|
||||
if res:
|
||||
return res
|
||||
task_state = get_retro_hunt_task_state(task_uuid)
|
||||
if not r_serv_tracker.sismember('tracker:retro_hunt:task:paused', task_uuid):
|
||||
return ({"status": "error", "reason": f"Task {task_uuid} not paused, current state: {get_retro_hunt_task_state(task_uuid)}"}, 400)
|
||||
resume_retro_hunt_task(task_uuid)
|
||||
return (task_uuid, 200)
|
||||
|
||||
def api_validate_rule_to_add(rule, rule_type):
|
||||
if rule_type=='yara_custom':
|
||||
if not is_valid_yara_rule(rule):
|
||||
return ({"status": "error", "reason": "Invalid custom Yara Rule"}, 400)
|
||||
elif rule_type=='yara_default':
|
||||
if not is_valid_default_yara_rule(rule):
|
||||
return ({"status": "error", "reason": "The Yara Rule doesn't exist"}, 400)
|
||||
else:
|
||||
return ({"status": "error", "reason": "Incorrect type"}, 400)
|
||||
return ({"status": "success", "rule": rule, "type": rule_type}, 200)
|
||||
|
||||
def api_create_retro_hunt_task(dict_input, creator):
|
||||
# # TODO: API: check mandatory arg
|
||||
# # TODO: TIMEOUT
|
||||
|
||||
# timeout=30
|
||||
rule = dict_input.get('rule', None)
|
||||
if not rule:
|
||||
return ({"status": "error", "reason": "Retro Hunt Rile not provided"}, 400)
|
||||
task_type = dict_input.get('type', None)
|
||||
if not task_type:
|
||||
return ({"status": "error", "reason": "type not provided"}, 400)
|
||||
|
||||
# # TODO: limit
|
||||
name = dict_input.get('name', '')
|
||||
name = escape(name)
|
||||
name = name[:60]
|
||||
# # TODO: limit
|
||||
description = dict_input.get('description', '')
|
||||
description = escape(description)
|
||||
description = description[:1000]
|
||||
|
||||
res = api_validate_rule_to_add(rule , task_type)
|
||||
if res[1]!=200:
|
||||
return res
|
||||
|
||||
tags = dict_input.get('tags', [])
|
||||
mails = dict_input.get('mails', [])
|
||||
res = verify_mail_list(mails)
|
||||
if res:
|
||||
return res
|
||||
|
||||
sources = dict_input.get('sources', [])
|
||||
res = item_basic.verify_sources_list(sources)
|
||||
if res:
|
||||
return res
|
||||
|
||||
date_from = dict_input.get('date_from', '')
|
||||
date_to = dict_input.get('date_to', '')
|
||||
res = Date.api_validate_str_date_range(date_from, date_to)
|
||||
if res:
|
||||
return res
|
||||
|
||||
task_uuid = str(uuid.uuid4())
|
||||
|
||||
# RULE
|
||||
rule = save_yara_rule(task_type, rule, tracker_uuid=task_uuid)
|
||||
task_type = 'yara'
|
||||
|
||||
task_uuid = create_retro_hunt_task(name, rule, date_from, date_to, creator, sources=sources,
|
||||
tags=tags, mails=mails, timeout=30, description=description, task_uuid=task_uuid)
|
||||
|
||||
return ({'name': name, 'rule': rule, 'type': task_type, 'uuid': task_uuid}, 200)
|
||||
|
||||
def api_delete_retro_hunt_task(task_uuid):
|
||||
res = api_check_retro_hunt_task_uuid(task_uuid)
|
||||
if res:
|
||||
return res
|
||||
if r_serv_tracker.sismember('tracker:retro_hunt:task:running', task_uuid):
|
||||
return ({"status": "error", "reason": "You can't delete a running task"}, 400)
|
||||
else:
|
||||
return (delete_retro_hunt_task(task_uuid), 200)
|
||||
|
||||
#if __name__ == '__main__':
|
||||
#res = is_valid_yara_rule('rule dummy { }')
|
||||
|
||||
# res = create_tracker('test', 'word', 'admin@admin.test', 1, [], [], None, sources=['crawled', 'pastebin.com', 'rt/pastebin.com'])
|
||||
res = create_tracker('circl\.lu', 'regex', 'admin@admin.test', 1, [], [], None, sources=['crawled','pastebin.com'])
|
||||
print(res)
|
||||
#res = create_tracker('circl\.lu', 'regex', 'admin@admin.test', 1, [], [], None, sources=['crawled','pastebin.com'])
|
||||
#print(res)
|
||||
|
||||
#t_uuid = '1c2d35b0-9330-4feb-b454-da13007aa9f7'
|
||||
#res = get_tracker_sources('ail-yara-rules/rules/crypto/certificate.yar', 'yara')
|
||||
|
|
@ -578,4 +1096,29 @@ if __name__ == '__main__':
|
|||
# Term.delete_term('074ab4be-6049-45b5-a20e-8125a4e4f500')
|
||||
|
||||
|
||||
#res = get_items_to_analyze('archive/pastebin.com_pro/2020/05/15', last='archive/pastebin.com_pro/2020/05/15/zkHEgqjQ.gz')
|
||||
#get_retro_hunt_task_progress('0', nb_src_done=2)
|
||||
|
||||
#res = set_cache_retro_hunt_task_progress('0', 100)
|
||||
#res = get_retro_hunt_task_nb_src_done('0', sources=['pastebin.com_pro', 'alerts/pastebin.com_pro', 'crawled'])
|
||||
#print(res)
|
||||
|
||||
# sources = ['pastebin.com_pro', 'alerts/pastebin.com_pro', 'crawled']
|
||||
# rule = 'custom-rules/4a8a3d04-f0b6-43ce-8e00-bdf47a8df241.yar'
|
||||
# name = 'retro_hunt_test_1'
|
||||
# description = 'circl retro hunt first test'
|
||||
# tags = ['retro_circl', 'circl']
|
||||
# creator = 'admin@admin.test'
|
||||
# date_from = '20200610'
|
||||
# date_to = '20210630'
|
||||
|
||||
#res = create_retro_hunt_task(name, rule, date_from, date_to, creator, sources=sources, tags=tags, description=description)
|
||||
|
||||
|
||||
#get_retro_hunt_nb_item_by_day(['80b402ef-a8a9-4e97-adb6-e090edcfd571'], date_from=None, date_to=None, num_day=31)
|
||||
|
||||
#res = get_retro_hunt_nb_item_by_day(['c625f971-16e6-4331-82a7-b1e1b9efdec1'], date_from='20200610', date_to='20210630')
|
||||
|
||||
#res = delete_retro_hunt_task('598687b6-f765-4f8b-861a-09ad76d0ab34')
|
||||
|
||||
#print(res)
|
||||
|
|
|
|||
|
|
@ -7,6 +7,9 @@ import gzip
|
|||
|
||||
import magic
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
import Tag
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
|
|
@ -41,7 +44,8 @@ def get_basename(item_id):
|
|||
return os.path.basename(item_id)
|
||||
|
||||
def get_source(item_id):
|
||||
return item_id.split('/')[-5]
|
||||
l_source = item_id.split('/')[:-4]
|
||||
return os.path.join(*l_source)
|
||||
|
||||
# # TODO: add an option to check the tag
|
||||
def is_crawled(item_id):
|
||||
|
|
@ -204,7 +208,7 @@ def _get_dir_source_name(directory, source_name=None, l_sources_name=set(), filt
|
|||
return l_sources_name
|
||||
|
||||
|
||||
def get_all_items_sources(filter_dir=True, r_list=False):
|
||||
def get_all_items_sources(filter_dir=False, r_list=False):
|
||||
res = _get_dir_source_name(PASTES_FOLDER, filter_dir=filter_dir)
|
||||
if r_list:
|
||||
res = list(res)
|
||||
|
|
@ -217,6 +221,12 @@ def verify_sources_list(sources):
|
|||
return ({'status': 'error', 'reason': 'Invalid source', 'value': source}, 400)
|
||||
return None
|
||||
|
||||
def get_all_items_metadata_dict(list_id):
|
||||
list_meta = []
|
||||
for item_id in list_id:
|
||||
list_meta.append( {'id': item_id, 'date': get_item_date(item_id), 'tags': Tag.get_obj_tag(item_id)} )
|
||||
return list_meta
|
||||
|
||||
##-- --##
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -110,6 +110,13 @@ def get_previous_date_list(num_day):
|
|||
date_list.append(date.substract_day(i))
|
||||
return list(reversed(date_list))
|
||||
|
||||
def get_nb_days_by_daterange(date_from, date_to):
|
||||
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
|
||||
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
|
||||
delta = date_to - date_from # timedelta
|
||||
return len(range(delta.days + 1))
|
||||
|
||||
|
||||
def substract_date(date_from, date_to):
|
||||
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
|
||||
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
|
||||
|
|
@ -129,6 +136,13 @@ def validate_str_date(str_date, separator=''):
|
|||
except TypeError:
|
||||
return False
|
||||
|
||||
def api_validate_str_date_range(date_from, date_to, separator=''):
|
||||
is_date = validate_str_date(date_from, separator=separator) and validate_str_date(date_from, separator=separator)
|
||||
if not is_date:
|
||||
return ({"status": "error", "reason": "Invalid Date"}, 400)
|
||||
if int(date_from) > int(date_to):
|
||||
return ({"status": "error", "reason": "Invalid Date range, Date from > Date to"}, 400)
|
||||
|
||||
def sanitise_date_range(date_from, date_to, separator='', date_type='str'):
|
||||
'''
|
||||
Check/Return a correct date_form and date_to
|
||||
|
|
|
|||
|
|
@ -26,6 +26,10 @@ publish = Redis_D4_client
|
|||
[D4_client]
|
||||
subscribe = Redis_D4_client
|
||||
|
||||
[Retro_Hunt]
|
||||
subscribe = Redis
|
||||
publish = Redis_Tags
|
||||
|
||||
[Tracker_Term]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_Tags
|
||||
|
|
|
|||
185
bin/trackers/Retro_Hunt.py
Executable file
185
bin/trackers/Retro_Hunt.py
Executable file
|
|
@ -0,0 +1,185 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The Retro_Hunt trackers module
|
||||
===================
|
||||
|
||||
"""
|
||||
|
||||
##################################
|
||||
# Import External packages
|
||||
##################################
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import yara
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from packages.Item import Item
|
||||
from packages.Item import Date
|
||||
from lib import Tracker
|
||||
|
||||
import NotificationHelper # # TODO: refractor
|
||||
|
||||
class Retro_Hunt(AbstractModule):
|
||||
|
||||
#mail_body_template = "AIL Framework,\nNew YARA match: {}\nitem id: {}\nurl: {}{}"
|
||||
|
||||
"""
|
||||
Retro_Hunt module for AIL framework
|
||||
"""
|
||||
def __init__(self):
|
||||
super(Retro_Hunt, self).__init__()
|
||||
self.pending_seconds = 5
|
||||
|
||||
self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
|
||||
|
||||
self.refresh_deleta = 10
|
||||
self.last_refresh = 0
|
||||
|
||||
# reset on each loop
|
||||
self.task_uuid = None
|
||||
self.date_from = 0
|
||||
self.date_to = 0
|
||||
self.nb_src_done = 0
|
||||
self.progress = 0
|
||||
self.item = None
|
||||
|
||||
self.redis_logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
|
||||
# # TODO: send mails
|
||||
# # TODO: # start_time
|
||||
# end_time
|
||||
|
||||
def compute(self, task_uuid):
|
||||
print(task_uuid)
|
||||
self.task_uuid = task_uuid
|
||||
self.progress = 0
|
||||
# First launch
|
||||
# restart
|
||||
rule = Tracker.get_retro_hunt_task_rule(task_uuid, r_compile=True)
|
||||
timeout = Tracker.get_retro_hunt_task_timeout(task_uuid)
|
||||
sources = Tracker.get_retro_hunt_task_sources(task_uuid, r_sort=True)
|
||||
|
||||
self.date_from = Tracker.get_retro_hunt_task_date_from(task_uuid)
|
||||
self.date_to = Tracker.get_retro_hunt_task_date_to(task_uuid)
|
||||
self.tags = Tracker.get_retro_hunt_task_tags(task_uuid)
|
||||
curr_date = Tracker.get_retro_hunt_task_current_date(task_uuid)
|
||||
self.nb_src_done = Tracker.get_retro_hunt_task_nb_src_done(task_uuid, sources=sources)
|
||||
self.progress = self.update_progress(sources, curr_date)
|
||||
# iterate on date
|
||||
filter_last = True
|
||||
while int(curr_date) <= int(self.date_to):
|
||||
print(curr_date)
|
||||
dirs_date = Tracker.get_retro_hunt_dir_day_to_analyze(task_uuid, curr_date, filter_last=filter_last, sources=sources)
|
||||
filter_last = False
|
||||
nb_id = 0
|
||||
self.nb_src_done = 0
|
||||
self.update_progress(sources, curr_date)
|
||||
# # TODO: Filter previous item
|
||||
for dir in dirs_date:
|
||||
print(dir)
|
||||
l_obj = Tracker.get_items_to_analyze(dir)
|
||||
for id in l_obj:
|
||||
#print(f'{dir} / {id}')
|
||||
self.item = Item(id)
|
||||
# save current item in cache
|
||||
Tracker.set_cache_retro_hunt_task_id(task_uuid, id)
|
||||
|
||||
yara_match = rule.match(data=self.item.get_content(), callback=self.yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=timeout)
|
||||
|
||||
# save last item
|
||||
if nb_id % 10 == 0: # # TODO: Add nb before save in DB
|
||||
Tracker.set_retro_hunt_last_analyzed(task_uuid, id)
|
||||
nb_id += 1
|
||||
self.update_progress(sources, curr_date)
|
||||
|
||||
# PAUSE
|
||||
self.update_progress(sources, curr_date)
|
||||
if Tracker.check_retro_hunt_pause(task_uuid):
|
||||
Tracker.set_retro_hunt_last_analyzed(task_uuid, id)
|
||||
#self.update_progress(sources, curr_date, save_db=True)
|
||||
Tracker.pause_retro_hunt_task(task_uuid)
|
||||
Tracker.clear_retro_hunt_task_cache(task_uuid)
|
||||
return None
|
||||
|
||||
self.nb_src_done += 1
|
||||
self.update_progress(sources, curr_date)
|
||||
curr_date = Date.date_add_day(curr_date)
|
||||
print('-----')
|
||||
|
||||
self.update_progress(sources, curr_date)
|
||||
|
||||
Tracker.set_retro_hunt_task_state(task_uuid, 'completed')
|
||||
Tracker.set_retro_hunt_nb_match(task_uuid)
|
||||
Tracker.clear_retro_hunt_task_cache(task_uuid)
|
||||
|
||||
|
||||
# # TODO: stop
|
||||
|
||||
def update_progress(self, sources, curr_date, save_db=False):
|
||||
progress = Tracker.compute_retro_hunt_task_progress(self.task_uuid, date_from=self.date_from, date_to=self.date_to,
|
||||
sources=sources, curr_date=curr_date, nb_src_done=self.nb_src_done)
|
||||
if self.progress != progress:
|
||||
Tracker.set_cache_retro_hunt_task_progress(self.task_uuid, progress)
|
||||
self.progress = progress
|
||||
# if save_db:
|
||||
# Tracker.set_retro_hunt_task_progress(task_uuid, progress)
|
||||
|
||||
def yara_rules_match(self, data):
|
||||
#print(data)
|
||||
|
||||
task_uuid = data['namespace']
|
||||
id = self.item.get_id()
|
||||
|
||||
Tracker.save_retro_hunt_match(task_uuid, id)
|
||||
|
||||
# Tags
|
||||
for tag in self.tags:
|
||||
msg = f'{tag};{id}'
|
||||
self.send_message_to_queue(msg, 'Tags')
|
||||
|
||||
# # Mails
|
||||
# mail_to_notify = Tracker.get_tracker_mails(tracker_uuid)
|
||||
# if mail_to_notify:
|
||||
# mail_subject = Tracker.get_email_subject(tracker_uuid)
|
||||
# mail_body = Tracker_Yara.mail_body_template.format(data['rule'], item_id, self.full_item_url, item_id)
|
||||
# for mail in mail_to_notify:
|
||||
# self.redis_logger.debug(f'Send Mail {mail_subject}')
|
||||
# print(f'Send Mail {mail_subject}')
|
||||
# NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
|
||||
return yara.CALLBACK_CONTINUE
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
Run Module endless process
|
||||
"""
|
||||
|
||||
# Endless loop processing messages from the input queue
|
||||
while self.proceed:
|
||||
task_uuid = Tracker.get_retro_hunt_task_to_start()
|
||||
if task_uuid:
|
||||
# Module processing with the message from the queue
|
||||
self.redis_logger.debug(task_uuid)
|
||||
#try:
|
||||
self.compute(task_uuid)
|
||||
#except Exception as err:
|
||||
# self.redis_logger.error(f'Error in module {self.module_name}: {err}')
|
||||
# # Remove uuid ref
|
||||
# self.remove_submit_uuid(uuid)
|
||||
else:
|
||||
# Wait before next process
|
||||
self.redis_logger.debug(f'{self.module_name}, waiting for new message, Idling {self.pending_seconds}s')
|
||||
time.sleep(self.pending_seconds)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
module = Retro_Hunt()
|
||||
module.run()
|
||||
|
|
@ -79,9 +79,7 @@ class Tracker_Regex(AbstractModule):
|
|||
if tracker_sources and item_source not in tracker_sources:
|
||||
continue
|
||||
|
||||
item_date = item.get_date()
|
||||
|
||||
Tracker.add_tracked_item(tracker_uuid, item_id, item_date)
|
||||
Tracker.add_tracked_item(tracker_uuid, item_id)
|
||||
|
||||
tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
|
||||
for tag in tags_to_add:
|
||||
|
|
|
|||
|
|
@ -111,7 +111,7 @@ class Tracker_Term(AbstractModule):
|
|||
if nb_uniq_word >= nb_words_threshold:
|
||||
self.new_term_found(word_set, 'set', item.get_id(), item_date, item_source)
|
||||
|
||||
def new_term_found(self, term, term_type, item_id, item_date, item_source):
|
||||
def new_term_found(self, term, term_type, item_id, item_source):
|
||||
uuid_list = Term.get_term_uuid_list(term, term_type)
|
||||
self.redis_logger.info(f'new tracked term found: {term} in {item_id}')
|
||||
print(f'new tracked term found: {term} in {item_id}')
|
||||
|
|
@ -120,7 +120,7 @@ class Tracker_Term(AbstractModule):
|
|||
tracker_sources = Tracker.get_tracker_uuid_sources(term_uuid)
|
||||
if not tracker_sources or item_source in tracker_sources:
|
||||
print(not tracker_sources or item_source in tracker_sources)
|
||||
Tracker.add_tracked_item(term_uuid, item_id, item_date)
|
||||
Tracker.add_tracked_item(term_uuid, item_id)
|
||||
|
||||
tags_to_add = Term.get_term_tags(term_uuid)
|
||||
for tag in tags_to_add:
|
||||
|
|
|
|||
|
|
@ -78,8 +78,7 @@ class Tracker_Yara(AbstractModule):
|
|||
print(f'Source Filtering: {data["rule"]}')
|
||||
return yara.CALLBACK_CONTINUE
|
||||
|
||||
item_date = self.item.get_date()
|
||||
Tracker.add_tracked_item(tracker_uuid, item_id, item_date)
|
||||
Tracker.add_tracked_item(tracker_uuid, item_id)
|
||||
|
||||
# Tags
|
||||
tags_to_add = Tracker.get_tracker_tags(tracker_uuid)
|
||||
|
|
|
|||
|
|
@ -43,6 +43,7 @@ from blueprints.correlation import correlation
|
|||
from blueprints.tags_ui import tags_ui
|
||||
from blueprints.import_export import import_export
|
||||
from blueprints.objects_item import objects_item
|
||||
from blueprints.hunters import hunters
|
||||
from blueprints.old_endpoints import old_endpoints
|
||||
|
||||
|
||||
|
|
@ -100,6 +101,7 @@ app.register_blueprint(correlation, url_prefix=baseUrl)
|
|||
app.register_blueprint(tags_ui, url_prefix=baseUrl)
|
||||
app.register_blueprint(import_export, url_prefix=baseUrl)
|
||||
app.register_blueprint(objects_item, url_prefix=baseUrl)
|
||||
app.register_blueprint(hunters, url_prefix=baseUrl)
|
||||
app.register_blueprint(old_endpoints, url_prefix=baseUrl)
|
||||
# ========= =========#
|
||||
|
||||
|
|
|
|||
199
var/www/blueprints/hunters.py
Normal file
199
var/www/blueprints/hunters.py
Normal file
|
|
@ -0,0 +1,199 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
|
||||
'''
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
import random
|
||||
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, make_response
|
||||
from flask_login import login_required, current_user, login_user, logout_user
|
||||
|
||||
sys.path.append('modules')
|
||||
import Flask_config
|
||||
|
||||
# Import Role_Manager
|
||||
from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
|
||||
from Role_Manager import login_admin, login_analyst, login_read_only
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||
import item_basic
|
||||
import Tracker
|
||||
|
||||
bootstrap_label = Flask_config.bootstrap_label
|
||||
|
||||
# ============ BLUEPRINT ============
|
||||
hunters = Blueprint('hunters', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/hunter'))
|
||||
|
||||
# ============ VARIABLES ============
|
||||
|
||||
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
def api_validator(api_response):
|
||||
if api_response:
|
||||
return Response(json.dumps(api_response[0], indent=2, sort_keys=True), mimetype='application/json'), api_response[1]
|
||||
|
||||
def create_json_response(data, status_code):
|
||||
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
|
||||
|
||||
# ============= ROUTES ==============
|
||||
|
||||
@hunters.route('/retro_hunt/tasks', methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def retro_hunt_all_tasks():
|
||||
retro_hunts = Tracker.get_all_retro_hunt_tasks_with_metadata()
|
||||
return render_template("retro_hunt_tasks.html", retro_hunts=retro_hunts, bootstrap_label=bootstrap_label)
|
||||
|
||||
@hunters.route('/retro_hunt/task/show', methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def retro_hunt_show_task():
|
||||
task_uuid = request.args.get('uuid', None)
|
||||
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
if date_from:
|
||||
date_from = date_from.replace('-', '')
|
||||
if date_to:
|
||||
date_to = date_to.replace('-', '')
|
||||
|
||||
res = Tracker.api_check_retro_hunt_task_uuid(task_uuid)
|
||||
if res:
|
||||
return create_json_response(res[0], res[1])
|
||||
|
||||
dict_task = Tracker.get_retro_hunt_task_metadata(task_uuid, date=True, progress=True, creator=True,
|
||||
sources=True, tags=True, description=True)
|
||||
rule_content = Tracker.get_yara_rule_content(dict_task['rule'])
|
||||
|
||||
if date_from:
|
||||
res = Tracker.api_get_retro_hunt_items({'uuid': task_uuid, 'date_from': date_from, 'date_to': date_to})
|
||||
if res[1] != 200:
|
||||
return create_json_response(res[0], res[1])
|
||||
dict_task['items'] = res[0]['items']
|
||||
dict_task['date_from_input'] = res[0]['date_from']
|
||||
dict_task['date_to_input'] = res[0]['date_to']
|
||||
else:
|
||||
dict_task['items'] = []
|
||||
dict_task['date_from_input'] = dict_task['date_from']
|
||||
dict_task['date_to_input'] = dict_task['date_to']
|
||||
|
||||
return render_template("show_retro_hunt.html", dict_task=dict_task,
|
||||
rule_content=rule_content,
|
||||
bootstrap_label=bootstrap_label)
|
||||
|
||||
|
||||
@hunters.route('/retro_hunt/task/add', methods=['GET', 'POST'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def retro_hunt_add_task():
|
||||
if request.method == 'POST':
|
||||
name = request.form.get("name", '')
|
||||
description = request.form.get("description", '')
|
||||
timeout = request.form.get("timeout", 30)
|
||||
tags = request.form.get("tags", [])
|
||||
if tags:
|
||||
tags = tags.split()
|
||||
# mails = request.form.get("mails", [])
|
||||
# if mails:
|
||||
# mails = mails.split()
|
||||
|
||||
sources = request.form.get("sources", [])
|
||||
if sources:
|
||||
sources = json.loads(sources)
|
||||
|
||||
date_from = request.form.get('date_from')
|
||||
date_to = request.form.get('date_to')
|
||||
if date_from:
|
||||
date_from = date_from.replace('-', '')
|
||||
if date_to:
|
||||
date_to = date_to.replace('-', '')
|
||||
|
||||
# YARA #
|
||||
yara_default_rule = request.form.get("yara_default_rule")
|
||||
yara_custom_rule = request.form.get("yara_custom_rule")
|
||||
if yara_custom_rule:
|
||||
rule = yara_custom_rule
|
||||
rule_type='yara_custom'
|
||||
else:
|
||||
rule = yara_default_rule
|
||||
rule_type='yara_default'
|
||||
|
||||
user_id = current_user.get_id()
|
||||
|
||||
input_dict = {"name": name, "description": description, "creator": user_id,
|
||||
"rule": rule, "type": rule_type,
|
||||
"tags": tags, "sources": sources, "timeout": timeout, #"mails": mails,
|
||||
"date_from": date_from, "date_to": date_to}
|
||||
|
||||
res = Tracker.api_create_retro_hunt_task(input_dict, user_id)
|
||||
if res[1] == 200:
|
||||
return redirect(url_for('hunters.retro_hunt_all_tasks'))
|
||||
else:
|
||||
## TODO: use modal
|
||||
return create_json_response(res[0], res[1])
|
||||
else:
|
||||
return render_template("add_retro_hunt_task.html",
|
||||
all_yara_files=Tracker.get_all_default_yara_files(),
|
||||
all_sources=item_basic.get_all_items_sources(r_list=True))
|
||||
|
||||
@hunters.route('/retro_hunt/task/pause', methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def retro_hunt_pause_task():
|
||||
task_uuid = request.args.get('uuid', None)
|
||||
res = Tracker.api_pause_retro_hunt_task(task_uuid)
|
||||
if res[1] != 200:
|
||||
return create_json_response(res[0], res[1])
|
||||
return redirect(url_for('hunters.retro_hunt_all_tasks'))
|
||||
|
||||
@hunters.route('/retro_hunt/task/resume', methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def retro_hunt_resume_task():
|
||||
task_uuid = request.args.get('uuid', None)
|
||||
res = Tracker.api_resume_retro_hunt_task(task_uuid)
|
||||
if res[1] != 200:
|
||||
return create_json_response(res[0], res[1])
|
||||
return redirect(url_for('hunters.retro_hunt_all_tasks'))
|
||||
|
||||
@hunters.route('/retro_hunt/task/delete', methods=['GET'])
|
||||
@login_required
|
||||
@login_analyst
|
||||
def retro_hunt_delete_task():
|
||||
task_uuid = request.args.get('uuid', None)
|
||||
res = Tracker.api_delete_retro_hunt_task(task_uuid)
|
||||
if res[1] != 200:
|
||||
return create_json_response(res[0], res[1])
|
||||
return redirect(url_for('hunters.retro_hunt_all_tasks'))
|
||||
|
||||
|
||||
#### JSON ####
|
||||
|
||||
@hunters.route("/tracker/get_json_retro_hunt_nb_items_by_date", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def get_json_retro_hunt_nb_items_by_date():
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
|
||||
if date_from:
|
||||
date_from = date_from.replace('-', '')
|
||||
if date_to:
|
||||
date_to = date_to.replace('-', '')
|
||||
|
||||
task_uuid = request.args.get('uuid')
|
||||
|
||||
if date_from and date_to:
|
||||
res = Tracker.get_retro_hunt_nb_item_by_day([task_uuid], date_from=date_from, date_to=date_to)
|
||||
else:
|
||||
res = Term.get_retro_hunt_nb_item_by_day([task_uuid])
|
||||
return jsonify(res)
|
||||
|
||||
|
||||
## - - ##
|
||||
|
|
@ -28,9 +28,6 @@ import Flask_config
|
|||
|
||||
app = Flask_config.app
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_term = Flask_config.r_serv_term
|
||||
r_serv_cred = Flask_config.r_serv_cred
|
||||
r_serv_db = Flask_config.r_serv_db
|
||||
bootstrap_label = Flask_config.bootstrap_label
|
||||
|
||||
hunter = Blueprint('hunter', __name__, template_folder='templates')
|
||||
|
|
|
|||
228
var/www/templates/hunter/add_retro_hunt_task.html
Normal file
228
var/www/templates/hunter/add_retro_hunt_task.html
Normal file
|
|
@ -0,0 +1,228 @@
|
|||
<!DOCTYPE html>
|
||||
|
||||
<html>
|
||||
<head>
|
||||
<title>AIL-Framework</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||
|
||||
</head>
|
||||
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'hunter/menu_sidebar.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="card my-3">
|
||||
<div class="card-header bg-dark text-white">
|
||||
<h5 class="card-title">Create a new Retro Hunt task</h5>
|
||||
</div>
|
||||
<div class="card-body">
|
||||
|
||||
<form action="{{ url_for('hunters.retro_hunt_add_task') }}" method='post'>
|
||||
|
||||
<div class="row">
|
||||
<div class="col-12 col-xl-9">
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-secondary text-white"><i class="fas fa-quote-right"></i></div>
|
||||
</div>
|
||||
<input id="name" name="name" class="form-control" placeholder="Retro Hunt Name" type="text" required>
|
||||
</div>
|
||||
|
||||
<!-- <div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-secondary text-white"><i class="fas fa-at"></i></div>
|
||||
</div>
|
||||
<input id="mails" name="mails" class="form-control" placeholder="E-Mails Notification (optional, space separated)" type="text"}>
|
||||
</div> -->
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-info text-white"><i class="fas fa-pencil-alt"></i></div>
|
||||
</div>
|
||||
<textarea id="description" name="description" class="form-control" placeholder="Retro Hunt Description (optional)" rows="3"></textarea>
|
||||
</div>
|
||||
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-danger text-white"><i class="fas fa-tag"></i></div>
|
||||
</div>
|
||||
<input id="tags" name="tags" class="form-control" placeholder="Tags (optional, space separated)" type="text">
|
||||
</div>
|
||||
|
||||
<div class="input-group mb-2 mr-sm-2">
|
||||
<div class="input-group-prepend">
|
||||
<div class="input-group-text bg-dark text-white"><i class="fas fa-folder"></i></div>
|
||||
</div>
|
||||
<input id="sources" class="form-control" type="text" name="sources" placeholder="Sources to track (ALL IF EMPTY)" autocomplete="off">
|
||||
</div>
|
||||
|
||||
<h6>Date range:</h6>
|
||||
<div class="row mb-2">
|
||||
<div class="col-lg-6">
|
||||
<div class="input-group" id="date-range-from">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from" autocomplete="off" required>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-6">
|
||||
<div class="input-group" id="date-range-to">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to" autocomplete="off" required>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<div class="col-12 col-xl-3">
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
|
||||
<div class="mb-2" id="yara_rule">
|
||||
<div class="" id="yara_default_rules">
|
||||
|
||||
<h6>Default YARA rules:</h6>
|
||||
<select class="custom-select w-100 mb-3" id="yara_default_rule" name="yara_default_rule" onchange="get_default_rule_content(this);">
|
||||
<option selected>Select a default rule</option>
|
||||
{% for yara_types in all_yara_files %}
|
||||
{% for yara_file_name in all_yara_files[yara_types] %}
|
||||
<option value="{{yara_types}}/{{yara_file_name}}">{{yara_types}} - {{yara_file_name}}</option>
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
</select>
|
||||
|
||||
<pre class="border bg-light" id="default_yara_rule_content"></pre>
|
||||
|
||||
</div>
|
||||
|
||||
<hr>
|
||||
|
||||
<h6>Custom YARA rules:</h6>
|
||||
<div class="row" id="textarea">
|
||||
<textarea class="form-control mx-3" id="text_input" name="yara_custom_rule" placeholder="Enter your own YARA rule" rows="5">{%if dict_tracker%}{%if dict_tracker['type']=='yara' and dict_tracker['content']%}{{dict_tracker['content']}}{%endif%}{%endif%}</textarea>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<br>
|
||||
<button class="btn btn-primary mt-2">
|
||||
<i class="fas fa-plus"></i> Create Retro Hunt Task
|
||||
</button>
|
||||
|
||||
</form>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</body>
|
||||
|
||||
<script>
|
||||
var ltags;
|
||||
var chart = {};
|
||||
$(document).ready(function(){
|
||||
$('#nav_title_retro_hunt').removeClass("text-muted");
|
||||
|
||||
$('#date-range-from').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function(){
|
||||
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
|
||||
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2){
|
||||
$('#date-range-from-input').val(s1);
|
||||
$('#date-range-to-input').val(s2);
|
||||
}
|
||||
});
|
||||
$('#date-range-to').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function(){
|
||||
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
|
||||
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2){
|
||||
$('#date-range-from-input').val(s1);
|
||||
$('#date-range-to-input').val(s2);
|
||||
}
|
||||
});
|
||||
|
||||
sources = $('#sources').tagSuggest({
|
||||
data: {{all_sources|safe}},
|
||||
{%if dict_tracker%}{%if dict_tracker['sources']%}value: {{dict_tracker['sources']|safe}},{%endif%}{%endif%}
|
||||
sortOrder: 'name',
|
||||
maxDropHeight: 200,
|
||||
name: 'sources',
|
||||
emptyText: 'Sources to track (ALL IF EMPTY)',
|
||||
});
|
||||
|
||||
{%if dict_tracker%}
|
||||
$('#tracker_type').val('{{dict_tracker['type']}}').change();
|
||||
|
||||
{%if dict_tracker['type']=='yara' and dict_tracker['yara_file']%}
|
||||
$('#yara_default_rule').val('{{dict_tracker['yara_file']}}').change();
|
||||
{%endif%}
|
||||
{%endif%}
|
||||
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
|
||||
function get_default_rule_content(selector){
|
||||
var yara_name = selector.value
|
||||
if (yara_name === "Select a default rule") {
|
||||
jQuery("#default_yara_rule_content").text("")
|
||||
} else {
|
||||
$.getJSON("{{ url_for('hunter.get_default_yara_rule_content') }}?rule_name=" + yara_name,
|
||||
function(data) {
|
||||
jQuery("#default_yara_rule_content").text(data['content'])
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
</script>
|
||||
|
|
@ -6,7 +6,7 @@
|
|||
</button>
|
||||
|
||||
<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
|
||||
<h5 class="d-flex text-muted w-100">
|
||||
<h5 class="d-flex text-muted w-100" id="nav_title_trackers">
|
||||
<span>Trackers </span>
|
||||
<a class="ml-auto" href="{{url_for('hunter.add_tracked_menu')}}">
|
||||
<i class="fas fa-plus-circle ml-auto"></i>
|
||||
|
|
@ -44,5 +44,19 @@
|
|||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
<h5 class="d-flex text-muted w-100" id="nav_title_retro_hunt">
|
||||
<span>Retro Hunt </span>
|
||||
<a class="ml-auto" href="{{url_for('hunters.retro_hunt_add_task')}}">
|
||||
<i class="fas fa-plus-circle ml-auto"></i>
|
||||
</a>
|
||||
</h5>
|
||||
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('hunters.retro_hunt_all_tasks')}}" id="nav_retro_hunts">
|
||||
<i class="fas fa-meteor"></i>
|
||||
<span>Retro Hunt</span>
|
||||
</a>
|
||||
</li>
|
||||
</ul>
|
||||
</nav>
|
||||
</div>
|
||||
|
|
|
|||
138
var/www/templates/hunter/retro_hunt_tasks.html
Normal file
138
var/www/templates/hunter/retro_hunt_tasks.html
Normal file
|
|
@ -0,0 +1,138 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Retro Hunt</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'hunter/menu_sidebar.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<a class="btn btn-primary my-4" href="{{url_for('hunters.retro_hunt_add_task')}}">
|
||||
<i class="fas fa-plus ml-auto"></i>
|
||||
Create New Retro Hunt
|
||||
</a>
|
||||
|
||||
<table id="table_user_trackers" class="table table-striped border-primary">
|
||||
<thead class="bg-dark text-white">
|
||||
<tr>
|
||||
<th>Name</th>
|
||||
<th>Date</th>
|
||||
<th>Nb Matches</th>
|
||||
<th></th>
|
||||
<th></th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody style="font-size: 15px;">
|
||||
{% for dict_task in retro_hunts %}
|
||||
<tr class="border-color: blue;">
|
||||
<td>
|
||||
<a href="{{ url_for('hunters.retro_hunt_show_task') }}?uuid={{ dict_task['uuid'] }}">
|
||||
<span>
|
||||
{{ dict_task['name']}}
|
||||
</span>
|
||||
</a>
|
||||
<div>
|
||||
{% for tag in dict_task['tags'] %}
|
||||
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item<ags={{ tag }}">
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</td>
|
||||
<td>{{ dict_task['date'][0:4]}}/{{ dict_task['date'][4:6]}}/{{ dict_task['date'][6:8]}}</td>
|
||||
<td>
|
||||
<b><h3 class="font-weight-bold text-primary">{{dict_task['nb_match']}}</h3></b>
|
||||
</td>
|
||||
<td>
|
||||
{%if dict_task['state']=='paused'%}
|
||||
<a href="{{ url_for('hunters.retro_hunt_resume_task') }}?uuid={{dict_task['uuid']}}" class="mx-1">
|
||||
<button class='btn btn-info'><i class="fas fa-play"></i></button>
|
||||
</a>
|
||||
{%endif%}
|
||||
{%if dict_task['state']=='running' or dict_task['state']=='pending'%}
|
||||
<!-- <a href="{{ url_for('hunters.retro_hunt_pause_task') }}?uuid={{dict_task['uuid']}}" class="mx-1">
|
||||
<button class='btn btn-secondary'><i class="fas fa-stop"></i></button>
|
||||
</a> -->
|
||||
<a href="{{ url_for('hunters.retro_hunt_pause_task') }}?uuid={{dict_task['uuid']}}" class="mx-1">
|
||||
<button class='btn btn-info'><i class="fas fa-pause"></i></button>
|
||||
</a>
|
||||
{%endif%}
|
||||
</td>
|
||||
|
||||
<td class="text-center">
|
||||
<span class="justify-content-end">
|
||||
{%if dict_task['state']=='pending'%}
|
||||
<span class="text-secondary"><i class="fas fa-ellipsis-h fa-3x"></i>pending</span>
|
||||
{%elif dict_task['state']=='completed'%}
|
||||
<span class="text-success"><i class="fas fa-check-square fa-3x"></i> completed</span>
|
||||
{%elif dict_task['state']=='paused'%}
|
||||
<span class="text-secondary"><i class="fas fa-pause fa-3x"></i> paused [{{ dict_task['progress']}}%]</span>
|
||||
{%elif dict_task['state']=='running'%}
|
||||
<span class="text-secondary"><i class="fas fa-sync-alt fa-3x fa-spin"></i>running [{{ dict_task['progress']}}%]</span>
|
||||
{%endif%}
|
||||
<span>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
<br>
|
||||
<br>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
$('#nav_title_retro_hunt').removeClass("text-muted");
|
||||
$("#nav_retro_hunts").addClass("active");
|
||||
|
||||
$('#table_user_trackers').DataTable({
|
||||
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||
"iDisplayLength": 10,
|
||||
"order": [[ 1, "desc" ],[ 4, "desc" ]]
|
||||
});
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
302
var/www/templates/hunter/show_retro_hunt.html
Normal file
302
var/www/templates/hunter/show_retro_hunt.html
Normal file
|
|
@ -0,0 +1,302 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
|
||||
<title>AIL Framework - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3/graphlinesgroup.js')}}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||
|
||||
<style>
|
||||
.btn-link {
|
||||
color: #17a2b8
|
||||
}
|
||||
.btn-link:hover {
|
||||
color: blue;
|
||||
}
|
||||
.mouse_pointer{
|
||||
cursor: pointer;
|
||||
}
|
||||
</style>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'hunter/menu_sidebar.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="row">
|
||||
<div class="col-lg-4">
|
||||
<div class="card my-2">
|
||||
<div class="card-header bg-dark text-white">
|
||||
<div class="">
|
||||
<span class="badge badge-pill badge-light lex-row-reverse float-right">
|
||||
{%if dict_task['state']=='pending'%}
|
||||
<span class="text-secondary"><i class="fas fa-ellipsis-h fa-3x"></i>pending</span>
|
||||
{%elif dict_task['state']=='completed'%}
|
||||
<span class="text-success"><i class="fas fa-check-square fa-3x"></i> completed</span>
|
||||
{%elif dict_task['state']=='paused'%}
|
||||
<span class="text-secondary"><i class="fas fa-pause fa-3x"></i> paused</span>
|
||||
{%elif dict_task['state']=='running'%}
|
||||
<span class="text-secondary"><i class="fas fa-sync-alt fa-3x fa-spin"></i>running</span>
|
||||
{%endif%}
|
||||
</span>
|
||||
|
||||
<h3 class="card-title">
|
||||
{{dict_task['name']}}
|
||||
{%if dict_task['state']!='completed'%}
|
||||
<span class="text-info">[{{ dict_task['progress']}}%]<span>
|
||||
{%endif%}
|
||||
</h3>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class="card-body bg-light pt-2">
|
||||
<div class="d-flex justify-content-end">
|
||||
{%if dict_task['state']=='paused'%}
|
||||
<a href="{{ url_for('hunters.retro_hunt_resume_task') }}?uuid={{dict_task['uuid']}}" class="mx-1">
|
||||
<button class='btn btn-info'><i class="fas fa-play"></i></button>
|
||||
</a>
|
||||
{%endif%}
|
||||
{%if dict_task['state']=='running' or dict_task['state']=='pending'%}
|
||||
<!-- <a href="{{ url_for('hunters.retro_hunt_pause_task') }}?uuid={{dict_task['uuid']}}" class="mx-1">
|
||||
<button class='btn btn-secondary'><i class="fas fa-stop"></i></button>
|
||||
</a> -->
|
||||
<a href="{{ url_for('hunters.retro_hunt_pause_task') }}?uuid={{dict_task['uuid']}}" class="mx-1">
|
||||
<button class='btn btn-info'><i class="fas fa-pause"></i></button>
|
||||
</a>
|
||||
{%endif%}
|
||||
<a href="{{ url_for('hunters.retro_hunt_delete_task') }}?uuid={{dict_task['uuid']}}" class="mx-1">
|
||||
<button class='btn btn-danger'><i class="fas fa-trash-alt"></i></button>
|
||||
</a>
|
||||
</div>
|
||||
<hr>
|
||||
<table class="table table-borderless">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td class="text-right"><b>Date</b></td>
|
||||
<td>
|
||||
{{dict_task['date'][0:4]}}/{{dict_task['date'][4:6]}}/{{dict_task['date'][6:8]}}
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="text-right"><b>Search date</b></td>
|
||||
<td>
|
||||
{{dict_task['date_from'][0:4]}}/{{dict_task['date_from'][4:6]}}/{{dict_task['date_from'][6:8]}} -
|
||||
{{dict_task['date_to'][0:4]}}/{{dict_task['date_to'][4:6]}}/{{dict_task['date_to'][6:8]}}
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="text-right"><b>Description</b></td>
|
||||
<td>{{dict_task['description']}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="text-right"><b>Tags</b></td>
|
||||
<td>
|
||||
{%for tag in dict_task['tags']%}
|
||||
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item<ags={{ tag }}">
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
|
||||
</a>
|
||||
{%endfor%}
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="text-right"><b>Creator</b></td>
|
||||
<td>{{dict_task['creator']}}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td class="text-right"><b>Sources</b></td>
|
||||
<td>
|
||||
<div class="">
|
||||
{%if not dict_task['sources']%}
|
||||
<span class="badge badge-secondary">All Souces</span>
|
||||
{%else%}
|
||||
{%for source in dict_task['sources']%}
|
||||
<span class="badge badge-secondary">{{source}}</span>
|
||||
{%endfor%}
|
||||
{%endif%}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="col-lg-8">
|
||||
|
||||
<div class="card border-dark my-2">
|
||||
<div class="card-body">
|
||||
|
||||
<div class="row mb-2">
|
||||
<div class="col-lg-6">
|
||||
<div class="input-group" id="date-range-from">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" name="date_from" autocomplete="off"
|
||||
value="{{ dict_task['date_from_input'][0:4] }}-{{ dict_task['date_from_input'][4:6] }}-{{ dict_task['date_from_input'][6:8] }}">
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-lg-6">
|
||||
<div class="input-group" id="date-range-to">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" name="date_to" autocomplete="off"
|
||||
value="{{ dict_task['date_to_input'][0:4] }}-{{ dict_task['date_to_input'][4:6] }}-{{ dict_task['date_to_input'][6:8] }}">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<button class="btn btn-info" type="button" id="button-search-tags" onclick="getItems();">
|
||||
<i class="fas fa-search"></i> Show Items
|
||||
</button>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<p class="my-0"><pre class="border bg-light">{{ rule_content }}</pre></p>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{%if dict_task['items']%}
|
||||
<div class="mt-4">
|
||||
<table class="table table-bordered table-hover" id="items_table">
|
||||
<thead class="thead-dark">
|
||||
<tr>
|
||||
<th>Date</th>
|
||||
<th>Item Id</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
|
||||
{% for item in dict_task['items'] %}
|
||||
<tr>
|
||||
<td>
|
||||
{{item['date'][0:4]}}/{{item['date'][4:6]}}/{{item['date'][6:8]}}
|
||||
</td>
|
||||
<td>
|
||||
<a class="text-secondary" target="_blank" href="{{ url_for('objects_item.showItem') }}?id={{item['id']}}">
|
||||
<div style="line-height:0.9;">{{ item['id'] }}</div>
|
||||
</a>
|
||||
<div class="mb-2">
|
||||
{% for tag in item['tags'] %}
|
||||
<a href="{{ url_for('tags_ui.get_obj_by_tags') }}?object_type=item<ags={{ tag }}">
|
||||
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
<hr>
|
||||
{% endif %}
|
||||
|
||||
|
||||
<div id="graphline" class="text-center"></div>
|
||||
<br>
|
||||
<br>
|
||||
<br>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
$(document).ready(function(){
|
||||
$('#nav_title_retro_hunt').removeClass("text-muted");
|
||||
|
||||
$('#date-range-from').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function(){
|
||||
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
|
||||
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2){
|
||||
$('#date-range-from-input').val(s1);
|
||||
$('#date-range-to-input').val(s2);
|
||||
}
|
||||
});
|
||||
$('#date-range-to').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function(){
|
||||
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
|
||||
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2){
|
||||
$('#date-range-from-input').val(s1);
|
||||
$('#date-range-to-input').val(s2);
|
||||
}
|
||||
});
|
||||
|
||||
$('#items_table').DataTable({
|
||||
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||
"iDisplayLength": 10,
|
||||
"order": [[ 0, "asc" ]]
|
||||
});
|
||||
|
||||
let div_width = $("#graphline").width();
|
||||
$.getJSON( "{{ url_for('hunters.get_json_retro_hunt_nb_items_by_date') }}?uuid={{ dict_task['uuid'] }}&date_from={{ dict_task['date_from_input'] }}&date_to={{ dict_task['date_to_input'] }}",
|
||||
function( data ) {multilines_group("graphline", data, {"width": div_width});}
|
||||
);
|
||||
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
|
||||
function getItems() {
|
||||
var date_from = $('#date-range-from-input').val();
|
||||
var date_to =$('#date-range-to-input').val();
|
||||
window.location.replace("{{ url_for('hunters.retro_hunt_show_task') }}?uuid={{ dict_task['uuid'] }}&date_from="+date_from+"&date_to="+date_to);
|
||||
}
|
||||
|
||||
</script>
|
||||
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
Loading…
Reference in a new issue