diff --git a/OVERVIEW.md b/OVERVIEW.md index a3425155..be19bbfd 100644 --- a/OVERVIEW.md +++ b/OVERVIEW.md @@ -109,63 +109,65 @@ Redis and ARDB overview | **uuid**:ltags | **tag** | | **uuid**:ltagsgalaxies | **tag** | -## DB2 - New TermFreq: +## DB2 - Leak Hunter: -##### Term Tracker metadata: +##### Tracker metadata: | Hset - Key | Field | Value | | ------ | ------ | ------ | -| tracked_term:**uuid** | tracked | **tacked word/set/regex** | -| | type | **term/set/regex** | +| tracker:**uuid** | tracked | **tacked word/set/regex** | +| | type | **word/set/regex** | | | date | **date added** | | | user_id | **created by user_id** | | | dashboard | **0/1 Display alert on dashboard** | | | level | **0/1 Tracker visibility** | -##### Term Tracked by user_id (visibility level: user only): +##### Tracker by user_id (visibility level: user only): | Set - Key | Value | | ------ | ------ | -| user:tracked_term:**user_id** | **uuid - tracked term uuid** | +| user:tracker:**user_id** | **uuid - tracker uuid** | +| user:tracker:**user_id**:**word/set/regex - tracker type** | **uuid - tracker uuid** | -##### Global Term Tracked (visibility level: all users): +##### Global Tracker (visibility level: all users): | Set - Key | Value | | ------ | ------ | -| gobal:tracked_term | **uuid - tracked term uuid** | +| gobal:tracker | **uuid - tracker uuid** | +| gobal:tracker:**word/set/regex - tracker type** | **uuid - tracker uuid** | -##### All Term Tracked by type: +##### All Tracker by type: | Set - Key | Value | | ------ | ------ | -| all:tracked_term:**word/set/regex - term type** | **tracked term** | +| all:tracker:**word/set/regex - tracker type** | **tracked item** | | Set - Key | Value | | ------ | ------ | -| all:tracked_term_uuid:**term type**:**tracked term** | **uuid - tracked term uuid** | +| all:tracker_uuid:**tracker type**:**tracked item** | **uuid - tracker uuid** | -##### All Term Tracked items: +##### All Tracked items: | Set - Key | Value | | ------ | ------ | -| tracked_term:item:**uuid**:**date** | **item_id** | +| tracker:item:**uuid**:**date** | **item_id** | -##### All Term Tracked tags: +##### All Tracked tags: | Set - Key | Value | | ------ | ------ | -| tracked_term:tags:**uuid** | **tag** | +| tracker:tags:**uuid** | **tag** | -##### All Term Tracked tags: +##### All Tracked mail: | Set - Key | Value | | ------ | ------ | -| tracked_term:mail:**uuid** | **mail** | +| tracker:mail:**uuid** | **mail** | -##### Refresh Tracked term: +##### Refresh Tracker: | Key | Value | | ------ | ------ | -| tracked_term:refresh:word | **last refreshed epoch** | -| tracked_term:refresh:set | - | -| tracked_term:refresh:regex | - | +| tracker:refresh:word | **last refreshed epoch** | +| tracker:refresh:set | - | +| tracker:refresh:regex | - | -##### Zset Stat Tracked term: +##### Zset Stat Tracker: | Key | Field | Value | | ------ | ------ | ------ | -| tracked_term:stat:**uuid** | **date** | **nb_seen** | +| tracker:stat:**uuid** | **date** | **nb_seen** | ##### Stat token: | Key | Field | Value | diff --git a/bin/DbCleaner.py b/bin/DbCleaner.py index ed2bb752..e0cf6512 100755 --- a/bin/DbCleaner.py +++ b/bin/DbCleaner.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* """ -The TermTracker Module +The DbCleaner Module =================== """ @@ -36,8 +36,6 @@ if __name__ == "__main__": publisher.channel = "Script" publisher.info("DbCleaner started") - config_section = 'TermTrackerMod' - # low priority time.sleep(180) diff --git a/bin/packages/Date.py b/bin/packages/Date.py index ccf59c54..c9997bab 100644 --- a/bin/packages/Date.py +++ b/bin/packages/Date.py @@ -60,3 +60,13 @@ def get_date_range(num_day): for i in range(0, num_day+1): date_list.append(date.substract_day(i)) return list(reversed(date_list)) + +def substract_date(date_from, date_to): + date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8])) + date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8])) + delta = date_to - date_from # timedelta + l_date = [] + for i in range(delta.days + 1): + date = date_from + datetime.timedelta(i) + l_date.append( date.strftime('%Y%m%d') ) + return l_date diff --git a/bin/packages/Item.py b/bin/packages/Item.py index aa2b92c2..2a464a5b 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -134,3 +134,13 @@ def get_item_pgp_name(item_id): def get_item_pgp_mail(item_id): return _get_item_correlation('pgpdump', 'mail', item_id) + + +### +### GET Internal Module DESC +### +def get_item_list_desc(list_item_id): + desc_list = [] + for item_id in list_item_id: + desc_list.append( {'id': item_id, 'date': get_item_date(item_id), 'tags': Tag.get_item_tags(item_id)} ) + return desc_list diff --git a/bin/packages/Term.py b/bin/packages/Term.py index 37fb661d..b4fcd832 100755 --- a/bin/packages/Term.py +++ b/bin/packages/Term.py @@ -18,6 +18,7 @@ sys.path.append(os.path.join(os.environ['AIL_FLASK'], 'modules')) import Flask_config import Date +import Item r_serv_term = Flask_config.r_serv_term email_regex = Flask_config.email_regex @@ -47,11 +48,11 @@ def is_in_role(user_id, role): def check_term_uuid_valid_access(term_uuid, user_id): if not is_valid_uuid_v4(term_uuid): return ({"status": "error", "reason": "Invalid uuid"}, 400) - level = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'level') + level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level') if not level: return ({"status": "error", "reason": "Unknown uuid"}, 404) if level == 0: - if r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'user_id') != user_id: + if r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id') != user_id: if not is_in_role(user_id, 'admin'): return ({"status": "error", "reason": "Unknown uuid"}, 404) return None @@ -91,10 +92,10 @@ def get_text_word_frequency(item_content, filtering=True): # # TODO: create all tracked words def get_tracked_words_list(): - return list(r_serv_term.smembers('all:tracked_term:word')) + return list(r_serv_term.smembers('all:tracker:word')) def get_set_tracked_words_list(): - set_list = r_serv_term.smembers('all:tracked_term:set') + set_list = r_serv_term.smembers('all:tracker:set') all_set_list = [] for elem in set_list: res = elem.split(';') @@ -104,7 +105,7 @@ def get_set_tracked_words_list(): return all_set_list def get_regex_tracked_words_dict(): - regex_list = r_serv_term.smembers('all:tracked_term:regex') + regex_list = r_serv_term.smembers('all:tracker:regex') dict_tracked_regex = {} for regex in regex_list: dict_tracked_regex[regex] = re.compile(regex) @@ -113,24 +114,24 @@ def get_regex_tracked_words_dict(): def get_tracked_term_list_item(term_uuid, date_from, date_to): all_item_id = [] if date_from and date_to: - for date in r_serv_term.zrangebyscore('tracked_term:stat:{}'.format(term_uuid), int(date_from), int(date_to)): - all_item_id = all_item_id + list(r_serv_term.smembers('tracked_term:item:{}:{}'.format(term_uuid, date))) + for date in r_serv_term.zrangebyscore('tracker:stat:{}'.format(term_uuid), int(date_from), int(date_to)): + all_item_id = all_item_id + list(r_serv_term.smembers('tracker:item:{}:{}'.format(term_uuid, date))) return all_item_id def is_term_tracked_in_global_level(term, term_type): - res = r_serv_term.smembers('all:tracked_term_uuid:{}:{}'.format(term_type, term)) + res = r_serv_term.smembers('all:tracker_uuid:{}:{}'.format(term_type, term)) if res: for elem_uuid in res: - if r_serv_term.hget('tracked_term:{}'.format(elem_uuid), 'level')=='1': + if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'level')=='1': return True return False def is_term_tracked_in_user_level(term, term_type, user_id): - res = r_serv_term.smembers('user:tracked_term:{}'.format(user_id)) + res = r_serv_term.smembers('user:tracker:{}'.format(user_id)) if res: for elem_uuid in res: - if r_serv_term.hget('tracked_term:{}'.format(elem_uuid), 'tracked')== term: - if r_serv_term.hget('tracked_term:{}'.format(elem_uuid), 'type')== term_type: + if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'tracked')== term: + if r_serv_term.hget('tracker:{}'.format(elem_uuid), 'type')== term_type: return True return False @@ -220,39 +221,42 @@ def add_tracked_term(term , term_type, user_id, level, tags, mails, dashboard=0) term_uuid = str(uuid.uuid4()) # create metadata - r_serv_term.hset('tracked_term:{}'.format(term_uuid), 'tracked',term) - r_serv_term.hset('tracked_term:{}'.format(term_uuid), 'type', term_type) - r_serv_term.hset('tracked_term:{}'.format(term_uuid), 'date', datetime.date.today().strftime("%Y%m%d")) - r_serv_term.hset('tracked_term:{}'.format(term_uuid), 'user_id', user_id) - r_serv_term.hset('tracked_term:{}'.format(term_uuid), 'level', level) - r_serv_term.hset('tracked_term:{}'.format(term_uuid), 'dashboard', dashboard) + r_serv_term.hset('tracker:{}'.format(term_uuid), 'tracked',term) + r_serv_term.hset('tracker:{}'.format(term_uuid), 'type', term_type) + r_serv_term.hset('tracker:{}'.format(term_uuid), 'date', datetime.date.today().strftime("%Y%m%d")) + r_serv_term.hset('tracker:{}'.format(term_uuid), 'user_id', user_id) + r_serv_term.hset('tracker:{}'.format(term_uuid), 'level', level) + r_serv_term.hset('tracker:{}'.format(term_uuid), 'dashboard', dashboard) # create all term set - r_serv_term.sadd('all:tracked_term:{}'.format(term_type), term) + r_serv_term.sadd('all:tracker:{}'.format(term_type), term) # create term - uuid map - r_serv_term.sadd('all:tracked_term_uuid:{}:{}'.format(term_type, term), term_uuid) + r_serv_term.sadd('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid) # add display level set if level == 0: # user only - r_serv_term.sadd('user:tracked_term:{}'.format(user_id), term_uuid) + r_serv_term.sadd('user:tracker:{}'.format(user_id), term_uuid) + r_serv_term.sadd('user:tracker:{}:{}'.format(user_id, term_type), term_uuid) elif level == 1: # global - r_serv_term.sadd('global:tracked_term', term_uuid) + r_serv_term.sadd('global:tracker', term_uuid) + r_serv_term.sadd('global:tracker:{}'.format(term_type), term_uuid) # create term tags list for tag in tags: - r_serv_term.sadd('tracked_term:tags:{}'.format(term_uuid), tag) + r_serv_term.sadd('tracker:tags:{}'.format(term_uuid), tag) # create term tags mail notification list for mail in mails: - r_serv_term.sadd('tracked_term:mail:{}'.format(term_uuid), mail) + r_serv_term.sadd('tracker:mail:{}'.format(term_uuid), mail) # toggle refresh module tracker list/set - r_serv_term.set('tracked_term:refresh:{}'.format(term_type), time.time()) + r_serv_term.set('tracker:refresh:{}'.format(term_type), time.time()) return term_uuid def parse_tracked_term_to_delete(dict_input, user_id): + term_uuid = dict_input.get("uuid", None) res = check_term_uuid_valid_access(term_uuid, user_id) if res: return res @@ -261,66 +265,68 @@ def parse_tracked_term_to_delete(dict_input, user_id): return ({"uuid": term_uuid}, 200) def delete_term(term_uuid): - term = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'tracked') - term_type = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'type') - level = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'level') - r_serv_term.srem('all:tracked_term_uuid:{}:{}'.format(term_type, term), term_uuid) + term = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked') + term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type') + level = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level') + r_serv_term.srem('all:tracker_uuid:{}:{}'.format(term_type, term), term_uuid) # Term not tracked by other users - if not r_serv_term.exists('all:tracked_term_uuid:{}:{}'.format(term_type, term)): - r_serv_term.srem('all:tracked_term:{}'.format(term_type), term) + if not r_serv_term.exists('all:tracker_uuid:{}:{}'.format(term_type, term)): + r_serv_term.srem('all:tracker:{}'.format(term_type), term) # toggle refresh module tracker list/set - r_serv_term.set('tracked_term:refresh:{}'.format(term_type), time.time()) + r_serv_term.set('tracker:refresh:{}'.format(term_type), time.time()) - if level == 0: # user only - user_id = term_type = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'user_id') - r_serv_term.srem('user:tracked_term:{}'.format(user_id), term_uuid) - elif level == 1: # global - r_serv_term.srem('global:tracked_term', term_uuid) + if level == '0': # user only + user_id = term_type = r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id') + r_serv_term.srem('user:tracker:{}'.format(user_id), term_uuid) + r_serv_term.srem('user:tracker:{}:{}'.format(user_id, term_type), term_uuid) + elif level == '1': # global + r_serv_term.srem('global:tracker', term_uuid) + r_serv_term.srem('global:tracker:{}'.format(term_type), term_uuid) # delete metatadata - r_serv_term.delete('tracked_term:{}'.format(term_uuid)) + r_serv_term.delete('tracker:{}'.format(term_uuid)) # remove tags - r_serv_term.delete('tracked_term:tags:{}'.format(term_uuid)) + r_serv_term.delete('tracker:tags:{}'.format(term_uuid)) # remove mails - r_serv_term.delete('tracked_term:mail:{}'.format(term_uuid)) + r_serv_term.delete('tracker:mail:{}'.format(term_uuid)) # remove item set - all_item_date = r_serv_term.zrange('tracked_term:stat:{}'.format(term_uuid), 0, -1) + all_item_date = r_serv_term.zrange('tracker:stat:{}'.format(term_uuid), 0, -1) for date in all_item_date: - r_serv_term.delete('tracked_term:item:{}:{}'.format(term_uuid, date)) - r_serv_term.delete('tracked_term:stat:{}'.format(term_uuid)) + r_serv_term.delete('tracker:item:{}:{}'.format(term_uuid, date)) + r_serv_term.delete('tracker:stat:{}'.format(term_uuid)) def replace_tracked_term_tags(term_uuid, tags): - r_serv_term.delete('tracked_term:tags:{}'.format(term_uuid)) + r_serv_term.delete('tracker:tags:{}'.format(term_uuid)) for tag in tags: - r_serv_term.sadd('tracked_term:tags:{}'.format(term_uuid), tag) + r_serv_term.sadd('tracker:tags:{}'.format(term_uuid), tag) def replace_tracked_term_mails(term_uuid, mails): res = verify_mail_list(mails) if res: return res else: - r_serv_term.delete('tracked_term:mail:{}'.format(term_uuid)) + r_serv_term.delete('tracker:mail:{}'.format(term_uuid)) for mail in mails: - r_serv_term.sadd('tracked_term:mail:{}'.format(term_uuid), mail) + r_serv_term.sadd('tracker:mail:{}'.format(term_uuid), mail) def get_term_uuid_list(term, term_type): - return list(r_serv_term.smembers('all:tracked_term_uuid:{}:{}'.format(term_type, term))) + return list(r_serv_term.smembers('all:tracker_uuid:{}:{}'.format(term_type, term))) def get_term_tags(term_uuid): - return list(r_serv_term.smembers('tracked_term:tags:{}'.format(term_uuid))) + return list(r_serv_term.smembers('tracker:tags:{}'.format(term_uuid))) def get_term_mails(term_uuid): - return list(r_serv_term.smembers('tracked_term:mail:{}'.format(term_uuid))) + return list(r_serv_term.smembers('tracker:mail:{}'.format(term_uuid))) def add_tracked_item(term_uuid, item_id, item_date): # track item - r_serv_term.sadd('tracked_term:item:{}:{}'.format(term_uuid, item_date), item_id) + r_serv_term.sadd('tracker:item:{}:{}'.format(term_uuid, item_date), item_id) # track nb item by date - r_serv_term.zadd('tracked_term:stat:{}'.format(term_uuid), item_date, int(item_date)) + r_serv_term.zadd('tracker:stat:{}'.format(term_uuid), item_date, int(item_date)) def create_token_statistics(item_date, word, nb): r_serv_term.zincrby('stat_token_per_item_by_day:{}'.format(item_date), word, 1) @@ -336,7 +342,7 @@ def get_all_token_stat_history(): return r_serv_term.smembers('stat_token_history') def get_tracked_term_last_updated_by_type(term_type): - epoch_update = r_serv_term.get('tracked_term:refresh:{}'.format(term_type)) + epoch_update = r_serv_term.get('tracker:refresh:{}'.format(term_type)) if not epoch_update: epoch_update = 0 return float(epoch_update) @@ -363,6 +369,7 @@ def parse_get_tracker_term_item(dict_input, user_id): date_from = date_to all_item_id = get_tracked_term_list_item(term_uuid, date_from, date_to) + all_item_id = Item.get_item_list_desc(all_item_id) res_dict = {} res_dict['uuid'] = term_uuid @@ -372,7 +379,7 @@ def parse_get_tracker_term_item(dict_input, user_id): return (res_dict, 200) def get_tracked_term_first_seen(term_uuid): - res = r_serv_term.zrange('tracked_term:stat:{}'.format(term_uuid), 0, 0) + res = r_serv_term.zrange('tracker:stat:{}'.format(term_uuid), 0, 0) if res: return res[0] else: @@ -380,7 +387,7 @@ def get_tracked_term_first_seen(term_uuid): def get_tracked_term_last_seen(term_uuid): - res = r_serv_term.zrevrange('tracked_term:stat:{}'.format(term_uuid), 0, 0) + res = r_serv_term.zrevrange('tracker:stat:{}'.format(term_uuid), 0, 0) if res: return res[0] else: @@ -388,15 +395,15 @@ def get_tracked_term_last_seen(term_uuid): def get_term_metedata(term_uuid, user_id=False, level=False, tags=False, mails=False, sparkline=False): dict_uuid = {} - dict_uuid['term'] = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'tracked') - dict_uuid['type'] = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'type') - dict_uuid['date'] = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'date') + dict_uuid['term'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'tracked') + dict_uuid['type'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'type') + dict_uuid['date'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'date') dict_uuid['first_seen'] = get_tracked_term_first_seen(term_uuid) dict_uuid['last_seen'] = get_tracked_term_last_seen(term_uuid) if user_id: - dict_uuid['user_id'] = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'user_id') + dict_uuid['user_id'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'user_id') if level: - dict_uuid['level'] = r_serv_term.hget('tracked_term:{}'.format(term_uuid), 'level') + dict_uuid['level'] = r_serv_term.hget('tracker:{}'.format(term_uuid), 'level') if mails: dict_uuid['mails'] = get_list_trackeed_term_mails(term_uuid) if tags: @@ -406,48 +413,71 @@ def get_term_metedata(term_uuid, user_id=False, level=False, tags=False, mails=F dict_uuid['uuid'] = term_uuid return dict_uuid -def get_tracked_term_sparkline(term_uuid, num_day=6): +def get_tracked_term_sparkline(tracker_uuid, num_day=6): date_range_sparkline = Date.get_date_range(num_day) sparklines_value = [] for date_day in date_range_sparkline: - nb_seen_this_day = r_serv_term.zscore('tracked_term:stat:{}'.format(term_uuid), date_day) + nb_seen_this_day = r_serv_term.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day)) if nb_seen_this_day is None: nb_seen_this_day = 0 sparklines_value.append(int(nb_seen_this_day)) return sparklines_value +def get_list_tracked_term_stats_by_day(list_tracker_uuid, num_day=31, date_from=None, date_to=None): + if date_from and date_to: + date_range = Date.substract_date(date_from, date_to) + else: + date_range = Date.get_date_range(num_day) + list_tracker_stats = [] + for tracker_uuid in list_tracker_uuid: + dict_tracker_data = [] + tracker = r_serv_term.hget('tracker:{}'.format(tracker_uuid), 'tracked') + for date_day in date_range: + nb_seen_this_day = r_serv_term.scard('tracker:item:{}:{}'.format(tracker_uuid, date_day)) + if nb_seen_this_day is None: + nb_seen_this_day = 0 + dict_tracker_data.append({"date": date_day,"value": int(nb_seen_this_day)}) + list_tracker_stats.append({"name": tracker,"Data": dict_tracker_data}) + return list_tracker_stats + def get_list_trackeed_term_tags(term_uuid): - res = r_serv_term.smembers('tracked_term:tags:{}'.format(term_uuid)) + res = r_serv_term.smembers('tracker:tags:{}'.format(term_uuid)) if res: return list(res) else: return [] def get_list_trackeed_term_mails(term_uuid): - res = r_serv_term.smembers('tracked_term:mail:{}'.format(term_uuid)) + res = r_serv_term.smembers('tracker:mail:{}'.format(term_uuid)) if res: return list(res) else: return [] -def get_user_tracked_term_uuid(user_id): - return list(r_serv_term.smembers('user:tracked_term:{}'.format(user_id))) +def get_user_tracked_term_uuid(user_id, filter_type=None): + if filter_type: + return list(r_serv_term.smembers('user:tracker:{}:{}'.format(user_id,filter_type))) + else: + return list(r_serv_term.smembers('user:tracker:{}'.format(user_id))) -def get_global_tracked_term_uuid(): - return list(r_serv_term.smembers('global:tracked_term')) +def get_global_tracked_term_uuid(filter_type=None): + if filter_type: + return list(r_serv_term.smembers('global:tracker:{}'.format(filter_type))) + else: + return list(r_serv_term.smembers('global:tracker')) -def get_all_user_tracked_terms(user_id): +def get_all_user_tracked_terms(user_id, filter_type=None): all_user_term = [] - all_user_term_uuid = get_user_tracked_term_uuid(user_id) + all_user_term_uuid = get_user_tracked_term_uuid(user_id, filter_type=filter_type) for term_uuid in all_user_term_uuid: - all_user_term.append(get_term_metedata(term_uuid, tags=True, mails=True)) + all_user_term.append(get_term_metedata(term_uuid, tags=True, mails=True, sparkline=True)) return all_user_term -def get_all_global_tracked_terms(): +def get_all_global_tracked_terms(filter_type=None): all_user_term = [] - all_user_term_uuid = get_global_tracked_term_uuid() + all_user_term_uuid = get_global_tracked_term_uuid(filter_type=filter_type) for term_uuid in all_user_term_uuid: - all_user_term.append(get_term_metedata(term_uuid, user_id=True, tags=True, mails=True)) + all_user_term.append(get_term_metedata(term_uuid, user_id=True, tags=True, mails=True, sparkline=True)) return all_user_term diff --git a/var/www/modules/hunter/Flask_hunter.py b/var/www/modules/hunter/Flask_hunter.py new file mode 100644 index 00000000..38376b67 --- /dev/null +++ b/var/www/modules/hunter/Flask_hunter.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +''' + Flask functions and routes for tracked items +''' +import json +import redis +import datetime +import calendar +import flask +from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect, Response + +from Role_Manager import login_admin, login_analyst +from flask_login import login_required, current_user + +import re +from pprint import pprint +import Levenshtein + +# --------------------------------------------------------------- + +import Paste +import Term + +# ============ VARIABLES ============ +import Flask_config + +app = Flask_config.app +cfg = Flask_config.cfg +baseUrl = Flask_config.baseUrl +r_serv_term = Flask_config.r_serv_term +r_serv_cred = Flask_config.r_serv_cred +r_serv_db = Flask_config.r_serv_db +bootstrap_label = Flask_config.bootstrap_label + +hunter = Blueprint('hunter', __name__, template_folder='templates') + +# ============ FUNCTIONS ============ + +# ============ ROUTES ============ + +@hunter.route("/trackers") +@login_required +@login_analyst +def tracked_menu(): + user_id = current_user.get_id() + user_term = Term.get_all_user_tracked_terms(user_id) + global_term = Term.get_all_global_tracked_terms() + return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label) + +@hunter.route("/trackers/word") +@login_required +@login_analyst +def tracked_menu_word(): + filter_type = 'word' + user_id = current_user.get_id() + user_term = Term.get_all_user_tracked_terms(user_id, filter_type='word') + global_term = Term.get_all_global_tracked_terms(filter_type='word') + return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) + +@hunter.route("/trackers/set") +@login_required +@login_analyst +def tracked_menu_set(): + filter_type = 'set' + user_id = current_user.get_id() + user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type) + global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) + return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) + +@hunter.route("/trackers/regex") +@login_required +@login_analyst +def tracked_menu_regex(): + filter_type = 'regex' + user_id = current_user.get_id() + user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type) + global_term = Term.get_all_global_tracked_terms(filter_type=filter_type) + return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type) + + +@hunter.route("/tracker/add", methods=['GET', 'POST']) +@login_required +@login_analyst +def add_tracked_menu(): + if request.method == 'POST': + term = request.form.get("term") + term_type = request.form.get("tracker_type") + nb_words = request.form.get("nb_word", 1) + level = request.form.get("level", 0) + tags = request.form.get("tags", []) + mails = request.form.get("mails", []) + + if level == 'on': + level = 1 + + if mails: + mails = mails.split() + if tags: + tags = tags.split() + + input_dict = {"term": term, "type": term_type, "nb_words": nb_words, "tags": tags, "mails": mails, "level": level} + user_id = current_user.get_id() + res = Term.parse_json_term_to_add(input_dict, user_id) + if res[1] == 200: + return redirect(url_for('hunter.tracked_menu')) + else: + ## TODO: use modal + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + else: + return render_template("Add_tracker.html") + +@hunter.route("/tracker/show_tracker") +@login_required +@login_analyst +def show_tracker(): + user_id = current_user.get_id() + term_uuid = request.args.get('uuid', None) + res = Term.check_term_uuid_valid_access(term_uuid, user_id) + if res: # invalid access + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + + if date_from: + date_from = date_from.replace('-', '') + if date_to: + date_to = date_to.replace('-', '') + + tracker_metadata = Term.get_term_metedata(term_uuid, user_id=True, level=True, tags=True, mails=True, sparkline=True) + + if date_from: + res = Term.parse_get_tracker_term_item({'uuid': term_uuid, 'date_from': date_from, 'date_to': date_to}, user_id) + if res[1] !=200: + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + tracker_metadata['items'] = res[0]['items'] + tracker_metadata['date_from'] = res[0]['date_from'] + tracker_metadata['date_to'] = res[0]['date_to'] + else: + tracker_metadata['items'] = [] + tracker_metadata['date_from'] = '' + tracker_metadata['date_to'] = '' + + return render_template("showTracker.html", tracker_metadata=tracker_metadata, bootstrap_label=bootstrap_label) + +@hunter.route("/tracker/update_tracker_tags", methods=['POST']) +@login_required +@login_analyst +def update_tracker_tags(): + user_id = current_user.get_id() + term_uuid = request.form.get('uuid') + res = Term.check_term_uuid_valid_access(term_uuid, user_id) + if res: # invalid access + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + tags = request.form.get('tags') + if tags: + tags = tags.split() + else: + tags = [] + Term.replace_tracked_term_tags(term_uuid, tags) + return redirect(url_for('hunter.show_tracker', uuid=term_uuid)) + +@hunter.route("/tracker/update_tracker_mails", methods=['POST']) +@login_required +@login_analyst +def update_tracker_mails(): + user_id = current_user.get_id() + term_uuid = request.form.get('uuid') + res = Term.check_term_uuid_valid_access(term_uuid, user_id) + if res: # invalid access + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + mails = request.form.get('mails') + if mails: + mails = mails.split() + else: + mails = [] + res = Term.replace_tracked_term_mails(term_uuid, mails) + if res: # invalid mail + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + return redirect(url_for('hunter.show_tracker', uuid=term_uuid)) + +@hunter.route("/tracker/delete", methods=['GET']) +@login_required +@login_analyst +def delete_tracker(): + user_id = current_user.get_id() + term_uuid = request.args.get('uuid') + res = Term.parse_tracked_term_to_delete({'uuid': term_uuid}, user_id) + if res[1] !=200: + return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] + return redirect(url_for('hunter.tracked_menu')) + +@hunter.route("/tracker/get_json_tracker_stats", methods=['GET']) +@login_required +@login_analyst +def get_json_tracker_stats(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + + if date_from: + date_from = date_from.replace('-', '') + if date_to: + date_to = date_to.replace('-', '') + + tracker_uuid = request.args.get('uuid') + + if date_from and date_to: + res = Term.get_list_tracked_term_stats_by_day([tracker_uuid], date_from=date_from, date_to=date_to) + else: + res = Term.get_list_tracked_term_stats_by_day([tracker_uuid]) + return jsonify(res) + +# ========= REGISTRATION ========= +app.register_blueprint(hunter, url_prefix=baseUrl) diff --git a/var/www/modules/terms/templates/Add_tracker.html b/var/www/modules/hunter/templates/Add_tracker.html similarity index 97% rename from var/www/modules/terms/templates/Add_tracker.html rename to var/www/modules/hunter/templates/Add_tracker.html index 8c0832d9..fc60d93c 100644 --- a/var/www/modules/terms/templates/Add_tracker.html +++ b/var/www/modules/hunter/templates/Add_tracker.html @@ -23,7 +23,7 @@
- {% include 'crawler/menu_sidebar.html' %} + {% include 'hunter/menu_sidebar.html' %}
@@ -34,7 +34,7 @@

Enter a domain and choose what kind of data you want.

-
+
diff --git a/var/www/modules/terms/templates/showTrackerTerm.html b/var/www/modules/hunter/templates/showTracker.html similarity index 62% rename from var/www/modules/terms/templates/showTrackerTerm.html rename to var/www/modules/hunter/templates/showTracker.html index 51d94699..a62be14c 100644 --- a/var/www/modules/terms/templates/showTrackerTerm.html +++ b/var/www/modules/hunter/templates/showTracker.html @@ -11,21 +11,21 @@ + + + + +