From 04bd78934d0c19d33c0e99e966eefb81461bd3d7 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 23 Feb 2021 14:07:57 +0100 Subject: [PATCH 1/6] fix: [API ACL] read_only user role --- var/www/modules/restApi/Flask_restApi.py | 32 ++++++++++++------------ 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 3b6363ab..8e1db66a 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -188,14 +188,14 @@ def one(): # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @restApi.route("api/v1/get/item", methods=['POST']) -@token_required('user') +@token_required('read_only') def get_item_id(): data = request.get_json() res = Item.get_item(data) return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] @restApi.route("api/v1/get/item/default", methods=['POST']) -@token_required('user') +@token_required('read_only') def get_item_id_basic(): data = request.get_json() @@ -218,7 +218,7 @@ def get_item_id_basic(): # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @restApi.route("api/v1/get/item/tag", methods=['POST']) -@token_required('user') +@token_required('read_only') def get_item_tag(): data = request.get_json() @@ -299,7 +299,7 @@ def delete_item_tags(): # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @restApi.route("api/v1/get/item/content", methods=['POST']) -@token_required('user') +@token_required('read_only') def get_item_content(): data = request.get_json() @@ -314,7 +314,7 @@ def get_item_content(): # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @restApi.route("api/v1/get/tag/metadata", methods=['POST']) -@token_required('user') +@token_required('read_only') def get_tag_metadata(): data = request.get_json() tag = data.get('tag', None) @@ -324,7 +324,7 @@ def get_tag_metadata(): return Response(json.dumps(metadata, indent=2, sort_keys=True), mimetype='application/json'), 200 @restApi.route("api/v1/get/tag/all", methods=['GET']) -@token_required('user') +@token_required('read_only') def get_all_tags(): res = {'tags': Tag.get_all_tags()} return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200 @@ -351,7 +351,7 @@ def delete_tracker_term(): return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] @restApi.route("api/v1/get/tracker/item", methods=['POST']) -@token_required('user') +@token_required('read_only') def get_tracker_term_item(): data = request.get_json() user_token = get_auth_from_header() @@ -364,7 +364,7 @@ def get_tracker_term_item(): # # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST']) -@token_required('user') +@token_required('read_only') def get_cryptocurrency_bitcoin_metadata(): data = request.get_json() crypto_address = data.get('bitcoin', None) @@ -373,7 +373,7 @@ def get_cryptocurrency_bitcoin_metadata(): return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] @restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST']) -@token_required('user') +@token_required('read_only') def get_cryptocurrency_bitcoin_item(): data = request.get_json() bitcoin_address = data.get('bitcoin', None) @@ -385,7 +385,7 @@ def get_cryptocurrency_bitcoin_item(): # # # # # # # # # # # # # # # PGP # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @restApi.route("api/v1/get/pgp/key/metadata", methods=['POST']) -@token_required('user') +@token_required('read_only') def get_pgp_key_metadata(): data = request.get_json() pgp_field = data.get('key', None) @@ -394,7 +394,7 @@ def get_pgp_key_metadata(): return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] @restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST']) -@token_required('user') +@token_required('read_only') def get_pgp_mail_metadata(): data = request.get_json() pgp_field = data.get('mail', None) @@ -403,7 +403,7 @@ def get_pgp_mail_metadata(): return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] @restApi.route("api/v1/get/pgp/name/metadata", methods=['POST']) -@token_required('user') +@token_required('read_only') def get_pgp_name_metadata(): data = request.get_json() pgp_field = data.get('name', None) @@ -412,7 +412,7 @@ def get_pgp_name_metadata(): return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] @restApi.route("api/v1/get/pgp/key/item", methods=['POST']) -@token_required('user') +@token_required('read_only') def get_pgp_key_item(): data = request.get_json() pgp_field = data.get('key', None) @@ -421,7 +421,7 @@ def get_pgp_key_item(): return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] @restApi.route("api/v1/get/pgp/mail/item", methods=['POST']) -@token_required('user') +@token_required('read_only') def get_pgp_mail_item(): data = request.get_json() pgp_mail = data.get('mail', None) @@ -430,7 +430,7 @@ def get_pgp_mail_item(): return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] @restApi.route("api/v1/get/pgp/name/item", methods=['POST']) -@token_required('user') +@token_required('read_only') def get_pgp_name_item(): data = request.get_json() pgp_name = data.get('name', None) @@ -609,7 +609,7 @@ def import_json_item(): # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # @restApi.route("api/v1/ping", methods=['GET']) -@token_required('user') +@token_required('read_only') def v1_ping(): return Response(json.dumps({'status': 'pong'}), mimetype='application/json'), 200 From 3bf42543086592968538cac807e229b2648a408f Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 23 Feb 2021 14:33:50 +0100 Subject: [PATCH 2/6] fix: [term Tracker] TimeoutException --- bin/TermTrackerMod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/TermTrackerMod.py b/bin/TermTrackerMod.py index f7abe4af..f3b333fc 100755 --- a/bin/TermTrackerMod.py +++ b/bin/TermTrackerMod.py @@ -81,7 +81,7 @@ if __name__ == "__main__": try: dict_words_freq = Term.get_text_word_frequency(item_content) except TimeoutException: - print ("{0} processing timeout".format(paste.p_rel_path)) + print ("{0} processing timeout".format(item_id)) continue else: signal.alarm(0) From 9ae21171cca2e72203c7c654e3bce4c52e21e1da Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 23 Feb 2021 15:05:00 +0100 Subject: [PATCH 3/6] fix: [Module Queue] module without subscriber --- bin/Helper.py | 9 ++++++--- bin/packages/modules.cfg | 1 - 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/bin/Helper.py b/bin/Helper.py index 31eb20e9..b23132f7 100755 --- a/bin/Helper.py +++ b/bin/Helper.py @@ -150,8 +150,11 @@ class Process(object): def populate_set_in(self): # monoproc - src = self.modules.get(self.subscriber_name, 'subscribe') - if src != 'Redis': + try: + src = self.modules.get(self.subscriber_name, 'subscribe') + except configparser.NoOptionError: #NoSectionError + src = None + if src != 'Redis' and src: self.pubsub.setup_subscribe(src) for msg in self.pubsub.subscribe(): in_set = self.subscriber_name + 'in' @@ -159,7 +162,7 @@ class Process(object): self.r_temp.hset('queues', self.subscriber_name, int(self.r_temp.scard(in_set))) else: - print('{} has no suscriber'.format(self.subscriber_name)) + print('{} has no subscriber'.format(self.subscriber_name)) def get_from_set(self): # multiproc diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index ed3c466e..302b68af 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -3,7 +3,6 @@ subscribe = ZMQ_Global publish = Redis_Mixer,Redis_preProcess1 [Importer_Json] -subscribe = ZMQ_JSON publish = Redis_Mixer,Redis_Tags [Global] From 1c991aa792acce111ca054dbe7d2dc2852feb3e6 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 23 Feb 2021 15:16:29 +0100 Subject: [PATCH 4/6] chg: [IPAddress] Add Debug + Check if option is empty --- bin/IPAddress.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/bin/IPAddress.py b/bin/IPAddress.py index e2ed59cf..4ec11647 100755 --- a/bin/IPAddress.py +++ b/bin/IPAddress.py @@ -62,9 +62,14 @@ if __name__ == '__main__': p = Process(config_section) ip_networks = [] + networks = p.config.get("IP", "networks") + if not networks: + print('No IP ranges provided') + sys.exit(0) try: - for network in p.config.get("IP", "networks").split(","): + for network in networks.split(","): ip_networks.append(IPv4Network(network)) + print(f'IP Range: {network}') except: print('Please provide a list of valid IP addresses') sys.exit(0) From e6a2655d312aa122eafea9b1d503a1ecda077d23 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 23 Feb 2021 15:28:15 +0100 Subject: [PATCH 5/6] fix: [API ACL] avoid user_no_api users to access the API --- var/www/modules/restApi/Flask_restApi.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index 8e1db66a..8e96bf67 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -68,6 +68,10 @@ def get_user_from_token(token): return r_serv_db.hget('user:tokens', token) def verify_user_role(role, token): + # User without API + if role == 'user_no_api': + return False + user_id = get_user_from_token(token) if user_id: if is_in_role(user_id, role): From 5ee1303db4768e943e47965e2e859081e7b89316 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 25 Feb 2021 11:24:08 +0100 Subject: [PATCH 6/6] fix: [Domain search] fix regex --- bin/lib/Domain.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/bin/lib/Domain.py b/bin/lib/Domain.py index 60d78967..d178cf29 100755 --- a/bin/lib/Domain.py +++ b/bin/lib/Domain.py @@ -246,10 +246,11 @@ def sanithyse_domain_name_to_search(name_to_search, domain_type): if domain_type == 'onion': r_name = r'[a-z0-9\.]+' else: - r_name = r'[a-zA-Z0-9\.-_]+' + r_name = r'[a-zA-Z0-9-_\.]+' # invalid domain name if not re.fullmatch(r_name, name_to_search): - return None + res = re.match(r_name, name_to_search) + return {'search': name_to_search, 'error': res.string.replace( res[0], '')} return name_to_search.replace('.', '\.') @@ -257,7 +258,7 @@ def search_domains_by_name(name_to_search, domain_types, r_pos=False): domains_dict = {} for domain_type in domain_types: r_name = sanithyse_domain_name_to_search(name_to_search, domain_type) - if not name_to_search: + if not name_to_search or isinstance(r_name, dict): break r_name = re.compile(r_name) for domain in get_all_domains_up(domain_type): @@ -269,6 +270,14 @@ def search_domains_by_name(name_to_search, domain_types, r_pos=False): domains_dict[domain]['hl-end'] = res.end() return domains_dict +def api_sanithyse_domain_name_to_search(name_to_search, domains_types): + domains_types = sanitize_domain_types(domains_types) + for domain_type in domains_types: + r_name = sanithyse_domain_name_to_search(name_to_search, domain_type) + if isinstance(r_name, dict): + return ({'error': 'Invalid'}, 400) + + def api_search_domains_by_name(name_to_search, domains_types, domains_metadata=False, page=1): domains_types = sanitize_domain_types(domains_types) domains_dict = search_domains_by_name(name_to_search, domains_types, r_pos=True)