Merge branch 'master' into crawler_manager

bin/Helper.py

@@ -150,8 +150,11 @@ class Process(object):
 
     def populate_set_in(self):
         # monoproc
-        src = self.modules.get(self.subscriber_name, 'subscribe')
+        try:
-        if src != 'Redis':
+            src = self.modules.get(self.subscriber_name, 'subscribe')
+        except configparser.NoOptionError: #NoSectionError
+            src = None
+        if src != 'Redis' and src:
             self.pubsub.setup_subscribe(src)
             for msg in self.pubsub.subscribe():
                 in_set = self.subscriber_name + 'in'
@@ -159,7 +162,7 @@ class Process(object):
                 self.r_temp.hset('queues', self.subscriber_name,
                                  int(self.r_temp.scard(in_set)))
         else:
-            print('{} has no suscriber'.format(self.subscriber_name))
+            print('{} has no subscriber'.format(self.subscriber_name))
 
     def get_from_set(self):
         # multiproc

bin/IPAddress.py

@@ -62,9 +62,14 @@ if __name__ == '__main__':
     p = Process(config_section)
 
     ip_networks = []
+    networks = p.config.get("IP", "networks")
+    if not networks:
+        print('No IP ranges provided')
+        sys.exit(0)
     try:
-        for network in p.config.get("IP", "networks").split(","):
+        for network in networks.split(","):
             ip_networks.append(IPv4Network(network))
+            print(f'IP Range: {network}')
     except:
         print('Please provide a list of valid IP addresses')
         sys.exit(0)

bin/TermTrackerMod.py

@@ -81,7 +81,7 @@ if __name__ == "__main__":
             try:
                 dict_words_freq = Term.get_text_word_frequency(item_content)
             except TimeoutException:
-                print ("{0} processing timeout".format(paste.p_rel_path))
+                print ("{0} processing timeout".format(item_id))
                 continue
             else:
                 signal.alarm(0)

bin/lib/Domain.py

@@ -246,10 +246,11 @@ def sanithyse_domain_name_to_search(name_to_search, domain_type):
     if domain_type == 'onion':
         r_name = r'[a-z0-9\.]+'
     else:
-        r_name = r'[a-zA-Z0-9\.-_]+'
+        r_name = r'[a-zA-Z0-9-_\.]+'
     # invalid domain name
     if not re.fullmatch(r_name, name_to_search):
-        return None
+        res = re.match(r_name, name_to_search)
+        return {'search': name_to_search, 'error': res.string.replace( res[0], '')}
     return name_to_search.replace('.', '\.')
 
 
@@ -257,7 +258,7 @@ def search_domains_by_name(name_to_search, domain_types, r_pos=False):
     domains_dict = {}
     for domain_type in domain_types:
         r_name = sanithyse_domain_name_to_search(name_to_search, domain_type)
-        if not name_to_search:
+        if not name_to_search or isinstance(r_name, dict):
             break
         r_name = re.compile(r_name)
         for domain in get_all_domains_up(domain_type):
@@ -269,6 +270,14 @@ def search_domains_by_name(name_to_search, domain_types, r_pos=False):
                     domains_dict[domain]['hl-end'] = res.end()
     return domains_dict
 
+def api_sanithyse_domain_name_to_search(name_to_search, domains_types):
+    domains_types = sanitize_domain_types(domains_types)
+    for domain_type in domains_types:
+        r_name = sanithyse_domain_name_to_search(name_to_search, domain_type)
+        if isinstance(r_name, dict):
+            return ({'error': 'Invalid'}, 400)
+
+
 def api_search_domains_by_name(name_to_search, domains_types, domains_metadata=False, page=1):
     domains_types = sanitize_domain_types(domains_types)
     domains_dict = search_domains_by_name(name_to_search, domains_types, r_pos=True)

bin/packages/modules.cfg

@@ -3,7 +3,6 @@ subscribe = ZMQ_Global
 publish = Redis_Mixer,Redis_preProcess1
 
 [Importer_Json]
-subscribe = ZMQ_JSON
 publish = Redis_Mixer,Redis_Tags
 
 [Global]

var/www/modules/restApi/Flask_restApi.py

@@ -68,6 +68,10 @@ def get_user_from_token(token):
     return r_serv_db.hget('user:tokens', token)
 
 def verify_user_role(role, token):
+    # User without API
+    if role == 'user_no_api':
+        return False
+        
     user_id = get_user_from_token(token)
     if user_id:
         if is_in_role(user_id, role):
@@ -188,14 +192,14 @@ def one():
 #
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 @restApi.route("api/v1/get/item", methods=['POST'])
-@token_required('user')
+@token_required('read_only')
 def get_item_id():
     data = request.get_json()
     res = Item.get_item(data)
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 @restApi.route("api/v1/get/item/default", methods=['POST'])
-@token_required('user')
+@token_required('read_only')
 def get_item_id_basic():
 
     data = request.get_json()
@@ -218,7 +222,7 @@ def get_item_id_basic():
 #
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 @restApi.route("api/v1/get/item/tag", methods=['POST'])
-@token_required('user')
+@token_required('read_only')
 def get_item_tag():
 
     data = request.get_json()
@@ -299,7 +303,7 @@ def delete_item_tags():
 #
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 @restApi.route("api/v1/get/item/content", methods=['POST'])
-@token_required('user')
+@token_required('read_only')
 def get_item_content():
 
     data = request.get_json()
@@ -314,7 +318,7 @@ def get_item_content():
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
 @restApi.route("api/v1/get/tag/metadata", methods=['POST'])
-@token_required('user')
+@token_required('read_only')
 def get_tag_metadata():
     data = request.get_json()
     tag = data.get('tag', None)
@@ -324,7 +328,7 @@ def get_tag_metadata():
     return Response(json.dumps(metadata, indent=2, sort_keys=True), mimetype='application/json'), 200
 
 @restApi.route("api/v1/get/tag/all", methods=['GET'])
-@token_required('user')
+@token_required('read_only')
 def get_all_tags():
     res = {'tags': Tag.get_all_tags()}
     return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
@@ -351,7 +355,7 @@ def delete_tracker_term():
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 @restApi.route("api/v1/get/tracker/item", methods=['POST'])
-@token_required('user')
+@token_required('read_only')
 def get_tracker_term_item():
     data = request.get_json()
     user_token = get_auth_from_header()
@@ -364,7 +368,7 @@ def get_tracker_term_item():
 # # # # # # # # # # # #        CRYPTOCURRENCY       # # # # # # # # # # # # # #
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 @restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST'])
-@token_required('user')
+@token_required('read_only')
 def get_cryptocurrency_bitcoin_metadata():
     data = request.get_json()
     crypto_address = data.get('bitcoin', None)
@@ -373,7 +377,7 @@ def get_cryptocurrency_bitcoin_metadata():
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 @restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST'])
-@token_required('user')
+@token_required('read_only')
 def get_cryptocurrency_bitcoin_item():
     data = request.get_json()
     bitcoin_address = data.get('bitcoin', None)
@@ -385,7 +389,7 @@ def get_cryptocurrency_bitcoin_item():
 # # # # # # # # # # # # # # #       PGP       # # # # # # # # # # # # # # # # #
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 @restApi.route("api/v1/get/pgp/key/metadata", methods=['POST'])
-@token_required('user')
+@token_required('read_only')
 def get_pgp_key_metadata():
     data = request.get_json()
     pgp_field = data.get('key', None)
@@ -394,7 +398,7 @@ def get_pgp_key_metadata():
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 @restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST'])
-@token_required('user')
+@token_required('read_only')
 def get_pgp_mail_metadata():
     data = request.get_json()
     pgp_field = data.get('mail', None)
@@ -403,7 +407,7 @@ def get_pgp_mail_metadata():
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 @restApi.route("api/v1/get/pgp/name/metadata", methods=['POST'])
-@token_required('user')
+@token_required('read_only')
 def get_pgp_name_metadata():
     data = request.get_json()
     pgp_field = data.get('name', None)
@@ -412,7 +416,7 @@ def get_pgp_name_metadata():
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 @restApi.route("api/v1/get/pgp/key/item", methods=['POST'])
-@token_required('user')
+@token_required('read_only')
 def get_pgp_key_item():
     data = request.get_json()
     pgp_field = data.get('key', None)
@@ -421,7 +425,7 @@ def get_pgp_key_item():
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 @restApi.route("api/v1/get/pgp/mail/item", methods=['POST'])
-@token_required('user')
+@token_required('read_only')
 def get_pgp_mail_item():
     data = request.get_json()
     pgp_mail = data.get('mail', None)
@@ -430,7 +434,7 @@ def get_pgp_mail_item():
     return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
 
 @restApi.route("api/v1/get/pgp/name/item", methods=['POST'])
-@token_required('user')
+@token_required('read_only')
 def get_pgp_name_item():
     data = request.get_json()
     pgp_name = data.get('name', None)
@@ -609,7 +613,7 @@ def import_json_item():
 #
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 @restApi.route("api/v1/ping", methods=['GET'])
-@token_required('user')
+@token_required('read_only')
 def v1_ping():
     return Response(json.dumps({'status': 'pong'}), mimetype='application/json'), 200