Merge branch 'master' into crawler_manager

This commit is contained in:
Terrtia 2021-03-05 18:02:08 +01:00
commit 2b082d01a0
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
6 changed files with 45 additions and 25 deletions

View file

@ -150,8 +150,11 @@ class Process(object):
def populate_set_in(self): def populate_set_in(self):
# monoproc # monoproc
src = self.modules.get(self.subscriber_name, 'subscribe') try:
if src != 'Redis': src = self.modules.get(self.subscriber_name, 'subscribe')
except configparser.NoOptionError: #NoSectionError
src = None
if src != 'Redis' and src:
self.pubsub.setup_subscribe(src) self.pubsub.setup_subscribe(src)
for msg in self.pubsub.subscribe(): for msg in self.pubsub.subscribe():
in_set = self.subscriber_name + 'in' in_set = self.subscriber_name + 'in'
@ -159,7 +162,7 @@ class Process(object):
self.r_temp.hset('queues', self.subscriber_name, self.r_temp.hset('queues', self.subscriber_name,
int(self.r_temp.scard(in_set))) int(self.r_temp.scard(in_set)))
else: else:
print('{} has no suscriber'.format(self.subscriber_name)) print('{} has no subscriber'.format(self.subscriber_name))
def get_from_set(self): def get_from_set(self):
# multiproc # multiproc

View file

@ -62,9 +62,14 @@ if __name__ == '__main__':
p = Process(config_section) p = Process(config_section)
ip_networks = [] ip_networks = []
networks = p.config.get("IP", "networks")
if not networks:
print('No IP ranges provided')
sys.exit(0)
try: try:
for network in p.config.get("IP", "networks").split(","): for network in networks.split(","):
ip_networks.append(IPv4Network(network)) ip_networks.append(IPv4Network(network))
print(f'IP Range: {network}')
except: except:
print('Please provide a list of valid IP addresses') print('Please provide a list of valid IP addresses')
sys.exit(0) sys.exit(0)

View file

@ -81,7 +81,7 @@ if __name__ == "__main__":
try: try:
dict_words_freq = Term.get_text_word_frequency(item_content) dict_words_freq = Term.get_text_word_frequency(item_content)
except TimeoutException: except TimeoutException:
print ("{0} processing timeout".format(paste.p_rel_path)) print ("{0} processing timeout".format(item_id))
continue continue
else: else:
signal.alarm(0) signal.alarm(0)

View file

@ -246,10 +246,11 @@ def sanithyse_domain_name_to_search(name_to_search, domain_type):
if domain_type == 'onion': if domain_type == 'onion':
r_name = r'[a-z0-9\.]+' r_name = r'[a-z0-9\.]+'
else: else:
r_name = r'[a-zA-Z0-9\.-_]+' r_name = r'[a-zA-Z0-9-_\.]+'
# invalid domain name # invalid domain name
if not re.fullmatch(r_name, name_to_search): if not re.fullmatch(r_name, name_to_search):
return None res = re.match(r_name, name_to_search)
return {'search': name_to_search, 'error': res.string.replace( res[0], '')}
return name_to_search.replace('.', '\.') return name_to_search.replace('.', '\.')
@ -257,7 +258,7 @@ def search_domains_by_name(name_to_search, domain_types, r_pos=False):
domains_dict = {} domains_dict = {}
for domain_type in domain_types: for domain_type in domain_types:
r_name = sanithyse_domain_name_to_search(name_to_search, domain_type) r_name = sanithyse_domain_name_to_search(name_to_search, domain_type)
if not name_to_search: if not name_to_search or isinstance(r_name, dict):
break break
r_name = re.compile(r_name) r_name = re.compile(r_name)
for domain in get_all_domains_up(domain_type): for domain in get_all_domains_up(domain_type):
@ -269,6 +270,14 @@ def search_domains_by_name(name_to_search, domain_types, r_pos=False):
domains_dict[domain]['hl-end'] = res.end() domains_dict[domain]['hl-end'] = res.end()
return domains_dict return domains_dict
def api_sanithyse_domain_name_to_search(name_to_search, domains_types):
domains_types = sanitize_domain_types(domains_types)
for domain_type in domains_types:
r_name = sanithyse_domain_name_to_search(name_to_search, domain_type)
if isinstance(r_name, dict):
return ({'error': 'Invalid'}, 400)
def api_search_domains_by_name(name_to_search, domains_types, domains_metadata=False, page=1): def api_search_domains_by_name(name_to_search, domains_types, domains_metadata=False, page=1):
domains_types = sanitize_domain_types(domains_types) domains_types = sanitize_domain_types(domains_types)
domains_dict = search_domains_by_name(name_to_search, domains_types, r_pos=True) domains_dict = search_domains_by_name(name_to_search, domains_types, r_pos=True)

View file

@ -3,7 +3,6 @@ subscribe = ZMQ_Global
publish = Redis_Mixer,Redis_preProcess1 publish = Redis_Mixer,Redis_preProcess1
[Importer_Json] [Importer_Json]
subscribe = ZMQ_JSON
publish = Redis_Mixer,Redis_Tags publish = Redis_Mixer,Redis_Tags
[Global] [Global]

View file

@ -68,6 +68,10 @@ def get_user_from_token(token):
return r_serv_db.hget('user:tokens', token) return r_serv_db.hget('user:tokens', token)
def verify_user_role(role, token): def verify_user_role(role, token):
# User without API
if role == 'user_no_api':
return False
user_id = get_user_from_token(token) user_id = get_user_from_token(token)
if user_id: if user_id:
if is_in_role(user_id, role): if is_in_role(user_id, role):
@ -188,14 +192,14 @@ def one():
# #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/get/item", methods=['POST']) @restApi.route("api/v1/get/item", methods=['POST'])
@token_required('user') @token_required('read_only')
def get_item_id(): def get_item_id():
data = request.get_json() data = request.get_json()
res = Item.get_item(data) res = Item.get_item(data)
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/item/default", methods=['POST']) @restApi.route("api/v1/get/item/default", methods=['POST'])
@token_required('user') @token_required('read_only')
def get_item_id_basic(): def get_item_id_basic():
data = request.get_json() data = request.get_json()
@ -218,7 +222,7 @@ def get_item_id_basic():
# #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/get/item/tag", methods=['POST']) @restApi.route("api/v1/get/item/tag", methods=['POST'])
@token_required('user') @token_required('read_only')
def get_item_tag(): def get_item_tag():
data = request.get_json() data = request.get_json()
@ -299,7 +303,7 @@ def delete_item_tags():
# #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/get/item/content", methods=['POST']) @restApi.route("api/v1/get/item/content", methods=['POST'])
@token_required('user') @token_required('read_only')
def get_item_content(): def get_item_content():
data = request.get_json() data = request.get_json()
@ -314,7 +318,7 @@ def get_item_content():
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/get/tag/metadata", methods=['POST']) @restApi.route("api/v1/get/tag/metadata", methods=['POST'])
@token_required('user') @token_required('read_only')
def get_tag_metadata(): def get_tag_metadata():
data = request.get_json() data = request.get_json()
tag = data.get('tag', None) tag = data.get('tag', None)
@ -324,7 +328,7 @@ def get_tag_metadata():
return Response(json.dumps(metadata, indent=2, sort_keys=True), mimetype='application/json'), 200 return Response(json.dumps(metadata, indent=2, sort_keys=True), mimetype='application/json'), 200
@restApi.route("api/v1/get/tag/all", methods=['GET']) @restApi.route("api/v1/get/tag/all", methods=['GET'])
@token_required('user') @token_required('read_only')
def get_all_tags(): def get_all_tags():
res = {'tags': Tag.get_all_tags()} res = {'tags': Tag.get_all_tags()}
return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200 return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
@ -351,7 +355,7 @@ def delete_tracker_term():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/tracker/item", methods=['POST']) @restApi.route("api/v1/get/tracker/item", methods=['POST'])
@token_required('user') @token_required('read_only')
def get_tracker_term_item(): def get_tracker_term_item():
data = request.get_json() data = request.get_json()
user_token = get_auth_from_header() user_token = get_auth_from_header()
@ -364,7 +368,7 @@ def get_tracker_term_item():
# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # # # # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST']) @restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST'])
@token_required('user') @token_required('read_only')
def get_cryptocurrency_bitcoin_metadata(): def get_cryptocurrency_bitcoin_metadata():
data = request.get_json() data = request.get_json()
crypto_address = data.get('bitcoin', None) crypto_address = data.get('bitcoin', None)
@ -373,7 +377,7 @@ def get_cryptocurrency_bitcoin_metadata():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST']) @restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST'])
@token_required('user') @token_required('read_only')
def get_cryptocurrency_bitcoin_item(): def get_cryptocurrency_bitcoin_item():
data = request.get_json() data = request.get_json()
bitcoin_address = data.get('bitcoin', None) bitcoin_address = data.get('bitcoin', None)
@ -385,7 +389,7 @@ def get_cryptocurrency_bitcoin_item():
# # # # # # # # # # # # # # # PGP # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # PGP # # # # # # # # # # # # # # # # #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/get/pgp/key/metadata", methods=['POST']) @restApi.route("api/v1/get/pgp/key/metadata", methods=['POST'])
@token_required('user') @token_required('read_only')
def get_pgp_key_metadata(): def get_pgp_key_metadata():
data = request.get_json() data = request.get_json()
pgp_field = data.get('key', None) pgp_field = data.get('key', None)
@ -394,7 +398,7 @@ def get_pgp_key_metadata():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST']) @restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST'])
@token_required('user') @token_required('read_only')
def get_pgp_mail_metadata(): def get_pgp_mail_metadata():
data = request.get_json() data = request.get_json()
pgp_field = data.get('mail', None) pgp_field = data.get('mail', None)
@ -403,7 +407,7 @@ def get_pgp_mail_metadata():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/pgp/name/metadata", methods=['POST']) @restApi.route("api/v1/get/pgp/name/metadata", methods=['POST'])
@token_required('user') @token_required('read_only')
def get_pgp_name_metadata(): def get_pgp_name_metadata():
data = request.get_json() data = request.get_json()
pgp_field = data.get('name', None) pgp_field = data.get('name', None)
@ -412,7 +416,7 @@ def get_pgp_name_metadata():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/pgp/key/item", methods=['POST']) @restApi.route("api/v1/get/pgp/key/item", methods=['POST'])
@token_required('user') @token_required('read_only')
def get_pgp_key_item(): def get_pgp_key_item():
data = request.get_json() data = request.get_json()
pgp_field = data.get('key', None) pgp_field = data.get('key', None)
@ -421,7 +425,7 @@ def get_pgp_key_item():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/pgp/mail/item", methods=['POST']) @restApi.route("api/v1/get/pgp/mail/item", methods=['POST'])
@token_required('user') @token_required('read_only')
def get_pgp_mail_item(): def get_pgp_mail_item():
data = request.get_json() data = request.get_json()
pgp_mail = data.get('mail', None) pgp_mail = data.get('mail', None)
@ -430,7 +434,7 @@ def get_pgp_mail_item():
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1] return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
@restApi.route("api/v1/get/pgp/name/item", methods=['POST']) @restApi.route("api/v1/get/pgp/name/item", methods=['POST'])
@token_required('user') @token_required('read_only')
def get_pgp_name_item(): def get_pgp_name_item():
data = request.get_json() data = request.get_json()
pgp_name = data.get('name', None) pgp_name = data.get('name', None)
@ -609,7 +613,7 @@ def import_json_item():
# #
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
@restApi.route("api/v1/ping", methods=['GET']) @restApi.route("api/v1/ping", methods=['GET'])
@token_required('user') @token_required('read_only')
def v1_ping(): def v1_ping():
return Response(json.dumps({'status': 'pong'}), mimetype='application/json'), 200 return Response(json.dumps({'status': 'pong'}), mimetype='application/json'), 200