mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-13 01:58:22 +00:00
Merge branch 'master' into crawler_manager
This commit is contained in:
commit
2b082d01a0
6 changed files with 45 additions and 25 deletions
|
@ -150,8 +150,11 @@ class Process(object):
|
||||||
|
|
||||||
def populate_set_in(self):
|
def populate_set_in(self):
|
||||||
# monoproc
|
# monoproc
|
||||||
src = self.modules.get(self.subscriber_name, 'subscribe')
|
try:
|
||||||
if src != 'Redis':
|
src = self.modules.get(self.subscriber_name, 'subscribe')
|
||||||
|
except configparser.NoOptionError: #NoSectionError
|
||||||
|
src = None
|
||||||
|
if src != 'Redis' and src:
|
||||||
self.pubsub.setup_subscribe(src)
|
self.pubsub.setup_subscribe(src)
|
||||||
for msg in self.pubsub.subscribe():
|
for msg in self.pubsub.subscribe():
|
||||||
in_set = self.subscriber_name + 'in'
|
in_set = self.subscriber_name + 'in'
|
||||||
|
@ -159,7 +162,7 @@ class Process(object):
|
||||||
self.r_temp.hset('queues', self.subscriber_name,
|
self.r_temp.hset('queues', self.subscriber_name,
|
||||||
int(self.r_temp.scard(in_set)))
|
int(self.r_temp.scard(in_set)))
|
||||||
else:
|
else:
|
||||||
print('{} has no suscriber'.format(self.subscriber_name))
|
print('{} has no subscriber'.format(self.subscriber_name))
|
||||||
|
|
||||||
def get_from_set(self):
|
def get_from_set(self):
|
||||||
# multiproc
|
# multiproc
|
||||||
|
|
|
@ -62,9 +62,14 @@ if __name__ == '__main__':
|
||||||
p = Process(config_section)
|
p = Process(config_section)
|
||||||
|
|
||||||
ip_networks = []
|
ip_networks = []
|
||||||
|
networks = p.config.get("IP", "networks")
|
||||||
|
if not networks:
|
||||||
|
print('No IP ranges provided')
|
||||||
|
sys.exit(0)
|
||||||
try:
|
try:
|
||||||
for network in p.config.get("IP", "networks").split(","):
|
for network in networks.split(","):
|
||||||
ip_networks.append(IPv4Network(network))
|
ip_networks.append(IPv4Network(network))
|
||||||
|
print(f'IP Range: {network}')
|
||||||
except:
|
except:
|
||||||
print('Please provide a list of valid IP addresses')
|
print('Please provide a list of valid IP addresses')
|
||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
|
@ -81,7 +81,7 @@ if __name__ == "__main__":
|
||||||
try:
|
try:
|
||||||
dict_words_freq = Term.get_text_word_frequency(item_content)
|
dict_words_freq = Term.get_text_word_frequency(item_content)
|
||||||
except TimeoutException:
|
except TimeoutException:
|
||||||
print ("{0} processing timeout".format(paste.p_rel_path))
|
print ("{0} processing timeout".format(item_id))
|
||||||
continue
|
continue
|
||||||
else:
|
else:
|
||||||
signal.alarm(0)
|
signal.alarm(0)
|
||||||
|
|
|
@ -246,10 +246,11 @@ def sanithyse_domain_name_to_search(name_to_search, domain_type):
|
||||||
if domain_type == 'onion':
|
if domain_type == 'onion':
|
||||||
r_name = r'[a-z0-9\.]+'
|
r_name = r'[a-z0-9\.]+'
|
||||||
else:
|
else:
|
||||||
r_name = r'[a-zA-Z0-9\.-_]+'
|
r_name = r'[a-zA-Z0-9-_\.]+'
|
||||||
# invalid domain name
|
# invalid domain name
|
||||||
if not re.fullmatch(r_name, name_to_search):
|
if not re.fullmatch(r_name, name_to_search):
|
||||||
return None
|
res = re.match(r_name, name_to_search)
|
||||||
|
return {'search': name_to_search, 'error': res.string.replace( res[0], '')}
|
||||||
return name_to_search.replace('.', '\.')
|
return name_to_search.replace('.', '\.')
|
||||||
|
|
||||||
|
|
||||||
|
@ -257,7 +258,7 @@ def search_domains_by_name(name_to_search, domain_types, r_pos=False):
|
||||||
domains_dict = {}
|
domains_dict = {}
|
||||||
for domain_type in domain_types:
|
for domain_type in domain_types:
|
||||||
r_name = sanithyse_domain_name_to_search(name_to_search, domain_type)
|
r_name = sanithyse_domain_name_to_search(name_to_search, domain_type)
|
||||||
if not name_to_search:
|
if not name_to_search or isinstance(r_name, dict):
|
||||||
break
|
break
|
||||||
r_name = re.compile(r_name)
|
r_name = re.compile(r_name)
|
||||||
for domain in get_all_domains_up(domain_type):
|
for domain in get_all_domains_up(domain_type):
|
||||||
|
@ -269,6 +270,14 @@ def search_domains_by_name(name_to_search, domain_types, r_pos=False):
|
||||||
domains_dict[domain]['hl-end'] = res.end()
|
domains_dict[domain]['hl-end'] = res.end()
|
||||||
return domains_dict
|
return domains_dict
|
||||||
|
|
||||||
|
def api_sanithyse_domain_name_to_search(name_to_search, domains_types):
|
||||||
|
domains_types = sanitize_domain_types(domains_types)
|
||||||
|
for domain_type in domains_types:
|
||||||
|
r_name = sanithyse_domain_name_to_search(name_to_search, domain_type)
|
||||||
|
if isinstance(r_name, dict):
|
||||||
|
return ({'error': 'Invalid'}, 400)
|
||||||
|
|
||||||
|
|
||||||
def api_search_domains_by_name(name_to_search, domains_types, domains_metadata=False, page=1):
|
def api_search_domains_by_name(name_to_search, domains_types, domains_metadata=False, page=1):
|
||||||
domains_types = sanitize_domain_types(domains_types)
|
domains_types = sanitize_domain_types(domains_types)
|
||||||
domains_dict = search_domains_by_name(name_to_search, domains_types, r_pos=True)
|
domains_dict = search_domains_by_name(name_to_search, domains_types, r_pos=True)
|
||||||
|
|
|
@ -3,7 +3,6 @@ subscribe = ZMQ_Global
|
||||||
publish = Redis_Mixer,Redis_preProcess1
|
publish = Redis_Mixer,Redis_preProcess1
|
||||||
|
|
||||||
[Importer_Json]
|
[Importer_Json]
|
||||||
subscribe = ZMQ_JSON
|
|
||||||
publish = Redis_Mixer,Redis_Tags
|
publish = Redis_Mixer,Redis_Tags
|
||||||
|
|
||||||
[Global]
|
[Global]
|
||||||
|
|
|
@ -68,6 +68,10 @@ def get_user_from_token(token):
|
||||||
return r_serv_db.hget('user:tokens', token)
|
return r_serv_db.hget('user:tokens', token)
|
||||||
|
|
||||||
def verify_user_role(role, token):
|
def verify_user_role(role, token):
|
||||||
|
# User without API
|
||||||
|
if role == 'user_no_api':
|
||||||
|
return False
|
||||||
|
|
||||||
user_id = get_user_from_token(token)
|
user_id = get_user_from_token(token)
|
||||||
if user_id:
|
if user_id:
|
||||||
if is_in_role(user_id, role):
|
if is_in_role(user_id, role):
|
||||||
|
@ -188,14 +192,14 @@ def one():
|
||||||
#
|
#
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
@restApi.route("api/v1/get/item", methods=['POST'])
|
@restApi.route("api/v1/get/item", methods=['POST'])
|
||||||
@token_required('user')
|
@token_required('read_only')
|
||||||
def get_item_id():
|
def get_item_id():
|
||||||
data = request.get_json()
|
data = request.get_json()
|
||||||
res = Item.get_item(data)
|
res = Item.get_item(data)
|
||||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
@restApi.route("api/v1/get/item/default", methods=['POST'])
|
@restApi.route("api/v1/get/item/default", methods=['POST'])
|
||||||
@token_required('user')
|
@token_required('read_only')
|
||||||
def get_item_id_basic():
|
def get_item_id_basic():
|
||||||
|
|
||||||
data = request.get_json()
|
data = request.get_json()
|
||||||
|
@ -218,7 +222,7 @@ def get_item_id_basic():
|
||||||
#
|
#
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
@restApi.route("api/v1/get/item/tag", methods=['POST'])
|
@restApi.route("api/v1/get/item/tag", methods=['POST'])
|
||||||
@token_required('user')
|
@token_required('read_only')
|
||||||
def get_item_tag():
|
def get_item_tag():
|
||||||
|
|
||||||
data = request.get_json()
|
data = request.get_json()
|
||||||
|
@ -299,7 +303,7 @@ def delete_item_tags():
|
||||||
#
|
#
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
@restApi.route("api/v1/get/item/content", methods=['POST'])
|
@restApi.route("api/v1/get/item/content", methods=['POST'])
|
||||||
@token_required('user')
|
@token_required('read_only')
|
||||||
def get_item_content():
|
def get_item_content():
|
||||||
|
|
||||||
data = request.get_json()
|
data = request.get_json()
|
||||||
|
@ -314,7 +318,7 @@ def get_item_content():
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
|
|
||||||
@restApi.route("api/v1/get/tag/metadata", methods=['POST'])
|
@restApi.route("api/v1/get/tag/metadata", methods=['POST'])
|
||||||
@token_required('user')
|
@token_required('read_only')
|
||||||
def get_tag_metadata():
|
def get_tag_metadata():
|
||||||
data = request.get_json()
|
data = request.get_json()
|
||||||
tag = data.get('tag', None)
|
tag = data.get('tag', None)
|
||||||
|
@ -324,7 +328,7 @@ def get_tag_metadata():
|
||||||
return Response(json.dumps(metadata, indent=2, sort_keys=True), mimetype='application/json'), 200
|
return Response(json.dumps(metadata, indent=2, sort_keys=True), mimetype='application/json'), 200
|
||||||
|
|
||||||
@restApi.route("api/v1/get/tag/all", methods=['GET'])
|
@restApi.route("api/v1/get/tag/all", methods=['GET'])
|
||||||
@token_required('user')
|
@token_required('read_only')
|
||||||
def get_all_tags():
|
def get_all_tags():
|
||||||
res = {'tags': Tag.get_all_tags()}
|
res = {'tags': Tag.get_all_tags()}
|
||||||
return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
|
return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
|
||||||
|
@ -351,7 +355,7 @@ def delete_tracker_term():
|
||||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
@restApi.route("api/v1/get/tracker/item", methods=['POST'])
|
@restApi.route("api/v1/get/tracker/item", methods=['POST'])
|
||||||
@token_required('user')
|
@token_required('read_only')
|
||||||
def get_tracker_term_item():
|
def get_tracker_term_item():
|
||||||
data = request.get_json()
|
data = request.get_json()
|
||||||
user_token = get_auth_from_header()
|
user_token = get_auth_from_header()
|
||||||
|
@ -364,7 +368,7 @@ def get_tracker_term_item():
|
||||||
# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # #
|
# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # #
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
@restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST'])
|
@restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST'])
|
||||||
@token_required('user')
|
@token_required('read_only')
|
||||||
def get_cryptocurrency_bitcoin_metadata():
|
def get_cryptocurrency_bitcoin_metadata():
|
||||||
data = request.get_json()
|
data = request.get_json()
|
||||||
crypto_address = data.get('bitcoin', None)
|
crypto_address = data.get('bitcoin', None)
|
||||||
|
@ -373,7 +377,7 @@ def get_cryptocurrency_bitcoin_metadata():
|
||||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
@restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST'])
|
@restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST'])
|
||||||
@token_required('user')
|
@token_required('read_only')
|
||||||
def get_cryptocurrency_bitcoin_item():
|
def get_cryptocurrency_bitcoin_item():
|
||||||
data = request.get_json()
|
data = request.get_json()
|
||||||
bitcoin_address = data.get('bitcoin', None)
|
bitcoin_address = data.get('bitcoin', None)
|
||||||
|
@ -385,7 +389,7 @@ def get_cryptocurrency_bitcoin_item():
|
||||||
# # # # # # # # # # # # # # # PGP # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # PGP # # # # # # # # # # # # # # # # #
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
@restApi.route("api/v1/get/pgp/key/metadata", methods=['POST'])
|
@restApi.route("api/v1/get/pgp/key/metadata", methods=['POST'])
|
||||||
@token_required('user')
|
@token_required('read_only')
|
||||||
def get_pgp_key_metadata():
|
def get_pgp_key_metadata():
|
||||||
data = request.get_json()
|
data = request.get_json()
|
||||||
pgp_field = data.get('key', None)
|
pgp_field = data.get('key', None)
|
||||||
|
@ -394,7 +398,7 @@ def get_pgp_key_metadata():
|
||||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
@restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST'])
|
@restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST'])
|
||||||
@token_required('user')
|
@token_required('read_only')
|
||||||
def get_pgp_mail_metadata():
|
def get_pgp_mail_metadata():
|
||||||
data = request.get_json()
|
data = request.get_json()
|
||||||
pgp_field = data.get('mail', None)
|
pgp_field = data.get('mail', None)
|
||||||
|
@ -403,7 +407,7 @@ def get_pgp_mail_metadata():
|
||||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
@restApi.route("api/v1/get/pgp/name/metadata", methods=['POST'])
|
@restApi.route("api/v1/get/pgp/name/metadata", methods=['POST'])
|
||||||
@token_required('user')
|
@token_required('read_only')
|
||||||
def get_pgp_name_metadata():
|
def get_pgp_name_metadata():
|
||||||
data = request.get_json()
|
data = request.get_json()
|
||||||
pgp_field = data.get('name', None)
|
pgp_field = data.get('name', None)
|
||||||
|
@ -412,7 +416,7 @@ def get_pgp_name_metadata():
|
||||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
@restApi.route("api/v1/get/pgp/key/item", methods=['POST'])
|
@restApi.route("api/v1/get/pgp/key/item", methods=['POST'])
|
||||||
@token_required('user')
|
@token_required('read_only')
|
||||||
def get_pgp_key_item():
|
def get_pgp_key_item():
|
||||||
data = request.get_json()
|
data = request.get_json()
|
||||||
pgp_field = data.get('key', None)
|
pgp_field = data.get('key', None)
|
||||||
|
@ -421,7 +425,7 @@ def get_pgp_key_item():
|
||||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
@restApi.route("api/v1/get/pgp/mail/item", methods=['POST'])
|
@restApi.route("api/v1/get/pgp/mail/item", methods=['POST'])
|
||||||
@token_required('user')
|
@token_required('read_only')
|
||||||
def get_pgp_mail_item():
|
def get_pgp_mail_item():
|
||||||
data = request.get_json()
|
data = request.get_json()
|
||||||
pgp_mail = data.get('mail', None)
|
pgp_mail = data.get('mail', None)
|
||||||
|
@ -430,7 +434,7 @@ def get_pgp_mail_item():
|
||||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||||
|
|
||||||
@restApi.route("api/v1/get/pgp/name/item", methods=['POST'])
|
@restApi.route("api/v1/get/pgp/name/item", methods=['POST'])
|
||||||
@token_required('user')
|
@token_required('read_only')
|
||||||
def get_pgp_name_item():
|
def get_pgp_name_item():
|
||||||
data = request.get_json()
|
data = request.get_json()
|
||||||
pgp_name = data.get('name', None)
|
pgp_name = data.get('name', None)
|
||||||
|
@ -609,7 +613,7 @@ def import_json_item():
|
||||||
#
|
#
|
||||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||||
@restApi.route("api/v1/ping", methods=['GET'])
|
@restApi.route("api/v1/ping", methods=['GET'])
|
||||||
@token_required('user')
|
@token_required('read_only')
|
||||||
def v1_ping():
|
def v1_ping():
|
||||||
return Response(json.dumps({'status': 'pong'}), mimetype='application/json'), 200
|
return Response(json.dumps({'status': 'pong'}), mimetype='application/json'), 200
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue