mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 22:27:17 +00:00
Merge branch 'master' into crawler_manager
This commit is contained in:
commit
2b082d01a0
6 changed files with 45 additions and 25 deletions
|
@ -150,8 +150,11 @@ class Process(object):
|
|||
|
||||
def populate_set_in(self):
|
||||
# monoproc
|
||||
src = self.modules.get(self.subscriber_name, 'subscribe')
|
||||
if src != 'Redis':
|
||||
try:
|
||||
src = self.modules.get(self.subscriber_name, 'subscribe')
|
||||
except configparser.NoOptionError: #NoSectionError
|
||||
src = None
|
||||
if src != 'Redis' and src:
|
||||
self.pubsub.setup_subscribe(src)
|
||||
for msg in self.pubsub.subscribe():
|
||||
in_set = self.subscriber_name + 'in'
|
||||
|
@ -159,7 +162,7 @@ class Process(object):
|
|||
self.r_temp.hset('queues', self.subscriber_name,
|
||||
int(self.r_temp.scard(in_set)))
|
||||
else:
|
||||
print('{} has no suscriber'.format(self.subscriber_name))
|
||||
print('{} has no subscriber'.format(self.subscriber_name))
|
||||
|
||||
def get_from_set(self):
|
||||
# multiproc
|
||||
|
|
|
@ -62,9 +62,14 @@ if __name__ == '__main__':
|
|||
p = Process(config_section)
|
||||
|
||||
ip_networks = []
|
||||
networks = p.config.get("IP", "networks")
|
||||
if not networks:
|
||||
print('No IP ranges provided')
|
||||
sys.exit(0)
|
||||
try:
|
||||
for network in p.config.get("IP", "networks").split(","):
|
||||
for network in networks.split(","):
|
||||
ip_networks.append(IPv4Network(network))
|
||||
print(f'IP Range: {network}')
|
||||
except:
|
||||
print('Please provide a list of valid IP addresses')
|
||||
sys.exit(0)
|
||||
|
|
|
@ -81,7 +81,7 @@ if __name__ == "__main__":
|
|||
try:
|
||||
dict_words_freq = Term.get_text_word_frequency(item_content)
|
||||
except TimeoutException:
|
||||
print ("{0} processing timeout".format(paste.p_rel_path))
|
||||
print ("{0} processing timeout".format(item_id))
|
||||
continue
|
||||
else:
|
||||
signal.alarm(0)
|
||||
|
|
|
@ -246,10 +246,11 @@ def sanithyse_domain_name_to_search(name_to_search, domain_type):
|
|||
if domain_type == 'onion':
|
||||
r_name = r'[a-z0-9\.]+'
|
||||
else:
|
||||
r_name = r'[a-zA-Z0-9\.-_]+'
|
||||
r_name = r'[a-zA-Z0-9-_\.]+'
|
||||
# invalid domain name
|
||||
if not re.fullmatch(r_name, name_to_search):
|
||||
return None
|
||||
res = re.match(r_name, name_to_search)
|
||||
return {'search': name_to_search, 'error': res.string.replace( res[0], '')}
|
||||
return name_to_search.replace('.', '\.')
|
||||
|
||||
|
||||
|
@ -257,7 +258,7 @@ def search_domains_by_name(name_to_search, domain_types, r_pos=False):
|
|||
domains_dict = {}
|
||||
for domain_type in domain_types:
|
||||
r_name = sanithyse_domain_name_to_search(name_to_search, domain_type)
|
||||
if not name_to_search:
|
||||
if not name_to_search or isinstance(r_name, dict):
|
||||
break
|
||||
r_name = re.compile(r_name)
|
||||
for domain in get_all_domains_up(domain_type):
|
||||
|
@ -269,6 +270,14 @@ def search_domains_by_name(name_to_search, domain_types, r_pos=False):
|
|||
domains_dict[domain]['hl-end'] = res.end()
|
||||
return domains_dict
|
||||
|
||||
def api_sanithyse_domain_name_to_search(name_to_search, domains_types):
|
||||
domains_types = sanitize_domain_types(domains_types)
|
||||
for domain_type in domains_types:
|
||||
r_name = sanithyse_domain_name_to_search(name_to_search, domain_type)
|
||||
if isinstance(r_name, dict):
|
||||
return ({'error': 'Invalid'}, 400)
|
||||
|
||||
|
||||
def api_search_domains_by_name(name_to_search, domains_types, domains_metadata=False, page=1):
|
||||
domains_types = sanitize_domain_types(domains_types)
|
||||
domains_dict = search_domains_by_name(name_to_search, domains_types, r_pos=True)
|
||||
|
|
|
@ -3,7 +3,6 @@ subscribe = ZMQ_Global
|
|||
publish = Redis_Mixer,Redis_preProcess1
|
||||
|
||||
[Importer_Json]
|
||||
subscribe = ZMQ_JSON
|
||||
publish = Redis_Mixer,Redis_Tags
|
||||
|
||||
[Global]
|
||||
|
|
|
@ -68,6 +68,10 @@ def get_user_from_token(token):
|
|||
return r_serv_db.hget('user:tokens', token)
|
||||
|
||||
def verify_user_role(role, token):
|
||||
# User without API
|
||||
if role == 'user_no_api':
|
||||
return False
|
||||
|
||||
user_id = get_user_from_token(token)
|
||||
if user_id:
|
||||
if is_in_role(user_id, role):
|
||||
|
@ -188,14 +192,14 @@ def one():
|
|||
#
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
@restApi.route("api/v1/get/item", methods=['POST'])
|
||||
@token_required('user')
|
||||
@token_required('read_only')
|
||||
def get_item_id():
|
||||
data = request.get_json()
|
||||
res = Item.get_item(data)
|
||||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
@restApi.route("api/v1/get/item/default", methods=['POST'])
|
||||
@token_required('user')
|
||||
@token_required('read_only')
|
||||
def get_item_id_basic():
|
||||
|
||||
data = request.get_json()
|
||||
|
@ -218,7 +222,7 @@ def get_item_id_basic():
|
|||
#
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
@restApi.route("api/v1/get/item/tag", methods=['POST'])
|
||||
@token_required('user')
|
||||
@token_required('read_only')
|
||||
def get_item_tag():
|
||||
|
||||
data = request.get_json()
|
||||
|
@ -299,7 +303,7 @@ def delete_item_tags():
|
|||
#
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
@restApi.route("api/v1/get/item/content", methods=['POST'])
|
||||
@token_required('user')
|
||||
@token_required('read_only')
|
||||
def get_item_content():
|
||||
|
||||
data = request.get_json()
|
||||
|
@ -314,7 +318,7 @@ def get_item_content():
|
|||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
|
||||
@restApi.route("api/v1/get/tag/metadata", methods=['POST'])
|
||||
@token_required('user')
|
||||
@token_required('read_only')
|
||||
def get_tag_metadata():
|
||||
data = request.get_json()
|
||||
tag = data.get('tag', None)
|
||||
|
@ -324,7 +328,7 @@ def get_tag_metadata():
|
|||
return Response(json.dumps(metadata, indent=2, sort_keys=True), mimetype='application/json'), 200
|
||||
|
||||
@restApi.route("api/v1/get/tag/all", methods=['GET'])
|
||||
@token_required('user')
|
||||
@token_required('read_only')
|
||||
def get_all_tags():
|
||||
res = {'tags': Tag.get_all_tags()}
|
||||
return Response(json.dumps(res, indent=2, sort_keys=True), mimetype='application/json'), 200
|
||||
|
@ -351,7 +355,7 @@ def delete_tracker_term():
|
|||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
@restApi.route("api/v1/get/tracker/item", methods=['POST'])
|
||||
@token_required('user')
|
||||
@token_required('read_only')
|
||||
def get_tracker_term_item():
|
||||
data = request.get_json()
|
||||
user_token = get_auth_from_header()
|
||||
|
@ -364,7 +368,7 @@ def get_tracker_term_item():
|
|||
# # # # # # # # # # # # CRYPTOCURRENCY # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
@restApi.route("api/v1/get/cryptocurrency/bitcoin/metadata", methods=['POST'])
|
||||
@token_required('user')
|
||||
@token_required('read_only')
|
||||
def get_cryptocurrency_bitcoin_metadata():
|
||||
data = request.get_json()
|
||||
crypto_address = data.get('bitcoin', None)
|
||||
|
@ -373,7 +377,7 @@ def get_cryptocurrency_bitcoin_metadata():
|
|||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
@restApi.route("api/v1/get/cryptocurrency/bitcoin/item", methods=['POST'])
|
||||
@token_required('user')
|
||||
@token_required('read_only')
|
||||
def get_cryptocurrency_bitcoin_item():
|
||||
data = request.get_json()
|
||||
bitcoin_address = data.get('bitcoin', None)
|
||||
|
@ -385,7 +389,7 @@ def get_cryptocurrency_bitcoin_item():
|
|||
# # # # # # # # # # # # # # # PGP # # # # # # # # # # # # # # # # #
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
@restApi.route("api/v1/get/pgp/key/metadata", methods=['POST'])
|
||||
@token_required('user')
|
||||
@token_required('read_only')
|
||||
def get_pgp_key_metadata():
|
||||
data = request.get_json()
|
||||
pgp_field = data.get('key', None)
|
||||
|
@ -394,7 +398,7 @@ def get_pgp_key_metadata():
|
|||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
@restApi.route("api/v1/get/pgp/mail/metadata", methods=['POST'])
|
||||
@token_required('user')
|
||||
@token_required('read_only')
|
||||
def get_pgp_mail_metadata():
|
||||
data = request.get_json()
|
||||
pgp_field = data.get('mail', None)
|
||||
|
@ -403,7 +407,7 @@ def get_pgp_mail_metadata():
|
|||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
@restApi.route("api/v1/get/pgp/name/metadata", methods=['POST'])
|
||||
@token_required('user')
|
||||
@token_required('read_only')
|
||||
def get_pgp_name_metadata():
|
||||
data = request.get_json()
|
||||
pgp_field = data.get('name', None)
|
||||
|
@ -412,7 +416,7 @@ def get_pgp_name_metadata():
|
|||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
@restApi.route("api/v1/get/pgp/key/item", methods=['POST'])
|
||||
@token_required('user')
|
||||
@token_required('read_only')
|
||||
def get_pgp_key_item():
|
||||
data = request.get_json()
|
||||
pgp_field = data.get('key', None)
|
||||
|
@ -421,7 +425,7 @@ def get_pgp_key_item():
|
|||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
@restApi.route("api/v1/get/pgp/mail/item", methods=['POST'])
|
||||
@token_required('user')
|
||||
@token_required('read_only')
|
||||
def get_pgp_mail_item():
|
||||
data = request.get_json()
|
||||
pgp_mail = data.get('mail', None)
|
||||
|
@ -430,7 +434,7 @@ def get_pgp_mail_item():
|
|||
return Response(json.dumps(res[0], indent=2, sort_keys=True), mimetype='application/json'), res[1]
|
||||
|
||||
@restApi.route("api/v1/get/pgp/name/item", methods=['POST'])
|
||||
@token_required('user')
|
||||
@token_required('read_only')
|
||||
def get_pgp_name_item():
|
||||
data = request.get_json()
|
||||
pgp_name = data.get('name', None)
|
||||
|
@ -609,7 +613,7 @@ def import_json_item():
|
|||
#
|
||||
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
|
||||
@restApi.route("api/v1/ping", methods=['GET'])
|
||||
@token_required('user')
|
||||
@token_required('read_only')
|
||||
def v1_ping():
|
||||
return Response(json.dumps({'status': 'pong'}), mimetype='application/json'), 200
|
||||
|
||||
|
|
Loading…
Reference in a new issue