diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 62bfd12d..3f79841c 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -215,6 +215,8 @@ function launching_scripts { # sleep 0.1 echo -e $GREEN"\t* Launching scripts"$DEFAULT + screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Mixer.py; read x" + sleep 0.1 screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Global.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Categ.py; read x" @@ -310,10 +312,8 @@ function launching_scripts { ################################## screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x" - sleep 0.1 - screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x" - sleep 0.1 +# screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x" +# sleep 0.1 screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./IPAddress.py; read x" @@ -610,7 +610,6 @@ function launch_all { checking_configuration; update; launch_redis; - launch_ardb; launch_kvrocks; launch_logs; launch_queues; diff --git a/bin/export/Export.py b/bin/export/Export.py index b974aabf..313bac4f 100755 --- a/bin/export/Export.py +++ b/bin/export/Export.py @@ -1,24 +1,24 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* -import os -import sys -import uuid - -sys.path.append(os.environ['AIL_BIN']) -################################## -# Import Project packages -################################## -from lib.ConfigLoader import ConfigLoader +# import os +# import sys +# import uuid +# +# sys.path.append(os.environ['AIL_BIN']) +# ################################## +# # Import Project packages +# ################################## +# from lib.ConfigLoader import ConfigLoader ## LOAD CONFIG ## -config_loader = ConfigLoader() - -r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") ###################################### -config_loader = None +# config_loader = ConfigLoader() +# +# r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") ###################################### +# config_loader = None ## -- ## -sys.path.append('../../configs/keys') +# sys.path.append('../../configs/keys') ################################## # Import Keys ################################## diff --git a/bin/export/MispImport.py b/bin/export/MispImport.py index a1c73509..74705c8f 100755 --- a/bin/export/MispImport.py +++ b/bin/export/MispImport.py @@ -1,248 +1,248 @@ #!/usr/bin/env python3 # -*-coding:UTF-8 -* -import os -import sys -import uuid - -from hashlib import sha1, sha256 - -sys.path.append(os.environ['AIL_BIN']) -from lib.objects import ail_objects - - -from lib.objects import Items - -# MISP -from pymisp import MISPEvent, MISPObject, PyMISP - -# # TODO: deplace me in another fil -def get_global_id(obj_type, obj_id, obj_subtype=None): - if obj_subtype: - return '{}:{}:{}'.format(obj_type, obj_subtype, obj_id) - else: - return '{}:{}'.format(obj_type, obj_id) - -# sub type -# obj type -# obj value -def get_global_id_from_id(global_id): - obj_meta = {} - global_id = global_id.split(':', 3) - if len(global_id) > 2: - obj_meta['type'] = global_id[0] - obj_meta['subtype'] = global_id[1] - obj_meta['id'] = global_id[2] - else: - obj_meta['type'] = global_id[0] - obj_meta['subtype'] = '' - obj_meta['id'] = global_id[1] - return obj_meta - -def get_import_dir(): - return os.path.join(os.environ['AIL_HOME'], 'temp/import') - -def sanitize_import_file_path(filename): - IMPORT_FOLDER = get_import_dir() - filename = os.path.join(IMPORT_FOLDER, filename) - filename = os.path.realpath(filename) - # path traversal - if not os.path.commonprefix([filename, IMPORT_FOLDER]) == IMPORT_FOLDER: - return os.path.join(IMPORT_FOLDER, str(uuid.uuid4()) + '.json') - # check if file already exist - if os.path.isfile(filename): - return os.path.join(IMPORT_FOLDER, str(uuid.uuid4()) + '.json') - return filename - -def get_misp_obj_tag(misp_obj): - if misp_obj.attributes: - misp_tags = misp_obj.attributes[0].tags - tags = [] - for misp_tag in misp_tags: - tags.append(misp_tag.name) - return tags - else: - return [] - -def get_object_metadata(misp_obj): - obj_meta = {} - if 'first_seen' in misp_obj.keys(): - obj_meta['first_seen'] = misp_obj.first_seen - if 'last_seen' in misp_obj.keys(): - obj_meta['last_seen'] = misp_obj.last_seen - obj_meta['tags'] = get_misp_obj_tag(misp_obj) - return obj_meta - -def unpack_item_obj(map_uuid_global_id, misp_obj): - obj_meta = get_object_metadata(misp_obj) - obj_id = None - io_content = None - - for attribute in misp_obj.attributes: - if attribute.object_relation == 'raw-data': - obj_id = attribute.value # # TODO: sanitize - io_content = attribute.data # # TODO: check if type == io - - if obj_id and io_content: - res = Items.create_item(obj_id, obj_meta, io_content) - - map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id) - - - -## TODO: handle multiple pgp in the same object -def unpack_obj_pgp(map_uuid_global_id, misp_obj): - # TODO ail_objects import_misp_object(misp_obj) - pass - # # get obj sub type - # obj_attr = misp_obj.attributes[0] - # obj_id = obj_attr.value - # if obj_attr.object_relation == 'key-id': - # obj_subtype = 'key' - # elif obj_attr.object_relation == 'user-id-name': - # obj_subtype = 'name' - # elif obj_attr.object_relation == 'user-id-email': - # obj_subtype = 'mail' - # - # if obj_id and obj_subtype: - # obj_meta = get_object_metadata(misp_obj) - # # res = Pgp.pgp.create_correlation(obj_subtype, obj_id, obj_meta) - # # TODO ail_objects import_misp_object(misp_obj) - # - # map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype) - - -def unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj): - # TODO ail_objects import_misp_object(misp_obj) - pass - # - # obj_id = None - # obj_subtype = None - # for attribute in misp_obj.attributes: - # if attribute.object_relation == 'address': # # TODO: handle xmr address field - # obj_id = attribute.value - # elif attribute.object_relation == 'symbol': - # obj_subtype = Cryptocurrency.get_cryptocurrency_type(attribute.value) - # - # # valid cryptocurrency type - # if obj_subtype and obj_id: - # obj_meta = get_object_metadata(misp_obj) - # # res = Cryptocurrency.cryptocurrency.create_correlation(obj_subtype, obj_id, obj_meta) - # - # map_uuid_global_id[misp_obj.uuid] = get_global_id('cryptocurrency', obj_id, obj_subtype=obj_subtype) - -def get_obj_type_from_relationship(misp_obj): - obj_uuid = misp_obj.uuid - obj_type = None - - for relation in misp_obj.ObjectReference: - if relation.object_uuid == obj_uuid: - if relation.relationship_type == "screenshot-of": - return 'screenshot' - if relation.relationship_type == "included-in": - obj_type = 'decoded' - return obj_type - - -# # TODO: covert md5 and sha1 to expected -def unpack_file(map_uuid_global_id, misp_obj): - - obj_type = get_obj_type_from_relationship(misp_obj) - if obj_type: - obj_id = None - io_content = None - for attribute in misp_obj.attributes: - # get file content - if attribute.object_relation == 'attachment': - io_content = attribute.data - elif attribute.object_relation == 'malware-sample': - io_content = attribute.data - - # # TODO: use/verify specified mimetype - elif attribute.object_relation == 'mimetype': - #print(attribute.value) - pass - - # # TODO: support more - elif attribute.object_relation == 'sha1' and obj_type == 'decoded': - obj_id = attribute.value - elif attribute.object_relation == 'sha256' and obj_type == 'screenshot': - obj_id = attribute.value - - # get SHA1/sha256 - if io_content and not obj_id: - if obj_type=='screenshot': - obj_id = sha256(io_content.getvalue()).hexdigest() - else: # decoded file - obj_id = sha1(io_content.getvalue()).hexdigest() - - if obj_id and io_content: - obj_meta = get_object_metadata(misp_obj) - if obj_type == 'screenshot': - # TODO MIGRATE + REFACTOR ME - # Screenshot.create_screenshot(obj_id, obj_meta, io_content) - map_uuid_global_id[misp_obj.uuid] = get_global_id('image', obj_id) - else: #decoded - # TODO MIGRATE + REFACTOR ME - # Decoded.create_decoded(obj_id, obj_meta, io_content) - map_uuid_global_id[misp_obj.uuid] = get_global_id('decoded', obj_id) - - -def get_misp_import_fct(map_uuid_global_id, misp_obj): - if misp_obj.name == 'ail-leak': - unpack_item_obj(map_uuid_global_id, misp_obj) - elif misp_obj.name == 'domain-crawled': - pass - elif misp_obj.name == 'pgp-meta': - unpack_obj_pgp(map_uuid_global_id, misp_obj) - elif misp_obj.name == 'coin-address': - unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj) - elif misp_obj.name == 'file': - unpack_file(map_uuid_global_id, misp_obj) - -# import relationship between objects -def create_obj_relationships(map_uuid_global_id, misp_obj): - if misp_obj.uuid in map_uuid_global_id: - for relationship in misp_obj.ObjectReference: - if relationship.referenced_uuid in map_uuid_global_id: - obj_meta_src = get_global_id_from_id(map_uuid_global_id[relationship.object_uuid]) - obj_meta_target = get_global_id_from_id(map_uuid_global_id[relationship.referenced_uuid]) - - if obj_meta_src == 'decoded' or obj_meta_src == 'item': - print('000000') - print(obj_meta_src) - print(obj_meta_target) - print('111111') - - # TODO CREATE OBJ RELATIONSHIP - -def import_objs_from_file(filepath): - map_uuid_global_id = {} - - event_to_import = MISPEvent() - try: - event_to_import.load_file(filepath) - except: - return map_uuid_global_id - - for misp_obj in event_to_import.objects: - get_misp_import_fct(map_uuid_global_id, misp_obj) - - for misp_obj in event_to_import.objects: - create_obj_relationships(map_uuid_global_id, misp_obj) - - return map_uuid_global_id - - -if __name__ == '__main__': - - # misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False) - - import_objs_from_file('ail_export_c777a4d1-5f63-4fa2-86c0-07da677bdac2.json') - - #Screenshot.delete_screenshot('a92d459f70c4dea8a14688f585a5e2364be8b91fbf924290ead361d9b909dcf1') - #Decoded.delete_decoded('d59a110ab233fe87cefaa0cf5603b047b432ee07') - #Pgp.pgp.delete_correlation('key', '0xA4BB02A75E6AF448') - - #Item.delete_item('submitted/2020/02/10/b2485894-4325-469b-bc8f-6ad1c2dbb202.gz') - #Item.delete_item('archive/pastebin.com_pro/2020/02/10/K2cerjP4.gz') +# import os +# import sys +# import uuid +# +# from hashlib import sha1, sha256 +# +# sys.path.append(os.environ['AIL_BIN']) +# from lib.objects import ail_objects +# +# +# from lib.objects import Items +# +# # MISP +# from pymisp import MISPEvent, MISPObject, PyMISP +# +# # # TODO: deplace me in another fil +# def get_global_id(obj_type, obj_id, obj_subtype=None): +# if obj_subtype: +# return '{}:{}:{}'.format(obj_type, obj_subtype, obj_id) +# else: +# return '{}:{}'.format(obj_type, obj_id) +# +# # sub type +# # obj type +# # obj value +# def get_global_id_from_id(global_id): +# obj_meta = {} +# global_id = global_id.split(':', 3) +# if len(global_id) > 2: +# obj_meta['type'] = global_id[0] +# obj_meta['subtype'] = global_id[1] +# obj_meta['id'] = global_id[2] +# else: +# obj_meta['type'] = global_id[0] +# obj_meta['subtype'] = '' +# obj_meta['id'] = global_id[1] +# return obj_meta +# +# def get_import_dir(): +# return os.path.join(os.environ['AIL_HOME'], 'temp/import') +# +# def sanitize_import_file_path(filename): +# IMPORT_FOLDER = get_import_dir() +# filename = os.path.join(IMPORT_FOLDER, filename) +# filename = os.path.realpath(filename) +# # path traversal +# if not os.path.commonprefix([filename, IMPORT_FOLDER]) == IMPORT_FOLDER: +# return os.path.join(IMPORT_FOLDER, str(uuid.uuid4()) + '.json') +# # check if file already exist +# if os.path.isfile(filename): +# return os.path.join(IMPORT_FOLDER, str(uuid.uuid4()) + '.json') +# return filename +# +# def get_misp_obj_tag(misp_obj): +# if misp_obj.attributes: +# misp_tags = misp_obj.attributes[0].tags +# tags = [] +# for misp_tag in misp_tags: +# tags.append(misp_tag.name) +# return tags +# else: +# return [] +# +# def get_object_metadata(misp_obj): +# obj_meta = {} +# if 'first_seen' in misp_obj.keys(): +# obj_meta['first_seen'] = misp_obj.first_seen +# if 'last_seen' in misp_obj.keys(): +# obj_meta['last_seen'] = misp_obj.last_seen +# obj_meta['tags'] = get_misp_obj_tag(misp_obj) +# return obj_meta +# +# def unpack_item_obj(map_uuid_global_id, misp_obj): +# obj_meta = get_object_metadata(misp_obj) +# obj_id = None +# io_content = None +# +# for attribute in misp_obj.attributes: +# if attribute.object_relation == 'raw-data': +# obj_id = attribute.value # # TODO: sanitize +# io_content = attribute.data # # TODO: check if type == io +# +# if obj_id and io_content: +# res = Items.create_item(obj_id, obj_meta, io_content) +# +# map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id) +# +# +# +# ## TODO: handle multiple pgp in the same object +# def unpack_obj_pgp(map_uuid_global_id, misp_obj): +# # TODO ail_objects import_misp_object(misp_obj) +# pass +# # # get obj sub type +# # obj_attr = misp_obj.attributes[0] +# # obj_id = obj_attr.value +# # if obj_attr.object_relation == 'key-id': +# # obj_subtype = 'key' +# # elif obj_attr.object_relation == 'user-id-name': +# # obj_subtype = 'name' +# # elif obj_attr.object_relation == 'user-id-email': +# # obj_subtype = 'mail' +# # +# # if obj_id and obj_subtype: +# # obj_meta = get_object_metadata(misp_obj) +# # # res = Pgp.pgp.create_correlation(obj_subtype, obj_id, obj_meta) +# # # TODO ail_objects import_misp_object(misp_obj) +# # +# # map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype) +# +# +# def unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj): +# # TODO ail_objects import_misp_object(misp_obj) +# pass +# # +# # obj_id = None +# # obj_subtype = None +# # for attribute in misp_obj.attributes: +# # if attribute.object_relation == 'address': # # TODO: handle xmr address field +# # obj_id = attribute.value +# # elif attribute.object_relation == 'symbol': +# # obj_subtype = Cryptocurrency.get_cryptocurrency_type(attribute.value) +# # +# # # valid cryptocurrency type +# # if obj_subtype and obj_id: +# # obj_meta = get_object_metadata(misp_obj) +# # # res = Cryptocurrency.cryptocurrency.create_correlation(obj_subtype, obj_id, obj_meta) +# # +# # map_uuid_global_id[misp_obj.uuid] = get_global_id('cryptocurrency', obj_id, obj_subtype=obj_subtype) +# +# def get_obj_type_from_relationship(misp_obj): +# obj_uuid = misp_obj.uuid +# obj_type = None +# +# for relation in misp_obj.ObjectReference: +# if relation.object_uuid == obj_uuid: +# if relation.relationship_type == "screenshot-of": +# return 'screenshot' +# if relation.relationship_type == "included-in": +# obj_type = 'decoded' +# return obj_type +# +# +# # # TODO: covert md5 and sha1 to expected +# def unpack_file(map_uuid_global_id, misp_obj): +# +# obj_type = get_obj_type_from_relationship(misp_obj) +# if obj_type: +# obj_id = None +# io_content = None +# for attribute in misp_obj.attributes: +# # get file content +# if attribute.object_relation == 'attachment': +# io_content = attribute.data +# elif attribute.object_relation == 'malware-sample': +# io_content = attribute.data +# +# # # TODO: use/verify specified mimetype +# elif attribute.object_relation == 'mimetype': +# #print(attribute.value) +# pass +# +# # # TODO: support more +# elif attribute.object_relation == 'sha1' and obj_type == 'decoded': +# obj_id = attribute.value +# elif attribute.object_relation == 'sha256' and obj_type == 'screenshot': +# obj_id = attribute.value +# +# # get SHA1/sha256 +# if io_content and not obj_id: +# if obj_type=='screenshot': +# obj_id = sha256(io_content.getvalue()).hexdigest() +# else: # decoded file +# obj_id = sha1(io_content.getvalue()).hexdigest() +# +# if obj_id and io_content: +# obj_meta = get_object_metadata(misp_obj) +# if obj_type == 'screenshot': +# # TODO MIGRATE + REFACTOR ME +# # Screenshot.create_screenshot(obj_id, obj_meta, io_content) +# map_uuid_global_id[misp_obj.uuid] = get_global_id('image', obj_id) +# else: #decoded +# # TODO MIGRATE + REFACTOR ME +# # Decoded.create_decoded(obj_id, obj_meta, io_content) +# map_uuid_global_id[misp_obj.uuid] = get_global_id('decoded', obj_id) +# +# +# def get_misp_import_fct(map_uuid_global_id, misp_obj): +# if misp_obj.name == 'ail-leak': +# unpack_item_obj(map_uuid_global_id, misp_obj) +# elif misp_obj.name == 'domain-crawled': +# pass +# elif misp_obj.name == 'pgp-meta': +# unpack_obj_pgp(map_uuid_global_id, misp_obj) +# elif misp_obj.name == 'coin-address': +# unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj) +# elif misp_obj.name == 'file': +# unpack_file(map_uuid_global_id, misp_obj) +# +# # import relationship between objects +# def create_obj_relationships(map_uuid_global_id, misp_obj): +# if misp_obj.uuid in map_uuid_global_id: +# for relationship in misp_obj.ObjectReference: +# if relationship.referenced_uuid in map_uuid_global_id: +# obj_meta_src = get_global_id_from_id(map_uuid_global_id[relationship.object_uuid]) +# obj_meta_target = get_global_id_from_id(map_uuid_global_id[relationship.referenced_uuid]) +# +# if obj_meta_src == 'decoded' or obj_meta_src == 'item': +# print('000000') +# print(obj_meta_src) +# print(obj_meta_target) +# print('111111') +# +# # TODO CREATE OBJ RELATIONSHIP +# +# def import_objs_from_file(filepath): +# map_uuid_global_id = {} +# +# event_to_import = MISPEvent() +# try: +# event_to_import.load_file(filepath) +# except: +# return map_uuid_global_id +# +# for misp_obj in event_to_import.objects: +# get_misp_import_fct(map_uuid_global_id, misp_obj) +# +# for misp_obj in event_to_import.objects: +# create_obj_relationships(map_uuid_global_id, misp_obj) +# +# return map_uuid_global_id +# +# +# if __name__ == '__main__': +# +# # misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False) +# +# import_objs_from_file('ail_export_c777a4d1-5f63-4fa2-86c0-07da677bdac2.json') +# +# #Screenshot.delete_screenshot('a92d459f70c4dea8a14688f585a5e2364be8b91fbf924290ead361d9b909dcf1') +# #Decoded.delete_decoded('d59a110ab233fe87cefaa0cf5603b047b432ee07') +# #Pgp.pgp.delete_correlation('key', '0xA4BB02A75E6AF448') +# +# #Item.delete_item('submitted/2020/02/10/b2485894-4325-469b-bc8f-6ad1c2dbb202.gz') +# #Item.delete_item('archive/pastebin.com_pro/2020/02/10/K2cerjP4.gz') diff --git a/bin/lib/ail_core.py b/bin/lib/ail_core.py index 26f52179..5c26501e 100755 --- a/bin/lib/ail_core.py +++ b/bin/lib/ail_core.py @@ -3,7 +3,7 @@ import os import sys -import redis +from uuid import uuid4 sys.path.append(os.environ['AIL_BIN']) ################################## @@ -20,6 +20,9 @@ AIL_OBJECTS = sorted({'cve', 'cryptocurrency', 'decoded', 'domain', 'item', 'pgp def get_ail_uuid(): return r_serv_db.get('ail:uuid') +def generate_uuid(): + return str(uuid4()) + #### AIL OBJECTS #### def get_all_objects(): diff --git a/bin/lib/item_basic.py b/bin/lib/item_basic.py index fb28639e..fdfe1059 100755 --- a/bin/lib/item_basic.py +++ b/bin/lib/item_basic.py @@ -17,7 +17,6 @@ from lib import Tag config_loader = ConfigLoader.ConfigLoader() r_cache = config_loader.get_redis_conn("Redis_Cache") -r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") r_object = config_loader.get_db_conn("Kvrocks_Objects") config_loader = None @@ -180,21 +179,21 @@ def get_all_domain_node_by_item_id(item_id, l_nodes=[]): # FIXME: #### UNKNOW SECTION #### -def get_obj_id_item_id(parent_type, parent_id): - all_parents_type = ['twitter_id', 'jabber_id', 'telegram_id'] - if parent_type in all_parents_type: - return r_serv_metadata.hget('map:{}:item_id'.format(parent_type), parent_id) - else: - return None +# def get_obj_id_item_id(parent_type, parent_id): +# all_parents_type = ['twitter_id', 'jabber_id', 'telegram_id'] +# if parent_type in all_parents_type: +# return r_serv_metadata.hget('map:{}:item_id'.format(parent_type), parent_id) +# else: +# return None -# # TODO: # FIXME: TO MIGRATE ?????? -def add_map_obj_id_item_id(obj_id, item_id, obj_type): - if obj_type == 'twitter_id': - r_serv_metadata.hset('map:twitter_id:item_id', obj_id, item_id) - if obj_type == 'jabber_id': - r_serv_metadata.hset('map:jabber_id:item_id', obj_id, item_id) - if obj_type == 'telegram_id': - r_serv_metadata.hset('map:telegram_id:item_id', obj_id, item_id) +# # # TODO: # FIXME: TO MIGRATE ?????? +# def add_map_obj_id_item_id(obj_id, item_id, obj_type): +# if obj_type == 'twitter_id': +# r_serv_metadata.hset('map:twitter_id:item_id', obj_id, item_id) +# if obj_type == 'jabber_id': +# r_serv_metadata.hset('map:jabber_id:item_id', obj_id, item_id) +# if obj_type == 'telegram_id': +# r_serv_metadata.hset('map:telegram_id:item_id', obj_id, item_id) # delete twitter id diff --git a/bin/modules/submit_paste.py b/bin/modules/submit_paste.py index b483a99d..eddd797b 100755 --- a/bin/modules/submit_paste.py +++ b/bin/modules/submit_paste.py @@ -5,8 +5,6 @@ The Submit paste module ================ -This module is taking paste in redis queue ARDB_DB and submit to global - """ ################################## @@ -15,7 +13,6 @@ This module is taking paste in redis queue ARDB_DB and submit to global import os import sys import gzip -import io import base64 import datetime import time @@ -51,11 +48,8 @@ class SubmitPaste(AbstractModule): super(SubmitPaste, self).__init__(queue_name='submit_paste') # TODO KVROCKS - self.r_serv_db = ConfigLoader.ConfigLoader().get_redis_conn("ARDB_DB") + self.r_serv_db = ConfigLoader.ConfigLoader().get_redis_conn("Kvrocks_DB") self.r_serv_log_submit = ConfigLoader.ConfigLoader().get_redis_conn("Redis_Log_submit") - self.r_serv_tags = ConfigLoader.ConfigLoader().get_redis_conn("ARDB_Tags") - self.r_serv_metadata = ConfigLoader.ConfigLoader().get_redis_conn("ARDB_Metadata") - self.serv_statistics = ConfigLoader.ConfigLoader().get_redis_conn("ARDB_Statistics") self.pending_seconds = 3 @@ -305,7 +299,6 @@ class SubmitPaste(AbstractModule): self.r_serv_log_submit.sadd(f'{uuid}:paste_submit_link', rel_item_path) curr_date = datetime.date.today() - self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'submit_paste', 1) self.redis_logger.debug("paste submitted") else: self.addError(uuid, f'File: {save_path} already exist in submitted pastes') @@ -335,7 +328,6 @@ class SubmitPaste(AbstractModule): self.addError(uuid, errorMessage) self.r_serv_log_submit.set(f'{uuid}:end', 1) curr_date = datetime.date.today() - self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"), 'submit_abord', 1) self.remove_submit_uuid(uuid) # # TODO: use Item function diff --git a/bin/packages/Import_helper.py b/bin/packages/Import_helper.py index b997d3e2..a3ce1699 100755 --- a/bin/packages/Import_helper.py +++ b/bin/packages/Import_helper.py @@ -16,7 +16,7 @@ from lib import ConfigLoader config_loader = ConfigLoader.ConfigLoader() -r_serv_db = config_loader.get_redis_conn("ARDB_DB") +r_serv_db = config_loader.get_redis_conn("Kvrocks_DB") r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit") config_loader = None @@ -62,11 +62,11 @@ def create_import_queue(tags, galaxy, paste_content, UUID, password=None, isfile def check_import_status(UUID): if not is_valid_uuid_v4(UUID): - return ({'status': 'error', 'reason': 'Invalid uuid'}, 400) + return {'status': 'error', 'reason': 'Invalid uuid'}, 400 processing = r_serv_log_submit.get(UUID + ':processing') if not processing: - return ({'status': 'error', 'reason': 'Unknown uuid'}, 404) + return {'status': 'error', 'reason': 'Unknown uuid'}, 404 # nb_total = r_serv_log_submit.get(UUID + ':nb_total') # nb_sucess = r_serv_log_submit.get(UUID + ':nb_sucess') @@ -90,4 +90,4 @@ def check_import_status(UUID): status = 'imported' dict_import_status['status'] = status - return (dict_import_status, 200) + return dict_import_status, 200 diff --git a/install_virtualenv.sh b/install_virtualenv.sh index c410069d..214b613c 100755 --- a/install_virtualenv.sh +++ b/install_virtualenv.sh @@ -14,7 +14,7 @@ if [ -z "$VIRTUAL_ENV" ]; then echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate echo export AIL_FLASK=$(pwd)/var/www/ >> ./AILENV/bin/activate echo export AIL_REDIS=$(pwd)/redis/src/ >> ./AILENV/bin/activate - echo export AIL_ARDB=$(pwd)/ardb/src/ >> ./AILENV/bin/activate + echo export AIL_KVROCKS=$(pwd)/kvrocks/src/ >> ./AILENV/bin/activate fi diff --git a/installing_deps.sh b/installing_deps.sh index c1513c39..77962dc2 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -84,10 +84,10 @@ sudo make install popd # ARDB # -test ! -d ardb/ && git clone https://github.com/ail-project/ardb.git -pushd ardb/ -make -popd +#test ! -d ardb/ && git clone https://github.com/ail-project/ardb.git +#pushd ardb/ +#make +#popd DEFAULT_HOME=$(pwd) @@ -148,8 +148,8 @@ echo "AIL current version:" git describe --abbrev=0 --tags popd -# LAUNCH ARDB -bash ${AIL_BIN}/LAUNCH.sh -lav & +# LAUNCH Kvrocks +bash ${AIL_BIN}/LAUNCH.sh -lkv & wait echo "" diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 0551119c..e0f6b418 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -25,9 +25,7 @@ from flask_login import login_required ################################## # Import Project packages ################################## -from export import Export from lib import Tag -from lib.objects.Items import Item from packages import Import_helper @@ -75,7 +73,7 @@ def limit_content_length(): # ============ FUNCTIONS ============ def allowed_file(filename): - if not '.' in filename: + if '.' not in filename: return True else: file_ext = filename.rsplit('.', 1)[1].lower() @@ -86,7 +84,7 @@ def allowed_file(filename): def clean_filename(filename, whitelist=valid_filename_chars, replace=' '): # replace characters for r in replace: - filename = filename.replace(r,'_') + filename = filename.replace(r, '_') # keep only valid ascii chars cleaned_filename = unicodedata.normalize('NFKD', filename).encode('ASCII', 'ignore').decode() @@ -116,8 +114,6 @@ def PasteSubmit_page(): @login_analyst @limit_content_length() def submit(): - - #paste_name = request.form['paste_name'] logger.debug('submit') password = request.form['archive_pass'] @@ -127,10 +123,10 @@ def submit(): paste_source = request.form['paste_source'] if paste_source: - # limit source length + # limit source length paste_source = paste_source.replace('/', '')[:80] if paste_source in ['crawled', 'tests']: - content = f'Invalid source' + content = 'Invalid source' logger.info(paste_source) return content, 400 @@ -150,9 +146,9 @@ def submit(): submitted_tag = 'infoleak:submission="manual"' - #active taxonomies + # active taxonomies active_taxonomies = Tag.get_active_taxonomies() - #active galaxies + # active galaxies active_galaxies = Tag.get_active_galaxies() if ltags or ltagsgalaxies: @@ -179,16 +175,12 @@ def submit(): # get UUID UUID = str(uuid.uuid4()) - '''if paste_name: - # clean file name - UUID = clean_filename(paste_name)''' - # create submitted dir if not os.path.exists(UPLOAD_FOLDER): logger.debug('create folder') os.makedirs(UPLOAD_FOLDER) - if not '.' in file_import.filename: + if '.' not in file_import.filename: logger.debug('add UUID to path') full_path = os.path.join(UPLOAD_FOLDER, UUID) else: @@ -202,23 +194,22 @@ def submit(): full_path = os.path.join(UPLOAD_FOLDER, name) logger.debug(f'full path {full_path}') - #Flask verify the file size + # Flask verify the file size file_import.save(full_path) logger.debug('file saved') Import_helper.create_import_queue(ltags, ltagsgalaxies, full_path, UUID, password, True) return render_template("submit_items.html", - active_taxonomies = active_taxonomies, - active_galaxies = active_galaxies, - UUID = UUID) + active_taxonomies=active_taxonomies, + active_galaxies=active_galaxies, + UUID=UUID) else: content = f'wrong file type, allowed_extensions: {allowed_extensions} or remove the extension' logger.info(content) return content, 400 - elif paste_content != '': logger.debug(f'entering text paste management') if sys.getsizeof(paste_content) < Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE: @@ -242,7 +233,6 @@ def submit(): logger.error(content) return content, 400 - return PasteSubmit_page() @PasteSubmit.route("/PasteSubmit/submit_status", methods=['GET']) @@ -278,10 +268,7 @@ def submit_status(): else: prog = 0 - if error: - isError = True - else: - isError = False + isError = bool(error) if end == '0': end = False @@ -327,6 +314,8 @@ def submit_status(): @login_required @login_analyst def edit_tag_export(): + return abort(404) + misp_auto_events = r_serv_db.get('misp:auto-events') hive_auto_alerts = r_serv_db.get('hive:auto-alerts') @@ -393,6 +382,9 @@ def edit_tag_export(): @login_required @login_analyst def tag_export_edited(): + return abort(404) + + tag_enabled_misp = request.form.getlist('tag_enabled_misp') tag_enabled_hive = request.form.getlist('tag_enabled_hive') @@ -419,6 +411,8 @@ def tag_export_edited(): @login_required @login_analyst def enable_misp_auto_event(): + return abort(404) + r_serv_db.set('misp:auto-events', 1) return edit_tag_export() @@ -426,6 +420,8 @@ def enable_misp_auto_event(): @login_required @login_analyst def disable_misp_auto_event(): + return abort(404) + r_serv_db.set('misp:auto-events', 0) return edit_tag_export() @@ -433,6 +429,8 @@ def disable_misp_auto_event(): @login_required @login_analyst def enable_hive_auto_alert(): + return abort(404) + r_serv_db.set('hive:auto-alerts', 1) return edit_tag_export() @@ -440,6 +438,8 @@ def enable_hive_auto_alert(): @login_required @login_analyst def disable_hive_auto_alert(): + return abort(404) + r_serv_db.set('hive:auto-alerts', 0) return edit_tag_export() @@ -447,6 +447,8 @@ def disable_hive_auto_alert(): @login_required @login_analyst def add_push_tag(): + return abort(404) + tag = request.args.get('tag') if tag is not None: @@ -466,6 +468,9 @@ def add_push_tag(): @login_required @login_analyst def delete_push_tag(): + return abort(404) + + tag = request.args.get('tag') infoleak_tags = Taxonomies().get('infoleak').machinetags() diff --git a/var/www/templates/sidebars/sidebar_objects.html b/var/www/templates/sidebars/sidebar_objects.html index 3b608fea..46fe58eb 100644 --- a/var/www/templates/sidebars/sidebar_objects.html +++ b/var/www/templates/sidebars/sidebar_objects.html @@ -69,11 +69,11 @@