AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-10 00:28:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [DB] remove ARDB + temp disable MISP import and auto export

Browse source
This commit is contained in:
Terrtia 2023-04-04 10:25:01 +02:00
parent 54a0bcb022
commit 80efc9cdbb
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
11 changed files with 328 additions and 330 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
bin/LAUNCH.sh
View file

@ -215,6 +215,8 @@ function launching_scripts {
# sleep 0.1 # sleep 0.1
echo -e $GREEN"\t* Launching scripts"$DEFAULT echo -e $GREEN"\t* Launching scripts"$DEFAULT
screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Mixer.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Global.py; read x" screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Global.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Categ.py; read x" screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Categ.py; read x"
@ -310,10 +312,8 @@ function launching_scripts {
################################## ##################################
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x" screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x" # screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
sleep 0.1 # sleep 0.1
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./IPAddress.py; read x" screen -S "Script_AIL" -X screen -t "IPAddress" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./IPAddress.py; read x"
@ -610,7 +610,6 @@ function launch_all {
checking_configuration; checking_configuration;
update; update;
launch_redis; launch_redis;
launch_ardb;
launch_kvrocks; launch_kvrocks;
launch_logs; launch_logs;
launch_queues; launch_queues;

28
bin/export/Export.py
View file

@ -1,24 +1,24 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import os # import os
import sys # import sys
import uuid # import uuid
#
sys.path.append(os.environ['AIL_BIN']) # sys.path.append(os.environ['AIL_BIN'])
################################## # ##################################
# Import Project packages # # Import Project packages
################################## # ##################################
from lib.ConfigLoader import ConfigLoader # from lib.ConfigLoader import ConfigLoader
## LOAD CONFIG ## ## LOAD CONFIG ##
config_loader = ConfigLoader() # config_loader = ConfigLoader()
#
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") ###################################### # r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") ######################################
config_loader = None # config_loader = None
## -- ## ## -- ##
sys.path.append('../../configs/keys') # sys.path.append('../../configs/keys')
################################## ##################################
# Import Keys # Import Keys
################################## ##################################

474
bin/export/MispImport.py
View file

@ -1,248 +1,248 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import os # import os
import sys # import sys
import uuid # import uuid
from hashlib import sha1, sha256
sys.path.append(os.environ['AIL_BIN'])
from lib.objects import ail_objects
from lib.objects import Items
# MISP
from pymisp import MISPEvent, MISPObject, PyMISP
# # TODO: deplace me in another fil
def get_global_id(obj_type, obj_id, obj_subtype=None):
if obj_subtype:
return '{}:{}:{}'.format(obj_type, obj_subtype, obj_id)
else:
return '{}:{}'.format(obj_type, obj_id)
# sub type
# obj type
# obj value
def get_global_id_from_id(global_id):
obj_meta = {}
global_id = global_id.split(':', 3)
if len(global_id) > 2:
obj_meta['type'] = global_id[0]
obj_meta['subtype'] = global_id[1]
obj_meta['id'] = global_id[2]
else:
obj_meta['type'] = global_id[0]
obj_meta['subtype'] = ''
obj_meta['id'] = global_id[1]
return obj_meta
def get_import_dir():
return os.path.join(os.environ['AIL_HOME'], 'temp/import')
def sanitize_import_file_path(filename):
IMPORT_FOLDER = get_import_dir()
filename = os.path.join(IMPORT_FOLDER, filename)
filename = os.path.realpath(filename)
# path traversal
if not os.path.commonprefix([filename, IMPORT_FOLDER]) == IMPORT_FOLDER:
return os.path.join(IMPORT_FOLDER, str(uuid.uuid4()) + '.json')
# check if file already exist
if os.path.isfile(filename):
return os.path.join(IMPORT_FOLDER, str(uuid.uuid4()) + '.json')
return filename
def get_misp_obj_tag(misp_obj):
if misp_obj.attributes:
misp_tags = misp_obj.attributes[0].tags
tags = []
for misp_tag in misp_tags:
tags.append(misp_tag.name)
return tags
else:
return []
def get_object_metadata(misp_obj):
obj_meta = {}
if 'first_seen' in misp_obj.keys():
obj_meta['first_seen'] = misp_obj.first_seen
if 'last_seen' in misp_obj.keys():
obj_meta['last_seen'] = misp_obj.last_seen
obj_meta['tags'] = get_misp_obj_tag(misp_obj)
return obj_meta
def unpack_item_obj(map_uuid_global_id, misp_obj):
obj_meta = get_object_metadata(misp_obj)
obj_id = None
io_content = None
for attribute in misp_obj.attributes:
if attribute.object_relation == 'raw-data':
obj_id = attribute.value # # TODO: sanitize
io_content = attribute.data # # TODO: check if type == io
if obj_id and io_content:
res = Items.create_item(obj_id, obj_meta, io_content)
map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id)
## TODO: handle multiple pgp in the same object
def unpack_obj_pgp(map_uuid_global_id, misp_obj):
# TODO ail_objects import_misp_object(misp_obj)
pass
# # get obj sub type
# obj_attr = misp_obj.attributes[0]
# obj_id = obj_attr.value
# if obj_attr.object_relation == 'key-id':
# obj_subtype = 'key'
# elif obj_attr.object_relation == 'user-id-name':
# obj_subtype = 'name'
# elif obj_attr.object_relation == 'user-id-email':
# obj_subtype = 'mail'
# #
# if obj_id and obj_subtype: # from hashlib import sha1, sha256
#
# sys.path.append(os.environ['AIL_BIN'])
# from lib.objects import ail_objects
#
#
# from lib.objects import Items
#
# # MISP
# from pymisp import MISPEvent, MISPObject, PyMISP
#
# # # TODO: deplace me in another fil
# def get_global_id(obj_type, obj_id, obj_subtype=None):
# if obj_subtype:
# return '{}:{}:{}'.format(obj_type, obj_subtype, obj_id)
# else:
# return '{}:{}'.format(obj_type, obj_id)
#
# # sub type
# # obj type
# # obj value
# def get_global_id_from_id(global_id):
# obj_meta = {}
# global_id = global_id.split(':', 3)
# if len(global_id) > 2:
# obj_meta['type'] = global_id[0]
# obj_meta['subtype'] = global_id[1]
# obj_meta['id'] = global_id[2]
# else:
# obj_meta['type'] = global_id[0]
# obj_meta['subtype'] = ''
# obj_meta['id'] = global_id[1]
# return obj_meta
#
# def get_import_dir():
# return os.path.join(os.environ['AIL_HOME'], 'temp/import')
#
# def sanitize_import_file_path(filename):
# IMPORT_FOLDER = get_import_dir()
# filename = os.path.join(IMPORT_FOLDER, filename)
# filename = os.path.realpath(filename)
# # path traversal
# if not os.path.commonprefix([filename, IMPORT_FOLDER]) == IMPORT_FOLDER:
# return os.path.join(IMPORT_FOLDER, str(uuid.uuid4()) + '.json')
# # check if file already exist
# if os.path.isfile(filename):
# return os.path.join(IMPORT_FOLDER, str(uuid.uuid4()) + '.json')
# return filename
#
# def get_misp_obj_tag(misp_obj):
# if misp_obj.attributes:
# misp_tags = misp_obj.attributes[0].tags
# tags = []
# for misp_tag in misp_tags:
# tags.append(misp_tag.name)
# return tags
# else:
# return []
#
# def get_object_metadata(misp_obj):
# obj_meta = {}
# if 'first_seen' in misp_obj.keys():
# obj_meta['first_seen'] = misp_obj.first_seen
# if 'last_seen' in misp_obj.keys():
# obj_meta['last_seen'] = misp_obj.last_seen
# obj_meta['tags'] = get_misp_obj_tag(misp_obj)
# return obj_meta
#
# def unpack_item_obj(map_uuid_global_id, misp_obj):
# obj_meta = get_object_metadata(misp_obj) # obj_meta = get_object_metadata(misp_obj)
# # res = Pgp.pgp.create_correlation(obj_subtype, obj_id, obj_meta)
# # TODO ail_objects import_misp_object(misp_obj)
#
# map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype)
def unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj):
# TODO ail_objects import_misp_object(misp_obj)
pass
#
# obj_id = None # obj_id = None
# obj_subtype = None # io_content = None
#
# for attribute in misp_obj.attributes: # for attribute in misp_obj.attributes:
# if attribute.object_relation == 'address': # # TODO: handle xmr address field # if attribute.object_relation == 'raw-data':
# obj_id = attribute.value # # TODO: sanitize
# io_content = attribute.data # # TODO: check if type == io
#
# if obj_id and io_content:
# res = Items.create_item(obj_id, obj_meta, io_content)
#
# map_uuid_global_id[misp_obj.uuid] = get_global_id('item', obj_id)
#
#
#
# ## TODO: handle multiple pgp in the same object
# def unpack_obj_pgp(map_uuid_global_id, misp_obj):
# # TODO ail_objects import_misp_object(misp_obj)
# pass
# # # get obj sub type
# # obj_attr = misp_obj.attributes[0]
# # obj_id = obj_attr.value
# # if obj_attr.object_relation == 'key-id':
# # obj_subtype = 'key'
# # elif obj_attr.object_relation == 'user-id-name':
# # obj_subtype = 'name'
# # elif obj_attr.object_relation == 'user-id-email':
# # obj_subtype = 'mail'
# #
# # if obj_id and obj_subtype:
# # obj_meta = get_object_metadata(misp_obj)
# # # res = Pgp.pgp.create_correlation(obj_subtype, obj_id, obj_meta)
# # # TODO ail_objects import_misp_object(misp_obj)
# #
# # map_uuid_global_id[misp_obj.uuid] = get_global_id('pgp', obj_id, obj_subtype=obj_subtype)
#
#
# def unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj):
# # TODO ail_objects import_misp_object(misp_obj)
# pass
# #
# # obj_id = None
# # obj_subtype = None
# # for attribute in misp_obj.attributes:
# # if attribute.object_relation == 'address': # # TODO: handle xmr address field
# # obj_id = attribute.value
# # elif attribute.object_relation == 'symbol':
# # obj_subtype = Cryptocurrency.get_cryptocurrency_type(attribute.value)
# #
# # # valid cryptocurrency type
# # if obj_subtype and obj_id:
# # obj_meta = get_object_metadata(misp_obj)
# # # res = Cryptocurrency.cryptocurrency.create_correlation(obj_subtype, obj_id, obj_meta)
# #
# # map_uuid_global_id[misp_obj.uuid] = get_global_id('cryptocurrency', obj_id, obj_subtype=obj_subtype)
#
# def get_obj_type_from_relationship(misp_obj):
# obj_uuid = misp_obj.uuid
# obj_type = None
#
# for relation in misp_obj.ObjectReference:
# if relation.object_uuid == obj_uuid:
# if relation.relationship_type == "screenshot-of":
# return 'screenshot'
# if relation.relationship_type == "included-in":
# obj_type = 'decoded'
# return obj_type
#
#
# # # TODO: covert md5 and sha1 to expected
# def unpack_file(map_uuid_global_id, misp_obj):
#
# obj_type = get_obj_type_from_relationship(misp_obj)
# if obj_type:
# obj_id = None
# io_content = None
# for attribute in misp_obj.attributes:
# # get file content
# if attribute.object_relation == 'attachment':
# io_content = attribute.data
# elif attribute.object_relation == 'malware-sample':
# io_content = attribute.data
#
# # # TODO: use/verify specified mimetype
# elif attribute.object_relation == 'mimetype':
# #print(attribute.value)
# pass
#
# # # TODO: support more
# elif attribute.object_relation == 'sha1' and obj_type == 'decoded':
# obj_id = attribute.value
# elif attribute.object_relation == 'sha256' and obj_type == 'screenshot':
# obj_id = attribute.value # obj_id = attribute.value
# elif attribute.object_relation == 'symbol':
# obj_subtype = Cryptocurrency.get_cryptocurrency_type(attribute.value)
# #
# # valid cryptocurrency type # # get SHA1/sha256
# if obj_subtype and obj_id: # if io_content and not obj_id:
# if obj_type=='screenshot':
# obj_id = sha256(io_content.getvalue()).hexdigest()
# else: # decoded file
# obj_id = sha1(io_content.getvalue()).hexdigest()
#
# if obj_id and io_content:
# obj_meta = get_object_metadata(misp_obj) # obj_meta = get_object_metadata(misp_obj)
# # res = Cryptocurrency.cryptocurrency.create_correlation(obj_subtype, obj_id, obj_meta) # if obj_type == 'screenshot':
# # TODO MIGRATE + REFACTOR ME
# # Screenshot.create_screenshot(obj_id, obj_meta, io_content)
# map_uuid_global_id[misp_obj.uuid] = get_global_id('image', obj_id)
# else: #decoded
# # TODO MIGRATE + REFACTOR ME
# # Decoded.create_decoded(obj_id, obj_meta, io_content)
# map_uuid_global_id[misp_obj.uuid] = get_global_id('decoded', obj_id)
# #
# map_uuid_global_id[misp_obj.uuid] = get_global_id('cryptocurrency', obj_id, obj_subtype=obj_subtype) #
# def get_misp_import_fct(map_uuid_global_id, misp_obj):
def get_obj_type_from_relationship(misp_obj): # if misp_obj.name == 'ail-leak':
obj_uuid = misp_obj.uuid # unpack_item_obj(map_uuid_global_id, misp_obj)
obj_type = None # elif misp_obj.name == 'domain-crawled':
# pass
for relation in misp_obj.ObjectReference: # elif misp_obj.name == 'pgp-meta':
if relation.object_uuid == obj_uuid: # unpack_obj_pgp(map_uuid_global_id, misp_obj)
if relation.relationship_type == "screenshot-of": # elif misp_obj.name == 'coin-address':
return 'screenshot' # unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj)
if relation.relationship_type == "included-in": # elif misp_obj.name == 'file':
obj_type = 'decoded' # unpack_file(map_uuid_global_id, misp_obj)
return obj_type #
# # import relationship between objects
# def create_obj_relationships(map_uuid_global_id, misp_obj):
# # TODO: covert md5 and sha1 to expected # if misp_obj.uuid in map_uuid_global_id:
def unpack_file(map_uuid_global_id, misp_obj): # for relationship in misp_obj.ObjectReference:
# if relationship.referenced_uuid in map_uuid_global_id:
obj_type = get_obj_type_from_relationship(misp_obj) # obj_meta_src = get_global_id_from_id(map_uuid_global_id[relationship.object_uuid])
if obj_type: # obj_meta_target = get_global_id_from_id(map_uuid_global_id[relationship.referenced_uuid])
obj_id = None #
io_content = None # if obj_meta_src == 'decoded' or obj_meta_src == 'item':
for attribute in misp_obj.attributes: # print('000000')
# get file content # print(obj_meta_src)
if attribute.object_relation == 'attachment': # print(obj_meta_target)
io_content = attribute.data # print('111111')
elif attribute.object_relation == 'malware-sample': #
io_content = attribute.data # # TODO CREATE OBJ RELATIONSHIP
#
# # TODO: use/verify specified mimetype # def import_objs_from_file(filepath):
elif attribute.object_relation == 'mimetype': # map_uuid_global_id = {}
#print(attribute.value) #
pass # event_to_import = MISPEvent()
# try:
# # TODO: support more # event_to_import.load_file(filepath)
elif attribute.object_relation == 'sha1' and obj_type == 'decoded': # except:
obj_id = attribute.value # return map_uuid_global_id
elif attribute.object_relation == 'sha256' and obj_type == 'screenshot': #
obj_id = attribute.value # for misp_obj in event_to_import.objects:
# get_misp_import_fct(map_uuid_global_id, misp_obj)
# get SHA1/sha256 #
if io_content and not obj_id: # for misp_obj in event_to_import.objects:
if obj_type=='screenshot': # create_obj_relationships(map_uuid_global_id, misp_obj)
obj_id = sha256(io_content.getvalue()).hexdigest() #
else: # decoded file # return map_uuid_global_id
obj_id = sha1(io_content.getvalue()).hexdigest() #
#
if obj_id and io_content: # if __name__ == '__main__':
obj_meta = get_object_metadata(misp_obj) #
if obj_type == 'screenshot': # # misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False)
# TODO MIGRATE + REFACTOR ME #
# Screenshot.create_screenshot(obj_id, obj_meta, io_content) # import_objs_from_file('ail_export_c777a4d1-5f63-4fa2-86c0-07da677bdac2.json')
map_uuid_global_id[misp_obj.uuid] = get_global_id('image', obj_id) #
else: #decoded # #Screenshot.delete_screenshot('a92d459f70c4dea8a14688f585a5e2364be8b91fbf924290ead361d9b909dcf1')
# TODO MIGRATE + REFACTOR ME # #Decoded.delete_decoded('d59a110ab233fe87cefaa0cf5603b047b432ee07')
# Decoded.create_decoded(obj_id, obj_meta, io_content) # #Pgp.pgp.delete_correlation('key', '0xA4BB02A75E6AF448')
map_uuid_global_id[misp_obj.uuid] = get_global_id('decoded', obj_id) #
# #Item.delete_item('submitted/2020/02/10/b2485894-4325-469b-bc8f-6ad1c2dbb202.gz')
# #Item.delete_item('archive/pastebin.com_pro/2020/02/10/K2cerjP4.gz')
def get_misp_import_fct(map_uuid_global_id, misp_obj):
if misp_obj.name == 'ail-leak':
unpack_item_obj(map_uuid_global_id, misp_obj)
elif misp_obj.name == 'domain-crawled':
pass
elif misp_obj.name == 'pgp-meta':
unpack_obj_pgp(map_uuid_global_id, misp_obj)
elif misp_obj.name == 'coin-address':
unpack_obj_cryptocurrency(map_uuid_global_id, misp_obj)
elif misp_obj.name == 'file':
unpack_file(map_uuid_global_id, misp_obj)
# import relationship between objects
def create_obj_relationships(map_uuid_global_id, misp_obj):
if misp_obj.uuid in map_uuid_global_id:
for relationship in misp_obj.ObjectReference:
if relationship.referenced_uuid in map_uuid_global_id:
obj_meta_src = get_global_id_from_id(map_uuid_global_id[relationship.object_uuid])
obj_meta_target = get_global_id_from_id(map_uuid_global_id[relationship.referenced_uuid])
if obj_meta_src == 'decoded' or obj_meta_src == 'item':
print('000000')
print(obj_meta_src)
print(obj_meta_target)
print('111111')
# TODO CREATE OBJ RELATIONSHIP
def import_objs_from_file(filepath):
map_uuid_global_id = {}
event_to_import = MISPEvent()
try:
event_to_import.load_file(filepath)
except:
return map_uuid_global_id
for misp_obj in event_to_import.objects:
get_misp_import_fct(map_uuid_global_id, misp_obj)
for misp_obj in event_to_import.objects:
create_obj_relationships(map_uuid_global_id, misp_obj)
return map_uuid_global_id
if __name__ == '__main__':
# misp = PyMISP('https://127.0.0.1:8443/', 'uXgcN42b7xuL88XqK5hubwD8Q8596VrrBvkHQzB0', False)
import_objs_from_file('ail_export_c777a4d1-5f63-4fa2-86c0-07da677bdac2.json')
#Screenshot.delete_screenshot('a92d459f70c4dea8a14688f585a5e2364be8b91fbf924290ead361d9b909dcf1')
#Decoded.delete_decoded('d59a110ab233fe87cefaa0cf5603b047b432ee07')
#Pgp.pgp.delete_correlation('key', '0xA4BB02A75E6AF448')
#Item.delete_item('submitted/2020/02/10/b2485894-4325-469b-bc8f-6ad1c2dbb202.gz')
#Item.delete_item('archive/pastebin.com_pro/2020/02/10/K2cerjP4.gz')

5
bin/lib/ail_core.py
View file

@ -3,7 +3,7 @@
import os import os
import sys import sys
import redis from uuid import uuid4
sys.path.append(os.environ['AIL_BIN']) sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
@ -20,6 +20,9 @@ AIL_OBJECTS = sorted({'cve', 'cryptocurrency', 'decoded', 'domain', 'item', 'pgp
def get_ail_uuid(): def get_ail_uuid():
return r_serv_db.get('ail:uuid') return r_serv_db.get('ail:uuid')
def generate_uuid():
return str(uuid4())
#### AIL OBJECTS #### #### AIL OBJECTS ####
def get_all_objects(): def get_all_objects():

29
bin/lib/item_basic.py
View file

@ -17,7 +17,6 @@ from lib import Tag
config_loader = ConfigLoader.ConfigLoader() config_loader = ConfigLoader.ConfigLoader()
r_cache = config_loader.get_redis_conn("Redis_Cache") r_cache = config_loader.get_redis_conn("Redis_Cache")
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
r_object = config_loader.get_db_conn("Kvrocks_Objects") r_object = config_loader.get_db_conn("Kvrocks_Objects")
config_loader = None config_loader = None
@ -180,21 +179,21 @@ def get_all_domain_node_by_item_id(item_id, l_nodes=[]):
# FIXME: # FIXME:
#### UNKNOW SECTION #### #### UNKNOW SECTION ####
def get_obj_id_item_id(parent_type, parent_id): # def get_obj_id_item_id(parent_type, parent_id):
all_parents_type = ['twitter_id', 'jabber_id', 'telegram_id'] # all_parents_type = ['twitter_id', 'jabber_id', 'telegram_id']
if parent_type in all_parents_type: # if parent_type in all_parents_type:
return r_serv_metadata.hget('map:{}:item_id'.format(parent_type), parent_id) # return r_serv_metadata.hget('map:{}:item_id'.format(parent_type), parent_id)
else: # else:
return None # return None
# # TODO: # FIXME: TO MIGRATE ?????? # # # TODO: # FIXME: TO MIGRATE ??????
def add_map_obj_id_item_id(obj_id, item_id, obj_type): # def add_map_obj_id_item_id(obj_id, item_id, obj_type):
if obj_type == 'twitter_id': # if obj_type == 'twitter_id':
r_serv_metadata.hset('map:twitter_id:item_id', obj_id, item_id) # r_serv_metadata.hset('map:twitter_id:item_id', obj_id, item_id)
if obj_type == 'jabber_id': # if obj_type == 'jabber_id':
r_serv_metadata.hset('map:jabber_id:item_id', obj_id, item_id) # r_serv_metadata.hset('map:jabber_id:item_id', obj_id, item_id)
if obj_type == 'telegram_id': # if obj_type == 'telegram_id':
r_serv_metadata.hset('map:telegram_id:item_id', obj_id, item_id) # r_serv_metadata.hset('map:telegram_id:item_id', obj_id, item_id)
# delete twitter id # delete twitter id

10
bin/modules/submit_paste.py
View file

@ -5,8 +5,6 @@
The Submit paste module The Submit paste module
================ ================
This module is taking paste in redis queue ARDB_DB and submit to global
""" """
################################## ##################################
@ -15,7 +13,6 @@ This module is taking paste in redis queue ARDB_DB and submit to global
import os import os
import sys import sys
import gzip import gzip
import io
import base64 import base64
import datetime import datetime
import time import time
@ -51,11 +48,8 @@ class SubmitPaste(AbstractModule):
super(SubmitPaste, self).__init__(queue_name='submit_paste') super(SubmitPaste, self).__init__(queue_name='submit_paste')
# TODO KVROCKS # TODO KVROCKS
self.r_serv_db = ConfigLoader.ConfigLoader().get_redis_conn("ARDB_DB") self.r_serv_db = ConfigLoader.ConfigLoader().get_redis_conn("Kvrocks_DB")
self.r_serv_log_submit = ConfigLoader.ConfigLoader().get_redis_conn("Redis_Log_submit") self.r_serv_log_submit = ConfigLoader.ConfigLoader().get_redis_conn("Redis_Log_submit")
self.r_serv_tags = ConfigLoader.ConfigLoader().get_redis_conn("ARDB_Tags")
self.r_serv_metadata = ConfigLoader.ConfigLoader().get_redis_conn("ARDB_Metadata")
self.serv_statistics = ConfigLoader.ConfigLoader().get_redis_conn("ARDB_Statistics")
self.pending_seconds = 3 self.pending_seconds = 3
@ -305,7 +299,6 @@ class SubmitPaste(AbstractModule):
self.r_serv_log_submit.sadd(f'{uuid}:paste_submit_link', rel_item_path) self.r_serv_log_submit.sadd(f'{uuid}:paste_submit_link', rel_item_path)
curr_date = datetime.date.today() curr_date = datetime.date.today()
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"),'submit_paste', 1)
self.redis_logger.debug("paste submitted") self.redis_logger.debug("paste submitted")
else: else:
self.addError(uuid, f'File: {save_path} already exist in submitted pastes') self.addError(uuid, f'File: {save_path} already exist in submitted pastes')
@ -335,7 +328,6 @@ class SubmitPaste(AbstractModule):
self.addError(uuid, errorMessage) self.addError(uuid, errorMessage)
self.r_serv_log_submit.set(f'{uuid}:end', 1) self.r_serv_log_submit.set(f'{uuid}:end', 1)
curr_date = datetime.date.today() curr_date = datetime.date.today()
self.serv_statistics.hincrby(curr_date.strftime("%Y%m%d"), 'submit_abord', 1)
self.remove_submit_uuid(uuid) self.remove_submit_uuid(uuid)
# # TODO: use Item function # # TODO: use Item function

8
bin/packages/Import_helper.py
View file

@ -16,7 +16,7 @@ from lib import ConfigLoader
config_loader = ConfigLoader.ConfigLoader() config_loader = ConfigLoader.ConfigLoader()
r_serv_db = config_loader.get_redis_conn("ARDB_DB") r_serv_db = config_loader.get_redis_conn("Kvrocks_DB")
r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit") r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit")
config_loader = None config_loader = None
@ -62,11 +62,11 @@ def create_import_queue(tags, galaxy, paste_content, UUID, password=None, isfile
def check_import_status(UUID): def check_import_status(UUID):
if not is_valid_uuid_v4(UUID): if not is_valid_uuid_v4(UUID):
return ({'status': 'error', 'reason': 'Invalid uuid'}, 400) return {'status': 'error', 'reason': 'Invalid uuid'}, 400
processing = r_serv_log_submit.get(UUID + ':processing') processing = r_serv_log_submit.get(UUID + ':processing')
if not processing: if not processing:
return ({'status': 'error', 'reason': 'Unknown uuid'}, 404) return {'status': 'error', 'reason': 'Unknown uuid'}, 404
# nb_total = r_serv_log_submit.get(UUID + ':nb_total') # nb_total = r_serv_log_submit.get(UUID + ':nb_total')
# nb_sucess = r_serv_log_submit.get(UUID + ':nb_sucess') # nb_sucess = r_serv_log_submit.get(UUID + ':nb_sucess')
@ -90,4 +90,4 @@ def check_import_status(UUID):
status = 'imported' status = 'imported'
dict_import_status['status'] = status dict_import_status['status'] = status
return (dict_import_status, 200) return dict_import_status, 200

2
install_virtualenv.sh
View file

@ -14,7 +14,7 @@ if [ -z "$VIRTUAL_ENV" ]; then
echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate echo export AIL_BIN=$(pwd)/bin/ >> ./AILENV/bin/activate
echo export AIL_FLASK=$(pwd)/var/www/ >> ./AILENV/bin/activate echo export AIL_FLASK=$(pwd)/var/www/ >> ./AILENV/bin/activate
echo export AIL_REDIS=$(pwd)/redis/src/ >> ./AILENV/bin/activate echo export AIL_REDIS=$(pwd)/redis/src/ >> ./AILENV/bin/activate
echo export AIL_ARDB=$(pwd)/ardb/src/ >> ./AILENV/bin/activate echo export AIL_KVROCKS=$(pwd)/kvrocks/src/ >> ./AILENV/bin/activate
fi fi

12
installing_deps.sh
View file

@ -84,10 +84,10 @@ sudo make install
popd popd
# ARDB # # ARDB #
test ! -d ardb/ && git clone https://github.com/ail-project/ardb.git #test ! -d ardb/ && git clone https://github.com/ail-project/ardb.git
pushd ardb/ #pushd ardb/
make #make
popd #popd
DEFAULT_HOME=$(pwd) DEFAULT_HOME=$(pwd)
@ -148,8 +148,8 @@ echo "AIL current version:"
git describe --abbrev=0 --tags git describe --abbrev=0 --tags
popd popd
# LAUNCH ARDB # LAUNCH Kvrocks
bash ${AIL_BIN}/LAUNCH.sh -lav & bash ${AIL_BIN}/LAUNCH.sh -lkv &
wait wait
echo "" echo ""

39
var/www/modules/PasteSubmit/Flask_PasteSubmit.py
View file

@ -25,9 +25,7 @@ from flask_login import login_required
################################## ##################################
# Import Project packages # Import Project packages
################################## ##################################
from export import Export
from lib import Tag from lib import Tag
from lib.objects.Items import Item
from packages import Import_helper from packages import Import_helper
@ -75,7 +73,7 @@ def limit_content_length():
# ============ FUNCTIONS ============ # ============ FUNCTIONS ============
def allowed_file(filename): def allowed_file(filename):
if not '.' in filename: if '.' not in filename:
return True return True
else: else:
file_ext = filename.rsplit('.', 1)[1].lower() file_ext = filename.rsplit('.', 1)[1].lower()
@ -116,8 +114,6 @@ def PasteSubmit_page():
@login_analyst @login_analyst
@limit_content_length() @limit_content_length()
def submit(): def submit():
#paste_name = request.form['paste_name']
logger.debug('submit') logger.debug('submit')
password = request.form['archive_pass'] password = request.form['archive_pass']
@ -130,7 +126,7 @@ def submit():
# limit source length # limit source length
paste_source = paste_source.replace('/', '')[:80] paste_source = paste_source.replace('/', '')[:80]
if paste_source in ['crawled', 'tests']: if paste_source in ['crawled', 'tests']:
content = f'Invalid source' content = 'Invalid source'
logger.info(paste_source) logger.info(paste_source)
return content, 400 return content, 400
@ -179,16 +175,12 @@ def submit():
# get UUID # get UUID
UUID = str(uuid.uuid4()) UUID = str(uuid.uuid4())
'''if paste_name:
# clean file name
UUID = clean_filename(paste_name)'''
# create submitted dir # create submitted dir
if not os.path.exists(UPLOAD_FOLDER): if not os.path.exists(UPLOAD_FOLDER):
logger.debug('create folder') logger.debug('create folder')
os.makedirs(UPLOAD_FOLDER) os.makedirs(UPLOAD_FOLDER)
if not '.' in file_import.filename: if '.' not in file_import.filename:
logger.debug('add UUID to path') logger.debug('add UUID to path')
full_path = os.path.join(UPLOAD_FOLDER, UUID) full_path = os.path.join(UPLOAD_FOLDER, UUID)
else: else:
@ -218,7 +210,6 @@ def submit():
logger.info(content) logger.info(content)
return content, 400 return content, 400
elif paste_content != '': elif paste_content != '':
logger.debug(f'entering text paste management') logger.debug(f'entering text paste management')
if sys.getsizeof(paste_content) < Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE: if sys.getsizeof(paste_content) < Flask_config.SUBMIT_PASTE_TEXT_MAX_SIZE:
@ -242,7 +233,6 @@ def submit():
logger.error(content) logger.error(content)
return content, 400 return content, 400
return PasteSubmit_page() return PasteSubmit_page()
@PasteSubmit.route("/PasteSubmit/submit_status", methods=['GET']) @PasteSubmit.route("/PasteSubmit/submit_status", methods=['GET'])
@ -278,10 +268,7 @@ def submit_status():
else: else:
prog = 0 prog = 0
if error: isError = bool(error)
isError = True
else:
isError = False
if end == '0': if end == '0':
end = False end = False
@ -327,6 +314,8 @@ def submit_status():
@login_required @login_required
@login_analyst @login_analyst
def edit_tag_export(): def edit_tag_export():
return abort(404)
misp_auto_events = r_serv_db.get('misp:auto-events') misp_auto_events = r_serv_db.get('misp:auto-events')
hive_auto_alerts = r_serv_db.get('hive:auto-alerts') hive_auto_alerts = r_serv_db.get('hive:auto-alerts')
@ -393,6 +382,9 @@ def edit_tag_export():
@login_required @login_required
@login_analyst @login_analyst
def tag_export_edited(): def tag_export_edited():
return abort(404)
tag_enabled_misp = request.form.getlist('tag_enabled_misp') tag_enabled_misp = request.form.getlist('tag_enabled_misp')
tag_enabled_hive = request.form.getlist('tag_enabled_hive') tag_enabled_hive = request.form.getlist('tag_enabled_hive')
@ -419,6 +411,8 @@ def tag_export_edited():
@login_required @login_required
@login_analyst @login_analyst
def enable_misp_auto_event(): def enable_misp_auto_event():
return abort(404)
r_serv_db.set('misp:auto-events', 1) r_serv_db.set('misp:auto-events', 1)
return edit_tag_export() return edit_tag_export()
@ -426,6 +420,8 @@ def enable_misp_auto_event():
@login_required @login_required
@login_analyst @login_analyst
def disable_misp_auto_event(): def disable_misp_auto_event():
return abort(404)
r_serv_db.set('misp:auto-events', 0) r_serv_db.set('misp:auto-events', 0)
return edit_tag_export() return edit_tag_export()
@ -433,6 +429,8 @@ def disable_misp_auto_event():
@login_required @login_required
@login_analyst @login_analyst
def enable_hive_auto_alert(): def enable_hive_auto_alert():
return abort(404)
r_serv_db.set('hive:auto-alerts', 1) r_serv_db.set('hive:auto-alerts', 1)
return edit_tag_export() return edit_tag_export()
@ -440,6 +438,8 @@ def enable_hive_auto_alert():
@login_required @login_required
@login_analyst @login_analyst
def disable_hive_auto_alert(): def disable_hive_auto_alert():
return abort(404)
r_serv_db.set('hive:auto-alerts', 0) r_serv_db.set('hive:auto-alerts', 0)
return edit_tag_export() return edit_tag_export()
@ -447,6 +447,8 @@ def disable_hive_auto_alert():
@login_required @login_required
@login_analyst @login_analyst
def add_push_tag(): def add_push_tag():
return abort(404)
tag = request.args.get('tag') tag = request.args.get('tag')
if tag is not None: if tag is not None:
@ -466,6 +468,9 @@ def add_push_tag():
@login_required @login_required
@login_analyst @login_analyst
def delete_push_tag(): def delete_push_tag():
return abort(404)
tag = request.args.get('tag') tag = request.args.get('tag')
infoleak_tags = Taxonomies().get('infoleak').machinetags() infoleak_tags = Taxonomies().get('infoleak').machinetags()

10
var/www/templates/sidebars/sidebar_objects.html
View file

@ -69,11 +69,11 @@
</span> </span>
</h5> </h5>
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100">
<li class="nav-item"> {# <li class="nav-item">#}
<a class="nav-link" href="{{url_for('import_export.import_object')}}" id="nav_misp_import"> {# <a class="nav-link" href="{{url_for('import_export.import_object')}}" id="nav_misp_import">#}
<b>Import</b> {# <b>Import</b>#}
</a> {# </a>#}
</li> {# </li>#}
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="{{url_for('import_export.objects_misp_export')}}" id="nav_misp_export"> <a class="nav-link" href="{{url_for('import_export.objects_misp_export')}}" id="nav_misp_export">
<b>Export</b> <b>Export</b>