AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-30 01:37:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [Item] change parent-child key

Browse source
This commit is contained in:
Terrtia 2023-01-10 10:32:01 +01:00
parent d295c084c6
commit b0891f45ec
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
3 changed files with 53 additions and 55 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
bin/DB_KVROCKS_MIGRATION.py
View file

@ -369,11 +369,11 @@ def get_item_duplicates_dict(item_id):
def items_migration(): def items_migration():
print('ITEMS MIGRATION...') print('ITEMS MIGRATION...')
# MIGRATE IMPORTED URLEXTRACT Father # MIGRATE IMPORTED URLEXTRACT Father
# for item_id in Items.get_items_by_source('urlextract'): for item_id in Items.get_items_by_source('urlextract'):
# father_id = get_item_father(item_id) father_id = get_item_father(item_id)
# if father_id: if father_id:
# item = Items.Item(item_id) item = Items.Item(item_id)
# item.set_father(father_id) item.set_parent(father_id)
# DUPLICATES # DUPLICATES
for tag in ['infoleak:automatic-detection="credential"']: # Creditcards, Mail, Keys ??????????????????????????????? for tag in ['infoleak:automatic-detection="credential"']: # Creditcards, Mail, Keys ???????????????????????????????
@ -871,10 +871,10 @@ if __name__ == '__main__':
#core_migration() #core_migration()
#user_migration() #user_migration()
#tags_migration() #tags_migration()
# items_migration() items_migration()
#crawler_migration() #crawler_migration()
# domain_migration() # TO TEST ########################### # domain_migration() # TO TEST ###########################
decodeds_migration() # decodeds_migration()
# screenshots_migration() # screenshots_migration()
# subtypes_obj_migration() # subtypes_obj_migration()
# ail_2_ail_migration() # ail_2_ail_migration()

4
bin/lib/item_basic.py
View file

@ -87,7 +87,7 @@ def get_item_mimetype(item_id):
# # # # TREE CHILD/FATHER # # # # # # # # TREE CHILD/FATHER # # # #
def is_parent(item_id): def is_parent(item_id):
return r_object.exists(f'obj:child:item::{item_id}') return r_object.exists(f'child:item::{item_id}')
def is_children(item_id): def is_children(item_id):
return r_object.hexists(f'meta:item::{item_id}' 'parent') return r_object.hexists(f'meta:item::{item_id}' 'parent')
@ -145,7 +145,7 @@ def get_item_parent(item_id):
return r_object.hget(f'meta:item::{item_id}', 'parent') return r_object.hget(f'meta:item::{item_id}', 'parent')
def get_item_children(item_id): def get_item_children(item_id):
return list(r_object.smembers(f'obj:child:item::{item_id}')) return list(r_object.smembers(f'child:item::{item_id}'))
# # TODO: handle domain last origin in domain lib # # TODO: handle domain last origin in domain lib
# def _delete_node(item_id): # def _delete_node(item_id):

90
bin/lib/objects/Items.py
View file

@ -40,10 +40,6 @@ baseurl = config_loader.get_config_str("Notifications", "ail_domain")
config_loader = None config_loader = None
################################################################################
################################################################################
################################################################################
class Item(AbstractObject): class Item(AbstractObject):
""" """
AIL Item Object. (strings) AIL Item Object. (strings)
@ -127,23 +123,27 @@ class Item(AbstractObject):
return item_basic.get_item_parent(self.id) return item_basic.get_item_parent(self.id)
def set_parent(self, parent_id): def set_parent(self, parent_id):
r_object.sadd(f'obj:child:item::{parent_id}', self.id) # TODO r_object.sadd(f'child:item::{parent_id}', self.id)
r_object.hset(f'meta:item::{self.id}', 'parent', parent_id) r_object.hset(f'meta:item::{self.id}', 'parent', parent_id)
def add_children(self, child_id): def add_children(self, child_id):
r_object.sadd(f'obj:child:item::{self.id}', child_id) # TODO r_object.sadd(f'child:item::{self.id}', child_id)
r_object.hset(f'meta:item::{child_id}', 'parent', self.id) r_object.hset(f'meta:item::{child_id}', 'parent', self.id)
####################################################################################
####################################################################################
def sanitize_id(self): def sanitize_id(self):
pass pass
# # TODO: sanitize_id # # TODO: sanitize_id
# # TODO: check if already exists ? # # TODO: check if already exists ?
# # TODO: check if duplicate # # TODO: check if duplicate
def save_on_disk(self, content, binary=True, compressed=False, base64=False): def save_on_disk(self, content, binary=True, compressed=False, b64=False):
if not binary: if not binary:
content = content.encode() content = content.encode()
if base64: if b64:
content = base64.standard_b64decode(content) content = base64.standard_b64decode(content)
if not compressed: if not compressed:
content = gzip.compress(content) content = gzip.compress(content)
@ -156,9 +156,8 @@ class Item(AbstractObject):
with open(filename, 'wb') as f: with open(filename, 'wb') as f:
f.write(content) f.write(content)
# # TODO:
# # TODO: correlations # correlations
#
# content # content
# tags # tags
# origin # origin
@ -180,10 +179,6 @@ class Item(AbstractObject):
self.add_duplicate(obj_id, dup['algo'], dup['similarity']) self.add_duplicate(obj_id, dup['algo'], dup['similarity'])
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\ # # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
# TODO: DELETE ITEM CORRELATION + TAGS + METADATA + ... # TODO: DELETE ITEM CORRELATION + TAGS + METADATA + ...
def delete(self): def delete(self):
@ -194,6 +189,9 @@ class Item(AbstractObject):
except FileNotFoundError: except FileNotFoundError:
return False return False
####################################################################################
####################################################################################
def get_link(self, flask_context=False): def get_link(self, flask_context=False):
if flask_context: if flask_context:
url = url_for('objects_item.showItem', id=self.id) url = url_for('objects_item.showItem', id=self.id)
@ -206,17 +204,16 @@ class Item(AbstractObject):
color = 'red' color = 'red'
else: else:
color = '#332288' color = '#332288'
return {'style': '', 'icon': '', 'color': color, 'radius':5} return {'style': '', 'icon': '', 'color': color, 'radius': 5}
def get_misp_object(self): def get_misp_object(self):
obj_date = self.get_date() obj_date = self.get_date()
obj = MISPObject('ail-leak', standalone=True) obj = MISPObject('ail-leak', standalone=True)
obj.first_seen = obj_date obj.first_seen = obj_date
obj_attrs = [] obj_attrs = [obj.add_attribute('first-seen', value=obj_date),
obj_attrs.append( obj.add_attribute('first-seen', value=obj_date) ) obj.add_attribute('raw-data', value=self.id, data=self.get_raw_content()),
obj_attrs.append( obj.add_attribute('raw-data', value=self.id, data=self.get_raw_content()) ) obj.add_attribute('sensor', value=get_ail_uuid())]
obj_attrs.append( obj.add_attribute('sensor', value=get_ail_uuid()) )
for obj_attr in obj_attrs: for obj_attr in obj_attrs:
for tag in self.get_tags(): for tag in self.get_tags():
obj_attr.add_tag(tag) obj_attr.add_tag(tag)
@ -253,12 +250,16 @@ class Item(AbstractObject):
self.set_parent(parent_id) self.set_parent(parent_id)
# options: set of optional meta fields # options: set of optional meta fields
def get_meta(self, options=set()): def get_meta(self, options=None):
meta = {} """
meta['id'] = self.id :type options: set
meta['date'] = self.get_date(separator=True) ############################ # TODO: """
meta['source'] = self.get_source() if options is None:
meta['tags'] = self.get_tags(r_list=True) options = set()
meta = {'id': self.id,
'date': self.get_date(separator=True),
'source': self.get_source(),
'tags': self.get_tags(r_list=True)}
# optional meta fields # optional meta fields
if 'content' in options: if 'content' in options:
meta['content'] = self.get_content() meta['content'] = self.get_content()
@ -282,7 +283,12 @@ class Item(AbstractObject):
# meta['encoding'] = None # meta['encoding'] = None
return meta return meta
def get_meta_crawler(self, tags=[]): def get_meta_crawler(self, tags=None):
"""
:type tags: list
"""
if tags is None:
tags = []
crawler = {} crawler = {}
if self.is_crawled(): if self.is_crawled():
crawler['domain'] = self.get_domain() crawler['domain'] = self.get_domain()
@ -330,20 +336,23 @@ class Item(AbstractObject):
############################################################################ ############################################################################
############################################################################ ############################################################################
def _get_dir_source_name(dir, source_name=None, l_sources_name=set(), filter_dir=False): def _get_dir_source_name(directory, source_name=None, l_sources_name=None, filter_dir=False):
"""
:type l_sources_name: set
"""
if not l_sources_name: if not l_sources_name:
l_sources_name = set() l_sources_name = set()
if source_name: if source_name:
l_dir = os.listdir(os.path.join(dir, source_name)) l_dir = os.listdir(os.path.join(directory, source_name))
else: else:
l_dir = os.listdir(dir) l_dir = os.listdir(directory)
# empty directory # empty directory
if not l_dir: if not l_dir:
return l_sources_name.add(source_name) return l_sources_name.add(source_name)
else: else:
for src_name in l_dir: for src_name in l_dir:
if len(src_name) == 4: if len(src_name) == 4:
#try: # try:
int(src_name) int(src_name)
to_add = os.path.join(source_name) to_add = os.path.join(source_name)
# filter sources, remove first directory # filter sources, remove first directory
@ -351,11 +360,11 @@ def _get_dir_source_name(dir, source_name=None, l_sources_name=set(), filter_dir
to_add = to_add.replace('archive/', '').replace('alerts/', '') to_add = to_add.replace('archive/', '').replace('alerts/', '')
l_sources_name.add(to_add) l_sources_name.add(to_add)
return l_sources_name return l_sources_name
#except: # except:
# pass # pass
if source_name: if source_name:
src_name = os.path.join(source_name, src_name) src_name = os.path.join(source_name, src_name)
l_sources_name = _get_dir_source_name(dir, source_name=src_name, l_sources_name=l_sources_name, filter_dir=filter_dir) l_sources_name = _get_dir_source_name(directory, source_name=src_name, l_sources_name=l_sources_name, filter_dir=filter_dir)
return l_sources_name return l_sources_name
def get_items_sources(filter_dir=False, r_list=False): def get_items_sources(filter_dir=False, r_list=False):
@ -397,6 +406,7 @@ def _manual_set_items_date_first_last():
################################################################################ ################################################################################
################################################################################ ################################################################################
################################################################################ ################################################################################
# TODO
def exist_item(item_id): def exist_item(item_id):
return item_basic.exist_item(item_id) return item_basic.exist_item(item_id)
@ -455,12 +465,6 @@ def get_item_metadata(item_id, item_content=None):
} }
return item_metadata return item_metadata
def get_item_parent(item_id):
return item_basic.get_item_parent(item_id)
def add_item_parent(item_parent, item_id):
return item_basic.add_item_parent(item_parent, item_id)
def get_item_content(item_id): def get_item_content(item_id):
return item_basic.get_item_content(item_id) return item_basic.get_item_content(item_id)
@ -646,6 +650,7 @@ def get_domain(item_id):
item_id = item_id[-1] item_id = item_id[-1]
return item_id[:-36] return item_id[:-36]
# TODO MOVE ME
def get_item_har_name(item_id): def get_item_har_name(item_id):
har_path = os.path.join(har_directory, item_id) + '.json' har_path = os.path.join(har_directory, item_id) + '.json'
if os.path.isfile(har_path): if os.path.isfile(har_path):
@ -653,9 +658,6 @@ def get_item_har_name(item_id):
else: else:
return None return None
def get_item_har(har_path):
pass
def get_item_filename(item_id): def get_item_filename(item_id):
# Creating the full filepath # Creating the full filepath
filename = os.path.join(ITEMS_FOLDER, item_id) filename = os.path.join(ITEMS_FOLDER, item_id)
@ -717,10 +719,6 @@ def create_item(obj_id, obj_metadata, io_content):
# Item not created # Item not created
return False return False
# # TODO:
def delete_item(obj_id):
pass
# # check if item exists # # check if item exists
# if not exist_item(obj_id): # if not exist_item(obj_id):
# return False # return False