chg: [correlation] new admin enpoint delete correlation + add screenshot correlation filter

This commit is contained in:
Terrtia 2023-04-06 15:13:27 +02:00
parent 657a39e629
commit c7eeae2a44
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
4 changed files with 36 additions and 10 deletions

View file

@ -108,7 +108,7 @@ class Crawler(AbstractModule):
if capture: if capture:
try: try:
status = self.lacus.get_capture_status(capture.uuid) status = self.lacus.get_capture_status(capture.uuid)
if status != crawlers.CaptureStatus.DONE: # TODO ADD GLOBAL TIMEOUT-> Save start time if status != crawlers.CaptureStatus.DONE: # TODO ADD GLOBAL TIMEOUT-> Save start time ### print start time
capture.update(status) capture.update(status)
print(capture.uuid, crawlers.CaptureStatus(status).name, int(time.time())) print(capture.uuid, crawlers.CaptureStatus(status).name, int(time.time()))
else: else:
@ -248,6 +248,8 @@ class Crawler(AbstractModule):
if 'png' in entries and entries['png']: if 'png' in entries and entries['png']:
screenshot = Screenshots.create_screenshot(entries['png'], b64=False) screenshot = Screenshots.create_screenshot(entries['png'], b64=False)
if screenshot: if screenshot:
# Remove Errors pages # TODO Replace with warning list ???
if screenshot.id not in ['27e14ace10b0f96acd2bd919aaa98a964597532c35b6409dff6cc8eec8214748']:
# Create Correlations # Create Correlations
screenshot.add_correlation('item', '', item_id) screenshot.add_correlation('item', '', item_id)
screenshot.add_correlation('domain', '', self.domain.id) screenshot.add_correlation('domain', '', self.domain.id)

View file

@ -118,6 +118,13 @@ def delete_obj_correlation(obj1_type, subtype1, obj1_id, obj2_type, subtype2, ob
r_metadata.srem(f'correlation:obj:{obj1_type}:{subtype1}:{obj2_type}:{obj1_id}', f'{subtype2}:{obj2_id}') r_metadata.srem(f'correlation:obj:{obj1_type}:{subtype1}:{obj2_type}:{obj1_id}', f'{subtype2}:{obj2_id}')
r_metadata.srem(f'correlation:obj:{obj2_type}:{subtype2}:{obj1_type}:{obj2_id}', f'{subtype1}:{obj1_id}') r_metadata.srem(f'correlation:obj:{obj2_type}:{subtype2}:{obj1_type}:{obj2_id}', f'{subtype1}:{obj1_id}')
def delete_obj_correlations(obj_type, subtype, obj_id):
obj_correlations = get_correlations(obj_type, subtype, obj_id)
for correl_type in obj_correlations:
for str_obj in obj_correlations[correl_type]:
subtype2, obj2_id = str_obj.split(':', 1)
delete_obj_correlation(obj_type, subtype, obj_id, correl_type, subtype2, obj2_id)
# # bypass max result/objects ??? # # bypass max result/objects ???
# def get_correlation_depht(obj_type, subtype, obj_id, filter_types=[], level=1, nb_max=300): # def get_correlation_depht(obj_type, subtype, obj_id, filter_types=[], level=1, nb_max=300):
# objs = set() # objs = set()

View file

@ -28,8 +28,6 @@ config_loader = ConfigLoader()
config_loader = None config_loader = None
def is_valid_object_type(obj_type): def is_valid_object_type(obj_type):
return obj_type in get_all_objects() return obj_type in get_all_objects()
@ -138,7 +136,7 @@ def get_object_meta(obj_type, subtype, id, options=set(), flask_context=False):
return meta return meta
def get_objects_meta(objs, options=[], flask_context=False): def get_objects_meta(objs, options=set(), flask_context=False):
metas = [] metas = []
for obj_dict in objs: for obj_dict in objs:
metas.append(get_object_meta(obj_dict['type'], obj_dict['subtype'], obj_dict['id'], options=options, metas.append(get_object_meta(obj_dict['type'], obj_dict['subtype'], obj_dict['id'], options=options,
@ -166,9 +164,9 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False):
def get_ui_obj_tag_table_keys(obj_type): def get_ui_obj_tag_table_keys(obj_type):
''' """
Warning: use only in flask (dynamic templates) Warning: use only in flask (dynamic templates)
''' """
if obj_type == "domain": if obj_type == "domain":
return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot
@ -304,6 +302,11 @@ def obj_correlations_objs_add_tags(obj_type, subtype, obj_id, tags, filter_types
################################################################################ ################################################################################
################################################################################ ################################################################################
def delete_obj_correlations(obj_type, subtype, obj_id):
obj = get_object(obj_type, subtype, obj_id)
if obj.exists():
return correlations_engine.delete_obj_correlations(obj_type, subtype, obj_id)
def delete_obj(obj_type, subtype, obj_id): def delete_obj(obj_type, subtype, obj_id):
obj = get_object(obj_type, subtype, obj_id) obj = get_object(obj_type, subtype, obj_id)
return obj.delete() return obj.delete()

View file

@ -174,9 +174,23 @@ def graph_node_json():
#json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes) #json_graph = Correlate_object.get_graph_node_object_correlation(obj_type, obj_id, 'union', correlation_names, correlation_objects, requested_correl_type=subtype, max_nodes=max_nodes)
return jsonify(json_graph) return jsonify(json_graph)
@correlation.route('/correlation/delete', methods=['GET'])
@login_required
@login_admin
def correlation_delete():
obj_type = request.args.get('type')
subtype = request.args.get('subtype', '')
obj_id = request.args.get('id')
if not ail_objects.exists_obj(obj_type, subtype, obj_id):
return abort(404)
ail_objects.delete_obj_correlations(obj_type, subtype, obj_id)
return redirect(url_for('correlation.show_correlation', type=obj_type, subtype=subtype, id=obj_id))
@correlation.route('/correlation/tags/add', methods=['POST']) @correlation.route('/correlation/tags/add', methods=['POST'])
@login_required @login_required
@login_read_only @login_analyst
def correlation_tags_add(): def correlation_tags_add():
obj_id = request.form.get('tag_obj_id') obj_id = request.form.get('tag_obj_id')
subtype = request.form.get('tag_subtype', '') subtype = request.form.get('tag_subtype', '')