From 5fce682541a34b019b662787b31f13a7d25f5a7f Mon Sep 17 00:00:00 2001
From: terrtia
Date: Wed, 27 Mar 2024 16:30:29 +0100
Subject: [PATCH] fix: [UI matches extractor] handle overlapping matches
---
bin/lib/module_extractor.py | 91 ++++++++++++-------
.../templates/chats_explorer/ChatMessage.html | 7 +-
.../chats_explorer/block_message.html | 2 +-
var/www/templates/objects/item/show_item.html | 10 +-
4 files changed, 66 insertions(+), 44 deletions(-)
diff --git a/bin/lib/module_extractor.py b/bin/lib/module_extractor.py
index c801168c..41aa78c9 100755
--- a/bin/lib/module_extractor.py
+++ b/bin/lib/module_extractor.py
@@ -62,6 +62,24 @@ tools = Tools(queue=False)
for tool_name in tools.get_tools():
MODULES[f'infoleak:automatic-detection="{tool_name}-tool"'] = tools
+def merge_overlap(extracted):
+ merged = []
+ curr_start, curr_end, curr_string_match, curr_obj_ref = extracted[0]
+ curr_obj_ref = [(curr_obj_ref, curr_string_match)]
+
+ for start, end, mstring, ref in extracted[1:]:
+ # overlap
+ if start <= curr_end:
+ curr_string_match += mstring[curr_end - start:]
+ curr_end = max(curr_end, end)
+ curr_obj_ref.append((ref, mstring))
+ else:
+ merged.append((curr_start, curr_end, curr_string_match, curr_obj_ref))
+ curr_start, curr_end, curr_string_match, curr_obj_ref = start, end, mstring, [(ref, mstring)]
+
+ merged.append((curr_start, curr_end, curr_string_match, curr_obj_ref))
+ return merged
+
def get_correl_match(extract_type, obj, content):
extracted = []
correl = correlations_engine.get_correlation_by_correl_type(obj.type, obj.get_subtype(r_str=True), obj.id, extract_type)
@@ -81,6 +99,8 @@ def get_correl_match(extract_type, obj, content):
map_value_id[sha256_val] = value
if to_extract:
objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj.get_global_id(), content)
+ if extract_type == 'title' and objs:
+ objs = [objs[0]]
for ob in objs:
if map_subtype.get(ob[2]):
subtype = map_subtype[ob[2]]
@@ -223,7 +243,7 @@ def extract(obj_type, subtype, obj_id, content=None):
# SORT By Start Pos
extracted = sorted(extracted, key=itemgetter(0))
- # print(extracted)
+ extracted = merge_overlap(extracted)
# Save In Cache
if extracted:
@@ -236,43 +256,46 @@ def extract(obj_type, subtype, obj_id, content=None):
# TODO ADD LINK UI
def get_extracted_by_match(extracted):
matches = {}
- for start, end, value, str_obj in extracted:
+ for start, end, value, raw_objs in extracted:
- if str_obj not in matches:
- matches[str_obj] = {}
- ob_type, row_id = str_obj.split(':', 1)
- if ob_type == 'tag': # TODO put me in object class
- matches[str_obj]['subtype'] = 'tag'
- matches[str_obj]['id'] = row_id
- matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf02b', 'color': '#28a745', 'radius': 5}
- matches[str_obj]['link'] = ''
- elif ob_type == 'tracker': # TODO put me in object class
- matches[str_obj]['subtype'] = 'tracker'
- matches[str_obj]['id'] = row_id
- matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#ffc107', 'radius': 5}
- matches[str_obj]['link'] = ''
- elif ob_type == 'retro_hunt': # TODO put me in object class
- matches[str_obj]['subtype'] = 'retro_hunt'
- matches[str_obj]['id'] = row_id
- matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#008107', 'radius': 5}
- matches[str_obj]['link'] = ''
- else:
- row_id = row_id.split(':', 1)
- if len(row_id) == 2:
- subtype = row_id[0]
- obj_id = row_id[1]
+ for raw in raw_objs:
+ str_obj, str_match = raw
+
+ if str_obj not in matches:
+ matches[str_obj] = {}
+ ob_type, row_id = str_obj.split(':', 1)
+ if ob_type == 'tag': # TODO put me in object class
+ matches[str_obj]['subtype'] = 'tag'
+ matches[str_obj]['id'] = row_id
+ matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf02b', 'color': '#28a745', 'radius': 5}
+ matches[str_obj]['link'] = ''
+ elif ob_type == 'tracker': # TODO put me in object class
+ matches[str_obj]['subtype'] = 'tracker'
+ matches[str_obj]['id'] = row_id
+ matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#ffc107', 'radius': 5}
+ matches[str_obj]['link'] = ''
+ elif ob_type == 'retro_hunt': # TODO put me in object class
+ matches[str_obj]['subtype'] = 'retro_hunt'
+ matches[str_obj]['id'] = row_id
+ matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#008107', 'radius': 5}
+ matches[str_obj]['link'] = ''
else:
- subtype = ''
- obj_id = row_id[0]
- matches[str_obj]['subtype'] = subtype
- matches[str_obj]['id'] = obj_id
- matches[str_obj]['icon'] = ail_objects.get_object_svg(ob_type, subtype, obj_id)
- matches[str_obj]['link'] = ail_objects.get_object_link(ob_type, subtype, obj_id)
+ row_id = row_id.split(':', 1)
+ if len(row_id) == 2:
+ subtype = row_id[0]
+ obj_id = row_id[1]
+ else:
+ subtype = ''
+ obj_id = row_id[0]
+ matches[str_obj]['subtype'] = subtype
+ matches[str_obj]['id'] = obj_id
+ matches[str_obj]['icon'] = ail_objects.get_object_svg(ob_type, subtype, obj_id)
+ matches[str_obj]['link'] = ail_objects.get_object_link(ob_type, subtype, obj_id)
- matches[str_obj]['matches'] = []
+ matches[str_obj]['matches'] = []
- match = [start, end, value]
- matches[str_obj]['matches'].append(match)
+ match = [start, end, str_match]
+ matches[str_obj]['matches'].append(match)
return matches
diff --git a/var/www/templates/chats_explorer/ChatMessage.html b/var/www/templates/chats_explorer/ChatMessage.html
index c853698f..8b402fed 100644
--- a/var/www/templates/chats_explorer/ChatMessage.html
+++ b/var/www/templates/chats_explorer/ChatMessage.html
@@ -225,13 +225,12 @@
$(document).ready(function(){
$("#page-Decoded").addClass("active");
$("#nav_chat").addClass("active");
+ $('[data-toggle="popover"]').popover({
+ boundary:'window',
+ });
});
-$(function () {
- $('[data-toggle="popover"]').popover()
-})
-
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
diff --git a/var/www/templates/chats_explorer/block_message.html b/var/www/templates/chats_explorer/block_message.html
index 90a0e76f..4cbdfc9d 100644
--- a/var/www/templates/chats_explorer/block_message.html
+++ b/var/www/templates/chats_explorer/block_message.html
@@ -79,7 +79,7 @@
{% if not message['extracted'] %}
{{ message['content'] }}
{% else %}
- {{ message['content'][:message['extracted'][0][0]] }}{% for row in message['extracted'] %}{{ message['content'][row[0]:row[1]] }}{% if loop.index + 1 > message['extracted']|length %}{{ message['content'][message['extracted'][-1][1]:] }}{% else %}{{ message['content'][row[1]:message['extracted'][loop.index][0]] }}{% endif %}{% endfor %}
+ {{ message['content'][:message['extracted'][0][0]] }}{% for row in message['extracted'] %}{{ message['content'][row[0]:row[1]] }}{% if loop.index + 1 > message['extracted']|length %}{{ message['content'][message['extracted'][-1][1]:] }}{% else %}{{ message['content'][row[1]:message['extracted'][loop.index][0]] }}{% endif %}{% endfor %}
{% endif %}
{% if message['translation'] %}
diff --git a/var/www/templates/objects/item/show_item.html b/var/www/templates/objects/item/show_item.html
index 5758a12e..26ae694f 100644
--- a/var/www/templates/objects/item/show_item.html
+++ b/var/www/templates/objects/item/show_item.html
@@ -547,7 +547,7 @@
{% if not extracted %}
{{ meta['content'] }}
{% else %}
-
{{ meta['content'][:extracted[0][0]] }}{% for row in extracted %}{{ meta['content'][row[0]:row[1]] }}{% if loop.index + 1 > extracted|length %}{{ meta['content'][extracted[-1][1]:] }}{% else %}{{ meta['content'][row[1]:extracted[loop.index][0]] }}{% endif %}{% endfor %}
+
{{ meta['content'][:extracted[0][0]] }}{% for row in extracted %}{{ meta['content'][row[0]:row[1]] }}{% if loop.index + 1 > extracted|length %}{{ meta['content'][extracted[-1][1]:] }}{% else %}{{ meta['content'][row[1]:extracted[loop.index][0]] }}{% endif %}{% endfor %}
{% endif %}
@@ -576,7 +576,10 @@
$(".rotate").click(function(){
$(this).toggleClass("down");
})
- });
+ $('[data-toggle="popover"]').popover({
+ boundary:'window',
+ })
+ });
$('#pills-html2text-tab').on('shown.bs.tab', function (e) {
if ($('#html2text-container').is(':empty')){
@@ -639,9 +642,6 @@
}
blocks.addEventListener('change', pixelate, false);
- $(function () {
- $('[data-toggle="popover"]').popover()
- })
{% endif %}