fix: [UI matches extractor] handle overlapping matches

This commit is contained in:
terrtia 2024-03-27 16:30:29 +01:00
parent 5ec0d7f0cf
commit 5fce682541
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
4 changed files with 66 additions and 44 deletions

View file

@ -62,6 +62,24 @@ tools = Tools(queue=False)
for tool_name in tools.get_tools(): for tool_name in tools.get_tools():
MODULES[f'infoleak:automatic-detection="{tool_name}-tool"'] = tools MODULES[f'infoleak:automatic-detection="{tool_name}-tool"'] = tools
def merge_overlap(extracted):
merged = []
curr_start, curr_end, curr_string_match, curr_obj_ref = extracted[0]
curr_obj_ref = [(curr_obj_ref, curr_string_match)]
for start, end, mstring, ref in extracted[1:]:
# overlap
if start <= curr_end:
curr_string_match += mstring[curr_end - start:]
curr_end = max(curr_end, end)
curr_obj_ref.append((ref, mstring))
else:
merged.append((curr_start, curr_end, curr_string_match, curr_obj_ref))
curr_start, curr_end, curr_string_match, curr_obj_ref = start, end, mstring, [(ref, mstring)]
merged.append((curr_start, curr_end, curr_string_match, curr_obj_ref))
return merged
def get_correl_match(extract_type, obj, content): def get_correl_match(extract_type, obj, content):
extracted = [] extracted = []
correl = correlations_engine.get_correlation_by_correl_type(obj.type, obj.get_subtype(r_str=True), obj.id, extract_type) correl = correlations_engine.get_correlation_by_correl_type(obj.type, obj.get_subtype(r_str=True), obj.id, extract_type)
@ -81,6 +99,8 @@ def get_correl_match(extract_type, obj, content):
map_value_id[sha256_val] = value map_value_id[sha256_val] = value
if to_extract: if to_extract:
objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj.get_global_id(), content) objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj.get_global_id(), content)
if extract_type == 'title' and objs:
objs = [objs[0]]
for ob in objs: for ob in objs:
if map_subtype.get(ob[2]): if map_subtype.get(ob[2]):
subtype = map_subtype[ob[2]] subtype = map_subtype[ob[2]]
@ -223,7 +243,7 @@ def extract(obj_type, subtype, obj_id, content=None):
# SORT By Start Pos # SORT By Start Pos
extracted = sorted(extracted, key=itemgetter(0)) extracted = sorted(extracted, key=itemgetter(0))
# print(extracted) extracted = merge_overlap(extracted)
# Save In Cache # Save In Cache
if extracted: if extracted:
@ -236,43 +256,46 @@ def extract(obj_type, subtype, obj_id, content=None):
# TODO ADD LINK UI # TODO ADD LINK UI
def get_extracted_by_match(extracted): def get_extracted_by_match(extracted):
matches = {} matches = {}
for start, end, value, str_obj in extracted: for start, end, value, raw_objs in extracted:
if str_obj not in matches: for raw in raw_objs:
matches[str_obj] = {} str_obj, str_match = raw
ob_type, row_id = str_obj.split(':', 1)
if ob_type == 'tag': # TODO put me in object class if str_obj not in matches:
matches[str_obj]['subtype'] = 'tag' matches[str_obj] = {}
matches[str_obj]['id'] = row_id ob_type, row_id = str_obj.split(':', 1)
matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf02b', 'color': '#28a745', 'radius': 5} if ob_type == 'tag': # TODO put me in object class
matches[str_obj]['link'] = '' matches[str_obj]['subtype'] = 'tag'
elif ob_type == 'tracker': # TODO put me in object class matches[str_obj]['id'] = row_id
matches[str_obj]['subtype'] = 'tracker' matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf02b', 'color': '#28a745', 'radius': 5}
matches[str_obj]['id'] = row_id matches[str_obj]['link'] = ''
matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#ffc107', 'radius': 5} elif ob_type == 'tracker': # TODO put me in object class
matches[str_obj]['link'] = '' matches[str_obj]['subtype'] = 'tracker'
elif ob_type == 'retro_hunt': # TODO put me in object class matches[str_obj]['id'] = row_id
matches[str_obj]['subtype'] = 'retro_hunt' matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#ffc107', 'radius': 5}
matches[str_obj]['id'] = row_id matches[str_obj]['link'] = ''
matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#008107', 'radius': 5} elif ob_type == 'retro_hunt': # TODO put me in object class
matches[str_obj]['link'] = '' matches[str_obj]['subtype'] = 'retro_hunt'
else: matches[str_obj]['id'] = row_id
row_id = row_id.split(':', 1) matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#008107', 'radius': 5}
if len(row_id) == 2: matches[str_obj]['link'] = ''
subtype = row_id[0]
obj_id = row_id[1]
else: else:
subtype = '' row_id = row_id.split(':', 1)
obj_id = row_id[0] if len(row_id) == 2:
matches[str_obj]['subtype'] = subtype subtype = row_id[0]
matches[str_obj]['id'] = obj_id obj_id = row_id[1]
matches[str_obj]['icon'] = ail_objects.get_object_svg(ob_type, subtype, obj_id) else:
matches[str_obj]['link'] = ail_objects.get_object_link(ob_type, subtype, obj_id) subtype = ''
obj_id = row_id[0]
matches[str_obj]['subtype'] = subtype
matches[str_obj]['id'] = obj_id
matches[str_obj]['icon'] = ail_objects.get_object_svg(ob_type, subtype, obj_id)
matches[str_obj]['link'] = ail_objects.get_object_link(ob_type, subtype, obj_id)
matches[str_obj]['matches'] = [] matches[str_obj]['matches'] = []
match = [start, end, value] match = [start, end, str_match]
matches[str_obj]['matches'].append(match) matches[str_obj]['matches'].append(match)
return matches return matches

View file

@ -225,13 +225,12 @@
$(document).ready(function(){ $(document).ready(function(){
$("#page-Decoded").addClass("active"); $("#page-Decoded").addClass("active");
$("#nav_chat").addClass("active"); $("#nav_chat").addClass("active");
$('[data-toggle="popover"]').popover({
boundary:'window',
});
}); });
$(function () {
$('[data-toggle="popover"]').popover()
})
function toggle_sidebar(){ function toggle_sidebar(){
if($('#nav_menu').is(':visible')){ if($('#nav_menu').is(':visible')){
$('#nav_menu').hide(); $('#nav_menu').hide();

View file

@ -79,7 +79,7 @@
{% if not message['extracted'] %} {% if not message['extracted'] %}
<pre class="my-0">{{ message['content'] }}</pre> <pre class="my-0">{{ message['content'] }}</pre>
{% else %} {% else %}
<pre class="my-0">{{ message['content'][:message['extracted'][0][0]] }}{% for row in message['extracted'] %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="<svg height=&quot;26&quot; width=&quot;26&quot;><g class=&quot;nodes&quot;><circle cx=&quot;13&quot; cy=&quot;13&quot; r=&quot;13&quot; fill=&quot;{{ message['extracted_matches'][row[3]]['icon']['color'] }}&quot;></circle><text x=&quot;13&quot; y=&quot;13&quot; text-anchor=&quot;middle&quot; dominant-baseline=&quot;central&quot; class=&quot;graph_node_icon {{ message['extracted_matches'][row[3]]['icon']['style'] }}&quot; font-size=&quot;16px&quot;>{{ message['extracted_matches'][row[3]]['icon']['icon'] }}</text></g></svg> {{ message['extracted_matches'][row[3]]['subtype'] }}" data-content="{{ message['extracted_matches'][row[3]]['id'] }}" id="{{ row[0] }}:{{ row[1] }}">{{ message['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > message['extracted']|length %}{{ message['content'][message['extracted'][-1][1]:] }}{% else %}{{ message['content'][row[1]:message['extracted'][loop.index][0]] }}{% endif %}{% endfor %}</pre> <pre class="my-0">{{ message['content'][:message['extracted'][0][0]] }}{% for row in message['extracted'] %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="Extracted:" data-content="<ul class=&quot;list-group&quot;>{% for r in row[3] %}<li class=&quot;list-group-item&quot;><div><svg height=&quot;26&quot; width=&quot;26&quot;><g class=&quot;nodes&quot;><circle cx=&quot;13&quot; cy=&quot;13&quot; r=&quot;13&quot; fill=&quot;{{ message['extracted_matches'][r[0]]['icon']['color'] }}&quot;></circle><text x=&quot;13&quot; y=&quot;13&quot; text-anchor=&quot;middle&quot; dominant-baseline=&quot;central&quot; class=&quot;{{ message['extracted_matches'][r[0]]['icon']['style'] }}&quot; font-size=&quot;16px&quot;>{{ message['extracted_matches'][r[0]]['icon']['icon'] }}</text></g></svg> {{ message['extracted_matches'][r[0]]['subtype'] }}</div>{{ message['extracted_matches'][r[0]]['id'] }} <div><b>{{ r[1] }}</b></div></li>{% endfor %}</ul>" id="{{ row[0] }}:{{ row[1] }}">{{ message['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > message['extracted']|length %}{{ message['content'][message['extracted'][-1][1]:] }}{% else %}{{ message['content'][row[1]:message['extracted'][loop.index][0]] }}{% endif %}{% endfor %}</pre>
{% endif %} {% endif %}
{% if message['translation'] %} {% if message['translation'] %}
<hr class="m-1"> <hr class="m-1">

View file

@ -547,7 +547,7 @@
{% if not extracted %} {% if not extracted %}
<p class="my-0"> <pre class="border">{{ meta['content'] }}</pre></p> <p class="my-0"> <pre class="border">{{ meta['content'] }}</pre></p>
{% else %} {% else %}
<p class="my-0"> <pre class="border">{{ meta['content'][:extracted[0][0]] }}{% for row in extracted %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="<svg height=&quot;26&quot; width=&quot;26&quot;><g class=&quot;nodes&quot;><circle cx=&quot;13&quot; cy=&quot;13&quot; r=&quot;13&quot; fill=&quot;{{ extracted_matches[row[3]]['icon']['color'] }}&quot;></circle><text x=&quot;13&quot; y=&quot;13&quot; text-anchor=&quot;middle&quot; dominant-baseline=&quot;central&quot; class=&quot;graph_node_icon {{ extracted_matches[row[3]]['icon']['style'] }}&quot; font-size=&quot;16px&quot;>{{ extracted_matches[row[3]]['icon']['icon'] }}</text></g></svg> {{ extracted_matches[row[3]]['subtype'] }}" data-content="{{ extracted_matches[row[3]]['id'] }}" id="{{ row[0] }}:{{ row[1] }}">{{ meta['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > extracted|length %}{{ meta['content'][extracted[-1][1]:] }}{% else %}{{ meta['content'][row[1]:extracted[loop.index][0]] }}{% endif %}{% endfor %}</pre></p> <p class="my-0"> <pre class="border">{{ meta['content'][:extracted[0][0]] }}{% for row in extracted %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="Extracted:" data-content="<ul class=&quot;list-group&quot;>{% for r in row[3] %}<li class=&quot;list-group-item&quot;><div><svg height=&quot;26&quot; width=&quot;26&quot;><g class=&quot;nodes&quot;><circle cx=&quot;13&quot; cy=&quot;13&quot; r=&quot;13&quot; fill=&quot;{{ extracted_matches[r[0]]['icon']['color'] }}&quot;></circle><text x=&quot;13&quot; y=&quot;13&quot; text-anchor=&quot;middle&quot; dominant-baseline=&quot;central&quot; class=&quot;{{ extracted_matches[r[0]]['icon']['style'] }}&quot; font-size=&quot;16px&quot;>{{ extracted_matches[r[0]]['icon']['icon'] }}</text></g></svg> {{ extracted_matches[r[0]]['subtype'] }}</div>{{ extracted_matches[r[0]]['id'] }} <div><b>{{ r[1] }}</b></div></li>{% endfor %}</ul>" id="{{ row[0] }}:{{ row[1] }}">{{ meta['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > extracted|length %}{{ meta['content'][extracted[-1][1]:] }}{% else %}{{ meta['content'][row[1]:extracted[loop.index][0]] }}{% endif %}{% endfor %}</pre></p>
{% endif %} {% endif %}
</div> </div>
<div class="tab-pane fade" id="pills-html2text" role="tabpanel" aria-labelledby="pills-html2text-tab"> <div class="tab-pane fade" id="pills-html2text" role="tabpanel" aria-labelledby="pills-html2text-tab">
@ -576,7 +576,10 @@
$(".rotate").click(function(){ $(".rotate").click(function(){
$(this).toggleClass("down"); $(this).toggleClass("down");
}) })
}); $('[data-toggle="popover"]').popover({
boundary:'window',
})
});
$('#pills-html2text-tab').on('shown.bs.tab', function (e) { $('#pills-html2text-tab').on('shown.bs.tab', function (e) {
if ($('#html2text-container').is(':empty')){ if ($('#html2text-container').is(':empty')){
@ -639,9 +642,6 @@
} }
blocks.addEventListener('change', pixelate, false); blocks.addEventListener('change', pixelate, false);
$(function () {
$('[data-toggle="popover"]').popover()
})
</script> </script>
{% endif %} {% endif %}