mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 22:27:17 +00:00
fix: [UI matches extractor] handle overlapping matches
This commit is contained in:
terrtia
parent
5ec0d7f0cf
commit
5fce682541
4 changed files with 66 additions and 44 deletions
|
|
@ -62,6 +62,24 @@ tools = Tools(queue=False)
|
|||
for tool_name in tools.get_tools():
|
||||
MODULES[f'infoleak:automatic-detection="{tool_name}-tool"'] = tools
|
||||
|
||||
def merge_overlap(extracted):
|
||||
merged = []
|
||||
curr_start, curr_end, curr_string_match, curr_obj_ref = extracted[0]
|
||||
curr_obj_ref = [(curr_obj_ref, curr_string_match)]
|
||||
|
||||
for start, end, mstring, ref in extracted[1:]:
|
||||
# overlap
|
||||
if start <= curr_end:
|
||||
curr_string_match += mstring[curr_end - start:]
|
||||
curr_end = max(curr_end, end)
|
||||
curr_obj_ref.append((ref, mstring))
|
||||
else:
|
||||
merged.append((curr_start, curr_end, curr_string_match, curr_obj_ref))
|
||||
curr_start, curr_end, curr_string_match, curr_obj_ref = start, end, mstring, [(ref, mstring)]
|
||||
|
||||
merged.append((curr_start, curr_end, curr_string_match, curr_obj_ref))
|
||||
return merged
|
||||
|
||||
def get_correl_match(extract_type, obj, content):
|
||||
extracted = []
|
||||
correl = correlations_engine.get_correlation_by_correl_type(obj.type, obj.get_subtype(r_str=True), obj.id, extract_type)
|
||||
|
|
@ -81,6 +99,8 @@ def get_correl_match(extract_type, obj, content):
|
|||
map_value_id[sha256_val] = value
|
||||
if to_extract:
|
||||
objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj.get_global_id(), content)
|
||||
if extract_type == 'title' and objs:
|
||||
objs = [objs[0]]
|
||||
for ob in objs:
|
||||
if map_subtype.get(ob[2]):
|
||||
subtype = map_subtype[ob[2]]
|
||||
|
|
@ -223,7 +243,7 @@ def extract(obj_type, subtype, obj_id, content=None):
|
|||
|
||||
# SORT By Start Pos
|
||||
extracted = sorted(extracted, key=itemgetter(0))
|
||||
# print(extracted)
|
||||
extracted = merge_overlap(extracted)
|
||||
|
||||
# Save In Cache
|
||||
if extracted:
|
||||
|
|
@ -236,43 +256,46 @@ def extract(obj_type, subtype, obj_id, content=None):
|
|||
# TODO ADD LINK UI
|
||||
def get_extracted_by_match(extracted):
|
||||
matches = {}
|
||||
for start, end, value, str_obj in extracted:
|
||||
for start, end, value, raw_objs in extracted:
|
||||
|
||||
if str_obj not in matches:
|
||||
matches[str_obj] = {}
|
||||
ob_type, row_id = str_obj.split(':', 1)
|
||||
if ob_type == 'tag': # TODO put me in object class
|
||||
matches[str_obj]['subtype'] = 'tag'
|
||||
matches[str_obj]['id'] = row_id
|
||||
matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf02b', 'color': '#28a745', 'radius': 5}
|
||||
matches[str_obj]['link'] = ''
|
||||
elif ob_type == 'tracker': # TODO put me in object class
|
||||
matches[str_obj]['subtype'] = 'tracker'
|
||||
matches[str_obj]['id'] = row_id
|
||||
matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#ffc107', 'radius': 5}
|
||||
matches[str_obj]['link'] = ''
|
||||
elif ob_type == 'retro_hunt': # TODO put me in object class
|
||||
matches[str_obj]['subtype'] = 'retro_hunt'
|
||||
matches[str_obj]['id'] = row_id
|
||||
matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#008107', 'radius': 5}
|
||||
matches[str_obj]['link'] = ''
|
||||
else:
|
||||
row_id = row_id.split(':', 1)
|
||||
if len(row_id) == 2:
|
||||
subtype = row_id[0]
|
||||
obj_id = row_id[1]
|
||||
for raw in raw_objs:
|
||||
str_obj, str_match = raw
|
||||
|
||||
if str_obj not in matches:
|
||||
matches[str_obj] = {}
|
||||
ob_type, row_id = str_obj.split(':', 1)
|
||||
if ob_type == 'tag': # TODO put me in object class
|
||||
matches[str_obj]['subtype'] = 'tag'
|
||||
matches[str_obj]['id'] = row_id
|
||||
matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf02b', 'color': '#28a745', 'radius': 5}
|
||||
matches[str_obj]['link'] = ''
|
||||
elif ob_type == 'tracker': # TODO put me in object class
|
||||
matches[str_obj]['subtype'] = 'tracker'
|
||||
matches[str_obj]['id'] = row_id
|
||||
matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#ffc107', 'radius': 5}
|
||||
matches[str_obj]['link'] = ''
|
||||
elif ob_type == 'retro_hunt': # TODO put me in object class
|
||||
matches[str_obj]['subtype'] = 'retro_hunt'
|
||||
matches[str_obj]['id'] = row_id
|
||||
matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#008107', 'radius': 5}
|
||||
matches[str_obj]['link'] = ''
|
||||
else:
|
||||
subtype = ''
|
||||
obj_id = row_id[0]
|
||||
matches[str_obj]['subtype'] = subtype
|
||||
matches[str_obj]['id'] = obj_id
|
||||
matches[str_obj]['icon'] = ail_objects.get_object_svg(ob_type, subtype, obj_id)
|
||||
matches[str_obj]['link'] = ail_objects.get_object_link(ob_type, subtype, obj_id)
|
||||
row_id = row_id.split(':', 1)
|
||||
if len(row_id) == 2:
|
||||
subtype = row_id[0]
|
||||
obj_id = row_id[1]
|
||||
else:
|
||||
subtype = ''
|
||||
obj_id = row_id[0]
|
||||
matches[str_obj]['subtype'] = subtype
|
||||
matches[str_obj]['id'] = obj_id
|
||||
matches[str_obj]['icon'] = ail_objects.get_object_svg(ob_type, subtype, obj_id)
|
||||
matches[str_obj]['link'] = ail_objects.get_object_link(ob_type, subtype, obj_id)
|
||||
|
||||
matches[str_obj]['matches'] = []
|
||||
matches[str_obj]['matches'] = []
|
||||
|
||||
match = [start, end, value]
|
||||
matches[str_obj]['matches'].append(match)
|
||||
match = [start, end, str_match]
|
||||
matches[str_obj]['matches'].append(match)
|
||||
return matches
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -225,13 +225,12 @@
|
|||
$(document).ready(function(){
|
||||
$("#page-Decoded").addClass("active");
|
||||
$("#nav_chat").addClass("active");
|
||||
$('[data-toggle="popover"]').popover({
|
||||
boundary:'window',
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
$(function () {
|
||||
$('[data-toggle="popover"]').popover()
|
||||
})
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
|
|
|
|||
|
|
@ -79,7 +79,7 @@
|
|||
{% if not message['extracted'] %}
|
||||
<pre class="my-0">{{ message['content'] }}</pre>
|
||||
{% else %}
|
||||
<pre class="my-0">{{ message['content'][:message['extracted'][0][0]] }}{% for row in message['extracted'] %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="<svg height="26" width="26"><g class="nodes"><circle cx="13" cy="13" r="13" fill="{{ message['extracted_matches'][row[3]]['icon']['color'] }}"></circle><text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ message['extracted_matches'][row[3]]['icon']['style'] }}" font-size="16px">{{ message['extracted_matches'][row[3]]['icon']['icon'] }}</text></g></svg> {{ message['extracted_matches'][row[3]]['subtype'] }}" data-content="{{ message['extracted_matches'][row[3]]['id'] }}" id="{{ row[0] }}:{{ row[1] }}">{{ message['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > message['extracted']|length %}{{ message['content'][message['extracted'][-1][1]:] }}{% else %}{{ message['content'][row[1]:message['extracted'][loop.index][0]] }}{% endif %}{% endfor %}</pre>
|
||||
<pre class="my-0">{{ message['content'][:message['extracted'][0][0]] }}{% for row in message['extracted'] %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="Extracted:" data-content="<ul class="list-group">{% for r in row[3] %}<li class="list-group-item"><div><svg height="26" width="26"><g class="nodes"><circle cx="13" cy="13" r="13" fill="{{ message['extracted_matches'][r[0]]['icon']['color'] }}"></circle><text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ message['extracted_matches'][r[0]]['icon']['style'] }}" font-size="16px">{{ message['extracted_matches'][r[0]]['icon']['icon'] }}</text></g></svg> {{ message['extracted_matches'][r[0]]['subtype'] }}</div>{{ message['extracted_matches'][r[0]]['id'] }} <div><b>{{ r[1] }}</b></div></li>{% endfor %}</ul>" id="{{ row[0] }}:{{ row[1] }}">{{ message['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > message['extracted']|length %}{{ message['content'][message['extracted'][-1][1]:] }}{% else %}{{ message['content'][row[1]:message['extracted'][loop.index][0]] }}{% endif %}{% endfor %}</pre>
|
||||
{% endif %}
|
||||
{% if message['translation'] %}
|
||||
<hr class="m-1">
|
||||
|
|
|
|||
|
|
@ -547,7 +547,7 @@
|
|||
{% if not extracted %}
|
||||
<p class="my-0"> <pre class="border">{{ meta['content'] }}</pre></p>
|
||||
{% else %}
|
||||
<p class="my-0"> <pre class="border">{{ meta['content'][:extracted[0][0]] }}{% for row in extracted %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="<svg height="26" width="26"><g class="nodes"><circle cx="13" cy="13" r="13" fill="{{ extracted_matches[row[3]]['icon']['color'] }}"></circle><text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ extracted_matches[row[3]]['icon']['style'] }}" font-size="16px">{{ extracted_matches[row[3]]['icon']['icon'] }}</text></g></svg> {{ extracted_matches[row[3]]['subtype'] }}" data-content="{{ extracted_matches[row[3]]['id'] }}" id="{{ row[0] }}:{{ row[1] }}">{{ meta['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > extracted|length %}{{ meta['content'][extracted[-1][1]:] }}{% else %}{{ meta['content'][row[1]:extracted[loop.index][0]] }}{% endif %}{% endfor %}</pre></p>
|
||||
<p class="my-0"> <pre class="border">{{ meta['content'][:extracted[0][0]] }}{% for row in extracted %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="Extracted:" data-content="<ul class="list-group">{% for r in row[3] %}<li class="list-group-item"><div><svg height="26" width="26"><g class="nodes"><circle cx="13" cy="13" r="13" fill="{{ extracted_matches[r[0]]['icon']['color'] }}"></circle><text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ extracted_matches[r[0]]['icon']['style'] }}" font-size="16px">{{ extracted_matches[r[0]]['icon']['icon'] }}</text></g></svg> {{ extracted_matches[r[0]]['subtype'] }}</div>{{ extracted_matches[r[0]]['id'] }} <div><b>{{ r[1] }}</b></div></li>{% endfor %}</ul>" id="{{ row[0] }}:{{ row[1] }}">{{ meta['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > extracted|length %}{{ meta['content'][extracted[-1][1]:] }}{% else %}{{ meta['content'][row[1]:extracted[loop.index][0]] }}{% endif %}{% endfor %}</pre></p>
|
||||
{% endif %}
|
||||
</div>
|
||||
<div class="tab-pane fade" id="pills-html2text" role="tabpanel" aria-labelledby="pills-html2text-tab">
|
||||
|
|
@ -576,7 +576,10 @@
|
|||
$(".rotate").click(function(){
|
||||
$(this).toggleClass("down");
|
||||
})
|
||||
});
|
||||
$('[data-toggle="popover"]').popover({
|
||||
boundary:'window',
|
||||
})
|
||||
});
|
||||
|
||||
$('#pills-html2text-tab').on('shown.bs.tab', function (e) {
|
||||
if ($('#html2text-container').is(':empty')){
|
||||
|
|
@ -639,9 +642,6 @@
|
|||
}
|
||||
|
||||
blocks.addEventListener('change', pixelate, false);
|
||||
$(function () {
|
||||
$('[data-toggle="popover"]').popover()
|
||||
})
|
||||
</script>
|
||||
{% endif %}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue