mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 22:27:17 +00:00
fix: [UI matches extractor] handle overlapping matches
This commit is contained in:
parent
5ec0d7f0cf
commit
5fce682541
4 changed files with 66 additions and 44 deletions
|
@ -62,6 +62,24 @@ tools = Tools(queue=False)
|
||||||
for tool_name in tools.get_tools():
|
for tool_name in tools.get_tools():
|
||||||
MODULES[f'infoleak:automatic-detection="{tool_name}-tool"'] = tools
|
MODULES[f'infoleak:automatic-detection="{tool_name}-tool"'] = tools
|
||||||
|
|
||||||
|
def merge_overlap(extracted):
|
||||||
|
merged = []
|
||||||
|
curr_start, curr_end, curr_string_match, curr_obj_ref = extracted[0]
|
||||||
|
curr_obj_ref = [(curr_obj_ref, curr_string_match)]
|
||||||
|
|
||||||
|
for start, end, mstring, ref in extracted[1:]:
|
||||||
|
# overlap
|
||||||
|
if start <= curr_end:
|
||||||
|
curr_string_match += mstring[curr_end - start:]
|
||||||
|
curr_end = max(curr_end, end)
|
||||||
|
curr_obj_ref.append((ref, mstring))
|
||||||
|
else:
|
||||||
|
merged.append((curr_start, curr_end, curr_string_match, curr_obj_ref))
|
||||||
|
curr_start, curr_end, curr_string_match, curr_obj_ref = start, end, mstring, [(ref, mstring)]
|
||||||
|
|
||||||
|
merged.append((curr_start, curr_end, curr_string_match, curr_obj_ref))
|
||||||
|
return merged
|
||||||
|
|
||||||
def get_correl_match(extract_type, obj, content):
|
def get_correl_match(extract_type, obj, content):
|
||||||
extracted = []
|
extracted = []
|
||||||
correl = correlations_engine.get_correlation_by_correl_type(obj.type, obj.get_subtype(r_str=True), obj.id, extract_type)
|
correl = correlations_engine.get_correlation_by_correl_type(obj.type, obj.get_subtype(r_str=True), obj.id, extract_type)
|
||||||
|
@ -81,6 +99,8 @@ def get_correl_match(extract_type, obj, content):
|
||||||
map_value_id[sha256_val] = value
|
map_value_id[sha256_val] = value
|
||||||
if to_extract:
|
if to_extract:
|
||||||
objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj.get_global_id(), content)
|
objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj.get_global_id(), content)
|
||||||
|
if extract_type == 'title' and objs:
|
||||||
|
objs = [objs[0]]
|
||||||
for ob in objs:
|
for ob in objs:
|
||||||
if map_subtype.get(ob[2]):
|
if map_subtype.get(ob[2]):
|
||||||
subtype = map_subtype[ob[2]]
|
subtype = map_subtype[ob[2]]
|
||||||
|
@ -223,7 +243,7 @@ def extract(obj_type, subtype, obj_id, content=None):
|
||||||
|
|
||||||
# SORT By Start Pos
|
# SORT By Start Pos
|
||||||
extracted = sorted(extracted, key=itemgetter(0))
|
extracted = sorted(extracted, key=itemgetter(0))
|
||||||
# print(extracted)
|
extracted = merge_overlap(extracted)
|
||||||
|
|
||||||
# Save In Cache
|
# Save In Cache
|
||||||
if extracted:
|
if extracted:
|
||||||
|
@ -236,43 +256,46 @@ def extract(obj_type, subtype, obj_id, content=None):
|
||||||
# TODO ADD LINK UI
|
# TODO ADD LINK UI
|
||||||
def get_extracted_by_match(extracted):
|
def get_extracted_by_match(extracted):
|
||||||
matches = {}
|
matches = {}
|
||||||
for start, end, value, str_obj in extracted:
|
for start, end, value, raw_objs in extracted:
|
||||||
|
|
||||||
if str_obj not in matches:
|
for raw in raw_objs:
|
||||||
matches[str_obj] = {}
|
str_obj, str_match = raw
|
||||||
ob_type, row_id = str_obj.split(':', 1)
|
|
||||||
if ob_type == 'tag': # TODO put me in object class
|
if str_obj not in matches:
|
||||||
matches[str_obj]['subtype'] = 'tag'
|
matches[str_obj] = {}
|
||||||
matches[str_obj]['id'] = row_id
|
ob_type, row_id = str_obj.split(':', 1)
|
||||||
matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf02b', 'color': '#28a745', 'radius': 5}
|
if ob_type == 'tag': # TODO put me in object class
|
||||||
matches[str_obj]['link'] = ''
|
matches[str_obj]['subtype'] = 'tag'
|
||||||
elif ob_type == 'tracker': # TODO put me in object class
|
matches[str_obj]['id'] = row_id
|
||||||
matches[str_obj]['subtype'] = 'tracker'
|
matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf02b', 'color': '#28a745', 'radius': 5}
|
||||||
matches[str_obj]['id'] = row_id
|
matches[str_obj]['link'] = ''
|
||||||
matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#ffc107', 'radius': 5}
|
elif ob_type == 'tracker': # TODO put me in object class
|
||||||
matches[str_obj]['link'] = ''
|
matches[str_obj]['subtype'] = 'tracker'
|
||||||
elif ob_type == 'retro_hunt': # TODO put me in object class
|
matches[str_obj]['id'] = row_id
|
||||||
matches[str_obj]['subtype'] = 'retro_hunt'
|
matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#ffc107', 'radius': 5}
|
||||||
matches[str_obj]['id'] = row_id
|
matches[str_obj]['link'] = ''
|
||||||
matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#008107', 'radius': 5}
|
elif ob_type == 'retro_hunt': # TODO put me in object class
|
||||||
matches[str_obj]['link'] = ''
|
matches[str_obj]['subtype'] = 'retro_hunt'
|
||||||
else:
|
matches[str_obj]['id'] = row_id
|
||||||
row_id = row_id.split(':', 1)
|
matches[str_obj]['icon'] = {'style': 'fas', 'icon': '\uf05b', 'color': '#008107', 'radius': 5}
|
||||||
if len(row_id) == 2:
|
matches[str_obj]['link'] = ''
|
||||||
subtype = row_id[0]
|
|
||||||
obj_id = row_id[1]
|
|
||||||
else:
|
else:
|
||||||
subtype = ''
|
row_id = row_id.split(':', 1)
|
||||||
obj_id = row_id[0]
|
if len(row_id) == 2:
|
||||||
matches[str_obj]['subtype'] = subtype
|
subtype = row_id[0]
|
||||||
matches[str_obj]['id'] = obj_id
|
obj_id = row_id[1]
|
||||||
matches[str_obj]['icon'] = ail_objects.get_object_svg(ob_type, subtype, obj_id)
|
else:
|
||||||
matches[str_obj]['link'] = ail_objects.get_object_link(ob_type, subtype, obj_id)
|
subtype = ''
|
||||||
|
obj_id = row_id[0]
|
||||||
|
matches[str_obj]['subtype'] = subtype
|
||||||
|
matches[str_obj]['id'] = obj_id
|
||||||
|
matches[str_obj]['icon'] = ail_objects.get_object_svg(ob_type, subtype, obj_id)
|
||||||
|
matches[str_obj]['link'] = ail_objects.get_object_link(ob_type, subtype, obj_id)
|
||||||
|
|
||||||
matches[str_obj]['matches'] = []
|
matches[str_obj]['matches'] = []
|
||||||
|
|
||||||
match = [start, end, value]
|
match = [start, end, str_match]
|
||||||
matches[str_obj]['matches'].append(match)
|
matches[str_obj]['matches'].append(match)
|
||||||
return matches
|
return matches
|
||||||
|
|
||||||
|
|
||||||
|
|
|
@ -225,13 +225,12 @@
|
||||||
$(document).ready(function(){
|
$(document).ready(function(){
|
||||||
$("#page-Decoded").addClass("active");
|
$("#page-Decoded").addClass("active");
|
||||||
$("#nav_chat").addClass("active");
|
$("#nav_chat").addClass("active");
|
||||||
|
$('[data-toggle="popover"]').popover({
|
||||||
|
boundary:'window',
|
||||||
|
});
|
||||||
|
|
||||||
});
|
});
|
||||||
|
|
||||||
$(function () {
|
|
||||||
$('[data-toggle="popover"]').popover()
|
|
||||||
})
|
|
||||||
|
|
||||||
function toggle_sidebar(){
|
function toggle_sidebar(){
|
||||||
if($('#nav_menu').is(':visible')){
|
if($('#nav_menu').is(':visible')){
|
||||||
$('#nav_menu').hide();
|
$('#nav_menu').hide();
|
||||||
|
|
|
@ -79,7 +79,7 @@
|
||||||
{% if not message['extracted'] %}
|
{% if not message['extracted'] %}
|
||||||
<pre class="my-0">{{ message['content'] }}</pre>
|
<pre class="my-0">{{ message['content'] }}</pre>
|
||||||
{% else %}
|
{% else %}
|
||||||
<pre class="my-0">{{ message['content'][:message['extracted'][0][0]] }}{% for row in message['extracted'] %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="<svg height="26" width="26"><g class="nodes"><circle cx="13" cy="13" r="13" fill="{{ message['extracted_matches'][row[3]]['icon']['color'] }}"></circle><text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ message['extracted_matches'][row[3]]['icon']['style'] }}" font-size="16px">{{ message['extracted_matches'][row[3]]['icon']['icon'] }}</text></g></svg> {{ message['extracted_matches'][row[3]]['subtype'] }}" data-content="{{ message['extracted_matches'][row[3]]['id'] }}" id="{{ row[0] }}:{{ row[1] }}">{{ message['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > message['extracted']|length %}{{ message['content'][message['extracted'][-1][1]:] }}{% else %}{{ message['content'][row[1]:message['extracted'][loop.index][0]] }}{% endif %}{% endfor %}</pre>
|
<pre class="my-0">{{ message['content'][:message['extracted'][0][0]] }}{% for row in message['extracted'] %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="Extracted:" data-content="<ul class="list-group">{% for r in row[3] %}<li class="list-group-item"><div><svg height="26" width="26"><g class="nodes"><circle cx="13" cy="13" r="13" fill="{{ message['extracted_matches'][r[0]]['icon']['color'] }}"></circle><text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ message['extracted_matches'][r[0]]['icon']['style'] }}" font-size="16px">{{ message['extracted_matches'][r[0]]['icon']['icon'] }}</text></g></svg> {{ message['extracted_matches'][r[0]]['subtype'] }}</div>{{ message['extracted_matches'][r[0]]['id'] }} <div><b>{{ r[1] }}</b></div></li>{% endfor %}</ul>" id="{{ row[0] }}:{{ row[1] }}">{{ message['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > message['extracted']|length %}{{ message['content'][message['extracted'][-1][1]:] }}{% else %}{{ message['content'][row[1]:message['extracted'][loop.index][0]] }}{% endif %}{% endfor %}</pre>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% if message['translation'] %}
|
{% if message['translation'] %}
|
||||||
<hr class="m-1">
|
<hr class="m-1">
|
||||||
|
|
|
@ -547,7 +547,7 @@
|
||||||
{% if not extracted %}
|
{% if not extracted %}
|
||||||
<p class="my-0"> <pre class="border">{{ meta['content'] }}</pre></p>
|
<p class="my-0"> <pre class="border">{{ meta['content'] }}</pre></p>
|
||||||
{% else %}
|
{% else %}
|
||||||
<p class="my-0"> <pre class="border">{{ meta['content'][:extracted[0][0]] }}{% for row in extracted %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="<svg height="26" width="26"><g class="nodes"><circle cx="13" cy="13" r="13" fill="{{ extracted_matches[row[3]]['icon']['color'] }}"></circle><text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ extracted_matches[row[3]]['icon']['style'] }}" font-size="16px">{{ extracted_matches[row[3]]['icon']['icon'] }}</text></g></svg> {{ extracted_matches[row[3]]['subtype'] }}" data-content="{{ extracted_matches[row[3]]['id'] }}" id="{{ row[0] }}:{{ row[1] }}">{{ meta['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > extracted|length %}{{ meta['content'][extracted[-1][1]:] }}{% else %}{{ meta['content'][row[1]:extracted[loop.index][0]] }}{% endif %}{% endfor %}</pre></p>
|
<p class="my-0"> <pre class="border">{{ meta['content'][:extracted[0][0]] }}{% for row in extracted %}<span class="hg-text" data-toggle="popover" data-trigger="hover" data-html="true" title="Extracted:" data-content="<ul class="list-group">{% for r in row[3] %}<li class="list-group-item"><div><svg height="26" width="26"><g class="nodes"><circle cx="13" cy="13" r="13" fill="{{ extracted_matches[r[0]]['icon']['color'] }}"></circle><text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="{{ extracted_matches[r[0]]['icon']['style'] }}" font-size="16px">{{ extracted_matches[r[0]]['icon']['icon'] }}</text></g></svg> {{ extracted_matches[r[0]]['subtype'] }}</div>{{ extracted_matches[r[0]]['id'] }} <div><b>{{ r[1] }}</b></div></li>{% endfor %}</ul>" id="{{ row[0] }}:{{ row[1] }}">{{ meta['content'][row[0]:row[1]] }}</span>{% if loop.index + 1 > extracted|length %}{{ meta['content'][extracted[-1][1]:] }}{% else %}{{ meta['content'][row[1]:extracted[loop.index][0]] }}{% endif %}{% endfor %}</pre></p>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
</div>
|
</div>
|
||||||
<div class="tab-pane fade" id="pills-html2text" role="tabpanel" aria-labelledby="pills-html2text-tab">
|
<div class="tab-pane fade" id="pills-html2text" role="tabpanel" aria-labelledby="pills-html2text-tab">
|
||||||
|
@ -576,7 +576,10 @@
|
||||||
$(".rotate").click(function(){
|
$(".rotate").click(function(){
|
||||||
$(this).toggleClass("down");
|
$(this).toggleClass("down");
|
||||||
})
|
})
|
||||||
});
|
$('[data-toggle="popover"]').popover({
|
||||||
|
boundary:'window',
|
||||||
|
})
|
||||||
|
});
|
||||||
|
|
||||||
$('#pills-html2text-tab').on('shown.bs.tab', function (e) {
|
$('#pills-html2text-tab').on('shown.bs.tab', function (e) {
|
||||||
if ($('#html2text-container').is(':empty')){
|
if ($('#html2text-container').is(':empty')){
|
||||||
|
@ -639,9 +642,6 @@
|
||||||
}
|
}
|
||||||
|
|
||||||
blocks.addEventListener('change', pixelate, false);
|
blocks.addEventListener('change', pixelate, false);
|
||||||
$(function () {
|
|
||||||
$('[data-toggle="popover"]').popover()
|
|
||||||
})
|
|
||||||
</script>
|
</script>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue