mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 00:28:22 +00:00
chg: [Correlation Graph] add correlation by pgp, cryptocurrency, decoded and item + request correlation by type(pgp, ...) + objects(domain, paste)
This commit is contained in:
Terrtia
parent
7d7c318534
commit
e42ec48380
6 changed files with 167 additions and 104 deletions
|
|
@ -16,11 +16,30 @@ import Domain
|
|||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
import Pgp
|
||||
import Cryptocurrency
|
||||
import Item
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
config_loader = None
|
||||
|
||||
|
||||
def get_object_correlation(object_type, value, correlation_names, correlation_objects, requested_correl_type=None):
|
||||
if object_type == 'domain':
|
||||
return Domain.get_domain_all_correlation(value, correlation_names=correlation_names)
|
||||
elif object_type == 'paste':
|
||||
return Item.get_item_all_correlation(value, correlation_names=correlation_names)
|
||||
elif object_type == 'decoded':
|
||||
return Decoded.get_decoded_correlated_object(value, correlation_objects)
|
||||
elif object_type == 'pgp':
|
||||
return Pgp.pgp.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects)
|
||||
elif object_type == 'cryptocurrency':
|
||||
return Cryptocurrency.cryptocurrency.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects)
|
||||
|
||||
return {}
|
||||
|
||||
|
||||
|
||||
|
||||
def get_correlation_node_icon(correlation_name, correlation_type=None, value=None):
|
||||
'''
|
||||
Used in UI Graph.
|
||||
|
|
@ -117,35 +136,6 @@ def get_item_url(correlation_name, value, correlation_type=None):
|
|||
url = url_for(endpoint, paste=value)
|
||||
return url
|
||||
|
||||
# # TODO: refractor
|
||||
# # get object description, return dict, used by correlation
|
||||
# def get_object_desc(object_type, item_value, correlation_name, correlation_type=None):
|
||||
# if object_type=="domain":
|
||||
# return Domain.get_object_desc(item_value)
|
||||
# if object_type=="correlation":
|
||||
# return Domain.get_object_desc(item_value)
|
||||
# {"name": self.correlation_name, "type": correlation_type, "id": correlation_value, "object": correl_object}
|
||||
#
|
||||
#
|
||||
# # # TODO: sanithyse dict_correlation_to_check
|
||||
# def get_object_correlation(object, object_value, mode, nb_max_elem=400, dict_correlation_to_check=[], depth_limit=1):
|
||||
# '''
|
||||
# Return all correlation of a given item id.
|
||||
#
|
||||
# :param l_items_to_correlate: list of dict
|
||||
# :type l_items_to_correlate: list
|
||||
# :param mode: correlation mode
|
||||
# mode == intersection, union
|
||||
# union: show all related objects
|
||||
# intersection: show only direct correlation
|
||||
# :type mode: str
|
||||
# :param nb_max_elem: max nb of nodes
|
||||
# :type nb_max_elem: int, optional
|
||||
#
|
||||
#
|
||||
# '''
|
||||
# dict_item_desc = {}
|
||||
# dict_correlation = object.get_correlation(value, dict_correlation_to_check)
|
||||
|
||||
def create_graph_links(links_set):
|
||||
graph_links_list = []
|
||||
|
|
@ -168,21 +158,25 @@ def create_graph_nodes(nodes_set, root_node_id):
|
|||
return graph_nodes_list
|
||||
|
||||
def create_node_id(correlation_name, value, correlation_type=''):
|
||||
if correlation_type is None:
|
||||
correlation_type = ''
|
||||
return '{};{};{}'.format(correlation_name, correlation_type, value)
|
||||
|
||||
|
||||
def get_graph_node_domain_correlation(domain, mode, max_nodes=50):
|
||||
|
||||
# # TODO: filter by correlation type => bitcoin, mail, ...
|
||||
def get_graph_node_object_correlation(object_type, domain, mode, correlation_names, correlation_objects, max_nodes=300, requested_correl_type=None):
|
||||
links = set()
|
||||
nodes = set()
|
||||
|
||||
root_node_id = create_node_id('domain', domain)
|
||||
root_node_id = create_node_id(object_type, domain, requested_correl_type)
|
||||
nodes.add(root_node_id)
|
||||
|
||||
domain_correlation = Domain.get_domain_all_correlation(domain)
|
||||
for correl in domain_correlation:
|
||||
root_correlation = get_object_correlation(object_type, domain, correlation_names, correlation_objects, requested_correl_type=requested_correl_type)
|
||||
for correl in root_correlation:
|
||||
if correl in ('pgp', 'cryptocurrency'):
|
||||
for correl_type in domain_correlation[correl]:
|
||||
for correl_val in domain_correlation[correl][correl_type]:
|
||||
for correl_type in root_correlation[correl]:
|
||||
for correl_val in root_correlation[correl][correl_type]:
|
||||
|
||||
# add correlation
|
||||
correl_node_id = create_node_id(correl, correl_val, correl_type)
|
||||
|
|
@ -193,33 +187,28 @@ def get_graph_node_domain_correlation(domain, mode, max_nodes=50):
|
|||
nodes.add(correl_node_id)
|
||||
links.add((root_node_id, correl_node_id))
|
||||
|
||||
# get PGP correlation
|
||||
if correl=='pgp':
|
||||
res = Pgp.pgp.get_correlation_obj_domain(correl_val, correlation_type=correl_type) # change function for item ?
|
||||
# get Cryptocurrency correlation
|
||||
else:
|
||||
res = Cryptocurrency.cryptocurrency.get_correlation_obj_domain(correl_val, correlation_type=correl_type)
|
||||
|
||||
# inter mode
|
||||
# get second correlation
|
||||
res = get_object_correlation(correl, correl_val, correlation_names, correlation_objects, requested_correl_type=correl_type)
|
||||
if res:
|
||||
for correl_key_val in res:
|
||||
#filter root domain
|
||||
if correl_key_val == domain:
|
||||
continue
|
||||
for corr_obj in res:
|
||||
for correl_key_val in res[corr_obj]:
|
||||
#filter root domain
|
||||
if correl_key_val == domain:
|
||||
continue
|
||||
|
||||
if len(nodes) > max_nodes:
|
||||
break
|
||||
new_corel_1 = create_node_id('domain', correl_key_val)
|
||||
new_corel_2 = create_node_id(correl, correl_val, correl_type)
|
||||
nodes.add(new_corel_1)
|
||||
nodes.add(new_corel_2)
|
||||
links.add((new_corel_1, new_corel_2))
|
||||
if len(nodes) > max_nodes:
|
||||
break
|
||||
new_corel_1 = create_node_id(corr_obj, correl_key_val)
|
||||
new_corel_2 = create_node_id(correl, correl_val, correl_type)
|
||||
nodes.add(new_corel_1)
|
||||
nodes.add(new_corel_2)
|
||||
links.add((new_corel_1, new_corel_2))
|
||||
|
||||
if mode=="inter":
|
||||
nodes.add(correl_node_id)
|
||||
links.add((root_node_id, correl_node_id))
|
||||
if correl=='decoded':
|
||||
for correl_val in domain_correlation[correl]:
|
||||
if mode=="inter":
|
||||
nodes.add(correl_node_id)
|
||||
links.add((root_node_id, correl_node_id))
|
||||
if correl in ('decoded', 'domain', 'paste'):
|
||||
for correl_val in root_correlation[correl]:
|
||||
|
||||
correl_node_id = create_node_id(correl, correl_val)
|
||||
if mode=="union":
|
||||
|
|
@ -228,24 +217,45 @@ def get_graph_node_domain_correlation(domain, mode, max_nodes=50):
|
|||
nodes.add(correl_node_id)
|
||||
links.add((root_node_id, correl_node_id))
|
||||
|
||||
res = Decoded.get_decoded_domain_item(correl_val)
|
||||
res = get_object_correlation(correl, correl_val, correlation_names, correlation_objects)
|
||||
if res:
|
||||
for correl_key_val in res:
|
||||
#filter root domain
|
||||
if correl_key_val == domain:
|
||||
continue
|
||||
for corr_obj in res:
|
||||
if corr_obj in ('decoded', 'domain', 'paste'):
|
||||
for correl_key_val in res[corr_obj]:
|
||||
#filter root domain
|
||||
if correl_key_val == domain:
|
||||
continue
|
||||
|
||||
if len(nodes) > max_nodes:
|
||||
break
|
||||
new_corel_1 = create_node_id('domain', correl_key_val)
|
||||
new_corel_2 = create_node_id(correl, correl_val)
|
||||
nodes.add(new_corel_1)
|
||||
nodes.add(new_corel_2)
|
||||
links.add((new_corel_1, new_corel_2))
|
||||
if len(nodes) > max_nodes:
|
||||
break
|
||||
new_corel_1 = create_node_id(corr_obj, correl_key_val)
|
||||
new_corel_2 = create_node_id(correl, correl_val)
|
||||
nodes.add(new_corel_1)
|
||||
nodes.add(new_corel_2)
|
||||
links.add((new_corel_1, new_corel_2))
|
||||
|
||||
if mode=="inter":
|
||||
nodes.add(correl_node_id)
|
||||
links.add((root_node_id, correl_node_id))
|
||||
if mode=="inter":
|
||||
nodes.add(correl_node_id)
|
||||
links.add((root_node_id, correl_node_id))
|
||||
|
||||
if corr_obj in ('pgp', 'cryptocurrency'):
|
||||
for correl_key_type in res[corr_obj]:
|
||||
for correl_key_val in res[corr_obj][correl_key_type]:
|
||||
#filter root domain
|
||||
if correl_key_val == domain:
|
||||
continue
|
||||
|
||||
if len(nodes) > max_nodes:
|
||||
break
|
||||
new_corel_1 = create_node_id(corr_obj, correl_key_val, correl_key_type)
|
||||
new_corel_2 = create_node_id(correl, correl_val)
|
||||
nodes.add(new_corel_1)
|
||||
nodes.add(new_corel_2)
|
||||
links.add((new_corel_1, new_corel_2))
|
||||
|
||||
if mode=="inter":
|
||||
nodes.add(correl_node_id)
|
||||
links.add((root_node_id, correl_node_id))
|
||||
|
||||
|
||||
return {"nodes": create_graph_nodes(nodes, root_node_id), "links": create_graph_links(links)}
|
||||
|
|
|
|||
|
|
@ -63,15 +63,30 @@ def get_decoded_domain_item(sha1_string):
|
|||
else:
|
||||
return []
|
||||
|
||||
def get_decoded_correlated_object(sha1_string, correlation_objects=[]):
|
||||
'''
|
||||
Retun all correlation of a given sha1.
|
||||
|
||||
:param sha1_string: sha1
|
||||
:type sha1_string: str
|
||||
|
||||
:return: a dict of all correlation for a given sha1
|
||||
:rtype: dict
|
||||
'''
|
||||
if correlation_objects is None:
|
||||
correlation_objects = Correlation.get_all_correlation_objects()
|
||||
decoded_correlation = {}
|
||||
for correlation_object in correlation_objects:
|
||||
if correlation_object == 'paste':
|
||||
res = get_decoded_items_list(sha1_string)
|
||||
elif correlation_object == 'domain':
|
||||
res = get_decoded_domain_item(sha1_string)
|
||||
else:
|
||||
res = None
|
||||
if res:
|
||||
decoded_correlation[correlation_object] = res
|
||||
return decoded_correlation
|
||||
|
||||
def save_domain_decoded(domain, sha1_string):
|
||||
r_serv_metadata.sadd('hash_domain:{}'.format(domain), sha1_string) # domain - hash map
|
||||
r_serv_metadata.sadd('domain_hash:{}'.format(sha1_string), domain) # hash - domain ma
|
||||
|
||||
if __name__ == "__main__":
|
||||
#sha1_str = '1e4db5adc1334ad2c9762db9ff6b845ee6ddc223'
|
||||
#res = _get_decoded_items_list(sha1_str)
|
||||
#print(res)
|
||||
#print(len(res))
|
||||
|
||||
res = get_domain_decoded_item('2222222dpg65ioqu.onion')
|
||||
print(res)
|
||||
|
|
|
|||
|
|
@ -209,7 +209,7 @@ def get_domain_decoded(domain):
|
|||
return Decoded.get_domain_decoded_item(domain)
|
||||
|
||||
|
||||
def get_domain_all_correlation(domain, correlation_type=None, get_nb=False):
|
||||
def get_domain_all_correlation(domain, correlation_names=[], get_nb=False):
|
||||
'''
|
||||
Retun all correlation of a given domain.
|
||||
|
||||
|
|
@ -219,16 +219,22 @@ def get_domain_all_correlation(domain, correlation_type=None, get_nb=False):
|
|||
:return: a dict of all correlation for a given domain
|
||||
:rtype: dict
|
||||
'''
|
||||
if not correlation_names:
|
||||
correlation_names = Correlation.get_all_correlation_names()
|
||||
domain_correl = {}
|
||||
res = get_domain_cryptocurrency(domain, get_nb=get_nb)
|
||||
if res:
|
||||
domain_correl['cryptocurrency'] = res
|
||||
res = get_domain_pgp(domain, get_nb=get_nb)
|
||||
if res:
|
||||
domain_correl['pgp'] = res
|
||||
res = get_domain_decoded(domain)
|
||||
if res:
|
||||
domain_correl['decoded'] = res
|
||||
for correlation_name in correlation_names:
|
||||
if correlation_name=='cryptocurrency':
|
||||
res = get_domain_cryptocurrency(domain, get_nb=get_nb)
|
||||
elif correlation_name=='pgp':
|
||||
res = get_domain_pgp(domain, get_nb=get_nb)
|
||||
elif correlation_name=='decoded':
|
||||
res = get_domain_decoded(domain)
|
||||
else:
|
||||
res = None
|
||||
# add correllation to dict
|
||||
if res:
|
||||
domain_correl[correlation_name] = res
|
||||
|
||||
return domain_correl
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -12,6 +12,18 @@ config_loader = ConfigLoader.ConfigLoader()
|
|||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
config_loader = None
|
||||
|
||||
def get_all_correlation_names():
|
||||
'''
|
||||
Return a list of all available correlations
|
||||
'''
|
||||
return ['pgp', 'cryptocurrency', 'decoded']
|
||||
|
||||
def get_all_correlation_objects():
|
||||
'''
|
||||
Return a list of all correllated objects
|
||||
'''
|
||||
return ['domain', 'paste']
|
||||
|
||||
class Correlation(object):
|
||||
|
||||
def __init__(self, correlation_name, all_correlation_types):
|
||||
|
|
@ -207,6 +219,20 @@ class Correlation(object):
|
|||
return dict_correlation
|
||||
|
||||
|
||||
def get_correlation_all_object(self, correlation_type, correlation_value, correlation_objects=[]):
|
||||
if correlation_objects is None:
|
||||
correlation_objects = get_all_correlation_objects()
|
||||
correlation_obj = {}
|
||||
for correlation_object in correlation_objects:
|
||||
if correlation_object == 'paste':
|
||||
res = self._get_items(correlation_type, correlation_value)
|
||||
elif correlation_object == 'domain':
|
||||
res = self.get_correlation_obj_domain(correlation_value, correlation_type=correlation_type)
|
||||
else:
|
||||
res = None
|
||||
if res:
|
||||
correlation_obj[correlation_object] = res
|
||||
return correlation_obj
|
||||
|
||||
def save_domain_correlation(self, domain, correlation_type, correlation_value):
|
||||
r_serv_metadata.sadd('domain_{}_{}:{}'.format(self.correlation_name, correlation_type, domain), correlation_value)
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import redis
|
|||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
import Date
|
||||
import Tag
|
||||
import Correlation
|
||||
import Cryptocurrency
|
||||
from Pgp import pgp
|
||||
|
||||
|
|
@ -159,7 +160,7 @@ def get_item_decoded(item_id):
|
|||
'''
|
||||
return Decoded.get_item_decoded(item_id)
|
||||
|
||||
def get_item_all_correlation(item_id, correlation_type=None, get_nb=False):
|
||||
def get_item_all_correlation(item_id, correlation_names=[], get_nb=False):
|
||||
'''
|
||||
Retun all correlation of a given item id.
|
||||
|
||||
|
|
@ -169,16 +170,21 @@ def get_item_all_correlation(item_id, correlation_type=None, get_nb=False):
|
|||
:return: a dict of all correlation for a item id
|
||||
:rtype: dict
|
||||
'''
|
||||
if not correlation_names:
|
||||
correlation_names = Correlation.get_all_correlation_names()
|
||||
item_correl = {}
|
||||
res = get_item_cryptocurrency(item_id, get_nb=get_nb)
|
||||
if res:
|
||||
item_correl['cryptocurrency'] = res
|
||||
res = get_item_pgp(item_id, get_nb=get_nb)
|
||||
if res:
|
||||
item_correl['pgp'] = res
|
||||
res = get_item_decoded(item_id)
|
||||
if res:
|
||||
item_correl['decoded'] = res
|
||||
for correlation_name in correlation_names:
|
||||
if correlation_name=='cryptocurrency':
|
||||
res = get_item_cryptocurrency(item_id, get_nb=get_nb)
|
||||
elif correlation_name=='pgp':
|
||||
res = get_item_pgp(item_id, get_nb=get_nb)
|
||||
elif correlation_name=='decoded':
|
||||
res = get_item_decoded(item_id)
|
||||
else:
|
||||
res = None
|
||||
# add correllation to dict
|
||||
if res:
|
||||
item_correl[correlation_name] = res
|
||||
return item_correl
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -23,7 +23,7 @@
|
|||
}
|
||||
line.link:hover{
|
||||
stroke: red;
|
||||
stroke-width: 2px
|
||||
stroke-width: 2px;
|
||||
}
|
||||
.node {
|
||||
pointer-events: all;
|
||||
|
|
|
|||
Loading…
Reference in a new issue