chg: [UI correlation] add username correlation graph

bin/lib/Correlate_object.py

@@ -13,6 +13,7 @@ import ConfigLoader
 import Decoded
 import Domain
 import Screenshot
+import telegram
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
 import Pgp
@@ -24,7 +25,7 @@ r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
 config_loader = None
 
 def is_valid_object_type(object_type):
-    if object_type in ['domain', 'item', 'image', 'decoded', 'pgp', 'cryptocurrency']:
+    if object_type in ['domain', 'item', 'image', 'decoded', 'pgp', 'cryptocurrency', 'username']:
         return True
     else:
         return False
@@ -33,25 +34,22 @@ def is_valid_object_subtype(object_type, object_subtype):
     if object_type == 'pgp':
         return Pgp.pgp.is_valid_obj_subtype(object_subtype)
     elif object_type == 'cryptocurrency':
-        return Pgp.pgp.is_valid_obj_subtype(object_subtype)
+        return Cryptocurrency.cryptocurrency.is_valid_obj_subtype(object_subtype)
+    elif object_type == 'username':
+        return telegram.correlation.is_valid_obj_subtype(object_subtype)
     elif object_subtype == None:
         return True
     else:
         return False
 
-    if object_type in ['domain', 'item', 'image', 'decoded', 'pgp', 'cryptocurrency']:
-        return True
-    else:
-        return False
-
 def get_all_objects():
-    return ['domain', 'paste', 'pgp', 'cryptocurrency', 'decoded', 'screenshot']
+    return ['domain', 'paste', 'pgp', 'cryptocurrency', 'decoded', 'screenshot', 'username']
 
 def get_all_correlation_names():
     '''
     Return a list of all available correlations
     '''
-    return ['pgp', 'cryptocurrency', 'decoded', 'screenshot']
+    return ['pgp', 'cryptocurrency', 'decoded', 'screenshot', 'username']
 
 def get_all_correlation_objects():
     '''
@@ -70,6 +68,8 @@ def exist_object(object_type, correlation_id, type_id=None): # => work on object
         return Pgp.pgp.exist_correlation(type_id, correlation_id)
     elif object_type == 'cryptocurrency':
         return Cryptocurrency.cryptocurrency.exist_correlation(type_id, correlation_id)
+    elif object_type == 'username':
+        return telegram.correlation.exist_correlation(type_id, correlation_id)
     elif object_type == 'screenshot' or object_type == 'image':
         return Screenshot.exist_screenshot(correlation_id)
     else:
@@ -87,6 +87,8 @@ def get_object_metadata(object_type, correlation_id, type_id=None):
         return Pgp.pgp.get_metadata(type_id, correlation_id)
     elif object_type == 'cryptocurrency':
         return Cryptocurrency.cryptocurrency.get_metadata(type_id, correlation_id)
+    elif object_type == 'username':
+        return telegram.correlation.get_metadata(type_id, correlation_id)
     elif object_type == 'screenshot' or object_type == 'image':
         return Screenshot.get_metadata(correlation_id)
 
@@ -101,6 +103,8 @@ def get_object_correlation(object_type, value, correlation_names=None, correlati
         return Pgp.pgp.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects)
     elif object_type == 'cryptocurrency':
         return Cryptocurrency.cryptocurrency.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects)
+    elif object_type == 'username':
+        return telegram.correlation.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects)
     elif object_type == 'screenshot' or object_type == 'image':
         return Screenshot.get_screenshot_correlated_object(value, correlation_objects=correlation_objects)
     return {}
@@ -118,6 +122,7 @@ def get_correlation_node_icon(correlation_name, correlation_type=None, value=Non
     :return: a dictionnary {font awesome class, icon_code}
     :rtype: dict
     '''
+
     icon_class = 'fas'
     icon_text = ''
     node_color = "#332288"
@@ -147,6 +152,14 @@ def get_correlation_node_icon(correlation_name, correlation_type=None, value=Non
         else:
             icon_text = '\uf51e'
 
+    elif correlation_name == 'username':
+        node_color = '#4dffff'
+        if correlation_type == 'telegram':
+            icon_class = 'fab'
+            icon_text = '\uf2c6'
+        else:
+            icon_text = '\uf007'
+
     elif correlation_name == 'decoded':
         node_color = '#88CCEE'
         print(Decoded.get_decoded_item_type(value))
@@ -196,6 +209,9 @@ def get_item_url(correlation_name, value, correlation_type=None):
     elif correlation_name == 'cryptocurrency':
         endpoint = 'correlation.show_correlation'
         url = url_for(endpoint, object_type="cryptocurrency", type_id=correlation_type, correlation_id=value)
+    elif correlation_name == 'username':
+        endpoint = 'correlation.show_correlation'
+        url = url_for(endpoint, object_type="username", type_id=correlation_type, correlation_id=value)
     elif correlation_name == 'decoded':
         endpoint = 'correlation.show_correlation'
         url = url_for(endpoint, object_type="decoded", correlation_id=value)
@@ -285,7 +301,7 @@ def get_graph_node_object_correlation(object_type, root_value, mode, correlation
 
     root_correlation = get_object_correlation(object_type, root_value, correlation_names, correlation_objects, requested_correl_type=requested_correl_type)
     for correl in root_correlation:
-        if correl in ('pgp', 'cryptocurrency'):
+        if correl in ('pgp', 'cryptocurrency', 'username'):
             for correl_type in root_correlation[correl]:
                 for correl_val in root_correlation[correl][correl_type]:
 
@@ -349,7 +365,7 @@ def get_graph_node_object_correlation(object_type, root_value, mode, correlation
                                     nodes.add(correl_node_id)
                                     links.add((root_node_id, correl_node_id))
 
-                        if corr_obj in ('pgp', 'cryptocurrency'):
+                        if corr_obj in ('pgp', 'cryptocurrency', 'username'):
                             for correl_key_type in res[corr_obj]:
                                 for correl_key_val in res[corr_obj][correl_key_type]:
                                     #filter root value

bin/lib/Domain.py

@@ -25,6 +25,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
 import ConfigLoader
 import Correlate_object
 import Screenshot
+import telegram
 
 config_loader = ConfigLoader.ConfigLoader()
 r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
@@ -555,6 +556,16 @@ def get_domain_pgp(domain, currencies_type=None, get_nb=False):
     '''
     return Pgp.pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb)
 
+def get_domain_username(domain, currencies_type=None, get_nb=False):
+    '''
+    Retun all pgp of a given domain.
+
+    :param domain: crawled domain
+    :param currencies_type: list of pgp type
+    :type currencies_type: list, optional
+    '''
+    return telegram.correlation.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb)
+
 def get_domain_decoded(domain):
     '''
     Retun all decoded item of a given domain.
@@ -590,6 +601,8 @@ def get_domain_all_correlation(domain, correlation_names=[], get_nb=False):
             res = get_domain_cryptocurrency(domain, get_nb=get_nb)
         elif correlation_name=='pgp':
             res = get_domain_pgp(domain, get_nb=get_nb)
+        elif correlation_name=='username':
+            res = get_domain_username(domain, get_nb=get_nb)
         elif correlation_name=='decoded':
             res = get_domain_decoded(domain)
         elif correlation_name=='screenshot':

bin/lib/telegram.py

@@ -15,10 +15,10 @@ config_loader = ConfigLoader.ConfigLoader()
 r_serv_crawler = config_loader.get_redis_conn("ARDB_Onion")
 config_loader = None
 
-correlaton = Correlation.Correlation('username', ['telegram'])
+correlation = Correlation.Correlation('username', ['telegram'])
 
 def save_item_correlation(username, item_id, item_date):
-    correlaton.save_item_correlation('telegram', username, item_id, item_date)
+    correlation.save_item_correlation('telegram', username, item_id, item_date)
 
 def save_telegram_invite_hash(invite_hash, item_id):
     r_serv_crawler.sadd('telegram:invite_code', '{};{}'.format(invite_hash, item_id))

bin/packages/Item.py

@@ -20,6 +20,7 @@ import ConfigLoader
 import Correlate_object
 import Decoded
 import Screenshot
+import telegram
 
 config_loader = ConfigLoader.ConfigLoader()
 # get and sanityze PASTE DIRECTORY
@@ -171,6 +172,16 @@ def get_item_pgp(item_id, currencies_type=None, get_nb=False):
     '''
     return Pgp.pgp.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb)
 
+def get_item_username(item_id, currencies_type=None, get_nb=False):
+    '''
+    Return all pgp of a given item.
+
+    :param item_id: item id
+    :param currencies_type: list of cryptocurrencies type
+    :type currencies_type: list, optional
+    '''
+    return telegram.correlation.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb)
+
 def get_item_decoded(item_id):
     '''
     Return all pgp of a given item.
@@ -207,6 +218,8 @@ def get_item_all_correlation(item_id, correlation_names=[], get_nb=False):
             res = get_item_cryptocurrency(item_id, get_nb=get_nb)
         elif correlation_name=='pgp':
             res = get_item_pgp(item_id, get_nb=get_nb)
+        elif correlation_name=='username':
+            res = get_item_username(item_id, get_nb=get_nb)
         elif correlation_name=='decoded':
             res = get_item_decoded(item_id)
         elif correlation_name=='screenshot':

var/www/blueprints/correlation.py

@@ -25,6 +25,7 @@ import Correlate_object
 import Domain
 import Screenshot
 import btc_ail
+import telegram
 
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
 import Cryptocurrency
@@ -108,6 +109,9 @@ def get_card_metadata(object_type, correlation_id, type_id=None, expand_card=Fal
     elif object_type == 'pgp':
         card_dict["sparkline"] = Pgp.pgp.get_list_nb_previous_correlation_object(type_id, correlation_id, 6)
         card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, type_id)
+    elif object_type == 'username':
+        card_dict["sparkline"] = telegram.correlation.get_list_nb_previous_correlation_object(type_id, correlation_id, 6)
+        card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, type_id)
     elif object_type == 'decoded':
         card_dict["sparkline"] = Decoded.get_list_nb_previous_hash(correlation_id, 6)
         card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, value=correlation_id)
@@ -149,6 +153,9 @@ def show_correlation():
         correl_option = request.form.get('PgpCheck')
         if correl_option:
             correlation_names.append('pgp')
+        correl_option = request.form.get('UsernameCheck')
+        if correl_option:
+            correlation_names.append('username')
         correl_option = request.form.get('DecodedCheck')
         if correl_option:
             correlation_names.append('decoded')

var/www/modules/hashDecoded/Flask_hashDecoded.py

@@ -146,7 +146,7 @@ def get_icon_text(correlation_type, type_id):
             icon_text = '\uf42e'
         else:
             icon_text = '\uf51e'
-    elif correlation_type == 'cryptocurrency':
+    elif correlation_type == 'username':
         if type_id == 'telegram':
             icon_text = '\uf2c6'
     return icon_text

var/www/templates/correlation/legend_graph_correlation.html

@@ -13,6 +13,9 @@
 										<th class="">
 											Pgp:
 										</th>
+										<th class="">
+											Username:
+										</th>
 										<th class="">
 											Domain:
 										</th>
@@ -139,6 +142,17 @@
 												mail
 											</div>
 										</td>
+										<td>
+											<div class="my-1">
+												<svg height="26" width="26">
+													<g class="nodes">
+														<circle cx="13" cy="13" r="13" fill="#4dffff"></circle>
+														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf2c6;</text>
+													</g>
+												</svg>
+												telegram
+											</div>
+										</td>
 										<td>
 											<div class="my-1">
 												<svg height="26" width="26">

var/www/templates/correlation/show_correlation.html

@@ -95,6 +95,8 @@
 						{% include 'correlation/metadata_card_pgp.html' %}
 					{% elif dict_object["object_type"] == "cryptocurrency" %}
 						{% include 'correlation/metadata_card_cryptocurrency.html' %}
+					{% elif dict_object["object_type"] == "username" %}
+							{% include 'correlation/metadata_card_username.html' %}
 					{% elif dict_object["object_type"] == "decoded" %}
 						{% include 'correlation/metadata_card_decoded.html' %}
 					{% elif dict_object["object_type"] == "domain" %}
@@ -112,9 +114,11 @@
 								<div class="card-header">
 									<i class="fas fa-project-diagram"></i> Graph
 									<span class="float-right">
-										{% with obj_type=dict_object["object_type"], obj_id=dict_object["correlation_id"], obj_subtype=dict_object["metadata"]["type_id"],obj_lvl=1%}
+										{% if dict_object["object_type"] != "username" %}
-				              {% include 'import_export/block_add_user_object_to_export.html' %}
+											{% with obj_type=dict_object["object_type"], obj_id=dict_object["correlation_id"], obj_subtype=dict_object["metadata"]["type_id"],obj_lvl=1%}
-				            {% endwith %}
+					              {% include 'import_export/block_add_user_object_to_export.html' %}
+					            {% endwith %}
+										{% endif %}
 									</span>
 									<span class="float-right">
 										<button class="btn btn-primary py-1" onclick="resize_graph();">
@@ -166,6 +170,10 @@
 												  <input class="form-check-input" type="checkbox" value="True" id="PgpCheck" name="PgpCheck" {%if "pgp" in dict_object["correlation_names"]%}checked{%endif%}>
 												  <label class="form-check-label" for="PgpCheck">PGP</label>
 												</div>
+												<div class="form-check">
+												  <input class="form-check-input" type="checkbox" value="True" id="UsernameCheck" name="UsernameCheck" {%if "username" in dict_object["correlation_names"]%}checked{%endif%}>
+												  <label class="form-check-label" for="UsernameCheck">Username</label>
+												</div>
 												<div class="form-check">
 												  <input class="form-check-input" type="checkbox" value="True" id="DomainCheck" name="DomainCheck" {%if "domain" in dict_object["correlation_objects"]%}checked{%endif%}>
 												  <label class="form-check-label" for="DomainCheck">Domain</label>