From 36772372b972cb666050456323fe60e1b2fa4132 Mon Sep 17 00:00:00 2001 From: Jean-Louis Huynen Date: Fri, 4 Mar 2022 11:18:36 +0100 Subject: [PATCH 1/2] add: [username] jabber support --- .../ail_json_importer/Ail_feeder_jabber.py | 50 +++ bin/lib/Correlate_object.py | 4 + bin/lib/Username.py | 2 +- bin/lib/item_basic.py | 6 +- .../modules/hashDecoded/Flask_hashDecoded.py | 6 +- .../correlation/legend_graph_correlation.html | 384 ++++++++++-------- 6 files changed, 269 insertions(+), 183 deletions(-) create mode 100755 bin/import/ail_json_importer/Ail_feeder_jabber.py diff --git a/bin/import/ail_json_importer/Ail_feeder_jabber.py b/bin/import/ail_json_importer/Ail_feeder_jabber.py new file mode 100755 index 00000000..37a4d95a --- /dev/null +++ b/bin/import/ail_json_importer/Ail_feeder_jabber.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +The JSON Receiver Module +================ + +Receiver Jabber Json Items + +""" +import os +import json +import sys +import time +import datetime + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) +import item_basic +import Username + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer')) +from Default_json import Default_json + +class Ail_feeder_jabber(Default_json): + """Jabber Feeder functions""" + + def __init__(self, name, json_item): + super().__init__(name, json_item) + + def get_feeder_name(self): + return 'jabber' + + # define item id + def get_item_id(self): + item_date = time.strptime(self.json_item['meta']['jabber:ts'], "%Y-%m-%dT%H:%M:%S.%f") + item_date_str = time.strftime("%Y/%m/%d", item_date) + item_id = str(self.json_item['meta']['jabber:id']) + return os.path.join('jabber', item_date_str, item_id) + '.gz' + + def process_json_meta(self, process, item_id): + ''' + Process JSON meta filed. + ''' + jabber_id = str(self.json_item['meta']['jabber:id']) + item_basic.add_map_obj_id_item_id(jabber_id, item_id, 'jabber_id') + to = str(self.json_item['meta']['jabber:to']) + fr = str(self.json_item['meta']['jabber:from']) + item_date = item_basic.get_item_date(item_id) + Username.save_item_correlation('jabber', to, item_id, item_date) + Username.save_item_correlation('jabber', fr, item_id, item_date) + return None \ No newline at end of file diff --git a/bin/lib/Correlate_object.py b/bin/lib/Correlate_object.py index 8bbc52e8..3d09b478 100755 --- a/bin/lib/Correlate_object.py +++ b/bin/lib/Correlate_object.py @@ -167,7 +167,11 @@ def get_correlation_node_icon(correlation_name, correlation_type=None, value=Non elif correlation_type == 'twitter': icon_class = 'fab' icon_text = '\uf099' + elif correlation_type == 'jabber': + icon_class = 'fa' + icon_text = '\uf007' else: + icon_class = 'fa' icon_text = '\uf007' elif correlation_name == 'decoded': diff --git a/bin/lib/Username.py b/bin/lib/Username.py index 0448a167..731ac904 100755 --- a/bin/lib/Username.py +++ b/bin/lib/Username.py @@ -15,7 +15,7 @@ config_loader = ConfigLoader.ConfigLoader() r_serv_crawler = config_loader.get_redis_conn("ARDB_Onion") config_loader = None -correlation = Correlation.Correlation('username', ['telegram', 'twitter']) +correlation = Correlation.Correlation('username', ['telegram', 'twitter', 'jabber']) def save_item_correlation(subtype, username, item_id, item_date): correlation.save_item_correlation(subtype, username, item_id, item_date) diff --git a/bin/lib/item_basic.py b/bin/lib/item_basic.py index 308dcb21..46c03a24 100755 --- a/bin/lib/item_basic.py +++ b/bin/lib/item_basic.py @@ -166,15 +166,17 @@ def add_item_parent(parent_item_id, item_id): #### UNKNOW SECTION #### def get_obj_id_item_id(parent_type, parent_id): - all_parents_type = ['twitter_id'] + all_parents_type = ['twitter_id', 'jabber_id'] if parent_type in all_parents_type: - return r_serv_metadata.hget('map:twitter_id:item_id', parent_id) + return r_serv_metadata.hget('map:{}:item_id'.format(parent_type), parent_id) else: return None def add_map_obj_id_item_id(obj_id, item_id, obj_type): if obj_type == 'twitter_id': r_serv_metadata.hset('map:twitter_id:item_id', obj_id, item_id) + if obj_type == 'jabber_id': + r_serv_metadata.hset('map:jabber_id:item_id', obj_id, item_id) # delete twitter id diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py index 73c5ca66..74d2d3e4 100644 --- a/var/www/modules/hashDecoded/Flask_hashDecoded.py +++ b/var/www/modules/hashDecoded/Flask_hashDecoded.py @@ -39,7 +39,7 @@ hashDecoded = Blueprint('hashDecoded', __name__, template_folder='templates') ## TODO: put me in option all_cryptocurrency = ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash'] all_pgpdump = ['key', 'name', 'mail'] -all_username = ['telegram', 'twitter'] +all_username = ['telegram', 'twitter', 'jabber'] # ============ FUNCTIONS ============ @@ -130,6 +130,8 @@ def get_icon(correlation_type, type_id): icon_text = 'fab fa-telegram-plane' elif type_id == 'twitter': icon_text = 'fab fa-twitter' + elif type_id == 'jabber': + icon_text = 'fa fa-user' return icon_text def get_icon_text(correlation_type, type_id): @@ -157,6 +159,8 @@ def get_icon_text(correlation_type, type_id): icon_text = '\uf2c6' elif type_id == 'twitter': icon_text = '\uf099' + elif type_id == 'jabber': + icon_text = '\uf007' return icon_text def get_all_types_id(correlation_type): diff --git a/var/www/templates/correlation/legend_graph_correlation.html b/var/www/templates/correlation/legend_graph_correlation.html index 6250ce10..f749dc69 100644 --- a/var/www/templates/correlation/legend_graph_correlation.html +++ b/var/www/templates/correlation/legend_graph_correlation.html @@ -1,6 +1,6 @@ - - - +
+ + @@ -27,179 +27,205 @@ - - - - - - - - -
Cryptocurrency:
-
- - - - - - - bitcoin -
-
- - - - - - - monero -
-
- - - - - - - etherum -
-
- - - - - - - other cryptocurrencies -
- 		-
- - - - - - - application -
-
- - - - - - - audio -
-
- - - - - - - text -
-
- - - - - - - other types of file -
- 		-
- - - - - - - screenshot -
- 		-
- - - - - - - key -
-
- - - - - - - name -
-
- - - - - - - mail -
- 		-
- - - - - - - telegram -
-
- - - - - - - twitter -
- 		-
- - - - - - - onion -
-
- - - - - - - web -
- 		-
- - - - - - crawled -
-
- - - - - - other -
-
+
+ + + + + + + bitcoin +
+
+ + + + + + + monero +
+
+ + + + + + + etherum +
+
+ + + + + + + other cryptocurrencies +
+ + +
+ + + + + + + application +
+
+ + + + + + + audio +
+
+ + + + + + + text +
+
+ + + + + + + other types of file +
+ + +
+ + + + + + + screenshot +
+ + +
+ + + + + + + key +
+
+ + + + + + + name +
+
+ + + + + + + mail +
+ + +
+ + + + + + + telegram +
+
+ + + + + + + twitter +
+
+ + + + + + + jabber +
+ + +
+ + + + + + + onion +
+
+ + + + + + + web +
+ + +
+ + + + + + crawled +
+
+ + + + + + other +
+ + + + \ No newline at end of file From e2a0ecda3b6ec7a33afd57c825b9f0bedf959c65 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thirion=20Aur=C3=A9lien?= Date: Fri, 4 Mar 2022 13:39:50 +0100 Subject: [PATCH 2/2] fix: [username] user icon --- var/www/modules/hashDecoded/Flask_hashDecoded.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py index 74d2d3e4..1470809b 100644 --- a/var/www/modules/hashDecoded/Flask_hashDecoded.py +++ b/var/www/modules/hashDecoded/Flask_hashDecoded.py @@ -131,7 +131,7 @@ def get_icon(correlation_type, type_id): elif type_id == 'twitter': icon_text = 'fab fa-twitter' elif type_id == 'jabber': - icon_text = 'fa fa-user' + icon_text = 'fas fa-user' return icon_text def get_icon_text(correlation_type, type_id):