Merge pull request #139 from gallypette/jabber-feeder

add: [username] jabber support

bin/import/ail_json_importer/Ail_feeder_jabber.py

@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+"""
+The JSON Receiver Module
+================
+
+Receiver Jabber Json Items
+
+"""
+import os
+import json
+import sys
+import time
+import datetime
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
+import item_basic
+import Username
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer'))
+from Default_json import Default_json
+
+class Ail_feeder_jabber(Default_json):
+    """Jabber Feeder functions"""
+
+    def __init__(self, name, json_item):
+        super().__init__(name, json_item)
+
+    def get_feeder_name(self):
+        return 'jabber'
+
+    # define item id
+    def get_item_id(self):
+        item_date = time.strptime(self.json_item['meta']['jabber:ts'], "%Y-%m-%dT%H:%M:%S.%f")
+        item_date_str = time.strftime("%Y/%m/%d", item_date)
+        item_id = str(self.json_item['meta']['jabber:id'])
+        return os.path.join('jabber', item_date_str, item_id) + '.gz'
+
+    def process_json_meta(self, process, item_id):
+        '''
+        Process JSON meta filed.
+        '''
+        jabber_id = str(self.json_item['meta']['jabber:id'])
+        item_basic.add_map_obj_id_item_id(jabber_id, item_id, 'jabber_id')
+        to = str(self.json_item['meta']['jabber:to'])
+        fr = str(self.json_item['meta']['jabber:from'])
+        item_date = item_basic.get_item_date(item_id)
+        Username.save_item_correlation('jabber', to, item_id, item_date)
+        Username.save_item_correlation('jabber', fr, item_id, item_date)
+        return None

bin/lib/Correlate_object.py

@@ -167,7 +167,11 @@ def get_correlation_node_icon(correlation_name, correlation_type=None, value=Non
         elif correlation_type == 'twitter':
             icon_class = 'fab'
             icon_text = '\uf099'
+        elif correlation_type == 'jabber':
+            icon_class = 'fa'
+            icon_text = '\uf007'
         else:
+            icon_class = 'fa'
             icon_text = '\uf007'
 
     elif correlation_name == 'decoded':

bin/lib/Username.py

@@ -15,7 +15,7 @@ config_loader = ConfigLoader.ConfigLoader()
 r_serv_crawler = config_loader.get_redis_conn("ARDB_Onion")
 config_loader = None
 
-correlation = Correlation.Correlation('username', ['telegram', 'twitter'])
+correlation = Correlation.Correlation('username', ['telegram', 'twitter', 'jabber'])
 
 def save_item_correlation(subtype, username, item_id, item_date):
     correlation.save_item_correlation(subtype, username, item_id, item_date)

bin/lib/item_basic.py

@@ -166,15 +166,17 @@ def add_item_parent(parent_item_id, item_id):
 #### UNKNOW SECTION ####
 
 def get_obj_id_item_id(parent_type, parent_id):
-    all_parents_type = ['twitter_id']
+    all_parents_type = ['twitter_id', 'jabber_id']
     if parent_type in all_parents_type:
-        return r_serv_metadata.hget('map:twitter_id:item_id', parent_id)
+        return r_serv_metadata.hget('map:{}:item_id'.format(parent_type), parent_id)
     else:
         return None
 
 def add_map_obj_id_item_id(obj_id, item_id, obj_type):
     if obj_type == 'twitter_id':
         r_serv_metadata.hset('map:twitter_id:item_id', obj_id, item_id)
+    if obj_type == 'jabber_id':
+        r_serv_metadata.hset('map:jabber_id:item_id', obj_id, item_id)
 
 # delete twitter id
 

var/www/modules/hashDecoded/Flask_hashDecoded.py

@@ -39,7 +39,7 @@ hashDecoded = Blueprint('hashDecoded', __name__, template_folder='templates')
 ## TODO: put me in option
 all_cryptocurrency = ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash']
 all_pgpdump = ['key', 'name', 'mail']
-all_username = ['telegram', 'twitter']
+all_username = ['telegram', 'twitter', 'jabber']
 
 # ============ FUNCTIONS ============
 
@@ -130,6 +130,8 @@ def get_icon(correlation_type, type_id):
             icon_text = 'fab fa-telegram-plane'
         elif type_id == 'twitter':
             icon_text = 'fab fa-twitter'
+        elif type_id == 'jabber':
+            icon_text = 'fas fa-user'
     return icon_text
 
 def get_icon_text(correlation_type, type_id):
@@ -157,6 +159,8 @@ def get_icon_text(correlation_type, type_id):
             icon_text = '\uf2c6'
         elif type_id == 'twitter':
             icon_text = '\uf099'
+        elif type_id == 'jabber':
+            icon_text = '\uf007'
     return icon_text
 
 def get_all_types_id(correlation_type):

var/www/templates/correlation/legend_graph_correlation.html

@@ -31,7 +31,8 @@
 			<svg height="26" width="26">
 				<g class="nodes">
 					<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
-														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf15a;</text>
+					<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab"
+						font-size="16px">&#xf15a;</text>
 				</g>
 			</svg>
 			bitcoin
@@ -40,7 +41,8 @@
 				<svg height="26" width="26">
 					<g class="nodes">
 						<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
-														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf3d0;</text>
+						<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab"
+							font-size="16px">&#xf3d0;</text>
 					</g>
 				</svg>
 				monero
@@ -49,7 +51,8 @@
 				<svg height="26" width="26">
 					<g class="nodes">
 						<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
-														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf42e;</text>
+						<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab"
+							font-size="16px">&#xf42e;</text>
 					</g>
 				</svg>
 				etherum
@@ -58,7 +61,8 @@
 				<svg height="26" width="26">
 					<g class="nodes">
 						<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
-														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf51e;</text>
+						<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas"
+							font-size="16px">&#xf51e;</text>
 					</g>
 				</svg>
 				other cryptocurrencies
@@ -69,7 +73,8 @@
 					<svg height="26" width="26">
 						<g class="nodes">
 							<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
-														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf15b;</text>
+							<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
+								class="graph_node_icon fas" font-size="16px">&#xf15b;</text>
 						</g>
 					</svg>
 					application
@@ -78,7 +83,8 @@
 					<svg height="26" width="26">
 						<g class="nodes">
 							<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
-														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf1c7;</text>
+							<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
+								class="graph_node_icon fas" font-size="16px">&#xf1c7;</text>
 						</g>
 					</svg>
 					audio
@@ -87,7 +93,8 @@
 					<svg height="26" width="26">
 						<g class="nodes">
 							<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
-														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf15c;</text>
+							<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
+								class="graph_node_icon fas" font-size="16px">&#xf15c;</text>
 						</g>
 					</svg>
 					text
@@ -96,7 +103,8 @@
 					<svg height="26" width="26">
 						<g class="nodes">
 							<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
-														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf249;</text>
+							<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
+								class="graph_node_icon fas" font-size="16px">&#xf249;</text>
 						</g>
 					</svg>
 					other types of file
@@ -107,7 +115,8 @@
 					<svg height="26" width="26">
 						<g class="nodes">
 							<circle cx="13" cy="13" r="13" fill="#E1F5DF"></circle>
-														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf03e;</text>
+							<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
+								class="graph_node_icon fas" font-size="16px">&#xf03e;</text>
 						</g>
 					</svg>
 					screenshot
@@ -118,7 +127,8 @@
 					<svg height="26" width="26">
 						<g class="nodes">
 							<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
-														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf084;</text>
+							<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
+								class="graph_node_icon fas" font-size="16px">&#xf084;</text>
 						</g>
 					</svg>
 					key
@@ -127,7 +137,8 @@
 					<svg height="26" width="26">
 						<g class="nodes">
 							<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
-														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf507;</text>
+							<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
+								class="graph_node_icon fas" font-size="16px">&#xf507;</text>
 						</g>
 					</svg>
 					name
@@ -136,7 +147,8 @@
 					<svg height="26" width="26">
 						<g class="nodes">
 							<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
-														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf1fa;</text>
+							<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
+								class="graph_node_icon fas" font-size="16px">&#xf1fa;</text>
 						</g>
 					</svg>
 					mail
@@ -147,7 +159,8 @@
 					<svg height="26" width="26">
 						<g class="nodes">
 							<circle cx="13" cy="13" r="13" fill="#4dffff"></circle>
-														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf2c6;</text>
+							<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
+								class="graph_node_icon fab" font-size="16px">&#xf2c6;</text>
 						</g>
 					</svg>
 					telegram
@@ -156,18 +169,30 @@
 					<svg height="26" width="26">
 						<g class="nodes">
 							<circle cx="13" cy="13" r="13" fill="#4dffff"></circle>
-														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf099;</text>
+							<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
+								class="graph_node_icon fab" font-size="16px">&#xf099;</text>
 						</g>
 					</svg>
 					twitter
 				</div>
+				<div class="my-1">
+					<svg height="26" width="26">
+						<g class="nodes">
+							<circle cx="13" cy="13" r="13" fill="#4dffff"></circle>
+							<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
+								class="graph_node_icon fa" font-size="16px">&#xf007;</text>
+						</g>
+					</svg>
+					jabber
+				</div>
 			</td>
 			<td>
 				<div class="my-1">
 					<svg height="26" width="26">
 						<g class="nodes">
 							<circle cx="13" cy="13" r="13" fill="#3DA760"></circle>
-														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf06e;</text>
+							<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
+								class="graph_node_icon fas" font-size="16px">&#xf06e;</text>
 						</g>
 					</svg>
 					onion
@@ -176,7 +201,8 @@
 					<svg height="26" width="26">
 						<g class="nodes">
 							<circle cx="13" cy="13" r="13" fill="#3DA760"></circle>
-														<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf13b;</text>
+							<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
+								class="graph_node_icon fab" font-size="16px">&#xf13b;</text>
 						</g>
 					</svg>
 					web