AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-30 01:37:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #139 from gallypette/jabber-feeder

Browse source 
add: [username] jabber support
This commit is contained in:
Thirion Aurélien 2022-03-04 13:41:11 +01:00 committed by GitHub
commit 9199cfe6df
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 269 additions and 183 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

50
bin/import/ail_json_importer/Ail_feeder_jabber.py Executable file
View file

@ -0,0 +1,50 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The JSON Receiver Module
================
Receiver Jabber Json Items
"""
import os
import json
import sys
import time
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import item_basic
import Username
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer'))
from Default_json import Default_json
class Ail_feeder_jabber(Default_json):
"""Jabber Feeder functions"""
def __init__(self, name, json_item):
super().__init__(name, json_item)
def get_feeder_name(self):
return 'jabber'
# define item id
def get_item_id(self):
item_date = time.strptime(self.json_item['meta']['jabber:ts'], "%Y-%m-%dT%H:%M:%S.%f")
item_date_str = time.strftime("%Y/%m/%d", item_date)
item_id = str(self.json_item['meta']['jabber:id'])
return os.path.join('jabber', item_date_str, item_id) + '.gz'
def process_json_meta(self, process, item_id):
'''
Process JSON meta filed.
'''
jabber_id = str(self.json_item['meta']['jabber:id'])
item_basic.add_map_obj_id_item_id(jabber_id, item_id, 'jabber_id')
to = str(self.json_item['meta']['jabber:to'])
fr = str(self.json_item['meta']['jabber:from'])
item_date = item_basic.get_item_date(item_id)
Username.save_item_correlation('jabber', to, item_id, item_date)
Username.save_item_correlation('jabber', fr, item_id, item_date)
return None

4
bin/lib/Correlate_object.py
View file

@ -167,7 +167,11 @@ def get_correlation_node_icon(correlation_name, correlation_type=None, value=Non
elif correlation_type == 'twitter': elif correlation_type == 'twitter':
icon_class = 'fab' icon_class = 'fab'
icon_text = '\uf099' icon_text = '\uf099'
elif correlation_type == 'jabber':
icon_class = 'fa'
icon_text = '\uf007'
else: else:
icon_class = 'fa'
icon_text = '\uf007' icon_text = '\uf007'
elif correlation_name == 'decoded': elif correlation_name == 'decoded':

2
bin/lib/Username.py
View file

@ -15,7 +15,7 @@ config_loader = ConfigLoader.ConfigLoader()
r_serv_crawler = config_loader.get_redis_conn("ARDB_Onion") r_serv_crawler = config_loader.get_redis_conn("ARDB_Onion")
config_loader = None config_loader = None
correlation = Correlation.Correlation('username', ['telegram', 'twitter']) correlation = Correlation.Correlation('username', ['telegram', 'twitter', 'jabber'])
def save_item_correlation(subtype, username, item_id, item_date): def save_item_correlation(subtype, username, item_id, item_date):
correlation.save_item_correlation(subtype, username, item_id, item_date) correlation.save_item_correlation(subtype, username, item_id, item_date)

6
bin/lib/item_basic.py
View file

@ -166,15 +166,17 @@ def add_item_parent(parent_item_id, item_id):
#### UNKNOW SECTION #### #### UNKNOW SECTION ####
def get_obj_id_item_id(parent_type, parent_id): def get_obj_id_item_id(parent_type, parent_id):
all_parents_type = ['twitter_id'] all_parents_type = ['twitter_id', 'jabber_id']
if parent_type in all_parents_type: if parent_type in all_parents_type:
return r_serv_metadata.hget('map:twitter_id:item_id', parent_id) return r_serv_metadata.hget('map:{}:item_id'.format(parent_type), parent_id)
else: else:
return None return None
def add_map_obj_id_item_id(obj_id, item_id, obj_type): def add_map_obj_id_item_id(obj_id, item_id, obj_type):
if obj_type == 'twitter_id': if obj_type == 'twitter_id':
r_serv_metadata.hset('map:twitter_id:item_id', obj_id, item_id) r_serv_metadata.hset('map:twitter_id:item_id', obj_id, item_id)
if obj_type == 'jabber_id':
r_serv_metadata.hset('map:jabber_id:item_id', obj_id, item_id)
# delete twitter id # delete twitter id

6
var/www/modules/hashDecoded/Flask_hashDecoded.py
View file

@ -39,7 +39,7 @@ hashDecoded = Blueprint('hashDecoded', __name__, template_folder='templates')
## TODO: put me in option ## TODO: put me in option
all_cryptocurrency = ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash'] all_cryptocurrency = ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash']
all_pgpdump = ['key', 'name', 'mail'] all_pgpdump = ['key', 'name', 'mail']
all_username = ['telegram', 'twitter'] all_username = ['telegram', 'twitter', 'jabber']
# ============ FUNCTIONS ============ # ============ FUNCTIONS ============
@ -130,6 +130,8 @@ def get_icon(correlation_type, type_id):
icon_text = 'fab fa-telegram-plane' icon_text = 'fab fa-telegram-plane'
elif type_id == 'twitter': elif type_id == 'twitter':
icon_text = 'fab fa-twitter' icon_text = 'fab fa-twitter'
elif type_id == 'jabber':
icon_text = 'fas fa-user'
return icon_text return icon_text
def get_icon_text(correlation_type, type_id): def get_icon_text(correlation_type, type_id):
@ -157,6 +159,8 @@ def get_icon_text(correlation_type, type_id):
icon_text = '\uf2c6' icon_text = '\uf2c6'
elif type_id == 'twitter': elif type_id == 'twitter':
icon_text = '\uf099' icon_text = '\uf099'
elif type_id == 'jabber':
icon_text = '\uf007'
return icon_text return icon_text
def get_all_types_id(correlation_type): def get_all_types_id(correlation_type):

384
var/www/templates/correlation/legend_graph_correlation.html
View file

@ -1,6 +1,6 @@
<table class="table table-bordered"> <table class="table table-bordered">
<thead> <thead>
<tr class="table-info""> <tr class="table-info"">
<th> <th>
Cryptocurrency: Cryptocurrency:
</th> </th>
@ -27,179 +27,205 @@
<tbody> <tbody>
<tr> <tr>
<td> <td>
<div class="my-1"> <div class=" my-1">
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle> <circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf15a;</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab"
</g> font-size="16px">&#xf15a;</text>
</svg> </g>
bitcoin </svg>
</div> bitcoin
<div class="my-1"> </div>
<svg height="26" width="26"> <div class="my-1">
<g class="nodes"> <svg height="26" width="26">
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle> <g class="nodes">
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf3d0;</text> <circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
</g> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab"
</svg> font-size="16px">&#xf3d0;</text>
monero </g>
</div> </svg>
<div class="my-1"> monero
<svg height="26" width="26"> </div>
<g class="nodes"> <div class="my-1">
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle> <svg height="26" width="26">
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf42e;</text> <g class="nodes">
</g> <circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
</svg> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab"
etherum font-size="16px">&#xf42e;</text>
</div> </g>
<div> </svg>
<svg height="26" width="26"> etherum
<g class="nodes"> </div>
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle> <div>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf51e;</text> <svg height="26" width="26">
</g> <g class="nodes">
</svg> <circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
other cryptocurrencies <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas"
</div> font-size="16px">&#xf51e;</text>
</td> </g>
<td> </svg>
<div class="my-1"> other cryptocurrencies
<svg height="26" width="26"> </div>
<g class="nodes"> </td>
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle> <td>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf15b;</text> <div class="my-1">
</g> <svg height="26" width="26">
</svg> <g class="nodes">
application <circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
</div> <text x="13" y="13" text-anchor="middle" dominant-baseline="central"
<div class="my-1"> class="graph_node_icon fas" font-size="16px">&#xf15b;</text>
<svg height="26" width="26"> </g>
<g class="nodes"> </svg>
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle> application
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf1c7;</text> </div>
</g> <div class="my-1">
</svg> <svg height="26" width="26">
audio <g class="nodes">
</div> <circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
<div class="my-1"> <text x="13" y="13" text-anchor="middle" dominant-baseline="central"
<svg height="26" width="26"> class="graph_node_icon fas" font-size="16px">&#xf1c7;</text>
<g class="nodes"> </g>
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle> </svg>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf15c;</text> audio
</g> </div>
</svg> <div class="my-1">
text <svg height="26" width="26">
</div> <g class="nodes">
<div> <circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
<svg height="26" width="26"> <text x="13" y="13" text-anchor="middle" dominant-baseline="central"
<g class="nodes"> class="graph_node_icon fas" font-size="16px">&#xf15c;</text>
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle> </g>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf249;</text> </svg>
</g> text
</svg> </div>
other types of file <div>
</div> <svg height="26" width="26">
</td> <g class="nodes">
<td> <circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
<div class="my-1"> <text x="13" y="13" text-anchor="middle" dominant-baseline="central"
<svg height="26" width="26"> class="graph_node_icon fas" font-size="16px">&#xf249;</text>
<g class="nodes"> </g>
<circle cx="13" cy="13" r="13" fill="#E1F5DF"></circle> </svg>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf03e;</text> other types of file
</g> </div>
</svg> </td>
screenshot <td>
</div> <div class="my-1">
</td> <svg height="26" width="26">
<td> <g class="nodes">
<div class="my-1"> <circle cx="13" cy="13" r="13" fill="#E1F5DF"></circle>
<svg height="26" width="26"> <text x="13" y="13" text-anchor="middle" dominant-baseline="central"
<g class="nodes"> class="graph_node_icon fas" font-size="16px">&#xf03e;</text>
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle> </g>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf084;</text> </svg>
</g> screenshot
</svg> </div>
key </td>
</div> <td>
<div class="my-1"> <div class="my-1">
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle> <circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf507;</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central"
</g> class="graph_node_icon fas" font-size="16px">&#xf084;</text>
</svg> </g>
name </svg>
</div> key
<div> </div>
<svg height="26" width="26"> <div class="my-1">
<g class="nodes"> <svg height="26" width="26">
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle> <g class="nodes">
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf1fa;</text> <circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
</g> <text x="13" y="13" text-anchor="middle" dominant-baseline="central"
</svg> class="graph_node_icon fas" font-size="16px">&#xf507;</text>
mail </g>
</div> </svg>
</td> name
<td> </div>
<div class="my-1"> <div>
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="#4dffff"></circle> <circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf2c6;</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central"
</g> class="graph_node_icon fas" font-size="16px">&#xf1fa;</text>
</svg> </g>
telegram </svg>
</div> mail
<div class="my-1"> </div>
<svg height="26" width="26"> </td>
<g class="nodes"> <td>
<circle cx="13" cy="13" r="13" fill="#4dffff"></circle> <div class="my-1">
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf099;</text> <svg height="26" width="26">
</g> <g class="nodes">
</svg> <circle cx="13" cy="13" r="13" fill="#4dffff"></circle>
twitter <text x="13" y="13" text-anchor="middle" dominant-baseline="central"
</div> class="graph_node_icon fab" font-size="16px">&#xf2c6;</text>
</td> </g>
<td> </svg>
<div class="my-1"> telegram
<svg height="26" width="26"> </div>
<g class="nodes"> <div class="my-1">
<circle cx="13" cy="13" r="13" fill="#3DA760"></circle> <svg height="26" width="26">
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px">&#xf06e;</text> <g class="nodes">
</g> <circle cx="13" cy="13" r="13" fill="#4dffff"></circle>
</svg> <text x="13" y="13" text-anchor="middle" dominant-baseline="central"
onion class="graph_node_icon fab" font-size="16px">&#xf099;</text>
</div> </g>
<div> </svg>
<svg height="26" width="26"> twitter
<g class="nodes"> </div>
<circle cx="13" cy="13" r="13" fill="#3DA760"></circle> <div class="my-1">
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px">&#xf13b;</text> <svg height="26" width="26">
</g> <g class="nodes">
</svg> <circle cx="13" cy="13" r="13" fill="#4dffff"></circle>
web <text x="13" y="13" text-anchor="middle" dominant-baseline="central"
</div> class="graph_node_icon fa" font-size="16px">&#xf007;</text>
</td> </g>
<td> </svg>
<div class="my-1"> jabber
<svg height="26" width="26"> </div>
<g class="nodes"> </td>
<circle cx="13" cy="13" r="13" fill="red"></circle> <td>
</g> <div class="my-1">
</svg> <svg height="26" width="26">
crawled <g class="nodes">
</div> <circle cx="13" cy="13" r="13" fill="#3DA760"></circle>
<div> <text x="13" y="13" text-anchor="middle" dominant-baseline="central"
<svg height="26" width="26"> class="graph_node_icon fas" font-size="16px">&#xf06e;</text>
<g class="nodes"> </g>
<circle cx="13" cy="13" r="13" fill="#332288"></circle> </svg>
</g> onion
</svg> </div>
other <div>
</div> <svg height="26" width="26">
</td> <g class="nodes">
</tr> <circle cx="13" cy="13" r="13" fill="#3DA760"></circle>
</tbody> <text x="13" y="13" text-anchor="middle" dominant-baseline="central"
</table> class="graph_node_icon fab" font-size="16px">&#xf13b;</text>
</g>
</svg>
web
</div>
</td>
<td>
<div class="my-1">
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="red"></circle>
</g>
</svg>
crawled
</div>
<div>
<svg height="26" width="26">
<g class="nodes">
<circle cx="13" cy="13" r="13" fill="#332288"></circle>
</g>
</svg>
other
</div>
</td>
</tr>
</tbody>
</table>