mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-29 17:27:17 +00:00
Merge branch 'master' of github.com:ail-project/ail-framework
This commit is contained in:
Terrtia
commit
7e6577961b
7 changed files with 270 additions and 184 deletions
50
bin/import/ail_json_importer/Ail_feeder_jabber.py
Executable file
50
bin/import/ail_json_importer/Ail_feeder_jabber.py
Executable file
|
|
@ -0,0 +1,50 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
The JSON Receiver Module
|
||||
================
|
||||
|
||||
Receiver Jabber Json Items
|
||||
|
||||
"""
|
||||
import os
|
||||
import json
|
||||
import sys
|
||||
import time
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||
import item_basic
|
||||
import Username
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer'))
|
||||
from Default_json import Default_json
|
||||
|
||||
class Ail_feeder_jabber(Default_json):
|
||||
"""Jabber Feeder functions"""
|
||||
|
||||
def __init__(self, name, json_item):
|
||||
super().__init__(name, json_item)
|
||||
|
||||
def get_feeder_name(self):
|
||||
return 'jabber'
|
||||
|
||||
# define item id
|
||||
def get_item_id(self):
|
||||
item_date = time.strptime(self.json_item['meta']['jabber:ts'], "%Y-%m-%dT%H:%M:%S.%f")
|
||||
item_date_str = time.strftime("%Y/%m/%d", item_date)
|
||||
item_id = str(self.json_item['meta']['jabber:id'])
|
||||
return os.path.join('jabber', item_date_str, item_id) + '.gz'
|
||||
|
||||
def process_json_meta(self, process, item_id):
|
||||
'''
|
||||
Process JSON meta filed.
|
||||
'''
|
||||
jabber_id = str(self.json_item['meta']['jabber:id'])
|
||||
item_basic.add_map_obj_id_item_id(jabber_id, item_id, 'jabber_id')
|
||||
to = str(self.json_item['meta']['jabber:to'])
|
||||
fr = str(self.json_item['meta']['jabber:from'])
|
||||
item_date = item_basic.get_item_date(item_id)
|
||||
Username.save_item_correlation('jabber', to, item_id, item_date)
|
||||
Username.save_item_correlation('jabber', fr, item_id, item_date)
|
||||
return None
|
||||
|
|
@ -167,7 +167,11 @@ def get_correlation_node_icon(correlation_name, correlation_type=None, value=Non
|
|||
elif correlation_type == 'twitter':
|
||||
icon_class = 'fab'
|
||||
icon_text = '\uf099'
|
||||
elif correlation_type == 'jabber':
|
||||
icon_class = 'fa'
|
||||
icon_text = '\uf007'
|
||||
else:
|
||||
icon_class = 'fa'
|
||||
icon_text = '\uf007'
|
||||
|
||||
elif correlation_name == 'decoded':
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ config_loader = ConfigLoader.ConfigLoader()
|
|||
r_serv_crawler = config_loader.get_redis_conn("ARDB_Onion")
|
||||
config_loader = None
|
||||
|
||||
correlation = Correlation.Correlation('username', ['telegram', 'twitter'])
|
||||
correlation = Correlation.Correlation('username', ['telegram', 'twitter', 'jabber'])
|
||||
|
||||
def save_item_correlation(subtype, username, item_id, item_date):
|
||||
correlation.save_item_correlation(subtype, username, item_id, item_date)
|
||||
|
|
|
|||
|
|
@ -166,15 +166,17 @@ def add_item_parent(parent_item_id, item_id):
|
|||
#### UNKNOW SECTION ####
|
||||
|
||||
def get_obj_id_item_id(parent_type, parent_id):
|
||||
all_parents_type = ['twitter_id']
|
||||
all_parents_type = ['twitter_id', 'jabber_id']
|
||||
if parent_type in all_parents_type:
|
||||
return r_serv_metadata.hget('map:twitter_id:item_id', parent_id)
|
||||
return r_serv_metadata.hget('map:{}:item_id'.format(parent_type), parent_id)
|
||||
else:
|
||||
return None
|
||||
|
||||
def add_map_obj_id_item_id(obj_id, item_id, obj_type):
|
||||
if obj_type == 'twitter_id':
|
||||
r_serv_metadata.hset('map:twitter_id:item_id', obj_id, item_id)
|
||||
if obj_type == 'jabber_id':
|
||||
r_serv_metadata.hset('map:jabber_id:item_id', obj_id, item_id)
|
||||
|
||||
# delete twitter id
|
||||
|
||||
|
|
|
|||
|
|
@ -66,7 +66,7 @@ pylibinjection>=0.2.4
|
|||
phonenumbers>8.12.1
|
||||
|
||||
# Web
|
||||
flask>1.1.2,<=1.1.4
|
||||
flask>=1.1.4
|
||||
flask-login
|
||||
bcrypt>3.1.6
|
||||
|
||||
|
|
|
|||
|
|
@ -39,7 +39,7 @@ hashDecoded = Blueprint('hashDecoded', __name__, template_folder='templates')
|
|||
## TODO: put me in option
|
||||
all_cryptocurrency = ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash']
|
||||
all_pgpdump = ['key', 'name', 'mail']
|
||||
all_username = ['telegram', 'twitter']
|
||||
all_username = ['telegram', 'twitter', 'jabber']
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
|
|
@ -130,6 +130,8 @@ def get_icon(correlation_type, type_id):
|
|||
icon_text = 'fab fa-telegram-plane'
|
||||
elif type_id == 'twitter':
|
||||
icon_text = 'fab fa-twitter'
|
||||
elif type_id == 'jabber':
|
||||
icon_text = 'fas fa-user'
|
||||
return icon_text
|
||||
|
||||
def get_icon_text(correlation_type, type_id):
|
||||
|
|
@ -157,6 +159,8 @@ def get_icon_text(correlation_type, type_id):
|
|||
icon_text = '\uf2c6'
|
||||
elif type_id == 'twitter':
|
||||
icon_text = '\uf099'
|
||||
elif type_id == 'jabber':
|
||||
icon_text = '\uf007'
|
||||
return icon_text
|
||||
|
||||
def get_all_types_id(correlation_type):
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
<table class="table table-bordered">
|
||||
<thead>
|
||||
<tr class="table-info"">
|
||||
<table class="table table-bordered">
|
||||
<thead>
|
||||
<tr class="table-info"">
|
||||
<th>
|
||||
Cryptocurrency:
|
||||
</th>
|
||||
|
|
@ -27,179 +27,205 @@
|
|||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
bitcoin
|
||||
</div>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
monero
|
||||
</div>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
etherum
|
||||
</div>
|
||||
<div>
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
other cryptocurrencies
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
application
|
||||
</div>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
audio
|
||||
</div>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
text
|
||||
</div>
|
||||
<div>
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
other types of file
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#E1F5DF"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
screenshot
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
key
|
||||
</div>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
name
|
||||
</div>
|
||||
<div>
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
mail
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#4dffff"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
telegram
|
||||
</div>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#4dffff"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
twitter
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#3DA760"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
onion
|
||||
</div>
|
||||
<div>
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#3DA760"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
web
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="red"></circle>
|
||||
</g>
|
||||
</svg>
|
||||
crawled
|
||||
</div>
|
||||
<div>
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#332288"></circle>
|
||||
</g>
|
||||
</svg>
|
||||
other
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
<div class=" my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab"
|
||||
font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
bitcoin
|
||||
</div>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab"
|
||||
font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
monero
|
||||
</div>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fab"
|
||||
font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
etherum
|
||||
</div>
|
||||
<div>
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#DDCC77"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon fas"
|
||||
font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
other cryptocurrencies
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
|
||||
class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
application
|
||||
</div>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
|
||||
class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
audio
|
||||
</div>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
|
||||
class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
text
|
||||
</div>
|
||||
<div>
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#88CCEE"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
|
||||
class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
other types of file
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#E1F5DF"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
|
||||
class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
screenshot
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
|
||||
class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
key
|
||||
</div>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
|
||||
class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
name
|
||||
</div>
|
||||
<div>
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#44AA99"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
|
||||
class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
mail
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#4dffff"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
|
||||
class="graph_node_icon fab" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
telegram
|
||||
</div>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#4dffff"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
|
||||
class="graph_node_icon fab" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
twitter
|
||||
</div>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#4dffff"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
|
||||
class="graph_node_icon fa" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
jabber
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#3DA760"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
|
||||
class="graph_node_icon fas" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
onion
|
||||
</div>
|
||||
<div>
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#3DA760"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central"
|
||||
class="graph_node_icon fab" font-size="16px"></text>
|
||||
</g>
|
||||
</svg>
|
||||
web
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<div class="my-1">
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="red"></circle>
|
||||
</g>
|
||||
</svg>
|
||||
crawled
|
||||
</div>
|
||||
<div>
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="#332288"></circle>
|
||||
</g>
|
||||
</svg>
|
||||
other
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
Loading…
Reference in a new issue