mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-09-20 07:58:29 +00:00
chg: [update + show decoded items] add background update
This commit is contained in:
Terrtia
parent
6b9ba9d377
commit
880c351c0c
11 changed files with 288 additions and 8 deletions
|
|
@ -17,6 +17,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
|||
import Correlation
|
||||
from Cryptocurrency import cryptocurrency
|
||||
from Pgp import pgp
|
||||
import Decoded
|
||||
import Item
|
||||
import Tag
|
||||
|
||||
|
|
@ -197,6 +198,14 @@ def get_domain_pgp(domain, currencies_type=None, get_nb=False):
|
|||
'''
|
||||
return pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb)
|
||||
|
||||
def get_domain_decoded(domain):
|
||||
'''
|
||||
Retun all decoded item of a given domain.
|
||||
|
||||
:param domain: crawled domain
|
||||
'''
|
||||
return Decoded.get_domain_decoded_item(domain)
|
||||
|
||||
def get_domain_all_correlation(domain, correlation_type=None, get_nb=False):
|
||||
'''
|
||||
Retun all correlation of a given domain.
|
||||
|
|
@ -214,6 +223,9 @@ def get_domain_all_correlation(domain, correlation_type=None, get_nb=False):
|
|||
res = get_domain_pgp(domain, get_nb=get_nb)
|
||||
if res:
|
||||
domain_correl['pgp'] = res
|
||||
res = get_domain_decoded(domain)
|
||||
if res:
|
||||
domain_correl['decoded'] = res
|
||||
return domain_correl
|
||||
|
||||
# TODO: handle port
|
||||
|
|
@ -271,6 +283,12 @@ class Domain(object):
|
|||
if self.is_domain_up():
|
||||
self.current_port = sanathyse_port(port, self.domain, self.type)
|
||||
|
||||
def get_domain_name(self):
|
||||
return self.domain
|
||||
|
||||
def get_domain_type(self):
|
||||
return self.type
|
||||
|
||||
def get_current_port(self):
|
||||
return self.current_port
|
||||
|
||||
|
|
@ -361,10 +379,16 @@ class Domain(object):
|
|||
'''
|
||||
return get_domain_all_correlation(self.domain, get_nb=True)
|
||||
|
||||
def get_domain_history_with_status(self):
|
||||
def get_domain_history(self):
|
||||
'''
|
||||
Retun the full history of a given domain and port.
|
||||
'''
|
||||
return get_domain_history(self.domain, self.type, 80)
|
||||
|
||||
def get_domain_history_with_status(self):
|
||||
'''
|
||||
Retun the full history (with status) of a given domain and port.
|
||||
'''
|
||||
return get_domain_history_with_status(self.domain, self.type, 80)
|
||||
|
||||
def get_domain_items_crawled(self, port=None, epoch=None, items_link=False, item_screenshot=False, item_tag=False):
|
||||
|
|
|
|||
|
|
@ -147,6 +147,50 @@ class Correlation(object):
|
|||
dict_correlation['nb'] = dict_correlation.get('nb', 0) + len(dict_correlation[correl])
|
||||
return dict_correlation
|
||||
|
||||
def _get_item_correlation_obj(self, item_id, correlation_type):
|
||||
'''
|
||||
Return correlation of a given item id.
|
||||
|
||||
:param item_id: item id
|
||||
:type item_id: str
|
||||
:param correlation_type: correlation type
|
||||
:type correlation_type: str
|
||||
|
||||
:return: a list of correlation
|
||||
:rtype: list
|
||||
'''
|
||||
res = r_serv_metadata.smembers('item_{}_{}:{}'.format(self.correlation_name, correlation_type, item_id))
|
||||
if res:
|
||||
return list(res)
|
||||
else:
|
||||
return []
|
||||
|
||||
def get_item_correlation_dict(self, item_id, correlation_type=None, get_nb=False):
|
||||
'''
|
||||
Return all correlation of a given item id.
|
||||
|
||||
:param item_id: item id
|
||||
:param correlation_type: list of correlation types
|
||||
:type correlation_type: list, optional
|
||||
|
||||
:return: a dictionnary of all the requested correlations
|
||||
:rtype: dict
|
||||
'''
|
||||
correlation_type = self.sanythise_correlation_types(correlation_type)
|
||||
dict_correlation = {}
|
||||
for correl in correlation_type:
|
||||
res = self._get_item_correlation_obj(item_id, correl)
|
||||
if res:
|
||||
dict_correlation[correl] = res
|
||||
if get_nb:
|
||||
dict_correlation['nb'] = dict_correlation.get('nb', 0) + len(dict_correlation[correl])
|
||||
return dict_correlation
|
||||
|
||||
|
||||
|
||||
def save_domain_correlation(self, domain, correlation_type, correlation_value):
|
||||
r_serv_metadata.sadd('domain_{}_{}:{}'.format(self.correlation_name, correlation_type, domain), correlation_value)
|
||||
r_serv_metadata.sadd('set_domain_{}_{}:{}'.format(self.correlation_name, correlation_type, correlation_value), domain)
|
||||
|
||||
######## API EXPOSED ########
|
||||
|
||||
|
|
|
|||
|
|
@ -61,6 +61,7 @@ def get_cryptocurrency(request_dict, cryptocurrency_type):
|
|||
|
||||
return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)
|
||||
|
||||
# # TODO: refractor/move me in Correlation
|
||||
def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrency_address):
|
||||
# create basic medata
|
||||
if not r_serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)):
|
||||
|
|
@ -89,7 +90,7 @@ def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrenc
|
|||
r_serv_metadata.sadd('item_cryptocurrency_{}:{}'.format(cryptocurrency_name, item_path), cryptocurrency_address)
|
||||
|
||||
# domain
|
||||
if Item.is_crawled(item_path):
|
||||
if Item.is_crawled(item_path): # # TODO: use save_domain_correlation
|
||||
domain = Item.get_item_domain(item_path)
|
||||
r_serv_metadata.sadd('domain_cryptocurrency_{}:{}'.format(cryptocurrency_name, domain), cryptocurrency_address)
|
||||
r_serv_metadata.sadd('set_domain_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), domain)
|
||||
|
|
|
|||
|
|
@ -8,10 +8,13 @@ import redis
|
|||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
import Decoded
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
import Date
|
||||
import Tag
|
||||
from Cryptocurrency import cryptocurrency
|
||||
from Pgp import pgp
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
|
||||
|
|
@ -126,7 +129,61 @@ def get_item(request_dict):
|
|||
###
|
||||
### correlation
|
||||
###
|
||||
def get_item_cryptocurrency(item_id, currencies_type=None, get_nb=False):
|
||||
'''
|
||||
Return all cryptocurrencies of a given item.
|
||||
|
||||
:param item_id: item id
|
||||
:param currencies_type: list of cryptocurrencies type
|
||||
:type currencies_type: list, optional
|
||||
'''
|
||||
return cryptocurrency.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb)
|
||||
|
||||
def get_item_pgp(item_id, currencies_type=None, get_nb=False):
|
||||
'''
|
||||
Return all pgp of a given item.
|
||||
|
||||
:param item_id: item id
|
||||
:param currencies_type: list of cryptocurrencies type
|
||||
:type currencies_type: list, optional
|
||||
'''
|
||||
return pgp.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb)
|
||||
|
||||
def get_item_decoded(item_id):
|
||||
'''
|
||||
Return all pgp of a given item.
|
||||
|
||||
:param item_id: item id
|
||||
:param currencies_type: list of cryptocurrencies type
|
||||
:type currencies_type: list, optional
|
||||
'''
|
||||
return Decoded.get_item_decoded(item_id)
|
||||
|
||||
def get_item_all_correlation(item_id, correlation_type=None, get_nb=False):
|
||||
'''
|
||||
Retun all correlation of a given item id.
|
||||
|
||||
:param item_id: item id
|
||||
:type domain: str
|
||||
|
||||
:return: a dict of all correlation for a item id
|
||||
:rtype: dict
|
||||
'''
|
||||
item_correl = {}
|
||||
res = get_item_cryptocurrency(item_id, get_nb=get_nb)
|
||||
if res:
|
||||
item_correl['cryptocurrency'] = res
|
||||
res = get_item_pgp(item_id, get_nb=get_nb)
|
||||
if res:
|
||||
item_correl['pgp'] = res
|
||||
res = get_item_decoded(item_id)
|
||||
if res:
|
||||
item_correl['decoded'] = res
|
||||
return item_correl
|
||||
|
||||
|
||||
|
||||
## TODO: REFRACTOR
|
||||
def _get_item_correlation(correlation_name, correlation_type, item_id):
|
||||
res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
|
||||
if res:
|
||||
|
|
@ -134,18 +191,23 @@ def _get_item_correlation(correlation_name, correlation_type, item_id):
|
|||
else:
|
||||
return []
|
||||
|
||||
## TODO: REFRACTOR
|
||||
def get_item_bitcoin(item_id):
|
||||
return _get_item_correlation('cryptocurrency', 'bitcoin', item_id)
|
||||
|
||||
## TODO: REFRACTOR
|
||||
def get_item_pgp_key(item_id):
|
||||
return _get_item_correlation('pgpdump', 'key', item_id)
|
||||
|
||||
## TODO: REFRACTOR
|
||||
def get_item_pgp_name(item_id):
|
||||
return _get_item_correlation('pgpdump', 'name', item_id)
|
||||
|
||||
## TODO: REFRACTOR
|
||||
def get_item_pgp_mail(item_id):
|
||||
return _get_item_correlation('pgpdump', 'mail', item_id)
|
||||
|
||||
## TODO: REFRACTOR
|
||||
def get_item_pgp_correlation(item_id):
|
||||
pass
|
||||
|
||||
|
|
|
|||
|
|
@ -172,8 +172,11 @@ def add_items_tags(tags=[], galaxy_tags=[], item_id=None, item_type="paste"):
|
|||
res_dict['type'] = item_type
|
||||
return (res_dict, 200)
|
||||
|
||||
def add_domain_tag(tag, domain, item_date):
|
||||
r_serv_metadata.sadd('tag:{}'.format(domain), tag)
|
||||
r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), domain)
|
||||
|
||||
def add_item_tag(tag, item_path, item_type="paste"):
|
||||
def add_item_tag(tag, item_path, item_type="paste", tag_date=None):
|
||||
|
||||
if item_type=="paste":
|
||||
item_date = int(Item.get_item_date(item_path))
|
||||
|
|
@ -189,8 +192,7 @@ def add_item_tag(tag, item_path, item_type="paste"):
|
|||
# domain item
|
||||
else:
|
||||
item_date = int(Domain.get_domain_last_check(item_path, r_format="int"))
|
||||
r_serv_metadata.sadd('tag:{}'.format(item_path), tag)
|
||||
r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), item_path)
|
||||
add_domain_tag(tag, item_path, item_date)
|
||||
|
||||
r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, 1)
|
||||
|
||||
|
|
|
|||
|
|
@ -54,5 +54,22 @@ if __name__ == "__main__":
|
|||
r_serv.delete('ail:current_background_script_stat')
|
||||
r_serv.delete('ail:current_background_update')
|
||||
|
||||
if r_serv.scard('ail:update_v2.4') != 1:
|
||||
pass
|
||||
if r_serv.get('ail:current_background_update') == 'v2.4':
|
||||
r_serv.delete('ail:update_error')
|
||||
r_serv.set('ail:update_in_progress', 'v2.4')
|
||||
r_serv.set('ail:current_background_update', 'v2.4')
|
||||
r_serv.set('ail:current_background_script', 'domain update')
|
||||
|
||||
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v2.4', 'Update_domain.py')
|
||||
process = subprocess.run(['python' ,update_file])
|
||||
|
||||
|
||||
if int(r_serv.get('ail:current_background_script_stat')) != 100:
|
||||
r_serv.set('ail:update_error', 'Update v2.4 Failed, please relaunch the bin/update-background.py script')
|
||||
else:
|
||||
r_serv.delete('ail:update_in_progress')
|
||||
r_serv.delete('ail:current_background_script')
|
||||
r_serv.delete('ail:current_background_script_stat')
|
||||
r_serv.delete('ail:current_background_update')
|
||||
r_serv.delete('update:nb_elem_to_convert')
|
||||
r_serv.delete('update:nb_elem_converted')
|
||||
|
|
|
|||
|
|
@ -19,12 +19,17 @@ if __name__ == '__main__':
|
|||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
config_loader = None
|
||||
|
||||
#Set current update_in_progress
|
||||
r_serv.set('ail:update_in_progress', new_version)
|
||||
r_serv.set('ail:current_background_update', new_version)
|
||||
|
||||
r_serv_onion.sunionstore('domain_update_v2.4', 'full_onion_up', 'full_regular_up')
|
||||
r_serv.set('update:nb_elem_to_convert', r_serv_onion.scard('domain_update_v2.4'))
|
||||
r_serv.set('update:nb_elem_converted',0)
|
||||
|
||||
#Set current ail version
|
||||
r_serv.set('ail:version', new_version)
|
||||
|
||||
|
|
|
|||
77
update/v2.4/Update_domain.py
Executable file
77
update/v2.4/Update_domain.py
Executable file
|
|
@ -0,0 +1,77 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import datetime
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||
import Item
|
||||
import Tag
|
||||
from Cryptocurrency import cryptocurrency
|
||||
from Pgp import pgp
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
import Decoded
|
||||
import Domain
|
||||
|
||||
def update_update_stats():
|
||||
nb_updated = int(r_serv_db.get('update:nb_elem_converted'))
|
||||
progress = int((nb_updated * 100) / nb_elem_to_update)
|
||||
print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress))
|
||||
r_serv_db.set('ail:current_background_script_stat', progress)
|
||||
|
||||
def update_domain_by_item(domain_obj, item_id):
|
||||
domain_name = domain_obj.get_domain_name()
|
||||
# update domain tags
|
||||
for tag in Tag.get_item_tags(item_id):
|
||||
if tag != 'infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
|
||||
Tag.add_domain_tag(tag, domain_name, Item.get_item_date(item_id))
|
||||
|
||||
# update domain correlation
|
||||
item_correlation = Item.get_item_all_correlation(item_id)
|
||||
|
||||
for correlation_name in item_correlation:
|
||||
for correlation_type in item_correlation[correlation_name]:
|
||||
if correlation_name in ('pgp', 'cryptocurrency'):
|
||||
for correl_value in item_correlation[correlation_name][correlation_type]:
|
||||
if correlation_name=='pgp':
|
||||
pgp.save_domain_correlation(domain_name, correlation_type, correl_value)
|
||||
if correlation_name=='cryptocurrency':
|
||||
cryptocurrency.save_domain_correlation(domain_name, correlation_type, correl_value)
|
||||
if correlation_name=='decoded':
|
||||
for decoded_item in item_correlation['decoded']:
|
||||
Decoded.save_domain_decoded(domain_name, decoded_item)
|
||||
|
||||
if __name__ == '__main__':
|
||||
|
||||
start_deb = time.time()
|
||||
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
config_loader = None
|
||||
|
||||
nb_elem_to_update = int( r_serv_db.get('update:nb_elem_to_convert') )
|
||||
|
||||
while True:
|
||||
domain = r_serv_onion.spop('domain_update_v2.4')
|
||||
if domain is not None:
|
||||
print(domain)
|
||||
domain = Domain.Domain(domain)
|
||||
for domain_history in domain.get_domain_history():
|
||||
|
||||
domain_item = domain.get_domain_items_crawled(epoch=domain_history[1]) # item_tag
|
||||
if "items" in domain_item:
|
||||
for item_dict in domain_item['items']:
|
||||
update_domain_by_item(domain, item_dict['id'])
|
||||
|
||||
r_serv_db.incr('update:nb_elem_converted')
|
||||
update_update_stats()
|
||||
|
||||
else:
|
||||
sys.exit(0)
|
||||
|
|
@ -69,5 +69,7 @@ def showDomain():
|
|||
dict_domain['crawler_history'] = domain.get_domain_items_crawled(items_link=True, epoch=epoch, item_screenshot=True, item_tag=True) # # TODO: handle multiple port
|
||||
dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items'])
|
||||
|
||||
print(dict_domain)
|
||||
|
||||
return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label,
|
||||
modal_add_tags=get_modal_add_tags(dict_domain['domain'], tag_type="domain"))
|
||||
|
|
|
|||
|
|
@ -86,7 +86,9 @@ DiffMaxLineLength = int(config_loader.get_config_str("Flask", "DiffMaxLineLengt
|
|||
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
|
||||
|
||||
dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_message': 'An Update is running on the background. Some informations like Tags, screenshot can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'}
|
||||
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||
'v2.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
|
||||
'update_warning_message_notice_me': 'missing from the UI.'}
|
||||
}
|
||||
|
||||
UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')
|
||||
|
|
|
|||
|
|
@ -102,6 +102,47 @@
|
|||
</div>
|
||||
|
||||
|
||||
{% if 'decoded' in dict_domain%}
|
||||
<div id="accordiondecoded" class="mt-3">
|
||||
<div class="card">
|
||||
<div class="card-header" id="headingDecoded">
|
||||
<div class="row">
|
||||
<div class="col-11">
|
||||
<div class="mt-2">
|
||||
<i class="fas fa-lock-open"></i> Decoded
|
||||
<div class="badge badge-warning">{{dict_domain['decoded']|length}}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-1">
|
||||
<button class="btn btn-link collapsed rotate" data-toggle="collapse" data-target="#collapseDecoded" aria-expanded="false" aria-controls="collapseDecoded">
|
||||
<i class="fas fa-chevron-circle-down"></i>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="collapseDecoded" class="collapse" aria-labelledby="headingDecoded" data-parent="#accordiondecoded">
|
||||
<div class="card-body">
|
||||
<table id="tabledecoded" class="table table-striped">
|
||||
<thead class="thead-dark">
|
||||
<tr>
|
||||
<th>Decoded</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for decoded in dict_domain['decoded']%}
|
||||
<tr>
|
||||
<td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ decoded }}">{{ decoded }}</a></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
|
||||
{% if 'pgp' in dict_domain%}
|
||||
<div id="accordionpgp" class="mt-3">
|
||||
<div class="card">
|
||||
|
|
@ -346,6 +387,9 @@
|
|||
<script>
|
||||
var table;
|
||||
$(document).ready(function(){
|
||||
{% if 'decoded' in dict_domain%}
|
||||
$('#tabledecoded').DataTable();
|
||||
{% endif %}
|
||||
{% if 'pgp' in dict_domain%}
|
||||
$('#tablepgp').DataTable();
|
||||
{% endif %}
|
||||
|
|
|
|||
Loading…
Reference in a new issue