mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-23 06:37:15 +00:00
chg: [update + show decoded items] add background update
This commit is contained in:
parent
6b9ba9d377
commit
880c351c0c
11 changed files with 288 additions and 8 deletions
|
@ -17,6 +17,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||||
import Correlation
|
import Correlation
|
||||||
from Cryptocurrency import cryptocurrency
|
from Cryptocurrency import cryptocurrency
|
||||||
from Pgp import pgp
|
from Pgp import pgp
|
||||||
|
import Decoded
|
||||||
import Item
|
import Item
|
||||||
import Tag
|
import Tag
|
||||||
|
|
||||||
|
@ -197,6 +198,14 @@ def get_domain_pgp(domain, currencies_type=None, get_nb=False):
|
||||||
'''
|
'''
|
||||||
return pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb)
|
return pgp.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb)
|
||||||
|
|
||||||
|
def get_domain_decoded(domain):
|
||||||
|
'''
|
||||||
|
Retun all decoded item of a given domain.
|
||||||
|
|
||||||
|
:param domain: crawled domain
|
||||||
|
'''
|
||||||
|
return Decoded.get_domain_decoded_item(domain)
|
||||||
|
|
||||||
def get_domain_all_correlation(domain, correlation_type=None, get_nb=False):
|
def get_domain_all_correlation(domain, correlation_type=None, get_nb=False):
|
||||||
'''
|
'''
|
||||||
Retun all correlation of a given domain.
|
Retun all correlation of a given domain.
|
||||||
|
@ -214,6 +223,9 @@ def get_domain_all_correlation(domain, correlation_type=None, get_nb=False):
|
||||||
res = get_domain_pgp(domain, get_nb=get_nb)
|
res = get_domain_pgp(domain, get_nb=get_nb)
|
||||||
if res:
|
if res:
|
||||||
domain_correl['pgp'] = res
|
domain_correl['pgp'] = res
|
||||||
|
res = get_domain_decoded(domain)
|
||||||
|
if res:
|
||||||
|
domain_correl['decoded'] = res
|
||||||
return domain_correl
|
return domain_correl
|
||||||
|
|
||||||
# TODO: handle port
|
# TODO: handle port
|
||||||
|
@ -271,6 +283,12 @@ class Domain(object):
|
||||||
if self.is_domain_up():
|
if self.is_domain_up():
|
||||||
self.current_port = sanathyse_port(port, self.domain, self.type)
|
self.current_port = sanathyse_port(port, self.domain, self.type)
|
||||||
|
|
||||||
|
def get_domain_name(self):
|
||||||
|
return self.domain
|
||||||
|
|
||||||
|
def get_domain_type(self):
|
||||||
|
return self.type
|
||||||
|
|
||||||
def get_current_port(self):
|
def get_current_port(self):
|
||||||
return self.current_port
|
return self.current_port
|
||||||
|
|
||||||
|
@ -361,10 +379,16 @@ class Domain(object):
|
||||||
'''
|
'''
|
||||||
return get_domain_all_correlation(self.domain, get_nb=True)
|
return get_domain_all_correlation(self.domain, get_nb=True)
|
||||||
|
|
||||||
def get_domain_history_with_status(self):
|
def get_domain_history(self):
|
||||||
'''
|
'''
|
||||||
Retun the full history of a given domain and port.
|
Retun the full history of a given domain and port.
|
||||||
'''
|
'''
|
||||||
|
return get_domain_history(self.domain, self.type, 80)
|
||||||
|
|
||||||
|
def get_domain_history_with_status(self):
|
||||||
|
'''
|
||||||
|
Retun the full history (with status) of a given domain and port.
|
||||||
|
'''
|
||||||
return get_domain_history_with_status(self.domain, self.type, 80)
|
return get_domain_history_with_status(self.domain, self.type, 80)
|
||||||
|
|
||||||
def get_domain_items_crawled(self, port=None, epoch=None, items_link=False, item_screenshot=False, item_tag=False):
|
def get_domain_items_crawled(self, port=None, epoch=None, items_link=False, item_screenshot=False, item_tag=False):
|
||||||
|
|
|
@ -147,6 +147,50 @@ class Correlation(object):
|
||||||
dict_correlation['nb'] = dict_correlation.get('nb', 0) + len(dict_correlation[correl])
|
dict_correlation['nb'] = dict_correlation.get('nb', 0) + len(dict_correlation[correl])
|
||||||
return dict_correlation
|
return dict_correlation
|
||||||
|
|
||||||
|
def _get_item_correlation_obj(self, item_id, correlation_type):
|
||||||
|
'''
|
||||||
|
Return correlation of a given item id.
|
||||||
|
|
||||||
|
:param item_id: item id
|
||||||
|
:type item_id: str
|
||||||
|
:param correlation_type: correlation type
|
||||||
|
:type correlation_type: str
|
||||||
|
|
||||||
|
:return: a list of correlation
|
||||||
|
:rtype: list
|
||||||
|
'''
|
||||||
|
res = r_serv_metadata.smembers('item_{}_{}:{}'.format(self.correlation_name, correlation_type, item_id))
|
||||||
|
if res:
|
||||||
|
return list(res)
|
||||||
|
else:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def get_item_correlation_dict(self, item_id, correlation_type=None, get_nb=False):
|
||||||
|
'''
|
||||||
|
Return all correlation of a given item id.
|
||||||
|
|
||||||
|
:param item_id: item id
|
||||||
|
:param correlation_type: list of correlation types
|
||||||
|
:type correlation_type: list, optional
|
||||||
|
|
||||||
|
:return: a dictionnary of all the requested correlations
|
||||||
|
:rtype: dict
|
||||||
|
'''
|
||||||
|
correlation_type = self.sanythise_correlation_types(correlation_type)
|
||||||
|
dict_correlation = {}
|
||||||
|
for correl in correlation_type:
|
||||||
|
res = self._get_item_correlation_obj(item_id, correl)
|
||||||
|
if res:
|
||||||
|
dict_correlation[correl] = res
|
||||||
|
if get_nb:
|
||||||
|
dict_correlation['nb'] = dict_correlation.get('nb', 0) + len(dict_correlation[correl])
|
||||||
|
return dict_correlation
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def save_domain_correlation(self, domain, correlation_type, correlation_value):
|
||||||
|
r_serv_metadata.sadd('domain_{}_{}:{}'.format(self.correlation_name, correlation_type, domain), correlation_value)
|
||||||
|
r_serv_metadata.sadd('set_domain_{}_{}:{}'.format(self.correlation_name, correlation_type, correlation_value), domain)
|
||||||
|
|
||||||
######## API EXPOSED ########
|
######## API EXPOSED ########
|
||||||
|
|
||||||
|
|
|
@ -61,6 +61,7 @@ def get_cryptocurrency(request_dict, cryptocurrency_type):
|
||||||
|
|
||||||
return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)
|
return cryptocurrency.get_correlation(request_dict, cryptocurrency_type, field_name)
|
||||||
|
|
||||||
|
# # TODO: refractor/move me in Correlation
|
||||||
def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrency_address):
|
def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrency_address):
|
||||||
# create basic medata
|
# create basic medata
|
||||||
if not r_serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)):
|
if not r_serv_metadata.exists('cryptocurrency_metadata_{}:{}'.format(cryptocurrency_name, cryptocurrency_address)):
|
||||||
|
@ -89,7 +90,7 @@ def save_cryptocurrency_data(cryptocurrency_name, date, item_path, cryptocurrenc
|
||||||
r_serv_metadata.sadd('item_cryptocurrency_{}:{}'.format(cryptocurrency_name, item_path), cryptocurrency_address)
|
r_serv_metadata.sadd('item_cryptocurrency_{}:{}'.format(cryptocurrency_name, item_path), cryptocurrency_address)
|
||||||
|
|
||||||
# domain
|
# domain
|
||||||
if Item.is_crawled(item_path):
|
if Item.is_crawled(item_path): # # TODO: use save_domain_correlation
|
||||||
domain = Item.get_item_domain(item_path)
|
domain = Item.get_item_domain(item_path)
|
||||||
r_serv_metadata.sadd('domain_cryptocurrency_{}:{}'.format(cryptocurrency_name, domain), cryptocurrency_address)
|
r_serv_metadata.sadd('domain_cryptocurrency_{}:{}'.format(cryptocurrency_name, domain), cryptocurrency_address)
|
||||||
r_serv_metadata.sadd('set_domain_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), domain)
|
r_serv_metadata.sadd('set_domain_cryptocurrency_{}:{}'.format(cryptocurrency_name, cryptocurrency_address), domain)
|
||||||
|
|
|
@ -8,10 +8,13 @@ import redis
|
||||||
|
|
||||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
import ConfigLoader
|
import ConfigLoader
|
||||||
|
import Decoded
|
||||||
|
|
||||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||||
import Date
|
import Date
|
||||||
import Tag
|
import Tag
|
||||||
|
from Cryptocurrency import cryptocurrency
|
||||||
|
from Pgp import pgp
|
||||||
|
|
||||||
config_loader = ConfigLoader.ConfigLoader()
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
|
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
|
||||||
|
@ -126,7 +129,61 @@ def get_item(request_dict):
|
||||||
###
|
###
|
||||||
### correlation
|
### correlation
|
||||||
###
|
###
|
||||||
|
def get_item_cryptocurrency(item_id, currencies_type=None, get_nb=False):
|
||||||
|
'''
|
||||||
|
Return all cryptocurrencies of a given item.
|
||||||
|
|
||||||
|
:param item_id: item id
|
||||||
|
:param currencies_type: list of cryptocurrencies type
|
||||||
|
:type currencies_type: list, optional
|
||||||
|
'''
|
||||||
|
return cryptocurrency.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb)
|
||||||
|
|
||||||
|
def get_item_pgp(item_id, currencies_type=None, get_nb=False):
|
||||||
|
'''
|
||||||
|
Return all pgp of a given item.
|
||||||
|
|
||||||
|
:param item_id: item id
|
||||||
|
:param currencies_type: list of cryptocurrencies type
|
||||||
|
:type currencies_type: list, optional
|
||||||
|
'''
|
||||||
|
return pgp.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb)
|
||||||
|
|
||||||
|
def get_item_decoded(item_id):
|
||||||
|
'''
|
||||||
|
Return all pgp of a given item.
|
||||||
|
|
||||||
|
:param item_id: item id
|
||||||
|
:param currencies_type: list of cryptocurrencies type
|
||||||
|
:type currencies_type: list, optional
|
||||||
|
'''
|
||||||
|
return Decoded.get_item_decoded(item_id)
|
||||||
|
|
||||||
|
def get_item_all_correlation(item_id, correlation_type=None, get_nb=False):
|
||||||
|
'''
|
||||||
|
Retun all correlation of a given item id.
|
||||||
|
|
||||||
|
:param item_id: item id
|
||||||
|
:type domain: str
|
||||||
|
|
||||||
|
:return: a dict of all correlation for a item id
|
||||||
|
:rtype: dict
|
||||||
|
'''
|
||||||
|
item_correl = {}
|
||||||
|
res = get_item_cryptocurrency(item_id, get_nb=get_nb)
|
||||||
|
if res:
|
||||||
|
item_correl['cryptocurrency'] = res
|
||||||
|
res = get_item_pgp(item_id, get_nb=get_nb)
|
||||||
|
if res:
|
||||||
|
item_correl['pgp'] = res
|
||||||
|
res = get_item_decoded(item_id)
|
||||||
|
if res:
|
||||||
|
item_correl['decoded'] = res
|
||||||
|
return item_correl
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## TODO: REFRACTOR
|
||||||
def _get_item_correlation(correlation_name, correlation_type, item_id):
|
def _get_item_correlation(correlation_name, correlation_type, item_id):
|
||||||
res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
|
res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
|
||||||
if res:
|
if res:
|
||||||
|
@ -134,18 +191,23 @@ def _get_item_correlation(correlation_name, correlation_type, item_id):
|
||||||
else:
|
else:
|
||||||
return []
|
return []
|
||||||
|
|
||||||
|
## TODO: REFRACTOR
|
||||||
def get_item_bitcoin(item_id):
|
def get_item_bitcoin(item_id):
|
||||||
return _get_item_correlation('cryptocurrency', 'bitcoin', item_id)
|
return _get_item_correlation('cryptocurrency', 'bitcoin', item_id)
|
||||||
|
|
||||||
|
## TODO: REFRACTOR
|
||||||
def get_item_pgp_key(item_id):
|
def get_item_pgp_key(item_id):
|
||||||
return _get_item_correlation('pgpdump', 'key', item_id)
|
return _get_item_correlation('pgpdump', 'key', item_id)
|
||||||
|
|
||||||
|
## TODO: REFRACTOR
|
||||||
def get_item_pgp_name(item_id):
|
def get_item_pgp_name(item_id):
|
||||||
return _get_item_correlation('pgpdump', 'name', item_id)
|
return _get_item_correlation('pgpdump', 'name', item_id)
|
||||||
|
|
||||||
|
## TODO: REFRACTOR
|
||||||
def get_item_pgp_mail(item_id):
|
def get_item_pgp_mail(item_id):
|
||||||
return _get_item_correlation('pgpdump', 'mail', item_id)
|
return _get_item_correlation('pgpdump', 'mail', item_id)
|
||||||
|
|
||||||
|
## TODO: REFRACTOR
|
||||||
def get_item_pgp_correlation(item_id):
|
def get_item_pgp_correlation(item_id):
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
|
@ -172,8 +172,11 @@ def add_items_tags(tags=[], galaxy_tags=[], item_id=None, item_type="paste"):
|
||||||
res_dict['type'] = item_type
|
res_dict['type'] = item_type
|
||||||
return (res_dict, 200)
|
return (res_dict, 200)
|
||||||
|
|
||||||
|
def add_domain_tag(tag, domain, item_date):
|
||||||
|
r_serv_metadata.sadd('tag:{}'.format(domain), tag)
|
||||||
|
r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), domain)
|
||||||
|
|
||||||
def add_item_tag(tag, item_path, item_type="paste"):
|
def add_item_tag(tag, item_path, item_type="paste", tag_date=None):
|
||||||
|
|
||||||
if item_type=="paste":
|
if item_type=="paste":
|
||||||
item_date = int(Item.get_item_date(item_path))
|
item_date = int(Item.get_item_date(item_path))
|
||||||
|
@ -189,8 +192,7 @@ def add_item_tag(tag, item_path, item_type="paste"):
|
||||||
# domain item
|
# domain item
|
||||||
else:
|
else:
|
||||||
item_date = int(Domain.get_domain_last_check(item_path, r_format="int"))
|
item_date = int(Domain.get_domain_last_check(item_path, r_format="int"))
|
||||||
r_serv_metadata.sadd('tag:{}'.format(item_path), tag)
|
add_domain_tag(tag, item_path, item_date)
|
||||||
r_serv_tags.sadd('domain:{}:{}'.format(tag, item_date), item_path)
|
|
||||||
|
|
||||||
r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, 1)
|
r_serv_tags.hincrby('daily_tags:{}'.format(item_date), tag, 1)
|
||||||
|
|
||||||
|
|
|
@ -54,5 +54,22 @@ if __name__ == "__main__":
|
||||||
r_serv.delete('ail:current_background_script_stat')
|
r_serv.delete('ail:current_background_script_stat')
|
||||||
r_serv.delete('ail:current_background_update')
|
r_serv.delete('ail:current_background_update')
|
||||||
|
|
||||||
if r_serv.scard('ail:update_v2.4') != 1:
|
if r_serv.get('ail:current_background_update') == 'v2.4':
|
||||||
pass
|
r_serv.delete('ail:update_error')
|
||||||
|
r_serv.set('ail:update_in_progress', 'v2.4')
|
||||||
|
r_serv.set('ail:current_background_update', 'v2.4')
|
||||||
|
r_serv.set('ail:current_background_script', 'domain update')
|
||||||
|
|
||||||
|
update_file = os.path.join(os.environ['AIL_HOME'], 'update', 'v2.4', 'Update_domain.py')
|
||||||
|
process = subprocess.run(['python' ,update_file])
|
||||||
|
|
||||||
|
|
||||||
|
if int(r_serv.get('ail:current_background_script_stat')) != 100:
|
||||||
|
r_serv.set('ail:update_error', 'Update v2.4 Failed, please relaunch the bin/update-background.py script')
|
||||||
|
else:
|
||||||
|
r_serv.delete('ail:update_in_progress')
|
||||||
|
r_serv.delete('ail:current_background_script')
|
||||||
|
r_serv.delete('ail:current_background_script_stat')
|
||||||
|
r_serv.delete('ail:current_background_update')
|
||||||
|
r_serv.delete('update:nb_elem_to_convert')
|
||||||
|
r_serv.delete('update:nb_elem_converted')
|
||||||
|
|
|
@ -19,12 +19,17 @@ if __name__ == '__main__':
|
||||||
|
|
||||||
config_loader = ConfigLoader.ConfigLoader()
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
r_serv = config_loader.get_redis_conn("ARDB_DB")
|
||||||
|
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||||
config_loader = None
|
config_loader = None
|
||||||
|
|
||||||
#Set current update_in_progress
|
#Set current update_in_progress
|
||||||
r_serv.set('ail:update_in_progress', new_version)
|
r_serv.set('ail:update_in_progress', new_version)
|
||||||
r_serv.set('ail:current_background_update', new_version)
|
r_serv.set('ail:current_background_update', new_version)
|
||||||
|
|
||||||
|
r_serv_onion.sunionstore('domain_update_v2.4', 'full_onion_up', 'full_regular_up')
|
||||||
|
r_serv.set('update:nb_elem_to_convert', r_serv_onion.scard('domain_update_v2.4'))
|
||||||
|
r_serv.set('update:nb_elem_converted',0)
|
||||||
|
|
||||||
#Set current ail version
|
#Set current ail version
|
||||||
r_serv.set('ail:version', new_version)
|
r_serv.set('ail:version', new_version)
|
||||||
|
|
||||||
|
|
77
update/v2.4/Update_domain.py
Executable file
77
update/v2.4/Update_domain.py
Executable file
|
@ -0,0 +1,77 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import redis
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||||
|
import Item
|
||||||
|
import Tag
|
||||||
|
from Cryptocurrency import cryptocurrency
|
||||||
|
from Pgp import pgp
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
|
import ConfigLoader
|
||||||
|
import Decoded
|
||||||
|
import Domain
|
||||||
|
|
||||||
|
def update_update_stats():
|
||||||
|
nb_updated = int(r_serv_db.get('update:nb_elem_converted'))
|
||||||
|
progress = int((nb_updated * 100) / nb_elem_to_update)
|
||||||
|
print('{}/{} updated {}%'.format(nb_updated, nb_elem_to_update, progress))
|
||||||
|
r_serv_db.set('ail:current_background_script_stat', progress)
|
||||||
|
|
||||||
|
def update_domain_by_item(domain_obj, item_id):
|
||||||
|
domain_name = domain_obj.get_domain_name()
|
||||||
|
# update domain tags
|
||||||
|
for tag in Tag.get_item_tags(item_id):
|
||||||
|
if tag != 'infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
|
||||||
|
Tag.add_domain_tag(tag, domain_name, Item.get_item_date(item_id))
|
||||||
|
|
||||||
|
# update domain correlation
|
||||||
|
item_correlation = Item.get_item_all_correlation(item_id)
|
||||||
|
|
||||||
|
for correlation_name in item_correlation:
|
||||||
|
for correlation_type in item_correlation[correlation_name]:
|
||||||
|
if correlation_name in ('pgp', 'cryptocurrency'):
|
||||||
|
for correl_value in item_correlation[correlation_name][correlation_type]:
|
||||||
|
if correlation_name=='pgp':
|
||||||
|
pgp.save_domain_correlation(domain_name, correlation_type, correl_value)
|
||||||
|
if correlation_name=='cryptocurrency':
|
||||||
|
cryptocurrency.save_domain_correlation(domain_name, correlation_type, correl_value)
|
||||||
|
if correlation_name=='decoded':
|
||||||
|
for decoded_item in item_correlation['decoded']:
|
||||||
|
Decoded.save_domain_decoded(domain_name, decoded_item)
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
|
||||||
|
start_deb = time.time()
|
||||||
|
|
||||||
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||||
|
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||||
|
config_loader = None
|
||||||
|
|
||||||
|
nb_elem_to_update = int( r_serv_db.get('update:nb_elem_to_convert') )
|
||||||
|
|
||||||
|
while True:
|
||||||
|
domain = r_serv_onion.spop('domain_update_v2.4')
|
||||||
|
if domain is not None:
|
||||||
|
print(domain)
|
||||||
|
domain = Domain.Domain(domain)
|
||||||
|
for domain_history in domain.get_domain_history():
|
||||||
|
|
||||||
|
domain_item = domain.get_domain_items_crawled(epoch=domain_history[1]) # item_tag
|
||||||
|
if "items" in domain_item:
|
||||||
|
for item_dict in domain_item['items']:
|
||||||
|
update_domain_by_item(domain, item_dict['id'])
|
||||||
|
|
||||||
|
r_serv_db.incr('update:nb_elem_converted')
|
||||||
|
update_update_stats()
|
||||||
|
|
||||||
|
else:
|
||||||
|
sys.exit(0)
|
|
@ -69,5 +69,7 @@ def showDomain():
|
||||||
dict_domain['crawler_history'] = domain.get_domain_items_crawled(items_link=True, epoch=epoch, item_screenshot=True, item_tag=True) # # TODO: handle multiple port
|
dict_domain['crawler_history'] = domain.get_domain_items_crawled(items_link=True, epoch=epoch, item_screenshot=True, item_tag=True) # # TODO: handle multiple port
|
||||||
dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items'])
|
dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items'])
|
||||||
|
|
||||||
|
print(dict_domain)
|
||||||
|
|
||||||
return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label,
|
return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label,
|
||||||
modal_add_tags=get_modal_add_tags(dict_domain['domain'], tag_type="domain"))
|
modal_add_tags=get_modal_add_tags(dict_domain['domain'], tag_type="domain"))
|
||||||
|
|
|
@ -86,6 +86,8 @@ DiffMaxLineLength = int(config_loader.get_config_str("Flask", "DiffMaxLineLengt
|
||||||
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
|
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
|
||||||
|
|
||||||
dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_message': 'An Update is running on the background. Some informations like Tags, screenshot can be',
|
dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_message': 'An Update is running on the background. Some informations like Tags, screenshot can be',
|
||||||
|
'update_warning_message_notice_me': 'missing from the UI.'},
|
||||||
|
'v2.4':{'nb_background_update': 1, 'update_warning_message': 'An Update is running on the background. Some informations like Domain Tags/Correlation can be',
|
||||||
'update_warning_message_notice_me': 'missing from the UI.'}
|
'update_warning_message_notice_me': 'missing from the UI.'}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -102,6 +102,47 @@
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
{% if 'decoded' in dict_domain%}
|
||||||
|
<div id="accordiondecoded" class="mt-3">
|
||||||
|
<div class="card">
|
||||||
|
<div class="card-header" id="headingDecoded">
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-11">
|
||||||
|
<div class="mt-2">
|
||||||
|
<i class="fas fa-lock-open"></i> Decoded
|
||||||
|
<div class="badge badge-warning">{{dict_domain['decoded']|length}}</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-1">
|
||||||
|
<button class="btn btn-link collapsed rotate" data-toggle="collapse" data-target="#collapseDecoded" aria-expanded="false" aria-controls="collapseDecoded">
|
||||||
|
<i class="fas fa-chevron-circle-down"></i>
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div id="collapseDecoded" class="collapse" aria-labelledby="headingDecoded" data-parent="#accordiondecoded">
|
||||||
|
<div class="card-body">
|
||||||
|
<table id="tabledecoded" class="table table-striped">
|
||||||
|
<thead class="thead-dark">
|
||||||
|
<tr>
|
||||||
|
<th>Decoded</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
{% for decoded in dict_domain['decoded']%}
|
||||||
|
<tr>
|
||||||
|
<td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ decoded }}">{{ decoded }}</a></td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
|
||||||
{% if 'pgp' in dict_domain%}
|
{% if 'pgp' in dict_domain%}
|
||||||
<div id="accordionpgp" class="mt-3">
|
<div id="accordionpgp" class="mt-3">
|
||||||
<div class="card">
|
<div class="card">
|
||||||
|
@ -346,6 +387,9 @@
|
||||||
<script>
|
<script>
|
||||||
var table;
|
var table;
|
||||||
$(document).ready(function(){
|
$(document).ready(function(){
|
||||||
|
{% if 'decoded' in dict_domain%}
|
||||||
|
$('#tabledecoded').DataTable();
|
||||||
|
{% endif %}
|
||||||
{% if 'pgp' in dict_domain%}
|
{% if 'pgp' in dict_domain%}
|
||||||
$('#tablepgp').DataTable();
|
$('#tablepgp').DataTable();
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
Loading…
Reference in a new issue