mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-12 17:48:22 +00:00
chg: [HiddenServices] get domain related to other domains
This commit is contained in:
parent
912f2352ab
commit
6edc1ddbeb
3 changed files with 47 additions and 1 deletions
|
@ -111,6 +111,41 @@ class HiddenServices(object):
|
||||||
l_crawled_pastes.extend(self.get_all_pastes_domain(children))
|
l_crawled_pastes.extend(self.get_all_pastes_domain(children))
|
||||||
return l_crawled_pastes
|
return l_crawled_pastes
|
||||||
|
|
||||||
|
def get_domain_son(self, l_paste):
|
||||||
|
if l_paste is None:
|
||||||
|
return None
|
||||||
|
|
||||||
|
set_domain = set()
|
||||||
|
for paste in l_paste:
|
||||||
|
paste_full = paste.replace(self.paste_directory+'/', '')
|
||||||
|
paste_childrens = self.r_serv_metadata.smembers('paste_children:{}'.format(paste_full))
|
||||||
|
## TODO: # FIXME: remove me
|
||||||
|
paste_children = self.r_serv_metadata.smembers('paste_children:{}'.format(paste))
|
||||||
|
paste_childrens = paste_childrens | paste_children
|
||||||
|
for children in paste_childrens:
|
||||||
|
if not self.domain in children:
|
||||||
|
print(children)
|
||||||
|
set_domain.add((children.split('.onion')[0]+'.onion').split('/')[-1])
|
||||||
|
|
||||||
|
return set_domain
|
||||||
|
|
||||||
|
def get_all_domain_son(self, father):
|
||||||
|
if father is None:
|
||||||
|
return []
|
||||||
|
l_crawled_pastes = []
|
||||||
|
paste_parent = father.replace(self.paste_directory+'/', '')
|
||||||
|
paste_childrens = self.r_serv_metadata.smembers('paste_children:{}'.format(paste_parent))
|
||||||
|
## TODO: # FIXME: remove me
|
||||||
|
paste_children = self.r_serv_metadata.smembers('paste_children:{}'.format(father))
|
||||||
|
paste_childrens = paste_childrens | paste_children
|
||||||
|
for children in paste_childrens:
|
||||||
|
if not self.domain in children:
|
||||||
|
l_crawled_pastes.append(children)
|
||||||
|
#self.update_domain_tags(children)
|
||||||
|
l_crawled_pastes.extend(self.get_all_domain_son(children))
|
||||||
|
|
||||||
|
return l_crawled_pastes
|
||||||
|
|
||||||
def get_domain_random_screenshot(self, l_crawled_pastes, num_screenshot = 1):
|
def get_domain_random_screenshot(self, l_crawled_pastes, num_screenshot = 1):
|
||||||
l_screenshot_paste = []
|
l_screenshot_paste = []
|
||||||
for paste in l_crawled_pastes:
|
for paste in l_crawled_pastes:
|
||||||
|
|
|
@ -89,7 +89,7 @@ def hiddenServices_page():
|
||||||
def onion_domain():
|
def onion_domain():
|
||||||
onion_domain = request.args.get('onion_domain')
|
onion_domain = request.args.get('onion_domain')
|
||||||
if onion_domain is None or not r_serv_onion.exists('onion_metadata:{}'.format(onion_domain)):
|
if onion_domain is None or not r_serv_onion.exists('onion_metadata:{}'.format(onion_domain)):
|
||||||
pass
|
return '404'
|
||||||
# # TODO: FIXME return 404
|
# # TODO: FIXME return 404
|
||||||
|
|
||||||
last_check = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'last_check')
|
last_check = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'last_check')
|
||||||
|
@ -127,6 +127,16 @@ def onion_domain():
|
||||||
origin_paste=origin_paste, origin_paste_name=origin_paste_name,
|
origin_paste=origin_paste, origin_paste_name=origin_paste_name,
|
||||||
domain_tags=domain_tags, screenshot=screenshot)
|
domain_tags=domain_tags, screenshot=screenshot)
|
||||||
|
|
||||||
|
@hiddenServices.route("/hiddenServices/onion_son", methods=['GET'])
|
||||||
|
def onion_son():
|
||||||
|
onion_domain = request.args.get('onion_domain')
|
||||||
|
|
||||||
|
h = HiddenServices(onion_domain, 'onion')
|
||||||
|
l_pastes = h.get_last_crawled_pastes()
|
||||||
|
l_son = h.get_domain_son(l_pastes)
|
||||||
|
print(l_son)
|
||||||
|
return 'l_son'
|
||||||
|
|
||||||
# ============= JSON ==============
|
# ============= JSON ==============
|
||||||
@hiddenServices.route("/hiddenServices/domain_crawled_7days_json", methods=['GET'])
|
@hiddenServices.route("/hiddenServices/domain_crawled_7days_json", methods=['GET'])
|
||||||
def domain_crawled_7days_json():
|
def domain_crawled_7days_json():
|
||||||
|
|
|
@ -81,6 +81,7 @@
|
||||||
</div>
|
</div>
|
||||||
</li>
|
</li>
|
||||||
</ul>
|
</ul>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
<div>
|
<div>
|
||||||
|
|
Loading…
Reference in a new issue