chg: [HiddenServices] get domain related to other domains

This commit is contained in:
Terrtia 2018-09-21 10:34:06 +02:00
parent 912f2352ab
commit 6edc1ddbeb
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
3 changed files with 47 additions and 1 deletions

View file

@ -111,6 +111,41 @@ class HiddenServices(object):
l_crawled_pastes.extend(self.get_all_pastes_domain(children))
return l_crawled_pastes
def get_domain_son(self, l_paste):
if l_paste is None:
return None
set_domain = set()
for paste in l_paste:
paste_full = paste.replace(self.paste_directory+'/', '')
paste_childrens = self.r_serv_metadata.smembers('paste_children:{}'.format(paste_full))
## TODO: # FIXME: remove me
paste_children = self.r_serv_metadata.smembers('paste_children:{}'.format(paste))
paste_childrens = paste_childrens | paste_children
for children in paste_childrens:
if not self.domain in children:
print(children)
set_domain.add((children.split('.onion')[0]+'.onion').split('/')[-1])
return set_domain
def get_all_domain_son(self, father):
if father is None:
return []
l_crawled_pastes = []
paste_parent = father.replace(self.paste_directory+'/', '')
paste_childrens = self.r_serv_metadata.smembers('paste_children:{}'.format(paste_parent))
## TODO: # FIXME: remove me
paste_children = self.r_serv_metadata.smembers('paste_children:{}'.format(father))
paste_childrens = paste_childrens | paste_children
for children in paste_childrens:
if not self.domain in children:
l_crawled_pastes.append(children)
#self.update_domain_tags(children)
l_crawled_pastes.extend(self.get_all_domain_son(children))
return l_crawled_pastes
def get_domain_random_screenshot(self, l_crawled_pastes, num_screenshot = 1):
l_screenshot_paste = []
for paste in l_crawled_pastes:

View file

@ -89,7 +89,7 @@ def hiddenServices_page():
def onion_domain():
onion_domain = request.args.get('onion_domain')
if onion_domain is None or not r_serv_onion.exists('onion_metadata:{}'.format(onion_domain)):
pass
return '404'
# # TODO: FIXME return 404
last_check = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'last_check')
@ -127,6 +127,16 @@ def onion_domain():
origin_paste=origin_paste, origin_paste_name=origin_paste_name,
domain_tags=domain_tags, screenshot=screenshot)
@hiddenServices.route("/hiddenServices/onion_son", methods=['GET'])
def onion_son():
onion_domain = request.args.get('onion_domain')
h = HiddenServices(onion_domain, 'onion')
l_pastes = h.get_last_crawled_pastes()
l_son = h.get_domain_son(l_pastes)
print(l_son)
return 'l_son'
# ============= JSON ==============
@hiddenServices.route("/hiddenServices/domain_crawled_7days_json", methods=['GET'])
def domain_crawled_7days_json():

View file

@ -81,6 +81,7 @@
</div>
</li>
</ul>
</div>
</div>
<div>