diff --git a/bin/packages/HiddenServices.py b/bin/packages/HiddenServices.py index 48f514fc..9f4e9302 100755 --- a/bin/packages/HiddenServices.py +++ b/bin/packages/HiddenServices.py @@ -19,6 +19,7 @@ Conditions to fulfill to be able to use this class correctly: import os import gzip import redis +import random import configparser import sys @@ -52,11 +53,19 @@ class HiddenServices(object): db=cfg.getint("ARDB_Onion", "db"), decode_responses=True) + self.r_serv_metadata = redis.StrictRedis( + host=cfg.get("ARDB_Metadata", "host"), + port=cfg.getint("ARDB_Metadata", "port"), + db=cfg.getint("ARDB_Metadata", "db"), + decode_responses=True) + self.domain = domain self.type = type if type == 'onion': - self.paste_directory = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes"), cfg.get("Directories", "crawled")) + self.paste_directory = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + self.paste_crawled_directory = os.path.join(self.paste_directory, cfg.get("Directories", "crawled")) + self.paste_crawled_directory_name = cfg.get("Directories", "crawled") self.screenshot_directory = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot")) elif type == 'i2p': self.paste_directory = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot")) @@ -65,15 +74,57 @@ class HiddenServices(object): ## TODO: # FIXME: add error pass - + #todo use the right paste def get_last_crawled_pastes(self): + paste_parent = self.r_serv_onion.hget('onion_metadata:{}'.format(self.domain), 'paste_parent') + #paste_parent = paste_parent.replace(self.paste_directory, '')[1:] + return self.get_all_pastes_domain(paste_parent) - last_check = self.r_serv_onion.hget('onion_metadata:{}'.format(self.domain), 'last_check') - return self.get_crawled_pastes_by_date(last_check) + def get_all_pastes_domain(self, father): + l_crawled_pastes = [] + paste_parent = father.replace(self.paste_directory, '')[1:] + paste_childrens = self.r_serv_metadata.smembers('paste_children:{}'.format(paste_parent)) + ## TODO: # FIXME: remove me + if not paste_childrens: + paste_childrens = self.r_serv_metadata.smembers('paste_children:{}'.format(father)) + for children in paste_childrens: + if self.domain in children: + l_crawled_pastes.append(children) + l_crawled_pastes.extend(self.get_all_pastes_domain(children)) + return l_crawled_pastes + + def get_domain_random_screenshot(self, l_crawled_pastes, num_screenshot = 1): + l_screenshot_paste = [] + for paste in l_crawled_pastes: + ## FIXME: # TODO: remove me + paste= paste.replace(self.paste_directory, '')[1:] + + paste = paste.replace(self.paste_crawled_directory_name, '') + if os.path.isfile( '{}{}.png'.format(self.screenshot_directory, paste) ): + l_screenshot_paste.append(paste[1:]) + + if len(l_screenshot_paste) > num_screenshot: + l_random_screenshot = [] + for index in random.sample( range(0, len(l_screenshot_paste)), num_screenshot ): + l_random_screenshot.append(l_screenshot_paste[index]) + return l_random_screenshot + else: + return l_screenshot_paste def get_crawled_pastes_by_date(self, date): - pastes_path = os.path.join(self.paste_directory, date[0:4], date[4:6], date[6:8]) - l_crawled_pastes = [f for f in os.listdir(pastes_path) if self.domain in f] - print(len(l_crawled_pastes)) - print(l_crawled_pastes) + + pastes_path = os.path.join(self.paste_crawled_directory, date[0:4], date[4:6], date[6:8]) + paste_parent = self.r_serv_onion.hget('onion_metadata:{}'.format(self.domain), 'last_check') + + l_crawled_pastes = [] + return l_crawled_pastes + + def get_last_crawled_pastes_fileSearch(self): + + last_check = self.r_serv_onion.hget('onion_metadata:{}'.format(self.domain), 'last_check') + return self.get_crawled_pastes_by_date_fileSearch(last_check) + + def get_crawled_pastes_by_date_fileSearch(self, date): + pastes_path = os.path.join(self.paste_crawled_directory, date[0:4], date[4:6], date[6:8]) + l_crawled_pastes = [f for f in os.listdir(pastes_path) if self.domain in f] return l_crawled_pastes diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py index 6d01bbbb..7969aae8 100644 --- a/var/www/modules/hiddenServices/Flask_hiddenServices.py +++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py @@ -6,10 +6,12 @@ ''' import redis import datetime +import sys +import os from flask import Flask, render_template, jsonify, request, Blueprint -import HiddenServices from Date import Date +from HiddenServices import HiddenServices # ============ VARIABLES ============ import Flask_config @@ -75,8 +77,12 @@ def onion_domain(): domain_paste = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'paste_parent') date_crawled = r_serv_onion.smembers('onion_history:{}'.format(onion_domain)) + h = HiddenServices(onion_domain, 'onion') + l_pastes = h.get_last_crawled_pastes() + screenshot = h.get_domain_random_screenshot(l_pastes)[0] + return render_template("showDomain.html", domain=onion_domain, last_check=last_check, first_seen=first_seen, - domain_paste=domain_paste) + domain_paste=domain_paste, screenshot=screenshot) # ============= JSON ============== @hiddenServices.route("/hiddenServices/domain_crawled_7days_json", methods=['GET']) diff --git a/var/www/modules/hiddenServices/templates/showDomain.html b/var/www/modules/hiddenServices/templates/showDomain.html index 88942c73..3f5b8736 100644 --- a/var/www/modules/hiddenServices/templates/showDomain.html +++ b/var/www/modules/hiddenServices/templates/showDomain.html @@ -62,7 +62,7 @@
- +