mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-23 22:47:15 +00:00
332 lines
14 KiB
Python
332 lines
14 KiB
Python
#!/usr/bin/env python3
|
|
# -*-coding:UTF-8 -*
|
|
|
|
'''
|
|
Flask functions and routes for the trending modules page
|
|
'''
|
|
import redis
|
|
import datetime
|
|
import sys
|
|
import os
|
|
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
|
|
|
|
from Date import Date
|
|
from HiddenServices import HiddenServices
|
|
|
|
# ============ VARIABLES ============
|
|
import Flask_config
|
|
|
|
app = Flask_config.app
|
|
cfg = Flask_config.cfg
|
|
baseUrl = Flask_config.baseUrl
|
|
r_cache = Flask_config.r_cache
|
|
r_serv_onion = Flask_config.r_serv_onion
|
|
r_serv_metadata = Flask_config.r_serv_metadata
|
|
bootstrap_label = Flask_config.bootstrap_label
|
|
PASTES_FOLDER = Flask_config.PASTES_FOLDER
|
|
|
|
hiddenServices = Blueprint('hiddenServices', __name__, template_folder='templates')
|
|
|
|
# ============ FUNCTIONS ============
|
|
def one():
|
|
return 1
|
|
|
|
def get_date_range(num_day):
|
|
curr_date = datetime.date.today()
|
|
date = Date( '{}{}{}'.format(str(curr_date.year), str(curr_date.month).zfill(2), str(curr_date.day).zfill(2)) )
|
|
date_list = []
|
|
|
|
for i in range(0, num_day):
|
|
date_list.append(date.substract_day(i))
|
|
|
|
return list(reversed(date_list))
|
|
|
|
def substract_date(date_from, date_to):
|
|
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
|
|
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
|
|
delta = date_to - date_from # timedelta
|
|
l_date = []
|
|
for i in range(delta.days + 1):
|
|
date = date_from + datetime.timedelta(i)
|
|
l_date.append( date.strftime('%Y%m%d') )
|
|
return l_date
|
|
|
|
def unpack_paste_tags(p_tags):
|
|
l_tags = []
|
|
for tag in p_tags:
|
|
complete_tag = tag
|
|
tag = tag.split('=')
|
|
if len(tag) > 1:
|
|
if tag[1] != '':
|
|
tag = tag[1][1:-1]
|
|
# no value
|
|
else:
|
|
tag = tag[0][1:-1]
|
|
# use for custom tags
|
|
else:
|
|
tag = tag[0]
|
|
l_tags.append( (tag, complete_tag) )
|
|
return l_tags
|
|
|
|
def get_onion_status(domain, date):
|
|
if r_serv_onion.sismember('onion_up:'+date , domain):
|
|
return True
|
|
else:
|
|
return False
|
|
# ============= ROUTES ==============
|
|
|
|
@hiddenServices.route("/hiddenServices/", methods=['GET'])
|
|
def hiddenServices_page():
|
|
last_onions = r_serv_onion.lrange('last_onion', 0 ,-1)
|
|
list_onion = []
|
|
|
|
now = datetime.datetime.now()
|
|
date = '{}{}{}'.format(now.strftime("%Y"), now.strftime("%m"), now.strftime("%d"))
|
|
statDomains = {}
|
|
statDomains['domains_up'] = r_serv_onion.scard('onion_up:{}'.format(date))
|
|
statDomains['domains_down'] = r_serv_onion.scard('onion_down:{}'.format(date))
|
|
statDomains['total'] = statDomains['domains_up'] + statDomains['domains_down']
|
|
statDomains['domains_queue'] = r_serv_onion.scard('onion_domain_crawler_queue')
|
|
|
|
for onion in last_onions:
|
|
metadata_onion = {}
|
|
metadata_onion['domain'] = onion
|
|
metadata_onion['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(onion), 'last_check')
|
|
metadata_onion['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(onion), 'first_seen')
|
|
if get_onion_status(onion, metadata_onion['last_check']):
|
|
metadata_onion['status_text'] = 'UP'
|
|
metadata_onion['status_color'] = 'Green'
|
|
metadata_onion['status_icon'] = 'fa-check-circle'
|
|
else:
|
|
metadata_onion['status_text'] = 'DOWN'
|
|
metadata_onion['status_color'] = 'Red'
|
|
metadata_onion['status_icon'] = 'fa-times-circle'
|
|
list_onion.append(metadata_onion)
|
|
|
|
crawler_metadata=[]
|
|
all_onion_crawler = r_cache.smembers('all_crawler:onion')
|
|
for crawler in all_onion_crawler:
|
|
crawling_domain = r_cache.hget('metadata_crawler:{}'.format(crawler), 'crawling_domain')
|
|
started_time = r_cache.hget('metadata_crawler:{}'.format(crawler), 'started_time')
|
|
status_info = r_cache.hget('metadata_crawler:{}'.format(crawler), 'status')
|
|
crawler_info = '{} - {}'.format(crawler, started_time)
|
|
if status_info=='Waiting' or status_info=='Crawling':
|
|
status=True
|
|
else:
|
|
status=False
|
|
crawler_metadata.append({'crawler_info': crawler_info, 'crawling_domain': crawling_domain, 'status_info': status_info, 'status': status})
|
|
|
|
date_string = '{}-{}-{}'.format(date[0:4], date[4:6], date[6:8])
|
|
return render_template("hiddenServices.html", last_onions=list_onion, statDomains=statDomains,
|
|
crawler_metadata=crawler_metadata, date_from=date_string, date_to=date_string)
|
|
|
|
@hiddenServices.route("/hiddenServices/last_crawled_domains_with_stats_json", methods=['GET'])
|
|
def last_crawled_domains_with_stats_json():
|
|
last_onions = r_serv_onion.lrange('last_onion', 0 ,-1)
|
|
list_onion = []
|
|
|
|
now = datetime.datetime.now()
|
|
date = '{}{}{}'.format(now.strftime("%Y"), now.strftime("%m"), now.strftime("%d"))
|
|
statDomains = {}
|
|
statDomains['domains_up'] = r_serv_onion.scard('onion_up:{}'.format(date))
|
|
statDomains['domains_down'] = r_serv_onion.scard('onion_down:{}'.format(date))
|
|
statDomains['total'] = statDomains['domains_up'] + statDomains['domains_down']
|
|
statDomains['domains_queue'] = r_serv_onion.scard('onion_domain_crawler_queue')
|
|
|
|
for onion in last_onions:
|
|
metadata_onion = {}
|
|
metadata_onion['domain'] = onion
|
|
metadata_onion['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(onion), 'last_check')
|
|
metadata_onion['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(onion), 'first_seen')
|
|
if get_onion_status(onion, metadata_onion['last_check']):
|
|
metadata_onion['status_text'] = 'UP'
|
|
metadata_onion['status_color'] = 'Green'
|
|
metadata_onion['status_icon'] = 'fa-check-circle'
|
|
else:
|
|
metadata_onion['status_text'] = 'DOWN'
|
|
metadata_onion['status_color'] = 'Red'
|
|
metadata_onion['status_icon'] = 'fa-times-circle'
|
|
list_onion.append(metadata_onion)
|
|
|
|
crawler_metadata=[]
|
|
all_onion_crawler = r_cache.smembers('all_crawler:onion')
|
|
for crawler in all_onion_crawler:
|
|
crawling_domain = r_cache.hget('metadata_crawler:{}'.format(crawler), 'crawling_domain')
|
|
started_time = r_cache.hget('metadata_crawler:{}'.format(crawler), 'started_time')
|
|
status_info = r_cache.hget('metadata_crawler:{}'.format(crawler), 'status')
|
|
crawler_info = '{} - {}'.format(crawler, started_time)
|
|
if status_info=='Waiting' or status_info=='Crawling':
|
|
status=True
|
|
else:
|
|
status=False
|
|
crawler_metadata.append({'crawler_info': crawler_info, 'crawling_domain': crawling_domain, 'status_info': status_info, 'status': status})
|
|
|
|
date_string = '{}-{}-{}'.format(date[0:4], date[4:6], date[6:8])
|
|
|
|
return jsonify({'last_onions': list_onion, 'statDomains': statDomains, 'crawler_metadata':crawler_metadata})
|
|
|
|
@hiddenServices.route("/hiddenServices/get_onions_by_daterange", methods=['POST'])
|
|
def get_onions_by_daterange():
|
|
date_from = request.form.get('date_from')
|
|
date_to = request.form.get('date_to')
|
|
domains_up = request.form.get('domains_up')
|
|
domains_down = request.form.get('domains_down')
|
|
domains_tags = request.form.get('domains_tags')
|
|
|
|
return redirect(url_for('hiddenServices.show_domains_by_daterange', date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down, domains_tags=domains_tags))
|
|
|
|
@hiddenServices.route("/hiddenServices/show_domains_by_daterange", methods=['GET'])
|
|
def show_domains_by_daterange():
|
|
date_from = request.args.get('date_from')
|
|
date_to = request.args.get('date_to')
|
|
domains_up = request.args.get('domains_up')
|
|
domains_down = request.args.get('domains_down')
|
|
domains_tags = request.args.get('domains_tags')
|
|
|
|
date_range = []
|
|
if date_from is not None and date_to is not None:
|
|
#change format
|
|
try:
|
|
if len(date_from) != 8:
|
|
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
|
|
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
|
|
date_range = substract_date(date_from, date_to)
|
|
except:
|
|
pass
|
|
|
|
if not date_range:
|
|
date_range.append(datetime.date.today().strftime("%Y%m%d"))
|
|
date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8]
|
|
date_to = date_from
|
|
|
|
else:
|
|
date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
|
|
date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
|
|
|
|
domains_by_day = {}
|
|
domain_metadata = {}
|
|
for date in date_range:
|
|
if domains_up:
|
|
domains_up = True
|
|
domains_by_day[date] = list(r_serv_onion.smembers('onion_up:{}'.format(date)))
|
|
for domain in domains_by_day[date]:
|
|
h = HiddenServices(domain, 'onion')
|
|
domain_metadata[domain] = {}
|
|
if domains_tags:
|
|
domains_tags = True
|
|
domain_metadata[domain]['tags'] = h.get_domain_tags(update=True)
|
|
|
|
domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check')
|
|
if domain_metadata[domain]['last_check'] is None:
|
|
domain_metadata[domain]['last_check'] = '********'
|
|
domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen')
|
|
if domain_metadata[domain]['first_seen'] is None:
|
|
domain_metadata[domain]['first_seen'] = '********'
|
|
domain_metadata[domain]['status_text'] = 'UP'
|
|
domain_metadata[domain]['status_color'] = 'Green'
|
|
domain_metadata[domain]['status_icon'] = 'fa-check-circle'
|
|
|
|
if domains_down:
|
|
domains_down = True
|
|
domains_by_day_down = list(r_serv_onion.smembers('onion_down:{}'.format(date)))
|
|
if domains_up:
|
|
domains_by_day[date].extend(domains_by_day_down)
|
|
else:
|
|
domains_by_day[date] = domains_by_day_down
|
|
for domain in domains_by_day_down:
|
|
#h = HiddenServices(onion_domain, 'onion')
|
|
domain_metadata[domain] = {}
|
|
#domain_metadata[domain]['tags'] = h.get_domain_tags()
|
|
|
|
domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check')
|
|
if domain_metadata[domain]['last_check'] is None:
|
|
domain_metadata[domain]['last_check'] = '********'
|
|
domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen')
|
|
if domain_metadata[domain]['first_seen'] is None:
|
|
domain_metadata[domain]['first_seen'] = '********'
|
|
|
|
domain_metadata[domain]['status_text'] = 'DOWN'
|
|
domain_metadata[domain]['status_color'] = 'Red'
|
|
domain_metadata[domain]['status_icon'] = 'fa-times-circle'
|
|
|
|
return render_template("domains.html", date_range=date_range, domains_by_day=domains_by_day, domain_metadata=domain_metadata,
|
|
date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down,
|
|
domains_tags=domains_tags, bootstrap_label=bootstrap_label)
|
|
|
|
@hiddenServices.route("/hiddenServices/onion_domain", methods=['GET'])
|
|
def onion_domain():
|
|
onion_domain = request.args.get('onion_domain')
|
|
if onion_domain is None or not r_serv_onion.exists('onion_metadata:{}'.format(onion_domain)):
|
|
return '404'
|
|
# # TODO: FIXME return 404
|
|
|
|
last_check = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'last_check')
|
|
if last_check is None:
|
|
last_check = '********'
|
|
last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
|
|
first_seen = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'first_seen')
|
|
if first_seen is None:
|
|
first_seen = '********'
|
|
first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8])
|
|
origin_paste = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'paste_parent')
|
|
|
|
h = HiddenServices(onion_domain, 'onion')
|
|
l_pastes = h.get_last_crawled_pastes()
|
|
if l_pastes:
|
|
status = True
|
|
else:
|
|
status = False
|
|
screenshot = h.get_domain_random_screenshot(l_pastes)
|
|
if screenshot:
|
|
screenshot = screenshot[0]
|
|
else:
|
|
screenshot = 'None'
|
|
|
|
domain_tags = h.get_domain_tags()
|
|
|
|
origin_paste_name = h.get_origin_paste_name()
|
|
origin_paste_tags = unpack_paste_tags(r_serv_metadata.smembers('tag:{}'.format(origin_paste)))
|
|
paste_tags = []
|
|
path_name = []
|
|
for path in l_pastes:
|
|
path_name.append(path.replace(PASTES_FOLDER+'/', ''))
|
|
p_tags = r_serv_metadata.smembers('tag:'+path)
|
|
paste_tags.append(unpack_paste_tags(p_tags))
|
|
|
|
return render_template("showDomain.html", domain=onion_domain, last_check=last_check, first_seen=first_seen,
|
|
l_pastes=l_pastes, paste_tags=paste_tags, bootstrap_label=bootstrap_label,
|
|
path_name=path_name, origin_paste_tags=origin_paste_tags, status=status,
|
|
origin_paste=origin_paste, origin_paste_name=origin_paste_name,
|
|
domain_tags=domain_tags, screenshot=screenshot)
|
|
|
|
@hiddenServices.route("/hiddenServices/onion_son", methods=['GET'])
|
|
def onion_son():
|
|
onion_domain = request.args.get('onion_domain')
|
|
|
|
h = HiddenServices(onion_domain, 'onion')
|
|
l_pastes = h.get_last_crawled_pastes()
|
|
l_son = h.get_domain_son(l_pastes)
|
|
print(l_son)
|
|
return 'l_son'
|
|
|
|
# ============= JSON ==============
|
|
@hiddenServices.route("/hiddenServices/domain_crawled_7days_json", methods=['GET'])
|
|
def domain_crawled_7days_json():
|
|
type = 'onion'
|
|
## TODO: # FIXME: 404 error
|
|
|
|
date_range = get_date_range(7)
|
|
json_domain_stats = []
|
|
#try:
|
|
for date in date_range:
|
|
nb_domain_up = r_serv_onion.scard('{}_up:{}'.format(type, date))
|
|
nb_domain_down = r_serv_onion.scard('{}_up:{}'.format(type, date))
|
|
date = date[0:4] + '-' + date[4:6] + '-' + date[6:8]
|
|
json_domain_stats.append({ 'date': date, 'value': int( nb_domain_up ), 'nb_domain_down': int( nb_domain_down )})
|
|
#except:
|
|
#return jsonify()
|
|
|
|
return jsonify(json_domain_stats)
|
|
|
|
# ========= REGISTRATION =========
|
|
app.register_blueprint(hiddenServices, url_prefix=baseUrl)
|