mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 22:27:17 +00:00
chg: [stats] disable statistics
This commit is contained in:
Terrtia
parent
5f150489b6
commit
ac484bed71
12 changed files with 270 additions and 268 deletions
|
|
@ -927,7 +927,7 @@ if __name__ == '__main__':
|
|||
# ail_2_ail_migration()
|
||||
# trackers_migration()
|
||||
# investigations_migration()
|
||||
# statistics_migration()
|
||||
## statistics_migration()
|
||||
|
||||
# cves_migration()
|
||||
|
||||
|
|
|
|||
|
|
@ -254,8 +254,6 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Mail.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./ModuleStats.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Onion.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./PgpDump.py; read x"
|
||||
|
|
@ -304,6 +302,8 @@ function launching_scripts {
|
|||
# sleep 0.1
|
||||
# screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x"
|
||||
# sleep 0.1
|
||||
# screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./ModuleStats.py; read x"
|
||||
# sleep 0.1
|
||||
|
||||
##################################
|
||||
# TO MIGRATE #
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from lib import ConfigLoader
|
||||
from lib import Statistics
|
||||
# from lib import Statistics
|
||||
|
||||
## LOAD CONFIG ##
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
|
|
@ -62,7 +62,7 @@ def regex_findall(module_name, redis_key, regex, item_id, item_content, max_time
|
|||
proc.join(max_time)
|
||||
if proc.is_alive():
|
||||
proc.terminate()
|
||||
Statistics.incr_module_timeout_statistic(module_name)
|
||||
# Statistics.incr_module_timeout_statistic(module_name)
|
||||
err_mess = f"{module_name}: processing timeout: {item_id}"
|
||||
print(err_mess)
|
||||
publisher.info(err_mess)
|
||||
|
|
@ -96,7 +96,7 @@ def regex_finditer(r_key, regex, item_id, content, max_time=30):
|
|||
proc.join(max_time)
|
||||
if proc.is_alive():
|
||||
proc.terminate()
|
||||
Statistics.incr_module_timeout_statistic(r_key)
|
||||
# Statistics.incr_module_timeout_statistic(r_key)
|
||||
err_mess = f"{r_key}: processing timeout: {item_id}"
|
||||
print(err_mess)
|
||||
publisher.info(err_mess)
|
||||
|
|
@ -127,7 +127,7 @@ def regex_search(r_key, regex, item_id, content, max_time=30):
|
|||
proc.join(max_time)
|
||||
if proc.is_alive():
|
||||
proc.terminate()
|
||||
Statistics.incr_module_timeout_statistic(r_key)
|
||||
# Statistics.incr_module_timeout_statistic(r_key)
|
||||
err_mess = f"{r_key}: processing timeout: {item_id}"
|
||||
print(err_mess)
|
||||
publisher.info(err_mess)
|
||||
|
|
|
|||
|
|
@ -40,7 +40,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
from modules.abstract_module import AbstractModule
|
||||
from lib.objects.Items import Item
|
||||
from lib import ConfigLoader
|
||||
from lib import Statistics
|
||||
# from lib import Statistics
|
||||
|
||||
|
||||
class Credential(AbstractModule):
|
||||
|
|
@ -160,8 +160,8 @@ class Credential(AbstractModule):
|
|||
except:
|
||||
pass
|
||||
nb_tlds[tld] = nb_tlds.get(tld, 0) + 1
|
||||
for tld in nb_tlds:
|
||||
Statistics.add_module_tld_stats_by_date('credential', date, tld, nb_tlds[tld])
|
||||
# for tld in nb_tlds:
|
||||
# Statistics.add_module_tld_stats_by_date('credential', date, tld, nb_tlds[tld])
|
||||
else:
|
||||
self.redis_logger.info(to_print)
|
||||
print(f'found {nb_cred} credentials')
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
from modules.abstract_module import AbstractModule
|
||||
from lib.objects.Items import Item
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib import Statistics
|
||||
# from lib import Statistics
|
||||
|
||||
class Iban(AbstractModule):
|
||||
"""
|
||||
|
|
@ -91,8 +91,8 @@ class Iban(AbstractModule):
|
|||
if valid_ibans:
|
||||
print(f'{valid_ibans} ibans {item_id}')
|
||||
date = datetime.datetime.now().strftime("%Y%m")
|
||||
for iban in valid_ibans:
|
||||
Statistics.add_module_tld_stats_by_date('iban', date, iban[0:2], 1)
|
||||
# for iban in valid_ibans:
|
||||
# Statistics.add_module_tld_stats_by_date('iban', date, iban[0:2], 1)
|
||||
|
||||
to_print = f'Iban;{item.get_source()};{item.get_date()};{item.get_basename()};'
|
||||
self.redis_logger.warning(f'{to_print}Checked found {len(valid_ibans)} IBAN;{item_id}')
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
from modules.abstract_module import AbstractModule
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.Items import Item
|
||||
from lib import Statistics
|
||||
# from lib import Statistics
|
||||
|
||||
class LibInjection(AbstractModule):
|
||||
"""docstring for LibInjection module."""
|
||||
|
|
@ -81,14 +81,14 @@ class LibInjection(AbstractModule):
|
|||
self.send_message_to_queue(msg, 'Tags')
|
||||
|
||||
# statistics
|
||||
# # TODO: # FIXME: remove me
|
||||
try:
|
||||
tld = url_parsed['tld'].decode()
|
||||
except:
|
||||
tld = url_parsed['tld']
|
||||
if tld is not None:
|
||||
date = datetime.now().strftime("%Y%m")
|
||||
Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
|
||||
# # # TODO: # FIXME: remove me
|
||||
# try:
|
||||
# tld = url_parsed['tld'].decode()
|
||||
# except:
|
||||
# tld = url_parsed['tld']
|
||||
# if tld is not None:
|
||||
# date = datetime.now().strftime("%Y%m")
|
||||
# Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
from modules.abstract_module import AbstractModule
|
||||
from lib.objects.Items import Item
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib import Statistics
|
||||
# from lib import Statistics
|
||||
|
||||
|
||||
class Mail(AbstractModule):
|
||||
|
|
@ -169,8 +169,8 @@ class Mail(AbstractModule):
|
|||
except:
|
||||
pass
|
||||
mx_tlds[tld] = mx_tlds.get(tld, 0) + nb_mails
|
||||
for tld in mx_tlds:
|
||||
Statistics.add_module_tld_stats_by_date('mail', item_date, tld, mx_tlds[tld])
|
||||
# for tld in mx_tlds:
|
||||
# Statistics.add_module_tld_stats_by_date('mail', item_date, tld, mx_tlds[tld])
|
||||
|
||||
msg = f'Mails;{item.get_source()};{item_date};{item.get_basename()};Checked {num_valid_email} e-mail(s);{item_id}'
|
||||
if num_valid_email > self.mail_threshold:
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
from modules.abstract_module import AbstractModule
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.Items import Item
|
||||
from lib import Statistics
|
||||
# from lib import Statistics
|
||||
|
||||
class SQLInjectionDetection(AbstractModule):
|
||||
"""docstring for SQLInjectionDetection module."""
|
||||
|
|
@ -62,15 +62,15 @@ class SQLInjectionDetection(AbstractModule):
|
|||
self.send_message_to_queue(msg, 'Tags')
|
||||
|
||||
# statistics
|
||||
tld = url_parsed['tld']
|
||||
if tld is not None:
|
||||
# # TODO: # FIXME: remove me
|
||||
try:
|
||||
tld = tld.decode()
|
||||
except:
|
||||
pass
|
||||
date = datetime.now().strftime("%Y%m")
|
||||
Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
|
||||
# tld = url_parsed['tld']
|
||||
# if tld is not None:
|
||||
# # # TODO: # FIXME: remove me
|
||||
# try:
|
||||
# tld = tld.decode()
|
||||
# except:
|
||||
# pass
|
||||
# date = datetime.now().strftime("%Y%m")
|
||||
# Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
|
||||
|
||||
# Try to detect if the url passed might be an sql injection by applying the regex
|
||||
# defined above on it.
|
||||
|
|
|
|||
|
|
@ -4,96 +4,96 @@
|
|||
'''
|
||||
Flask functions and routes for the trending charts page
|
||||
'''
|
||||
import os
|
||||
import sys
|
||||
import datetime
|
||||
import flask
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint
|
||||
|
||||
from Role_Manager import login_admin, login_analyst, login_read_only
|
||||
from flask_login import login_required
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from packages.Date import Date
|
||||
|
||||
# ============ VARIABLES ============
|
||||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
config_loader = Flask_config.config_loader
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_charts = Flask_config.r_serv_charts
|
||||
|
||||
trendings = Blueprint('trendings', __name__, template_folder='templates')
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
def get_date_range(num_day):
|
||||
curr_date = datetime.date.today()
|
||||
date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
|
||||
date_list = []
|
||||
|
||||
for i in range(0, num_day+1):
|
||||
date_list.append(date.substract_day(i))
|
||||
|
||||
return date_list
|
||||
|
||||
|
||||
# ============ ROUTES ============
|
||||
|
||||
@trendings.route("/_progressionCharts", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def progressionCharts():
|
||||
attribute_name = request.args.get('attributeName')
|
||||
trending_name = request.args.get('trendingName')
|
||||
bar_requested = True if request.args.get('bar') == "true" else False
|
||||
|
||||
if (bar_requested):
|
||||
num_day = int(request.args.get('days'))
|
||||
bar_values = []
|
||||
|
||||
date_range = get_date_range(num_day)
|
||||
# Retreive all data from the last num_day
|
||||
for date in date_range:
|
||||
|
||||
curr_value = r_serv_charts.hget(attribute_name, date)
|
||||
bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
|
||||
bar_values.insert(0, attribute_name)
|
||||
return jsonify(bar_values)
|
||||
|
||||
else:
|
||||
redis_progression_name = "z_top_progression_" + trending_name
|
||||
keyw_value = r_serv_charts.zrevrangebyscore(redis_progression_name, '+inf', '-inf', withscores=True, start=0, num=10)
|
||||
|
||||
return jsonify(keyw_value)
|
||||
|
||||
@trendings.route("/wordstrending/")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def wordstrending():
|
||||
default_display = config_loader.get_config_str("Flask", "default_display")
|
||||
return render_template("Wordstrending.html", default_display = default_display)
|
||||
|
||||
|
||||
@trendings.route("/protocolstrending/")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def protocolstrending():
|
||||
default_display = config_loader.get_config_str("Flask", "default_display")
|
||||
return render_template("Protocolstrending.html", default_display = default_display)
|
||||
|
||||
|
||||
@trendings.route("/trending/")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def trending():
|
||||
default_display = config_loader.get_config_str("Flask", "default_display")
|
||||
return render_template("Trending.html", default_display = default_display)
|
||||
|
||||
|
||||
# ========= REGISTRATION =========
|
||||
app.register_blueprint(trendings, url_prefix=baseUrl)
|
||||
# import os
|
||||
# import sys
|
||||
# import datetime
|
||||
# import flask
|
||||
# from flask import Flask, render_template, jsonify, request, Blueprint
|
||||
#
|
||||
# from Role_Manager import login_admin, login_analyst, login_read_only
|
||||
# from flask_login import login_required
|
||||
#
|
||||
# sys.path.append(os.environ['AIL_BIN'])
|
||||
# ##################################
|
||||
# # Import Project packages
|
||||
# ##################################
|
||||
# from packages.Date import Date
|
||||
#
|
||||
# # ============ VARIABLES ============
|
||||
# import Flask_config
|
||||
#
|
||||
# app = Flask_config.app
|
||||
# config_loader = Flask_config.config_loader
|
||||
# baseUrl = Flask_config.baseUrl
|
||||
# r_serv_charts = Flask_config.r_serv_charts
|
||||
#
|
||||
# trendings = Blueprint('trendings', __name__, template_folder='templates')
|
||||
#
|
||||
# # ============ FUNCTIONS ============
|
||||
#
|
||||
# def get_date_range(num_day):
|
||||
# curr_date = datetime.date.today()
|
||||
# date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
|
||||
# date_list = []
|
||||
#
|
||||
# for i in range(0, num_day+1):
|
||||
# date_list.append(date.substract_day(i))
|
||||
#
|
||||
# return date_list
|
||||
#
|
||||
#
|
||||
# # ============ ROUTES ============
|
||||
#
|
||||
# @trendings.route("/_progressionCharts", methods=['GET'])
|
||||
# @login_required
|
||||
# @login_read_only
|
||||
# def progressionCharts():
|
||||
# attribute_name = request.args.get('attributeName')
|
||||
# trending_name = request.args.get('trendingName')
|
||||
# bar_requested = True if request.args.get('bar') == "true" else False
|
||||
#
|
||||
# if (bar_requested):
|
||||
# num_day = int(request.args.get('days'))
|
||||
# bar_values = []
|
||||
#
|
||||
# date_range = get_date_range(num_day)
|
||||
# # Retreive all data from the last num_day
|
||||
# for date in date_range:
|
||||
#
|
||||
# curr_value = r_serv_charts.hget(attribute_name, date)
|
||||
# bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
|
||||
# bar_values.insert(0, attribute_name)
|
||||
# return jsonify(bar_values)
|
||||
#
|
||||
# else:
|
||||
# redis_progression_name = "z_top_progression_" + trending_name
|
||||
# keyw_value = r_serv_charts.zrevrangebyscore(redis_progression_name, '+inf', '-inf', withscores=True, start=0, num=10)
|
||||
#
|
||||
# return jsonify(keyw_value)
|
||||
#
|
||||
# @trendings.route("/wordstrending/")
|
||||
# @login_required
|
||||
# @login_read_only
|
||||
# def wordstrending():
|
||||
# default_display = config_loader.get_config_str("Flask", "default_display")
|
||||
# return render_template("Wordstrending.html", default_display = default_display)
|
||||
#
|
||||
#
|
||||
# @trendings.route("/protocolstrending/")
|
||||
# @login_required
|
||||
# @login_read_only
|
||||
# def protocolstrending():
|
||||
# default_display = config_loader.get_config_str("Flask", "default_display")
|
||||
# return render_template("Protocolstrending.html", default_display = default_display)
|
||||
#
|
||||
#
|
||||
# @trendings.route("/trending/")
|
||||
# @login_required
|
||||
# @login_read_only
|
||||
# def trending():
|
||||
# default_display = config_loader.get_config_str("Flask", "default_display")
|
||||
# return render_template("Trending.html", default_display = default_display)
|
||||
#
|
||||
#
|
||||
# # ========= REGISTRATION =========
|
||||
# app.register_blueprint(trendings, url_prefix=baseUrl)
|
||||
|
|
|
|||
|
|
@ -4,140 +4,140 @@
|
|||
'''
|
||||
Flask functions and routes for the trending modules page
|
||||
'''
|
||||
import os
|
||||
import sys
|
||||
import datetime
|
||||
import flask
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint
|
||||
|
||||
from Role_Manager import login_admin, login_analyst, login_read_only
|
||||
from flask_login import login_required
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from packages.Date import Date
|
||||
|
||||
# ============ VARIABLES ============
|
||||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_charts = Flask_config.r_serv_charts
|
||||
|
||||
trendingmodules = Blueprint('trendingmodules', __name__, template_folder='templates')
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
|
||||
# Iterate over elements in the module provided and return the today data or the last data
|
||||
# return format: [('passed_days', num_of_passed_days), ('elem_name1', elem_value1), ('elem_name2', elem_value2)]]
|
||||
def get_top_relevant_data(server, module_name):
|
||||
days = 0
|
||||
for date in get_date_range(15):
|
||||
redis_progression_name_set = 'top_'+ module_name +'_set_' + date
|
||||
member_set = server.zrevrangebyscore(redis_progression_name_set, '+inf', '-inf', withscores=True)
|
||||
|
||||
if len(member_set) == 0: #No data for this date
|
||||
days += 1
|
||||
else:
|
||||
member_set.insert(0, ("passed_days", days))
|
||||
return member_set
|
||||
|
||||
|
||||
def get_date_range(num_day):
|
||||
curr_date = datetime.date.today()
|
||||
date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
|
||||
date_list = []
|
||||
|
||||
for i in range(0, num_day+1):
|
||||
date_list.append(date.substract_day(i))
|
||||
return date_list
|
||||
|
||||
# ============ ROUTES ============
|
||||
|
||||
@trendingmodules.route("/_moduleCharts", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def modulesCharts():
|
||||
keyword_name = request.args.get('keywordName')
|
||||
module_name = request.args.get('moduleName')
|
||||
bar_requested = True if request.args.get('bar') == "true" else False
|
||||
|
||||
if (bar_requested):
|
||||
num_day = int(request.args.get('days'))
|
||||
bar_values = []
|
||||
|
||||
date_range = get_date_range(num_day)
|
||||
# Retreive all data from the last num_day
|
||||
for date in date_range:
|
||||
curr_value = r_serv_charts.hget(date, module_name+'-'+keyword_name)
|
||||
bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
|
||||
bar_values.insert(0, keyword_name)
|
||||
return jsonify(bar_values)
|
||||
|
||||
else:
|
||||
member_set = get_top_relevant_data(r_serv_charts, module_name)
|
||||
member_set = member_set if member_set is not None else []
|
||||
if len(member_set) == 0:
|
||||
member_set.append(("No relevant data", int(100)))
|
||||
return jsonify(member_set)
|
||||
|
||||
|
||||
@trendingmodules.route("/_providersChart", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def providersChart():
|
||||
keyword_name = request.args.get('keywordName')
|
||||
module_name = request.args.get('moduleName')
|
||||
bar_requested = True if request.args.get('bar') == "true" else False
|
||||
|
||||
if (bar_requested):
|
||||
num_day = int(request.args.get('days'))
|
||||
bar_values = []
|
||||
|
||||
date_range = get_date_range(num_day)
|
||||
# Retreive all data from the last num_day
|
||||
for date in date_range:
|
||||
curr_value_size = ( r_serv_charts.hget(keyword_name+'_'+'size', date) )
|
||||
if curr_value_size is not None:
|
||||
curr_value_size = curr_value_size
|
||||
|
||||
curr_value_num = r_serv_charts.hget(keyword_name+'_'+'num', date)
|
||||
|
||||
curr_value_size_avg = r_serv_charts.hget(keyword_name+'_'+'avg', date)
|
||||
if curr_value_size_avg is not None:
|
||||
curr_value_size_avg = curr_value_size_avg
|
||||
|
||||
|
||||
if module_name == "size":
|
||||
curr_value = float(curr_value_size_avg if curr_value_size_avg is not None else 0)
|
||||
else:
|
||||
curr_value = float(curr_value_num if curr_value_num is not None else 0.0)
|
||||
|
||||
bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], curr_value])
|
||||
bar_values.insert(0, keyword_name)
|
||||
return jsonify(bar_values)
|
||||
|
||||
else:
|
||||
#redis_provider_name_set = 'top_size_set' if module_name == "size" else 'providers_set'
|
||||
redis_provider_name_set = 'top_avg_size_set_' if module_name == "size" else 'providers_set_'
|
||||
redis_provider_name_set = redis_provider_name_set + get_date_range(0)[0]
|
||||
|
||||
member_set = r_serv_charts.zrevrangebyscore(redis_provider_name_set, '+inf', '-inf', withscores=True, start=0, num=8)
|
||||
|
||||
# Member set is a list of (value, score) pairs
|
||||
if len(member_set) == 0:
|
||||
member_set.append(("No relevant data", float(100)))
|
||||
return jsonify(member_set)
|
||||
|
||||
|
||||
@trendingmodules.route("/moduletrending/")
|
||||
@login_required
|
||||
@login_read_only
|
||||
def moduletrending():
|
||||
return render_template("Moduletrending.html")
|
||||
|
||||
|
||||
# ========= REGISTRATION =========
|
||||
app.register_blueprint(trendingmodules, url_prefix=baseUrl)
|
||||
# import os
|
||||
# import sys
|
||||
# import datetime
|
||||
# import flask
|
||||
# from flask import Flask, render_template, jsonify, request, Blueprint
|
||||
#
|
||||
# from Role_Manager import login_admin, login_analyst, login_read_only
|
||||
# from flask_login import login_required
|
||||
#
|
||||
# sys.path.append(os.environ['AIL_BIN'])
|
||||
# ##################################
|
||||
# # Import Project packages
|
||||
# ##################################
|
||||
# from packages.Date import Date
|
||||
#
|
||||
# # ============ VARIABLES ============
|
||||
# import Flask_config
|
||||
#
|
||||
# app = Flask_config.app
|
||||
# baseUrl = Flask_config.baseUrl
|
||||
# r_serv_charts = Flask_config.r_serv_charts
|
||||
#
|
||||
# trendingmodules = Blueprint('trendingmodules', __name__, template_folder='templates')
|
||||
#
|
||||
# # ============ FUNCTIONS ============
|
||||
#
|
||||
# # Iterate over elements in the module provided and return the today data or the last data
|
||||
# # return format: [('passed_days', num_of_passed_days), ('elem_name1', elem_value1), ('elem_name2', elem_value2)]]
|
||||
# def get_top_relevant_data(server, module_name):
|
||||
# days = 0
|
||||
# for date in get_date_range(15):
|
||||
# redis_progression_name_set = 'top_'+ module_name +'_set_' + date
|
||||
# member_set = server.zrevrangebyscore(redis_progression_name_set, '+inf', '-inf', withscores=True)
|
||||
#
|
||||
# if len(member_set) == 0: #No data for this date
|
||||
# days += 1
|
||||
# else:
|
||||
# member_set.insert(0, ("passed_days", days))
|
||||
# return member_set
|
||||
#
|
||||
#
|
||||
# def get_date_range(num_day):
|
||||
# curr_date = datetime.date.today()
|
||||
# date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
|
||||
# date_list = []
|
||||
#
|
||||
# for i in range(0, num_day+1):
|
||||
# date_list.append(date.substract_day(i))
|
||||
# return date_list
|
||||
#
|
||||
# # ============ ROUTES ============
|
||||
#
|
||||
# @trendingmodules.route("/_moduleCharts", methods=['GET'])
|
||||
# @login_required
|
||||
# @login_read_only
|
||||
# def modulesCharts():
|
||||
# keyword_name = request.args.get('keywordName')
|
||||
# module_name = request.args.get('moduleName')
|
||||
# bar_requested = True if request.args.get('bar') == "true" else False
|
||||
#
|
||||
# if (bar_requested):
|
||||
# num_day = int(request.args.get('days'))
|
||||
# bar_values = []
|
||||
#
|
||||
# date_range = get_date_range(num_day)
|
||||
# # Retreive all data from the last num_day
|
||||
# for date in date_range:
|
||||
# curr_value = r_serv_charts.hget(date, module_name+'-'+keyword_name)
|
||||
# bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
|
||||
# bar_values.insert(0, keyword_name)
|
||||
# return jsonify(bar_values)
|
||||
#
|
||||
# else:
|
||||
# member_set = get_top_relevant_data(r_serv_charts, module_name)
|
||||
# member_set = member_set if member_set is not None else []
|
||||
# if len(member_set) == 0:
|
||||
# member_set.append(("No relevant data", int(100)))
|
||||
# return jsonify(member_set)
|
||||
#
|
||||
#
|
||||
# @trendingmodules.route("/_providersChart", methods=['GET'])
|
||||
# @login_required
|
||||
# @login_read_only
|
||||
# def providersChart():
|
||||
# keyword_name = request.args.get('keywordName')
|
||||
# module_name = request.args.get('moduleName')
|
||||
# bar_requested = True if request.args.get('bar') == "true" else False
|
||||
#
|
||||
# if (bar_requested):
|
||||
# num_day = int(request.args.get('days'))
|
||||
# bar_values = []
|
||||
#
|
||||
# date_range = get_date_range(num_day)
|
||||
# # Retreive all data from the last num_day
|
||||
# for date in date_range:
|
||||
# curr_value_size = ( r_serv_charts.hget(keyword_name+'_'+'size', date) )
|
||||
# if curr_value_size is not None:
|
||||
# curr_value_size = curr_value_size
|
||||
#
|
||||
# curr_value_num = r_serv_charts.hget(keyword_name+'_'+'num', date)
|
||||
#
|
||||
# curr_value_size_avg = r_serv_charts.hget(keyword_name+'_'+'avg', date)
|
||||
# if curr_value_size_avg is not None:
|
||||
# curr_value_size_avg = curr_value_size_avg
|
||||
#
|
||||
#
|
||||
# if module_name == "size":
|
||||
# curr_value = float(curr_value_size_avg if curr_value_size_avg is not None else 0)
|
||||
# else:
|
||||
# curr_value = float(curr_value_num if curr_value_num is not None else 0.0)
|
||||
#
|
||||
# bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], curr_value])
|
||||
# bar_values.insert(0, keyword_name)
|
||||
# return jsonify(bar_values)
|
||||
#
|
||||
# else:
|
||||
# #redis_provider_name_set = 'top_size_set' if module_name == "size" else 'providers_set'
|
||||
# redis_provider_name_set = 'top_avg_size_set_' if module_name == "size" else 'providers_set_'
|
||||
# redis_provider_name_set = redis_provider_name_set + get_date_range(0)[0]
|
||||
#
|
||||
# member_set = r_serv_charts.zrevrangebyscore(redis_provider_name_set, '+inf', '-inf', withscores=True, start=0, num=8)
|
||||
#
|
||||
# # Member set is a list of (value, score) pairs
|
||||
# if len(member_set) == 0:
|
||||
# member_set.append(("No relevant data", float(100)))
|
||||
# return jsonify(member_set)
|
||||
#
|
||||
#
|
||||
# @trendingmodules.route("/moduletrending/")
|
||||
# @login_required
|
||||
# @login_read_only
|
||||
# def moduletrending():
|
||||
# return render_template("Moduletrending.html")
|
||||
#
|
||||
#
|
||||
# # ========= REGISTRATION =========
|
||||
# app.register_blueprint(trendingmodules, url_prefix=baseUrl)
|
||||
|
|
|
|||
|
|
@ -1 +1,3 @@
|
|||
{#
|
||||
<li id='page-modulestats'><a href="{{ url_for('trendingmodules.moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>
|
||||
#}
|
||||
|
|
|
|||
|
|
@ -27,9 +27,9 @@
|
|||
<li class="nav-item mr-3">
|
||||
<a class="nav-link" id="page-Decoded" href="{{ url_for('investigations_b.investigations_dashboard') }}" aria-disabled="true"><i class="fas fa-cube"></i> Objects</a>
|
||||
</li>
|
||||
<li class="nav-item mr-3">
|
||||
<a class="nav-link" href="{{ url_for('trendingmodules.moduletrending') }}" aria-disabled="true"><i class="fas fa-chart-bar"></i> Statistics</a>
|
||||
</li>
|
||||
{# <li class="nav-item mr-3">#}
|
||||
{# <a class="nav-link" href="{{ url_for('trendingmodules.moduletrending') }}" aria-disabled="true"><i class="fas fa-chart-bar"></i> Statistics</a>#}
|
||||
{# </li>#}
|
||||
<li class="nav-item mr-3">
|
||||
<a class="nav-link" id="page-options" href="{{ url_for('settings_b.settings_page') }}" aria-disabled="true"><i class="fas fa-cog"></i> Server Management</a>
|
||||
</li>
|
||||
|
|
|
|||
Loading…
Reference in a new issue