mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
chg: [stats] disable statistics
This commit is contained in:
parent
5f150489b6
commit
ac484bed71
12 changed files with 270 additions and 268 deletions
|
@ -927,7 +927,7 @@ if __name__ == '__main__':
|
||||||
# ail_2_ail_migration()
|
# ail_2_ail_migration()
|
||||||
# trackers_migration()
|
# trackers_migration()
|
||||||
# investigations_migration()
|
# investigations_migration()
|
||||||
# statistics_migration()
|
## statistics_migration()
|
||||||
|
|
||||||
# cves_migration()
|
# cves_migration()
|
||||||
|
|
||||||
|
|
|
@ -254,8 +254,6 @@ function launching_scripts {
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Mail.py; read x"
|
screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Mail.py; read x"
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./ModuleStats.py; read x"
|
|
||||||
sleep 0.1
|
|
||||||
screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Onion.py; read x"
|
screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Onion.py; read x"
|
||||||
sleep 0.1
|
sleep 0.1
|
||||||
screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./PgpDump.py; read x"
|
screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./PgpDump.py; read x"
|
||||||
|
@ -304,6 +302,8 @@ function launching_scripts {
|
||||||
# sleep 0.1
|
# sleep 0.1
|
||||||
# screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x"
|
# screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x"
|
||||||
# sleep 0.1
|
# sleep 0.1
|
||||||
|
# screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./ModuleStats.py; read x"
|
||||||
|
# sleep 0.1
|
||||||
|
|
||||||
##################################
|
##################################
|
||||||
# TO MIGRATE #
|
# TO MIGRATE #
|
||||||
|
|
|
@ -20,7 +20,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
||||||
# Import Project packages
|
# Import Project packages
|
||||||
##################################
|
##################################
|
||||||
from lib import ConfigLoader
|
from lib import ConfigLoader
|
||||||
from lib import Statistics
|
# from lib import Statistics
|
||||||
|
|
||||||
## LOAD CONFIG ##
|
## LOAD CONFIG ##
|
||||||
config_loader = ConfigLoader.ConfigLoader()
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
@ -62,7 +62,7 @@ def regex_findall(module_name, redis_key, regex, item_id, item_content, max_time
|
||||||
proc.join(max_time)
|
proc.join(max_time)
|
||||||
if proc.is_alive():
|
if proc.is_alive():
|
||||||
proc.terminate()
|
proc.terminate()
|
||||||
Statistics.incr_module_timeout_statistic(module_name)
|
# Statistics.incr_module_timeout_statistic(module_name)
|
||||||
err_mess = f"{module_name}: processing timeout: {item_id}"
|
err_mess = f"{module_name}: processing timeout: {item_id}"
|
||||||
print(err_mess)
|
print(err_mess)
|
||||||
publisher.info(err_mess)
|
publisher.info(err_mess)
|
||||||
|
@ -96,7 +96,7 @@ def regex_finditer(r_key, regex, item_id, content, max_time=30):
|
||||||
proc.join(max_time)
|
proc.join(max_time)
|
||||||
if proc.is_alive():
|
if proc.is_alive():
|
||||||
proc.terminate()
|
proc.terminate()
|
||||||
Statistics.incr_module_timeout_statistic(r_key)
|
# Statistics.incr_module_timeout_statistic(r_key)
|
||||||
err_mess = f"{r_key}: processing timeout: {item_id}"
|
err_mess = f"{r_key}: processing timeout: {item_id}"
|
||||||
print(err_mess)
|
print(err_mess)
|
||||||
publisher.info(err_mess)
|
publisher.info(err_mess)
|
||||||
|
@ -127,7 +127,7 @@ def regex_search(r_key, regex, item_id, content, max_time=30):
|
||||||
proc.join(max_time)
|
proc.join(max_time)
|
||||||
if proc.is_alive():
|
if proc.is_alive():
|
||||||
proc.terminate()
|
proc.terminate()
|
||||||
Statistics.incr_module_timeout_statistic(r_key)
|
# Statistics.incr_module_timeout_statistic(r_key)
|
||||||
err_mess = f"{r_key}: processing timeout: {item_id}"
|
err_mess = f"{r_key}: processing timeout: {item_id}"
|
||||||
print(err_mess)
|
print(err_mess)
|
||||||
publisher.info(err_mess)
|
publisher.info(err_mess)
|
||||||
|
|
|
@ -40,7 +40,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
||||||
from modules.abstract_module import AbstractModule
|
from modules.abstract_module import AbstractModule
|
||||||
from lib.objects.Items import Item
|
from lib.objects.Items import Item
|
||||||
from lib import ConfigLoader
|
from lib import ConfigLoader
|
||||||
from lib import Statistics
|
# from lib import Statistics
|
||||||
|
|
||||||
|
|
||||||
class Credential(AbstractModule):
|
class Credential(AbstractModule):
|
||||||
|
@ -160,8 +160,8 @@ class Credential(AbstractModule):
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
nb_tlds[tld] = nb_tlds.get(tld, 0) + 1
|
nb_tlds[tld] = nb_tlds.get(tld, 0) + 1
|
||||||
for tld in nb_tlds:
|
# for tld in nb_tlds:
|
||||||
Statistics.add_module_tld_stats_by_date('credential', date, tld, nb_tlds[tld])
|
# Statistics.add_module_tld_stats_by_date('credential', date, tld, nb_tlds[tld])
|
||||||
else:
|
else:
|
||||||
self.redis_logger.info(to_print)
|
self.redis_logger.info(to_print)
|
||||||
print(f'found {nb_cred} credentials')
|
print(f'found {nb_cred} credentials')
|
||||||
|
|
|
@ -26,7 +26,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
||||||
from modules.abstract_module import AbstractModule
|
from modules.abstract_module import AbstractModule
|
||||||
from lib.objects.Items import Item
|
from lib.objects.Items import Item
|
||||||
from lib.ConfigLoader import ConfigLoader
|
from lib.ConfigLoader import ConfigLoader
|
||||||
from lib import Statistics
|
# from lib import Statistics
|
||||||
|
|
||||||
class Iban(AbstractModule):
|
class Iban(AbstractModule):
|
||||||
"""
|
"""
|
||||||
|
@ -91,8 +91,8 @@ class Iban(AbstractModule):
|
||||||
if valid_ibans:
|
if valid_ibans:
|
||||||
print(f'{valid_ibans} ibans {item_id}')
|
print(f'{valid_ibans} ibans {item_id}')
|
||||||
date = datetime.datetime.now().strftime("%Y%m")
|
date = datetime.datetime.now().strftime("%Y%m")
|
||||||
for iban in valid_ibans:
|
# for iban in valid_ibans:
|
||||||
Statistics.add_module_tld_stats_by_date('iban', date, iban[0:2], 1)
|
# Statistics.add_module_tld_stats_by_date('iban', date, iban[0:2], 1)
|
||||||
|
|
||||||
to_print = f'Iban;{item.get_source()};{item.get_date()};{item.get_basename()};'
|
to_print = f'Iban;{item.get_source()};{item.get_date()};{item.get_basename()};'
|
||||||
self.redis_logger.warning(f'{to_print}Checked found {len(valid_ibans)} IBAN;{item_id}')
|
self.redis_logger.warning(f'{to_print}Checked found {len(valid_ibans)} IBAN;{item_id}')
|
||||||
|
|
|
@ -27,7 +27,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
||||||
from modules.abstract_module import AbstractModule
|
from modules.abstract_module import AbstractModule
|
||||||
from lib.ConfigLoader import ConfigLoader
|
from lib.ConfigLoader import ConfigLoader
|
||||||
from lib.objects.Items import Item
|
from lib.objects.Items import Item
|
||||||
from lib import Statistics
|
# from lib import Statistics
|
||||||
|
|
||||||
class LibInjection(AbstractModule):
|
class LibInjection(AbstractModule):
|
||||||
"""docstring for LibInjection module."""
|
"""docstring for LibInjection module."""
|
||||||
|
@ -81,14 +81,14 @@ class LibInjection(AbstractModule):
|
||||||
self.send_message_to_queue(msg, 'Tags')
|
self.send_message_to_queue(msg, 'Tags')
|
||||||
|
|
||||||
# statistics
|
# statistics
|
||||||
# # TODO: # FIXME: remove me
|
# # # TODO: # FIXME: remove me
|
||||||
try:
|
# try:
|
||||||
tld = url_parsed['tld'].decode()
|
# tld = url_parsed['tld'].decode()
|
||||||
except:
|
# except:
|
||||||
tld = url_parsed['tld']
|
# tld = url_parsed['tld']
|
||||||
if tld is not None:
|
# if tld is not None:
|
||||||
date = datetime.now().strftime("%Y%m")
|
# date = datetime.now().strftime("%Y%m")
|
||||||
Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
|
# Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
|
|
|
@ -28,7 +28,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
||||||
from modules.abstract_module import AbstractModule
|
from modules.abstract_module import AbstractModule
|
||||||
from lib.objects.Items import Item
|
from lib.objects.Items import Item
|
||||||
from lib.ConfigLoader import ConfigLoader
|
from lib.ConfigLoader import ConfigLoader
|
||||||
from lib import Statistics
|
# from lib import Statistics
|
||||||
|
|
||||||
|
|
||||||
class Mail(AbstractModule):
|
class Mail(AbstractModule):
|
||||||
|
@ -169,8 +169,8 @@ class Mail(AbstractModule):
|
||||||
except:
|
except:
|
||||||
pass
|
pass
|
||||||
mx_tlds[tld] = mx_tlds.get(tld, 0) + nb_mails
|
mx_tlds[tld] = mx_tlds.get(tld, 0) + nb_mails
|
||||||
for tld in mx_tlds:
|
# for tld in mx_tlds:
|
||||||
Statistics.add_module_tld_stats_by_date('mail', item_date, tld, mx_tlds[tld])
|
# Statistics.add_module_tld_stats_by_date('mail', item_date, tld, mx_tlds[tld])
|
||||||
|
|
||||||
msg = f'Mails;{item.get_source()};{item_date};{item.get_basename()};Checked {num_valid_email} e-mail(s);{item_id}'
|
msg = f'Mails;{item.get_source()};{item_date};{item.get_basename()};Checked {num_valid_email} e-mail(s);{item_id}'
|
||||||
if num_valid_email > self.mail_threshold:
|
if num_valid_email > self.mail_threshold:
|
||||||
|
|
|
@ -27,7 +27,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
||||||
from modules.abstract_module import AbstractModule
|
from modules.abstract_module import AbstractModule
|
||||||
from lib.ConfigLoader import ConfigLoader
|
from lib.ConfigLoader import ConfigLoader
|
||||||
from lib.objects.Items import Item
|
from lib.objects.Items import Item
|
||||||
from lib import Statistics
|
# from lib import Statistics
|
||||||
|
|
||||||
class SQLInjectionDetection(AbstractModule):
|
class SQLInjectionDetection(AbstractModule):
|
||||||
"""docstring for SQLInjectionDetection module."""
|
"""docstring for SQLInjectionDetection module."""
|
||||||
|
@ -62,15 +62,15 @@ class SQLInjectionDetection(AbstractModule):
|
||||||
self.send_message_to_queue(msg, 'Tags')
|
self.send_message_to_queue(msg, 'Tags')
|
||||||
|
|
||||||
# statistics
|
# statistics
|
||||||
tld = url_parsed['tld']
|
# tld = url_parsed['tld']
|
||||||
if tld is not None:
|
# if tld is not None:
|
||||||
# # TODO: # FIXME: remove me
|
# # # TODO: # FIXME: remove me
|
||||||
try:
|
# try:
|
||||||
tld = tld.decode()
|
# tld = tld.decode()
|
||||||
except:
|
# except:
|
||||||
pass
|
# pass
|
||||||
date = datetime.now().strftime("%Y%m")
|
# date = datetime.now().strftime("%Y%m")
|
||||||
Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
|
# Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
|
||||||
|
|
||||||
# Try to detect if the url passed might be an sql injection by applying the regex
|
# Try to detect if the url passed might be an sql injection by applying the regex
|
||||||
# defined above on it.
|
# defined above on it.
|
||||||
|
|
|
@ -4,96 +4,96 @@
|
||||||
'''
|
'''
|
||||||
Flask functions and routes for the trending charts page
|
Flask functions and routes for the trending charts page
|
||||||
'''
|
'''
|
||||||
import os
|
# import os
|
||||||
import sys
|
# import sys
|
||||||
import datetime
|
# import datetime
|
||||||
import flask
|
# import flask
|
||||||
from flask import Flask, render_template, jsonify, request, Blueprint
|
# from flask import Flask, render_template, jsonify, request, Blueprint
|
||||||
|
#
|
||||||
from Role_Manager import login_admin, login_analyst, login_read_only
|
# from Role_Manager import login_admin, login_analyst, login_read_only
|
||||||
from flask_login import login_required
|
# from flask_login import login_required
|
||||||
|
#
|
||||||
sys.path.append(os.environ['AIL_BIN'])
|
# sys.path.append(os.environ['AIL_BIN'])
|
||||||
##################################
|
# ##################################
|
||||||
# Import Project packages
|
# # Import Project packages
|
||||||
##################################
|
# ##################################
|
||||||
from packages.Date import Date
|
# from packages.Date import Date
|
||||||
|
#
|
||||||
# ============ VARIABLES ============
|
# # ============ VARIABLES ============
|
||||||
import Flask_config
|
# import Flask_config
|
||||||
|
#
|
||||||
app = Flask_config.app
|
# app = Flask_config.app
|
||||||
config_loader = Flask_config.config_loader
|
# config_loader = Flask_config.config_loader
|
||||||
baseUrl = Flask_config.baseUrl
|
# baseUrl = Flask_config.baseUrl
|
||||||
r_serv_charts = Flask_config.r_serv_charts
|
# r_serv_charts = Flask_config.r_serv_charts
|
||||||
|
#
|
||||||
trendings = Blueprint('trendings', __name__, template_folder='templates')
|
# trendings = Blueprint('trendings', __name__, template_folder='templates')
|
||||||
|
#
|
||||||
# ============ FUNCTIONS ============
|
# # ============ FUNCTIONS ============
|
||||||
|
#
|
||||||
def get_date_range(num_day):
|
# def get_date_range(num_day):
|
||||||
curr_date = datetime.date.today()
|
# curr_date = datetime.date.today()
|
||||||
date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
|
# date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
|
||||||
date_list = []
|
# date_list = []
|
||||||
|
#
|
||||||
for i in range(0, num_day+1):
|
# for i in range(0, num_day+1):
|
||||||
date_list.append(date.substract_day(i))
|
# date_list.append(date.substract_day(i))
|
||||||
|
#
|
||||||
return date_list
|
# return date_list
|
||||||
|
#
|
||||||
|
#
|
||||||
# ============ ROUTES ============
|
# # ============ ROUTES ============
|
||||||
|
#
|
||||||
@trendings.route("/_progressionCharts", methods=['GET'])
|
# @trendings.route("/_progressionCharts", methods=['GET'])
|
||||||
@login_required
|
# @login_required
|
||||||
@login_read_only
|
# @login_read_only
|
||||||
def progressionCharts():
|
# def progressionCharts():
|
||||||
attribute_name = request.args.get('attributeName')
|
# attribute_name = request.args.get('attributeName')
|
||||||
trending_name = request.args.get('trendingName')
|
# trending_name = request.args.get('trendingName')
|
||||||
bar_requested = True if request.args.get('bar') == "true" else False
|
# bar_requested = True if request.args.get('bar') == "true" else False
|
||||||
|
#
|
||||||
if (bar_requested):
|
# if (bar_requested):
|
||||||
num_day = int(request.args.get('days'))
|
# num_day = int(request.args.get('days'))
|
||||||
bar_values = []
|
# bar_values = []
|
||||||
|
#
|
||||||
date_range = get_date_range(num_day)
|
# date_range = get_date_range(num_day)
|
||||||
# Retreive all data from the last num_day
|
# # Retreive all data from the last num_day
|
||||||
for date in date_range:
|
# for date in date_range:
|
||||||
|
#
|
||||||
curr_value = r_serv_charts.hget(attribute_name, date)
|
# curr_value = r_serv_charts.hget(attribute_name, date)
|
||||||
bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
|
# bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
|
||||||
bar_values.insert(0, attribute_name)
|
# bar_values.insert(0, attribute_name)
|
||||||
return jsonify(bar_values)
|
# return jsonify(bar_values)
|
||||||
|
#
|
||||||
else:
|
# else:
|
||||||
redis_progression_name = "z_top_progression_" + trending_name
|
# redis_progression_name = "z_top_progression_" + trending_name
|
||||||
keyw_value = r_serv_charts.zrevrangebyscore(redis_progression_name, '+inf', '-inf', withscores=True, start=0, num=10)
|
# keyw_value = r_serv_charts.zrevrangebyscore(redis_progression_name, '+inf', '-inf', withscores=True, start=0, num=10)
|
||||||
|
#
|
||||||
return jsonify(keyw_value)
|
# return jsonify(keyw_value)
|
||||||
|
#
|
||||||
@trendings.route("/wordstrending/")
|
# @trendings.route("/wordstrending/")
|
||||||
@login_required
|
# @login_required
|
||||||
@login_read_only
|
# @login_read_only
|
||||||
def wordstrending():
|
# def wordstrending():
|
||||||
default_display = config_loader.get_config_str("Flask", "default_display")
|
# default_display = config_loader.get_config_str("Flask", "default_display")
|
||||||
return render_template("Wordstrending.html", default_display = default_display)
|
# return render_template("Wordstrending.html", default_display = default_display)
|
||||||
|
#
|
||||||
|
#
|
||||||
@trendings.route("/protocolstrending/")
|
# @trendings.route("/protocolstrending/")
|
||||||
@login_required
|
# @login_required
|
||||||
@login_read_only
|
# @login_read_only
|
||||||
def protocolstrending():
|
# def protocolstrending():
|
||||||
default_display = config_loader.get_config_str("Flask", "default_display")
|
# default_display = config_loader.get_config_str("Flask", "default_display")
|
||||||
return render_template("Protocolstrending.html", default_display = default_display)
|
# return render_template("Protocolstrending.html", default_display = default_display)
|
||||||
|
#
|
||||||
|
#
|
||||||
@trendings.route("/trending/")
|
# @trendings.route("/trending/")
|
||||||
@login_required
|
# @login_required
|
||||||
@login_read_only
|
# @login_read_only
|
||||||
def trending():
|
# def trending():
|
||||||
default_display = config_loader.get_config_str("Flask", "default_display")
|
# default_display = config_loader.get_config_str("Flask", "default_display")
|
||||||
return render_template("Trending.html", default_display = default_display)
|
# return render_template("Trending.html", default_display = default_display)
|
||||||
|
#
|
||||||
|
#
|
||||||
# ========= REGISTRATION =========
|
# # ========= REGISTRATION =========
|
||||||
app.register_blueprint(trendings, url_prefix=baseUrl)
|
# app.register_blueprint(trendings, url_prefix=baseUrl)
|
||||||
|
|
|
@ -4,140 +4,140 @@
|
||||||
'''
|
'''
|
||||||
Flask functions and routes for the trending modules page
|
Flask functions and routes for the trending modules page
|
||||||
'''
|
'''
|
||||||
import os
|
# import os
|
||||||
import sys
|
# import sys
|
||||||
import datetime
|
# import datetime
|
||||||
import flask
|
# import flask
|
||||||
from flask import Flask, render_template, jsonify, request, Blueprint
|
# from flask import Flask, render_template, jsonify, request, Blueprint
|
||||||
|
#
|
||||||
from Role_Manager import login_admin, login_analyst, login_read_only
|
# from Role_Manager import login_admin, login_analyst, login_read_only
|
||||||
from flask_login import login_required
|
# from flask_login import login_required
|
||||||
|
#
|
||||||
sys.path.append(os.environ['AIL_BIN'])
|
# sys.path.append(os.environ['AIL_BIN'])
|
||||||
##################################
|
# ##################################
|
||||||
# Import Project packages
|
# # Import Project packages
|
||||||
##################################
|
# ##################################
|
||||||
from packages.Date import Date
|
# from packages.Date import Date
|
||||||
|
#
|
||||||
# ============ VARIABLES ============
|
# # ============ VARIABLES ============
|
||||||
import Flask_config
|
# import Flask_config
|
||||||
|
#
|
||||||
app = Flask_config.app
|
# app = Flask_config.app
|
||||||
baseUrl = Flask_config.baseUrl
|
# baseUrl = Flask_config.baseUrl
|
||||||
r_serv_charts = Flask_config.r_serv_charts
|
# r_serv_charts = Flask_config.r_serv_charts
|
||||||
|
#
|
||||||
trendingmodules = Blueprint('trendingmodules', __name__, template_folder='templates')
|
# trendingmodules = Blueprint('trendingmodules', __name__, template_folder='templates')
|
||||||
|
#
|
||||||
# ============ FUNCTIONS ============
|
# # ============ FUNCTIONS ============
|
||||||
|
#
|
||||||
# Iterate over elements in the module provided and return the today data or the last data
|
# # Iterate over elements in the module provided and return the today data or the last data
|
||||||
# return format: [('passed_days', num_of_passed_days), ('elem_name1', elem_value1), ('elem_name2', elem_value2)]]
|
# # return format: [('passed_days', num_of_passed_days), ('elem_name1', elem_value1), ('elem_name2', elem_value2)]]
|
||||||
def get_top_relevant_data(server, module_name):
|
# def get_top_relevant_data(server, module_name):
|
||||||
days = 0
|
# days = 0
|
||||||
for date in get_date_range(15):
|
# for date in get_date_range(15):
|
||||||
redis_progression_name_set = 'top_'+ module_name +'_set_' + date
|
# redis_progression_name_set = 'top_'+ module_name +'_set_' + date
|
||||||
member_set = server.zrevrangebyscore(redis_progression_name_set, '+inf', '-inf', withscores=True)
|
# member_set = server.zrevrangebyscore(redis_progression_name_set, '+inf', '-inf', withscores=True)
|
||||||
|
#
|
||||||
if len(member_set) == 0: #No data for this date
|
# if len(member_set) == 0: #No data for this date
|
||||||
days += 1
|
# days += 1
|
||||||
else:
|
# else:
|
||||||
member_set.insert(0, ("passed_days", days))
|
# member_set.insert(0, ("passed_days", days))
|
||||||
return member_set
|
# return member_set
|
||||||
|
#
|
||||||
|
#
|
||||||
def get_date_range(num_day):
|
# def get_date_range(num_day):
|
||||||
curr_date = datetime.date.today()
|
# curr_date = datetime.date.today()
|
||||||
date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
|
# date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
|
||||||
date_list = []
|
# date_list = []
|
||||||
|
#
|
||||||
for i in range(0, num_day+1):
|
# for i in range(0, num_day+1):
|
||||||
date_list.append(date.substract_day(i))
|
# date_list.append(date.substract_day(i))
|
||||||
return date_list
|
# return date_list
|
||||||
|
#
|
||||||
# ============ ROUTES ============
|
# # ============ ROUTES ============
|
||||||
|
#
|
||||||
@trendingmodules.route("/_moduleCharts", methods=['GET'])
|
# @trendingmodules.route("/_moduleCharts", methods=['GET'])
|
||||||
@login_required
|
# @login_required
|
||||||
@login_read_only
|
# @login_read_only
|
||||||
def modulesCharts():
|
# def modulesCharts():
|
||||||
keyword_name = request.args.get('keywordName')
|
# keyword_name = request.args.get('keywordName')
|
||||||
module_name = request.args.get('moduleName')
|
# module_name = request.args.get('moduleName')
|
||||||
bar_requested = True if request.args.get('bar') == "true" else False
|
# bar_requested = True if request.args.get('bar') == "true" else False
|
||||||
|
#
|
||||||
if (bar_requested):
|
# if (bar_requested):
|
||||||
num_day = int(request.args.get('days'))
|
# num_day = int(request.args.get('days'))
|
||||||
bar_values = []
|
# bar_values = []
|
||||||
|
#
|
||||||
date_range = get_date_range(num_day)
|
# date_range = get_date_range(num_day)
|
||||||
# Retreive all data from the last num_day
|
# # Retreive all data from the last num_day
|
||||||
for date in date_range:
|
# for date in date_range:
|
||||||
curr_value = r_serv_charts.hget(date, module_name+'-'+keyword_name)
|
# curr_value = r_serv_charts.hget(date, module_name+'-'+keyword_name)
|
||||||
bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
|
# bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
|
||||||
bar_values.insert(0, keyword_name)
|
# bar_values.insert(0, keyword_name)
|
||||||
return jsonify(bar_values)
|
# return jsonify(bar_values)
|
||||||
|
#
|
||||||
else:
|
# else:
|
||||||
member_set = get_top_relevant_data(r_serv_charts, module_name)
|
# member_set = get_top_relevant_data(r_serv_charts, module_name)
|
||||||
member_set = member_set if member_set is not None else []
|
# member_set = member_set if member_set is not None else []
|
||||||
if len(member_set) == 0:
|
# if len(member_set) == 0:
|
||||||
member_set.append(("No relevant data", int(100)))
|
# member_set.append(("No relevant data", int(100)))
|
||||||
return jsonify(member_set)
|
# return jsonify(member_set)
|
||||||
|
#
|
||||||
|
#
|
||||||
@trendingmodules.route("/_providersChart", methods=['GET'])
|
# @trendingmodules.route("/_providersChart", methods=['GET'])
|
||||||
@login_required
|
# @login_required
|
||||||
@login_read_only
|
# @login_read_only
|
||||||
def providersChart():
|
# def providersChart():
|
||||||
keyword_name = request.args.get('keywordName')
|
# keyword_name = request.args.get('keywordName')
|
||||||
module_name = request.args.get('moduleName')
|
# module_name = request.args.get('moduleName')
|
||||||
bar_requested = True if request.args.get('bar') == "true" else False
|
# bar_requested = True if request.args.get('bar') == "true" else False
|
||||||
|
#
|
||||||
if (bar_requested):
|
# if (bar_requested):
|
||||||
num_day = int(request.args.get('days'))
|
# num_day = int(request.args.get('days'))
|
||||||
bar_values = []
|
# bar_values = []
|
||||||
|
#
|
||||||
date_range = get_date_range(num_day)
|
# date_range = get_date_range(num_day)
|
||||||
# Retreive all data from the last num_day
|
# # Retreive all data from the last num_day
|
||||||
for date in date_range:
|
# for date in date_range:
|
||||||
curr_value_size = ( r_serv_charts.hget(keyword_name+'_'+'size', date) )
|
# curr_value_size = ( r_serv_charts.hget(keyword_name+'_'+'size', date) )
|
||||||
if curr_value_size is not None:
|
# if curr_value_size is not None:
|
||||||
curr_value_size = curr_value_size
|
# curr_value_size = curr_value_size
|
||||||
|
#
|
||||||
curr_value_num = r_serv_charts.hget(keyword_name+'_'+'num', date)
|
# curr_value_num = r_serv_charts.hget(keyword_name+'_'+'num', date)
|
||||||
|
#
|
||||||
curr_value_size_avg = r_serv_charts.hget(keyword_name+'_'+'avg', date)
|
# curr_value_size_avg = r_serv_charts.hget(keyword_name+'_'+'avg', date)
|
||||||
if curr_value_size_avg is not None:
|
# if curr_value_size_avg is not None:
|
||||||
curr_value_size_avg = curr_value_size_avg
|
# curr_value_size_avg = curr_value_size_avg
|
||||||
|
#
|
||||||
|
#
|
||||||
if module_name == "size":
|
# if module_name == "size":
|
||||||
curr_value = float(curr_value_size_avg if curr_value_size_avg is not None else 0)
|
# curr_value = float(curr_value_size_avg if curr_value_size_avg is not None else 0)
|
||||||
else:
|
# else:
|
||||||
curr_value = float(curr_value_num if curr_value_num is not None else 0.0)
|
# curr_value = float(curr_value_num if curr_value_num is not None else 0.0)
|
||||||
|
#
|
||||||
bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], curr_value])
|
# bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], curr_value])
|
||||||
bar_values.insert(0, keyword_name)
|
# bar_values.insert(0, keyword_name)
|
||||||
return jsonify(bar_values)
|
# return jsonify(bar_values)
|
||||||
|
#
|
||||||
else:
|
# else:
|
||||||
#redis_provider_name_set = 'top_size_set' if module_name == "size" else 'providers_set'
|
# #redis_provider_name_set = 'top_size_set' if module_name == "size" else 'providers_set'
|
||||||
redis_provider_name_set = 'top_avg_size_set_' if module_name == "size" else 'providers_set_'
|
# redis_provider_name_set = 'top_avg_size_set_' if module_name == "size" else 'providers_set_'
|
||||||
redis_provider_name_set = redis_provider_name_set + get_date_range(0)[0]
|
# redis_provider_name_set = redis_provider_name_set + get_date_range(0)[0]
|
||||||
|
#
|
||||||
member_set = r_serv_charts.zrevrangebyscore(redis_provider_name_set, '+inf', '-inf', withscores=True, start=0, num=8)
|
# member_set = r_serv_charts.zrevrangebyscore(redis_provider_name_set, '+inf', '-inf', withscores=True, start=0, num=8)
|
||||||
|
#
|
||||||
# Member set is a list of (value, score) pairs
|
# # Member set is a list of (value, score) pairs
|
||||||
if len(member_set) == 0:
|
# if len(member_set) == 0:
|
||||||
member_set.append(("No relevant data", float(100)))
|
# member_set.append(("No relevant data", float(100)))
|
||||||
return jsonify(member_set)
|
# return jsonify(member_set)
|
||||||
|
#
|
||||||
|
#
|
||||||
@trendingmodules.route("/moduletrending/")
|
# @trendingmodules.route("/moduletrending/")
|
||||||
@login_required
|
# @login_required
|
||||||
@login_read_only
|
# @login_read_only
|
||||||
def moduletrending():
|
# def moduletrending():
|
||||||
return render_template("Moduletrending.html")
|
# return render_template("Moduletrending.html")
|
||||||
|
#
|
||||||
|
#
|
||||||
# ========= REGISTRATION =========
|
# # ========= REGISTRATION =========
|
||||||
app.register_blueprint(trendingmodules, url_prefix=baseUrl)
|
# app.register_blueprint(trendingmodules, url_prefix=baseUrl)
|
||||||
|
|
|
@ -1 +1,3 @@
|
||||||
|
{#
|
||||||
<li id='page-modulestats'><a href="{{ url_for('trendingmodules.moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>
|
<li id='page-modulestats'><a href="{{ url_for('trendingmodules.moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>
|
||||||
|
#}
|
||||||
|
|
|
@ -27,9 +27,9 @@
|
||||||
<li class="nav-item mr-3">
|
<li class="nav-item mr-3">
|
||||||
<a class="nav-link" id="page-Decoded" href="{{ url_for('investigations_b.investigations_dashboard') }}" aria-disabled="true"><i class="fas fa-cube"></i> Objects</a>
|
<a class="nav-link" id="page-Decoded" href="{{ url_for('investigations_b.investigations_dashboard') }}" aria-disabled="true"><i class="fas fa-cube"></i> Objects</a>
|
||||||
</li>
|
</li>
|
||||||
<li class="nav-item mr-3">
|
{# <li class="nav-item mr-3">#}
|
||||||
<a class="nav-link" href="{{ url_for('trendingmodules.moduletrending') }}" aria-disabled="true"><i class="fas fa-chart-bar"></i> Statistics</a>
|
{# <a class="nav-link" href="{{ url_for('trendingmodules.moduletrending') }}" aria-disabled="true"><i class="fas fa-chart-bar"></i> Statistics</a>#}
|
||||||
</li>
|
{# </li>#}
|
||||||
<li class="nav-item mr-3">
|
<li class="nav-item mr-3">
|
||||||
<a class="nav-link" id="page-options" href="{{ url_for('settings_b.settings_page') }}" aria-disabled="true"><i class="fas fa-cog"></i> Server Management</a>
|
<a class="nav-link" id="page-options" href="{{ url_for('settings_b.settings_page') }}" aria-disabled="true"><i class="fas fa-cog"></i> Server Management</a>
|
||||||
</li>
|
</li>
|
||||||
|
|
Loading…
Reference in a new issue