chg: [stats] disable statistics

This commit is contained in:
Terrtia 2023-03-30 15:23:41 +02:00
parent 5f150489b6
commit ac484bed71
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
12 changed files with 270 additions and 268 deletions

View file

@ -927,7 +927,7 @@ if __name__ == '__main__':
# ail_2_ail_migration()
# trackers_migration()
# investigations_migration()
# statistics_migration()
## statistics_migration()
# cves_migration()

View file

@ -254,8 +254,6 @@ function launching_scripts {
sleep 0.1
screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Mail.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./ModuleStats.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Onion.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "PgpDump" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./PgpDump.py; read x"
@ -304,6 +302,8 @@ function launching_scripts {
# sleep 0.1
# screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x"
# sleep 0.1
# screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./ModuleStats.py; read x"
# sleep 0.1
##################################
# TO MIGRATE #

View file

@ -20,7 +20,7 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from lib import ConfigLoader
from lib import Statistics
# from lib import Statistics
## LOAD CONFIG ##
config_loader = ConfigLoader.ConfigLoader()
@ -62,7 +62,7 @@ def regex_findall(module_name, redis_key, regex, item_id, item_content, max_time
proc.join(max_time)
if proc.is_alive():
proc.terminate()
Statistics.incr_module_timeout_statistic(module_name)
# Statistics.incr_module_timeout_statistic(module_name)
err_mess = f"{module_name}: processing timeout: {item_id}"
print(err_mess)
publisher.info(err_mess)
@ -96,7 +96,7 @@ def regex_finditer(r_key, regex, item_id, content, max_time=30):
proc.join(max_time)
if proc.is_alive():
proc.terminate()
Statistics.incr_module_timeout_statistic(r_key)
# Statistics.incr_module_timeout_statistic(r_key)
err_mess = f"{r_key}: processing timeout: {item_id}"
print(err_mess)
publisher.info(err_mess)
@ -127,7 +127,7 @@ def regex_search(r_key, regex, item_id, content, max_time=30):
proc.join(max_time)
if proc.is_alive():
proc.terminate()
Statistics.incr_module_timeout_statistic(r_key)
# Statistics.incr_module_timeout_statistic(r_key)
err_mess = f"{r_key}: processing timeout: {item_id}"
print(err_mess)
publisher.info(err_mess)

View file

@ -40,7 +40,7 @@ sys.path.append(os.environ['AIL_BIN'])
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
from lib import ConfigLoader
from lib import Statistics
# from lib import Statistics
class Credential(AbstractModule):
@ -160,8 +160,8 @@ class Credential(AbstractModule):
except:
pass
nb_tlds[tld] = nb_tlds.get(tld, 0) + 1
for tld in nb_tlds:
Statistics.add_module_tld_stats_by_date('credential', date, tld, nb_tlds[tld])
# for tld in nb_tlds:
# Statistics.add_module_tld_stats_by_date('credential', date, tld, nb_tlds[tld])
else:
self.redis_logger.info(to_print)
print(f'found {nb_cred} credentials')

View file

@ -26,7 +26,7 @@ sys.path.append(os.environ['AIL_BIN'])
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
from lib.ConfigLoader import ConfigLoader
from lib import Statistics
# from lib import Statistics
class Iban(AbstractModule):
"""
@ -91,8 +91,8 @@ class Iban(AbstractModule):
if valid_ibans:
print(f'{valid_ibans} ibans {item_id}')
date = datetime.datetime.now().strftime("%Y%m")
for iban in valid_ibans:
Statistics.add_module_tld_stats_by_date('iban', date, iban[0:2], 1)
# for iban in valid_ibans:
# Statistics.add_module_tld_stats_by_date('iban', date, iban[0:2], 1)
to_print = f'Iban;{item.get_source()};{item.get_date()};{item.get_basename()};'
self.redis_logger.warning(f'{to_print}Checked found {len(valid_ibans)} IBAN;{item_id}')

View file

@ -27,7 +27,7 @@ sys.path.append(os.environ['AIL_BIN'])
from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item
from lib import Statistics
# from lib import Statistics
class LibInjection(AbstractModule):
"""docstring for LibInjection module."""
@ -81,14 +81,14 @@ class LibInjection(AbstractModule):
self.send_message_to_queue(msg, 'Tags')
# statistics
# # TODO: # FIXME: remove me
try:
tld = url_parsed['tld'].decode()
except:
tld = url_parsed['tld']
if tld is not None:
date = datetime.now().strftime("%Y%m")
Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
# # # TODO: # FIXME: remove me
# try:
# tld = url_parsed['tld'].decode()
# except:
# tld = url_parsed['tld']
# if tld is not None:
# date = datetime.now().strftime("%Y%m")
# Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
if __name__ == "__main__":

View file

@ -28,7 +28,7 @@ sys.path.append(os.environ['AIL_BIN'])
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item
from lib.ConfigLoader import ConfigLoader
from lib import Statistics
# from lib import Statistics
class Mail(AbstractModule):
@ -169,8 +169,8 @@ class Mail(AbstractModule):
except:
pass
mx_tlds[tld] = mx_tlds.get(tld, 0) + nb_mails
for tld in mx_tlds:
Statistics.add_module_tld_stats_by_date('mail', item_date, tld, mx_tlds[tld])
# for tld in mx_tlds:
# Statistics.add_module_tld_stats_by_date('mail', item_date, tld, mx_tlds[tld])
msg = f'Mails;{item.get_source()};{item_date};{item.get_basename()};Checked {num_valid_email} e-mail(s);{item_id}'
if num_valid_email > self.mail_threshold:

View file

@ -27,7 +27,7 @@ sys.path.append(os.environ['AIL_BIN'])
from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
from lib.objects.Items import Item
from lib import Statistics
# from lib import Statistics
class SQLInjectionDetection(AbstractModule):
"""docstring for SQLInjectionDetection module."""
@ -62,15 +62,15 @@ class SQLInjectionDetection(AbstractModule):
self.send_message_to_queue(msg, 'Tags')
# statistics
tld = url_parsed['tld']
if tld is not None:
# # TODO: # FIXME: remove me
try:
tld = tld.decode()
except:
pass
date = datetime.now().strftime("%Y%m")
Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
# tld = url_parsed['tld']
# if tld is not None:
# # # TODO: # FIXME: remove me
# try:
# tld = tld.decode()
# except:
# pass
# date = datetime.now().strftime("%Y%m")
# Statistics.add_module_tld_stats_by_date(self.module_name, date, tld, 1)
# Try to detect if the url passed might be an sql injection by applying the regex
# defined above on it.

View file

@ -4,96 +4,96 @@
'''
Flask functions and routes for the trending charts page
'''
import os
import sys
import datetime
import flask
from flask import Flask, render_template, jsonify, request, Blueprint
from Role_Manager import login_admin, login_analyst, login_read_only
from flask_login import login_required
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from packages.Date import Date
# ============ VARIABLES ============
import Flask_config
app = Flask_config.app
config_loader = Flask_config.config_loader
baseUrl = Flask_config.baseUrl
r_serv_charts = Flask_config.r_serv_charts
trendings = Blueprint('trendings', __name__, template_folder='templates')
# ============ FUNCTIONS ============
def get_date_range(num_day):
curr_date = datetime.date.today()
date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
date_list = []
for i in range(0, num_day+1):
date_list.append(date.substract_day(i))
return date_list
# ============ ROUTES ============
@trendings.route("/_progressionCharts", methods=['GET'])
@login_required
@login_read_only
def progressionCharts():
attribute_name = request.args.get('attributeName')
trending_name = request.args.get('trendingName')
bar_requested = True if request.args.get('bar') == "true" else False
if (bar_requested):
num_day = int(request.args.get('days'))
bar_values = []
date_range = get_date_range(num_day)
# Retreive all data from the last num_day
for date in date_range:
curr_value = r_serv_charts.hget(attribute_name, date)
bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
bar_values.insert(0, attribute_name)
return jsonify(bar_values)
else:
redis_progression_name = "z_top_progression_" + trending_name
keyw_value = r_serv_charts.zrevrangebyscore(redis_progression_name, '+inf', '-inf', withscores=True, start=0, num=10)
return jsonify(keyw_value)
@trendings.route("/wordstrending/")
@login_required
@login_read_only
def wordstrending():
default_display = config_loader.get_config_str("Flask", "default_display")
return render_template("Wordstrending.html", default_display = default_display)
@trendings.route("/protocolstrending/")
@login_required
@login_read_only
def protocolstrending():
default_display = config_loader.get_config_str("Flask", "default_display")
return render_template("Protocolstrending.html", default_display = default_display)
@trendings.route("/trending/")
@login_required
@login_read_only
def trending():
default_display = config_loader.get_config_str("Flask", "default_display")
return render_template("Trending.html", default_display = default_display)
# ========= REGISTRATION =========
app.register_blueprint(trendings, url_prefix=baseUrl)
# import os
# import sys
# import datetime
# import flask
# from flask import Flask, render_template, jsonify, request, Blueprint
#
# from Role_Manager import login_admin, login_analyst, login_read_only
# from flask_login import login_required
#
# sys.path.append(os.environ['AIL_BIN'])
# ##################################
# # Import Project packages
# ##################################
# from packages.Date import Date
#
# # ============ VARIABLES ============
# import Flask_config
#
# app = Flask_config.app
# config_loader = Flask_config.config_loader
# baseUrl = Flask_config.baseUrl
# r_serv_charts = Flask_config.r_serv_charts
#
# trendings = Blueprint('trendings', __name__, template_folder='templates')
#
# # ============ FUNCTIONS ============
#
# def get_date_range(num_day):
# curr_date = datetime.date.today()
# date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
# date_list = []
#
# for i in range(0, num_day+1):
# date_list.append(date.substract_day(i))
#
# return date_list
#
#
# # ============ ROUTES ============
#
# @trendings.route("/_progressionCharts", methods=['GET'])
# @login_required
# @login_read_only
# def progressionCharts():
# attribute_name = request.args.get('attributeName')
# trending_name = request.args.get('trendingName')
# bar_requested = True if request.args.get('bar') == "true" else False
#
# if (bar_requested):
# num_day = int(request.args.get('days'))
# bar_values = []
#
# date_range = get_date_range(num_day)
# # Retreive all data from the last num_day
# for date in date_range:
#
# curr_value = r_serv_charts.hget(attribute_name, date)
# bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
# bar_values.insert(0, attribute_name)
# return jsonify(bar_values)
#
# else:
# redis_progression_name = "z_top_progression_" + trending_name
# keyw_value = r_serv_charts.zrevrangebyscore(redis_progression_name, '+inf', '-inf', withscores=True, start=0, num=10)
#
# return jsonify(keyw_value)
#
# @trendings.route("/wordstrending/")
# @login_required
# @login_read_only
# def wordstrending():
# default_display = config_loader.get_config_str("Flask", "default_display")
# return render_template("Wordstrending.html", default_display = default_display)
#
#
# @trendings.route("/protocolstrending/")
# @login_required
# @login_read_only
# def protocolstrending():
# default_display = config_loader.get_config_str("Flask", "default_display")
# return render_template("Protocolstrending.html", default_display = default_display)
#
#
# @trendings.route("/trending/")
# @login_required
# @login_read_only
# def trending():
# default_display = config_loader.get_config_str("Flask", "default_display")
# return render_template("Trending.html", default_display = default_display)
#
#
# # ========= REGISTRATION =========
# app.register_blueprint(trendings, url_prefix=baseUrl)

View file

@ -4,140 +4,140 @@
'''
Flask functions and routes for the trending modules page
'''
import os
import sys
import datetime
import flask
from flask import Flask, render_template, jsonify, request, Blueprint
from Role_Manager import login_admin, login_analyst, login_read_only
from flask_login import login_required
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from packages.Date import Date
# ============ VARIABLES ============
import Flask_config
app = Flask_config.app
baseUrl = Flask_config.baseUrl
r_serv_charts = Flask_config.r_serv_charts
trendingmodules = Blueprint('trendingmodules', __name__, template_folder='templates')
# ============ FUNCTIONS ============
# Iterate over elements in the module provided and return the today data or the last data
# return format: [('passed_days', num_of_passed_days), ('elem_name1', elem_value1), ('elem_name2', elem_value2)]]
def get_top_relevant_data(server, module_name):
days = 0
for date in get_date_range(15):
redis_progression_name_set = 'top_'+ module_name +'_set_' + date
member_set = server.zrevrangebyscore(redis_progression_name_set, '+inf', '-inf', withscores=True)
if len(member_set) == 0: #No data for this date
days += 1
else:
member_set.insert(0, ("passed_days", days))
return member_set
def get_date_range(num_day):
curr_date = datetime.date.today()
date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
date_list = []
for i in range(0, num_day+1):
date_list.append(date.substract_day(i))
return date_list
# ============ ROUTES ============
@trendingmodules.route("/_moduleCharts", methods=['GET'])
@login_required
@login_read_only
def modulesCharts():
keyword_name = request.args.get('keywordName')
module_name = request.args.get('moduleName')
bar_requested = True if request.args.get('bar') == "true" else False
if (bar_requested):
num_day = int(request.args.get('days'))
bar_values = []
date_range = get_date_range(num_day)
# Retreive all data from the last num_day
for date in date_range:
curr_value = r_serv_charts.hget(date, module_name+'-'+keyword_name)
bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
bar_values.insert(0, keyword_name)
return jsonify(bar_values)
else:
member_set = get_top_relevant_data(r_serv_charts, module_name)
member_set = member_set if member_set is not None else []
if len(member_set) == 0:
member_set.append(("No relevant data", int(100)))
return jsonify(member_set)
@trendingmodules.route("/_providersChart", methods=['GET'])
@login_required
@login_read_only
def providersChart():
keyword_name = request.args.get('keywordName')
module_name = request.args.get('moduleName')
bar_requested = True if request.args.get('bar') == "true" else False
if (bar_requested):
num_day = int(request.args.get('days'))
bar_values = []
date_range = get_date_range(num_day)
# Retreive all data from the last num_day
for date in date_range:
curr_value_size = ( r_serv_charts.hget(keyword_name+'_'+'size', date) )
if curr_value_size is not None:
curr_value_size = curr_value_size
curr_value_num = r_serv_charts.hget(keyword_name+'_'+'num', date)
curr_value_size_avg = r_serv_charts.hget(keyword_name+'_'+'avg', date)
if curr_value_size_avg is not None:
curr_value_size_avg = curr_value_size_avg
if module_name == "size":
curr_value = float(curr_value_size_avg if curr_value_size_avg is not None else 0)
else:
curr_value = float(curr_value_num if curr_value_num is not None else 0.0)
bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], curr_value])
bar_values.insert(0, keyword_name)
return jsonify(bar_values)
else:
#redis_provider_name_set = 'top_size_set' if module_name == "size" else 'providers_set'
redis_provider_name_set = 'top_avg_size_set_' if module_name == "size" else 'providers_set_'
redis_provider_name_set = redis_provider_name_set + get_date_range(0)[0]
member_set = r_serv_charts.zrevrangebyscore(redis_provider_name_set, '+inf', '-inf', withscores=True, start=0, num=8)
# Member set is a list of (value, score) pairs
if len(member_set) == 0:
member_set.append(("No relevant data", float(100)))
return jsonify(member_set)
@trendingmodules.route("/moduletrending/")
@login_required
@login_read_only
def moduletrending():
return render_template("Moduletrending.html")
# ========= REGISTRATION =========
app.register_blueprint(trendingmodules, url_prefix=baseUrl)
# import os
# import sys
# import datetime
# import flask
# from flask import Flask, render_template, jsonify, request, Blueprint
#
# from Role_Manager import login_admin, login_analyst, login_read_only
# from flask_login import login_required
#
# sys.path.append(os.environ['AIL_BIN'])
# ##################################
# # Import Project packages
# ##################################
# from packages.Date import Date
#
# # ============ VARIABLES ============
# import Flask_config
#
# app = Flask_config.app
# baseUrl = Flask_config.baseUrl
# r_serv_charts = Flask_config.r_serv_charts
#
# trendingmodules = Blueprint('trendingmodules', __name__, template_folder='templates')
#
# # ============ FUNCTIONS ============
#
# # Iterate over elements in the module provided and return the today data or the last data
# # return format: [('passed_days', num_of_passed_days), ('elem_name1', elem_value1), ('elem_name2', elem_value2)]]
# def get_top_relevant_data(server, module_name):
# days = 0
# for date in get_date_range(15):
# redis_progression_name_set = 'top_'+ module_name +'_set_' + date
# member_set = server.zrevrangebyscore(redis_progression_name_set, '+inf', '-inf', withscores=True)
#
# if len(member_set) == 0: #No data for this date
# days += 1
# else:
# member_set.insert(0, ("passed_days", days))
# return member_set
#
#
# def get_date_range(num_day):
# curr_date = datetime.date.today()
# date = Date(str(curr_date.year)+str(curr_date.month).zfill(2)+str(curr_date.day).zfill(2))
# date_list = []
#
# for i in range(0, num_day+1):
# date_list.append(date.substract_day(i))
# return date_list
#
# # ============ ROUTES ============
#
# @trendingmodules.route("/_moduleCharts", methods=['GET'])
# @login_required
# @login_read_only
# def modulesCharts():
# keyword_name = request.args.get('keywordName')
# module_name = request.args.get('moduleName')
# bar_requested = True if request.args.get('bar') == "true" else False
#
# if (bar_requested):
# num_day = int(request.args.get('days'))
# bar_values = []
#
# date_range = get_date_range(num_day)
# # Retreive all data from the last num_day
# for date in date_range:
# curr_value = r_serv_charts.hget(date, module_name+'-'+keyword_name)
# bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
# bar_values.insert(0, keyword_name)
# return jsonify(bar_values)
#
# else:
# member_set = get_top_relevant_data(r_serv_charts, module_name)
# member_set = member_set if member_set is not None else []
# if len(member_set) == 0:
# member_set.append(("No relevant data", int(100)))
# return jsonify(member_set)
#
#
# @trendingmodules.route("/_providersChart", methods=['GET'])
# @login_required
# @login_read_only
# def providersChart():
# keyword_name = request.args.get('keywordName')
# module_name = request.args.get('moduleName')
# bar_requested = True if request.args.get('bar') == "true" else False
#
# if (bar_requested):
# num_day = int(request.args.get('days'))
# bar_values = []
#
# date_range = get_date_range(num_day)
# # Retreive all data from the last num_day
# for date in date_range:
# curr_value_size = ( r_serv_charts.hget(keyword_name+'_'+'size', date) )
# if curr_value_size is not None:
# curr_value_size = curr_value_size
#
# curr_value_num = r_serv_charts.hget(keyword_name+'_'+'num', date)
#
# curr_value_size_avg = r_serv_charts.hget(keyword_name+'_'+'avg', date)
# if curr_value_size_avg is not None:
# curr_value_size_avg = curr_value_size_avg
#
#
# if module_name == "size":
# curr_value = float(curr_value_size_avg if curr_value_size_avg is not None else 0)
# else:
# curr_value = float(curr_value_num if curr_value_num is not None else 0.0)
#
# bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], curr_value])
# bar_values.insert(0, keyword_name)
# return jsonify(bar_values)
#
# else:
# #redis_provider_name_set = 'top_size_set' if module_name == "size" else 'providers_set'
# redis_provider_name_set = 'top_avg_size_set_' if module_name == "size" else 'providers_set_'
# redis_provider_name_set = redis_provider_name_set + get_date_range(0)[0]
#
# member_set = r_serv_charts.zrevrangebyscore(redis_provider_name_set, '+inf', '-inf', withscores=True, start=0, num=8)
#
# # Member set is a list of (value, score) pairs
# if len(member_set) == 0:
# member_set.append(("No relevant data", float(100)))
# return jsonify(member_set)
#
#
# @trendingmodules.route("/moduletrending/")
# @login_required
# @login_read_only
# def moduletrending():
# return render_template("Moduletrending.html")
#
#
# # ========= REGISTRATION =========
# app.register_blueprint(trendingmodules, url_prefix=baseUrl)

View file

@ -1 +1,3 @@
{#
<li id='page-modulestats'><a href="{{ url_for('trendingmodules.moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>
#}

View file

@ -27,9 +27,9 @@
<li class="nav-item mr-3">
<a class="nav-link" id="page-Decoded" href="{{ url_for('investigations_b.investigations_dashboard') }}" aria-disabled="true"><i class="fas fa-cube"></i> Objects</a>
</li>
<li class="nav-item mr-3">
<a class="nav-link" href="{{ url_for('trendingmodules.moduletrending') }}" aria-disabled="true"><i class="fas fa-chart-bar"></i> Statistics</a>
</li>
{# <li class="nav-item mr-3">#}
{# <a class="nav-link" href="{{ url_for('trendingmodules.moduletrending') }}" aria-disabled="true"><i class="fas fa-chart-bar"></i> Statistics</a>#}
{# </li>#}
<li class="nav-item mr-3">
<a class="nav-link" id="page-options" href="{{ url_for('settings_b.settings_page') }}" aria-disabled="true"><i class="fas fa-cog"></i> Server Management</a>
</li>