mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
chg: [lib ConfigLoader] add configLoader class
This commit is contained in:
parent
48abb89d28
commit
f1def65c89
19 changed files with 97 additions and 145 deletions
|
@ -58,7 +58,6 @@ class PubSub(object):
|
|||
for address in addresses.split(','):
|
||||
new_sub = context.socket(zmq.SUB)
|
||||
new_sub.connect(address)
|
||||
# bytes64 encode bytes to ascii only bytes
|
||||
new_sub.setsockopt_string(zmq.SUBSCRIBE, channel)
|
||||
self.subscribers.append(new_sub)
|
||||
|
||||
|
|
48
bin/lib/ConfigLoader.py
Executable file
48
bin/lib/ConfigLoader.py
Executable file
|
@ -0,0 +1,48 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
"""
|
||||
The ``Domain``
|
||||
===================
|
||||
|
||||
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import redis
|
||||
import configparser
|
||||
|
||||
# Get Config file
|
||||
config_dir = os.path.join(os.environ['AIL_HOME'], 'configs')
|
||||
config_file = os.path.join(config_dir, 'core.cfg')
|
||||
if not os.path.exists(config_file):
|
||||
raise Exception('Unable to find the configuration file. \
|
||||
Did you set environment variables? \
|
||||
Or activate the virtualenv.')
|
||||
|
||||
# # TODO: create sphinx doc
|
||||
|
||||
# # TODO: add config_field to reload
|
||||
|
||||
class ConfigLoader(object):
|
||||
"""docstring for Config_Loader."""
|
||||
|
||||
def __init__(self):
|
||||
self.cfg = configparser.ConfigParser()
|
||||
self.cfg.read(config_file)
|
||||
|
||||
def get_redis_conn(self, redis_name, decode_responses=True): ## TODO: verify redis name
|
||||
return redis.StrictRedis( host=self.cfg.get(redis_name, "host"),
|
||||
port=self.cfg.getint(redis_name, "port"),
|
||||
db=self.cfg.getint(redis_name, "db"),
|
||||
decode_responses=decode_responses )
|
||||
|
||||
def get_config_str(self, section, key_name):
|
||||
return self.cfg.get(section, key_name)
|
||||
|
||||
def get_config_int(self, section, key_name):
|
||||
return self.cfg.getint(section, key_name)
|
||||
|
||||
def get_config_boolean(self, section, key_name):
|
||||
return self.cfg.getboolean(section, key_name)
|
|
@ -24,6 +24,10 @@ sys.path.append('./modules/')
|
|||
|
||||
from User import User
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
|
||||
from pytaxonomies import Taxonomies
|
||||
|
||||
# Import config
|
||||
|
@ -31,33 +35,21 @@ import Flask_config
|
|||
|
||||
# Import Blueprint
|
||||
from blueprints.root import root
|
||||
from blueprints.crawler_splash import crawler_splash
|
||||
|
||||
Flask_dir = os.environ['AIL_FLASK']
|
||||
|
||||
# CONFIG #
|
||||
cfg = Flask_config.cfg
|
||||
baseUrl = cfg.get("Flask", "baseurl")
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
baseUrl = config_loader.get_config_str("Flask", "baseurl")
|
||||
baseUrl = baseUrl.replace('/', '')
|
||||
if baseUrl != '':
|
||||
baseUrl = '/'+baseUrl
|
||||
|
||||
# ========= REDIS =========#
|
||||
r_serv_db = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_DB", "host"),
|
||||
port=cfg.getint("ARDB_DB", "port"),
|
||||
db=cfg.getint("ARDB_DB", "db"),
|
||||
decode_responses=True)
|
||||
r_serv_tags = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_Tags", "host"),
|
||||
port=cfg.getint("ARDB_Tags", "port"),
|
||||
db=cfg.getint("ARDB_Tags", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_cache = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Cache", "host"),
|
||||
port=cfg.getint("Redis_Cache", "port"),
|
||||
db=cfg.getint("Redis_Cache", "db"),
|
||||
decode_responses=True)
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
|
||||
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||
|
||||
# logs
|
||||
log_dir = os.path.join(os.environ['AIL_HOME'], 'logs')
|
||||
|
@ -88,6 +80,7 @@ app.config['MAX_CONTENT_LENGTH'] = 900 * 1024 * 1024
|
|||
|
||||
# ========= BLUEPRINT =========#
|
||||
app.register_blueprint(root, url_prefix=baseUrl)
|
||||
app.register_blueprint(crawler_splash, url_prefix=baseUrl)
|
||||
# ========= =========#
|
||||
|
||||
# ========= session ========
|
||||
|
|
|
@ -4,109 +4,34 @@
|
|||
'''
|
||||
Flask global variables shared accross modules
|
||||
'''
|
||||
import configparser
|
||||
import redis
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
|
||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||
import ConfigLoader
|
||||
|
||||
# FLASK #
|
||||
app = None
|
||||
|
||||
# CONFIG #
|
||||
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
|
||||
if not os.path.exists(configfile):
|
||||
raise Exception('Unable to find the configuration file. \
|
||||
Did you set environment variables? \
|
||||
Or activate the virtualenv.')
|
||||
|
||||
cfg = configparser.ConfigParser()
|
||||
cfg.read(configfile)
|
||||
config_loader = ConfigLoader.ConfigLoader()
|
||||
|
||||
# REDIS #
|
||||
r_serv = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Queues", "host"),
|
||||
port=cfg.getint("Redis_Queues", "port"),
|
||||
db=cfg.getint("Redis_Queues", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_cache = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Cache", "host"),
|
||||
port=cfg.getint("Redis_Cache", "port"),
|
||||
db=cfg.getint("Redis_Cache", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_log = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Log", "host"),
|
||||
port=cfg.getint("Redis_Log", "port"),
|
||||
db=cfg.getint("Redis_Log", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_log_submit = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Log_submit", "host"),
|
||||
port=cfg.getint("Redis_Log_submit", "port"),
|
||||
db=cfg.getint("Redis_Log_submit", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_charts = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_Trending", "host"),
|
||||
port=cfg.getint("ARDB_Trending", "port"),
|
||||
db=cfg.getint("ARDB_Trending", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_sentiment = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_Sentiment", "host"),
|
||||
port=cfg.getint("ARDB_Sentiment", "port"),
|
||||
db=cfg.getint("ARDB_Sentiment", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_term = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_Tracker", "host"),
|
||||
port=cfg.getint("ARDB_Tracker", "port"),
|
||||
db=cfg.getint("ARDB_Tracker", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_cred = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_TermCred", "host"),
|
||||
port=cfg.getint("ARDB_TermCred", "port"),
|
||||
db=cfg.getint("ARDB_TermCred", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_pasteName = redis.StrictRedis(
|
||||
host=cfg.get("Redis_Paste_Name", "host"),
|
||||
port=cfg.getint("Redis_Paste_Name", "port"),
|
||||
db=cfg.getint("Redis_Paste_Name", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_tags = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_Tags", "host"),
|
||||
port=cfg.getint("ARDB_Tags", "port"),
|
||||
db=cfg.getint("ARDB_Tags", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_metadata = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_Metadata", "host"),
|
||||
port=cfg.getint("ARDB_Metadata", "port"),
|
||||
db=cfg.getint("ARDB_Metadata", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_db = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_DB", "host"),
|
||||
port=cfg.getint("ARDB_DB", "port"),
|
||||
db=cfg.getint("ARDB_DB", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_statistics = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_Statistics", "host"),
|
||||
port=cfg.getint("ARDB_Statistics", "port"),
|
||||
db=cfg.getint("ARDB_Statistics", "db"),
|
||||
decode_responses=True)
|
||||
|
||||
r_serv_onion = redis.StrictRedis(
|
||||
host=cfg.get("ARDB_Onion", "host"),
|
||||
port=cfg.getint("ARDB_Onion", "port"),
|
||||
db=cfg.getint("ARDB_Onion", "db"),
|
||||
decode_responses=True)
|
||||
r_serv = config_loader.get_redis_conn("Redis_Queues")
|
||||
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||
r_serv_log = config_loader.get_redis_conn("Redis_Log")
|
||||
r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit")
|
||||
r_serv_charts = config_loader.get_redis_conn("ARDB_Trending")
|
||||
r_serv_sentiment = config_loader.get_redis_conn("ARDB_Sentiment")
|
||||
r_serv_term = config_loader.get_redis_conn("ARDB_Tracker")
|
||||
r_serv_cred = config_loader.get_redis_conn("ARDB_TermCred")
|
||||
r_serv_pasteName = config_loader.get_redis_conn("Redis_Paste_Name")
|
||||
r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
|
||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||
r_serv_statistics = config_loader.get_redis_conn("ARDB_Statistics")
|
||||
r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
|
||||
|
||||
sys.path.append('../../configs/keys')
|
||||
# MISP #
|
||||
|
@ -146,17 +71,17 @@ if HiveApi != False:
|
|||
print('The Hive not connected')
|
||||
|
||||
#### VARIABLES ####
|
||||
baseUrl = cfg.get("Flask", "baseurl")
|
||||
baseUrl = config_loader.get_config_str("Flask", "baseurl")
|
||||
baseUrl = baseUrl.replace('/', '')
|
||||
if baseUrl != '':
|
||||
baseUrl = '/'+baseUrl
|
||||
|
||||
max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
|
||||
max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
|
||||
max_preview_char = int(config_loader.get_config_str("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
|
||||
max_preview_modal = int(config_loader.get_config_str("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
|
||||
|
||||
max_tags_result = 50
|
||||
|
||||
DiffMaxLineLength = int(cfg.get("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value
|
||||
DiffMaxLineLength = int(config_loader.get_config_str("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value
|
||||
|
||||
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
|
||||
|
||||
|
@ -166,14 +91,14 @@ dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_me
|
|||
|
||||
UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')
|
||||
|
||||
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + '/'
|
||||
SCREENSHOT_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot"), 'screenshot')
|
||||
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
|
||||
SCREENSHOT_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "crawled_screenshot"), 'screenshot')
|
||||
|
||||
REPO_ORIGIN = 'https://github.com/CIRCL/AIL-framework.git'
|
||||
|
||||
max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs"))
|
||||
max_dashboard_logs = int(config_loader.get_config_str("Flask", "max_dashboard_logs"))
|
||||
|
||||
crawler_enabled = cfg.getboolean("Crawler", "activate_crawler")
|
||||
crawler_enabled = config_loader.get_config_boolean("Crawler", "activate_crawler")
|
||||
|
||||
email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
|
||||
email_regex = re.compile(email_regex)
|
||||
|
@ -191,6 +116,6 @@ try:
|
|||
vt_enabled = False
|
||||
print('VT submission is disabled')
|
||||
except:
|
||||
vt_auth = {'apikey': cfg.get("Flask", "max_preview_char")}
|
||||
vt_auth = {'apikey': config_loader.get_config_str("Flask", "max_preview_char")}
|
||||
vt_enabled = False
|
||||
print('VT submission is disabled')
|
||||
|
|
|
@ -44,7 +44,6 @@ except:
|
|||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_tags = Flask_config.r_serv_tags
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
|
|
|
@ -23,7 +23,6 @@ import Flask_config
|
|||
import Tag
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_tags = Flask_config.r_serv_tags
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
|
|
|
@ -21,7 +21,7 @@ from flask_login import login_required
|
|||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
config_loader = Flask_config.config_loader
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv = Flask_config.r_serv
|
||||
r_serv_log = Flask_config.r_serv_log
|
||||
|
@ -171,8 +171,8 @@ def stuff():
|
|||
@login_required
|
||||
@login_analyst
|
||||
def index():
|
||||
default_minute = cfg.get("Flask", "minute_processed_paste")
|
||||
threshold_stucked_module = cfg.getint("Module_ModuleInformation", "threshold_stucked_module")
|
||||
default_minute = config_loader.get_config_str("Flask", "minute_processed_paste")
|
||||
threshold_stucked_module = config_loader.get_config_int("Module_ModuleInformation", "threshold_stucked_module")
|
||||
log_select = {10, 25, 50, 100}
|
||||
log_select.add(max_dashboard_logs)
|
||||
log_select = list(log_select)
|
||||
|
|
|
@ -24,7 +24,6 @@ from flask_login import login_required
|
|||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
vt_enabled = Flask_config.vt_enabled
|
||||
|
|
|
@ -23,7 +23,6 @@ from HiddenServices import HiddenServices
|
|||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_cache = Flask_config.r_cache
|
||||
r_serv_onion = Flask_config.r_serv_onion
|
||||
|
|
|
@ -27,7 +27,6 @@ import Term
|
|||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_term = Flask_config.r_serv_term
|
||||
r_serv_cred = Flask_config.r_serv_cred
|
||||
|
|
|
@ -14,7 +14,6 @@ from flask_login import login_required
|
|||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
|
||||
rawSkeleton = Blueprint('rawSkeleton', __name__, template_folder='templates')
|
||||
|
||||
|
|
|
@ -31,7 +31,6 @@ import Flask_config
|
|||
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_cache = Flask_config.r_cache
|
||||
r_serv_db = Flask_config.r_serv_db
|
||||
|
|
|
@ -25,7 +25,7 @@ import time
|
|||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
config_loader = Flask_config.config_loader
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_pasteName = Flask_config.r_serv_pasteName
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
|
@ -34,9 +34,8 @@ max_preview_modal = Flask_config.max_preview_modal
|
|||
bootstrap_label = Flask_config.bootstrap_label
|
||||
PASTES_FOLDER = Flask_config.PASTES_FOLDER
|
||||
|
||||
baseindexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path"))
|
||||
indexRegister_path = os.path.join(os.environ['AIL_HOME'],
|
||||
cfg.get("Indexer", "register"))
|
||||
baseindexpath = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Indexer", "path"))
|
||||
indexRegister_path = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Indexer", "register"))
|
||||
|
||||
searches = Blueprint('searches', __name__, template_folder='templates')
|
||||
|
||||
|
|
|
@ -20,7 +20,6 @@ import Paste
|
|||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_charts = Flask_config.r_serv_charts
|
||||
r_serv_sentiment = Flask_config.r_serv_sentiment
|
||||
|
|
|
@ -19,7 +19,6 @@ import git_status
|
|||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_db = Flask_config.r_serv_db
|
||||
max_preview_char = Flask_config.max_preview_char
|
||||
|
|
|
@ -23,7 +23,6 @@ import requests
|
|||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_pasteName = Flask_config.r_serv_pasteName
|
||||
r_serv_metadata = Flask_config.r_serv_metadata
|
||||
|
|
|
@ -29,7 +29,6 @@ import Term
|
|||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_term = Flask_config.r_serv_term
|
||||
r_serv_cred = Flask_config.r_serv_cred
|
||||
|
|
|
@ -17,7 +17,7 @@ from flask_login import login_required
|
|||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
config_loader = Flask_config.config_loader
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_charts = Flask_config.r_serv_charts
|
||||
|
||||
|
@ -69,7 +69,7 @@ def progressionCharts():
|
|||
@login_required
|
||||
@login_analyst
|
||||
def wordstrending():
|
||||
default_display = cfg.get("Flask", "default_display")
|
||||
default_display = config_loader.get_config_str("Flask", "default_display")
|
||||
return render_template("Wordstrending.html", default_display = default_display)
|
||||
|
||||
|
||||
|
@ -77,7 +77,7 @@ def wordstrending():
|
|||
@login_required
|
||||
@login_analyst
|
||||
def protocolstrending():
|
||||
default_display = cfg.get("Flask", "default_display")
|
||||
default_display = config_loader.get_config_str("Flask", "default_display")
|
||||
return render_template("Protocolstrending.html", default_display = default_display)
|
||||
|
||||
|
||||
|
@ -85,7 +85,7 @@ def protocolstrending():
|
|||
@login_required
|
||||
@login_analyst
|
||||
def trending():
|
||||
default_display = cfg.get("Flask", "default_display")
|
||||
default_display = config_loader.get_config_str("Flask", "default_display")
|
||||
return render_template("Trending.html", default_display = default_display)
|
||||
|
||||
|
||||
|
|
|
@ -17,7 +17,6 @@ from flask_login import login_required
|
|||
import Flask_config
|
||||
|
||||
app = Flask_config.app
|
||||
cfg = Flask_config.cfg
|
||||
baseUrl = Flask_config.baseUrl
|
||||
r_serv_charts = Flask_config.r_serv_charts
|
||||
|
||||
|
|
Loading…
Reference in a new issue