diff --git a/bin/Helper.py b/bin/Helper.py index 52097ef6..2942b415 100755 --- a/bin/Helper.py +++ b/bin/Helper.py @@ -58,7 +58,6 @@ class PubSub(object): for address in addresses.split(','): new_sub = context.socket(zmq.SUB) new_sub.connect(address) - # bytes64 encode bytes to ascii only bytes new_sub.setsockopt_string(zmq.SUBSCRIBE, channel) self.subscribers.append(new_sub) diff --git a/bin/lib/ConfigLoader.py b/bin/lib/ConfigLoader.py new file mode 100755 index 00000000..51508daa --- /dev/null +++ b/bin/lib/ConfigLoader.py @@ -0,0 +1,48 @@ +#!/usr/bin/python3 + +""" +The ``Domain`` +=================== + + +""" + +import os +import sys +import time +import redis +import configparser + +# Get Config file +config_dir = os.path.join(os.environ['AIL_HOME'], 'configs') +config_file = os.path.join(config_dir, 'core.cfg') +if not os.path.exists(config_file): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + + # # TODO: create sphinx doc + + # # TODO: add config_field to reload + +class ConfigLoader(object): + """docstring for Config_Loader.""" + + def __init__(self): + self.cfg = configparser.ConfigParser() + self.cfg.read(config_file) + + def get_redis_conn(self, redis_name, decode_responses=True): ## TODO: verify redis name + return redis.StrictRedis( host=self.cfg.get(redis_name, "host"), + port=self.cfg.getint(redis_name, "port"), + db=self.cfg.getint(redis_name, "db"), + decode_responses=decode_responses ) + + def get_config_str(self, section, key_name): + return self.cfg.get(section, key_name) + + def get_config_int(self, section, key_name): + return self.cfg.getint(section, key_name) + + def get_config_boolean(self, section, key_name): + return self.cfg.getboolean(section, key_name) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 0b6b7fa6..a4518a00 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -24,6 +24,10 @@ sys.path.append('./modules/') from User import User +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + + from pytaxonomies import Taxonomies # Import config @@ -31,33 +35,21 @@ import Flask_config # Import Blueprint from blueprints.root import root +from blueprints.crawler_splash import crawler_splash Flask_dir = os.environ['AIL_FLASK'] # CONFIG # -cfg = Flask_config.cfg -baseUrl = cfg.get("Flask", "baseurl") +config_loader = ConfigLoader.ConfigLoader() +baseUrl = config_loader.get_config_str("Flask", "baseurl") baseUrl = baseUrl.replace('/', '') if baseUrl != '': baseUrl = '/'+baseUrl # ========= REDIS =========# -r_serv_db = redis.StrictRedis( - host=cfg.get("ARDB_DB", "host"), - port=cfg.getint("ARDB_DB", "port"), - db=cfg.getint("ARDB_DB", "db"), - decode_responses=True) -r_serv_tags = redis.StrictRedis( - host=cfg.get("ARDB_Tags", "host"), - port=cfg.getint("ARDB_Tags", "port"), - db=cfg.getint("ARDB_Tags", "db"), - decode_responses=True) - -r_cache = redis.StrictRedis( - host=cfg.get("Redis_Cache", "host"), - port=cfg.getint("Redis_Cache", "port"), - db=cfg.getint("Redis_Cache", "db"), - decode_responses=True) +r_serv_db = config_loader.get_redis_conn("ARDB_DB") +r_serv_tags = config_loader.get_redis_conn("ARDB_Tags") +r_cache = config_loader.get_redis_conn("Redis_Cache") # logs log_dir = os.path.join(os.environ['AIL_HOME'], 'logs') @@ -88,6 +80,7 @@ app.config['MAX_CONTENT_LENGTH'] = 900 * 1024 * 1024 # ========= BLUEPRINT =========# app.register_blueprint(root, url_prefix=baseUrl) +app.register_blueprint(crawler_splash, url_prefix=baseUrl) # ========= =========# # ========= session ======== @@ -199,7 +192,7 @@ def add_header(response): @app.errorhandler(405) def _handle_client_error(e): - if request.path.startswith('/api/'): ## # TODO: add baseUrl + if request.path.startswith('/api/'): ## # TODO: add baseUrl res_dict = {"status": "error", "reason": "Method Not Allowed: The method is not allowed for the requested URL"} anchor_id = request.path[8:] anchor_id = anchor_id.replace('/', '_') diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index 49f2919c..6525cb5e 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -4,109 +4,34 @@ ''' Flask global variables shared accross modules ''' -import configparser -import redis import os import re import sys +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + # FLASK # app = None # CONFIG # -configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') -if not os.path.exists(configfile): - raise Exception('Unable to find the configuration file. \ - Did you set environment variables? \ - Or activate the virtualenv.') - -cfg = configparser.ConfigParser() -cfg.read(configfile) +config_loader = ConfigLoader.ConfigLoader() # REDIS # -r_serv = redis.StrictRedis( - host=cfg.get("Redis_Queues", "host"), - port=cfg.getint("Redis_Queues", "port"), - db=cfg.getint("Redis_Queues", "db"), - decode_responses=True) - -r_cache = redis.StrictRedis( - host=cfg.get("Redis_Cache", "host"), - port=cfg.getint("Redis_Cache", "port"), - db=cfg.getint("Redis_Cache", "db"), - decode_responses=True) - -r_serv_log = redis.StrictRedis( - host=cfg.get("Redis_Log", "host"), - port=cfg.getint("Redis_Log", "port"), - db=cfg.getint("Redis_Log", "db"), - decode_responses=True) - -r_serv_log_submit = redis.StrictRedis( - host=cfg.get("Redis_Log_submit", "host"), - port=cfg.getint("Redis_Log_submit", "port"), - db=cfg.getint("Redis_Log_submit", "db"), - decode_responses=True) - -r_serv_charts = redis.StrictRedis( - host=cfg.get("ARDB_Trending", "host"), - port=cfg.getint("ARDB_Trending", "port"), - db=cfg.getint("ARDB_Trending", "db"), - decode_responses=True) - -r_serv_sentiment = redis.StrictRedis( - host=cfg.get("ARDB_Sentiment", "host"), - port=cfg.getint("ARDB_Sentiment", "port"), - db=cfg.getint("ARDB_Sentiment", "db"), - decode_responses=True) - -r_serv_term = redis.StrictRedis( - host=cfg.get("ARDB_Tracker", "host"), - port=cfg.getint("ARDB_Tracker", "port"), - db=cfg.getint("ARDB_Tracker", "db"), - decode_responses=True) - -r_serv_cred = redis.StrictRedis( - host=cfg.get("ARDB_TermCred", "host"), - port=cfg.getint("ARDB_TermCred", "port"), - db=cfg.getint("ARDB_TermCred", "db"), - decode_responses=True) - -r_serv_pasteName = redis.StrictRedis( - host=cfg.get("Redis_Paste_Name", "host"), - port=cfg.getint("Redis_Paste_Name", "port"), - db=cfg.getint("Redis_Paste_Name", "db"), - decode_responses=True) - -r_serv_tags = redis.StrictRedis( - host=cfg.get("ARDB_Tags", "host"), - port=cfg.getint("ARDB_Tags", "port"), - db=cfg.getint("ARDB_Tags", "db"), - decode_responses=True) - -r_serv_metadata = redis.StrictRedis( - host=cfg.get("ARDB_Metadata", "host"), - port=cfg.getint("ARDB_Metadata", "port"), - db=cfg.getint("ARDB_Metadata", "db"), - decode_responses=True) - -r_serv_db = redis.StrictRedis( - host=cfg.get("ARDB_DB", "host"), - port=cfg.getint("ARDB_DB", "port"), - db=cfg.getint("ARDB_DB", "db"), - decode_responses=True) - -r_serv_statistics = redis.StrictRedis( - host=cfg.get("ARDB_Statistics", "host"), - port=cfg.getint("ARDB_Statistics", "port"), - db=cfg.getint("ARDB_Statistics", "db"), - decode_responses=True) - -r_serv_onion = redis.StrictRedis( - host=cfg.get("ARDB_Onion", "host"), - port=cfg.getint("ARDB_Onion", "port"), - db=cfg.getint("ARDB_Onion", "db"), - decode_responses=True) +r_serv = config_loader.get_redis_conn("Redis_Queues") +r_cache = config_loader.get_redis_conn("Redis_Cache") +r_serv_log = config_loader.get_redis_conn("Redis_Log") +r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit") +r_serv_charts = config_loader.get_redis_conn("ARDB_Trending") +r_serv_sentiment = config_loader.get_redis_conn("ARDB_Sentiment") +r_serv_term = config_loader.get_redis_conn("ARDB_Tracker") +r_serv_cred = config_loader.get_redis_conn("ARDB_TermCred") +r_serv_pasteName = config_loader.get_redis_conn("Redis_Paste_Name") +r_serv_tags = config_loader.get_redis_conn("ARDB_Tags") +r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") +r_serv_db = config_loader.get_redis_conn("ARDB_DB") +r_serv_statistics = config_loader.get_redis_conn("ARDB_Statistics") +r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") sys.path.append('../../configs/keys') # MISP # @@ -146,17 +71,17 @@ if HiveApi != False: print('The Hive not connected') #### VARIABLES #### -baseUrl = cfg.get("Flask", "baseurl") +baseUrl = config_loader.get_config_str("Flask", "baseurl") baseUrl = baseUrl.replace('/', '') if baseUrl != '': baseUrl = '/'+baseUrl -max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip -max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal +max_preview_char = int(config_loader.get_config_str("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip +max_preview_modal = int(config_loader.get_config_str("Flask", "max_preview_modal")) # Maximum number of character to display in the modal max_tags_result = 50 -DiffMaxLineLength = int(cfg.get("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value +DiffMaxLineLength = int(config_loader.get_config_str("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] @@ -166,14 +91,14 @@ dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_me UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted') -PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + '/' -SCREENSHOT_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot"), 'screenshot') +PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' +SCREENSHOT_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "crawled_screenshot"), 'screenshot') REPO_ORIGIN = 'https://github.com/CIRCL/AIL-framework.git' -max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs")) +max_dashboard_logs = int(config_loader.get_config_str("Flask", "max_dashboard_logs")) -crawler_enabled = cfg.getboolean("Crawler", "activate_crawler") +crawler_enabled = config_loader.get_config_boolean("Crawler", "activate_crawler") email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}' email_regex = re.compile(email_regex) @@ -191,6 +116,6 @@ try: vt_enabled = False print('VT submission is disabled') except: - vt_auth = {'apikey': cfg.get("Flask", "max_preview_char")} + vt_auth = {'apikey': config_loader.get_config_str("Flask", "max_preview_char")} vt_enabled = False print('VT submission is disabled') diff --git a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py index 76bae898..4af8fa0b 100644 --- a/var/www/modules/PasteSubmit/Flask_PasteSubmit.py +++ b/var/www/modules/PasteSubmit/Flask_PasteSubmit.py @@ -44,7 +44,6 @@ except: import Flask_config app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_tags = Flask_config.r_serv_tags r_serv_metadata = Flask_config.r_serv_metadata diff --git a/var/www/modules/Tags/Flask_Tags.py b/var/www/modules/Tags/Flask_Tags.py index d15b78a8..4772e82e 100644 --- a/var/www/modules/Tags/Flask_Tags.py +++ b/var/www/modules/Tags/Flask_Tags.py @@ -23,7 +23,6 @@ import Flask_config import Tag app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_tags = Flask_config.r_serv_tags r_serv_metadata = Flask_config.r_serv_metadata diff --git a/var/www/modules/dashboard/Flask_dashboard.py b/var/www/modules/dashboard/Flask_dashboard.py index 160d9edb..29def6cf 100644 --- a/var/www/modules/dashboard/Flask_dashboard.py +++ b/var/www/modules/dashboard/Flask_dashboard.py @@ -21,7 +21,7 @@ from flask_login import login_required import Flask_config app = Flask_config.app -cfg = Flask_config.cfg +config_loader = Flask_config.config_loader baseUrl = Flask_config.baseUrl r_serv = Flask_config.r_serv r_serv_log = Flask_config.r_serv_log @@ -171,8 +171,8 @@ def stuff(): @login_required @login_analyst def index(): - default_minute = cfg.get("Flask", "minute_processed_paste") - threshold_stucked_module = cfg.getint("Module_ModuleInformation", "threshold_stucked_module") + default_minute = config_loader.get_config_str("Flask", "minute_processed_paste") + threshold_stucked_module = config_loader.get_config_int("Module_ModuleInformation", "threshold_stucked_module") log_select = {10, 25, 50, 100} log_select.add(max_dashboard_logs) log_select = list(log_select) diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py index f6073afb..ae3ee48b 100644 --- a/var/www/modules/hashDecoded/Flask_hashDecoded.py +++ b/var/www/modules/hashDecoded/Flask_hashDecoded.py @@ -24,7 +24,6 @@ from flask_login import login_required import Flask_config app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_metadata = Flask_config.r_serv_metadata vt_enabled = Flask_config.vt_enabled diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py index e3ee2bcb..f64df4e1 100644 --- a/var/www/modules/hiddenServices/Flask_hiddenServices.py +++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py @@ -23,7 +23,6 @@ from HiddenServices import HiddenServices import Flask_config app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_cache = Flask_config.r_cache r_serv_onion = Flask_config.r_serv_onion diff --git a/var/www/modules/hunter/Flask_hunter.py b/var/www/modules/hunter/Flask_hunter.py index f86d3403..e9028b36 100644 --- a/var/www/modules/hunter/Flask_hunter.py +++ b/var/www/modules/hunter/Flask_hunter.py @@ -27,7 +27,6 @@ import Term import Flask_config app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_term = Flask_config.r_serv_term r_serv_cred = Flask_config.r_serv_cred diff --git a/var/www/modules/rawSkeleton/Flask_rawSkeleton.py b/var/www/modules/rawSkeleton/Flask_rawSkeleton.py index d767a83c..dca8f331 100644 --- a/var/www/modules/rawSkeleton/Flask_rawSkeleton.py +++ b/var/www/modules/rawSkeleton/Flask_rawSkeleton.py @@ -14,7 +14,6 @@ from flask_login import login_required import Flask_config app = Flask_config.app -cfg = Flask_config.cfg rawSkeleton = Blueprint('rawSkeleton', __name__, template_folder='templates') diff --git a/var/www/modules/restApi/Flask_restApi.py b/var/www/modules/restApi/Flask_restApi.py index cc32afd6..cbd93dd6 100644 --- a/var/www/modules/restApi/Flask_restApi.py +++ b/var/www/modules/restApi/Flask_restApi.py @@ -31,7 +31,6 @@ import Flask_config app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_cache = Flask_config.r_cache r_serv_db = Flask_config.r_serv_db diff --git a/var/www/modules/search/Flask_search.py b/var/www/modules/search/Flask_search.py index 67a518fb..ff5395e6 100644 --- a/var/www/modules/search/Flask_search.py +++ b/var/www/modules/search/Flask_search.py @@ -25,7 +25,7 @@ import time import Flask_config app = Flask_config.app -cfg = Flask_config.cfg +config_loader = Flask_config.config_loader baseUrl = Flask_config.baseUrl r_serv_pasteName = Flask_config.r_serv_pasteName r_serv_metadata = Flask_config.r_serv_metadata @@ -34,9 +34,8 @@ max_preview_modal = Flask_config.max_preview_modal bootstrap_label = Flask_config.bootstrap_label PASTES_FOLDER = Flask_config.PASTES_FOLDER -baseindexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path")) -indexRegister_path = os.path.join(os.environ['AIL_HOME'], - cfg.get("Indexer", "register")) +baseindexpath = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Indexer", "path")) +indexRegister_path = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Indexer", "register")) searches = Blueprint('searches', __name__, template_folder='templates') diff --git a/var/www/modules/sentiment/Flask_sentiment.py b/var/www/modules/sentiment/Flask_sentiment.py index af6c220c..895bd0ee 100644 --- a/var/www/modules/sentiment/Flask_sentiment.py +++ b/var/www/modules/sentiment/Flask_sentiment.py @@ -20,7 +20,6 @@ import Paste import Flask_config app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_charts = Flask_config.r_serv_charts r_serv_sentiment = Flask_config.r_serv_sentiment diff --git a/var/www/modules/settings/Flask_settings.py b/var/www/modules/settings/Flask_settings.py index a569cbbb..0ad1f43c 100644 --- a/var/www/modules/settings/Flask_settings.py +++ b/var/www/modules/settings/Flask_settings.py @@ -19,7 +19,6 @@ import git_status import Flask_config app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_db = Flask_config.r_serv_db max_preview_char = Flask_config.max_preview_char diff --git a/var/www/modules/showpaste/Flask_showpaste.py b/var/www/modules/showpaste/Flask_showpaste.py index 10519d53..a972a346 100644 --- a/var/www/modules/showpaste/Flask_showpaste.py +++ b/var/www/modules/showpaste/Flask_showpaste.py @@ -23,7 +23,6 @@ import requests import Flask_config app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_pasteName = Flask_config.r_serv_pasteName r_serv_metadata = Flask_config.r_serv_metadata diff --git a/var/www/modules/terms/Flask_terms.py b/var/www/modules/terms/Flask_terms.py index c594839a..3e166063 100644 --- a/var/www/modules/terms/Flask_terms.py +++ b/var/www/modules/terms/Flask_terms.py @@ -29,7 +29,6 @@ import Term import Flask_config app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_term = Flask_config.r_serv_term r_serv_cred = Flask_config.r_serv_cred diff --git a/var/www/modules/trendingcharts/Flask_trendingcharts.py b/var/www/modules/trendingcharts/Flask_trendingcharts.py index a037e171..b2dfa68a 100644 --- a/var/www/modules/trendingcharts/Flask_trendingcharts.py +++ b/var/www/modules/trendingcharts/Flask_trendingcharts.py @@ -17,7 +17,7 @@ from flask_login import login_required import Flask_config app = Flask_config.app -cfg = Flask_config.cfg +config_loader = Flask_config.config_loader baseUrl = Flask_config.baseUrl r_serv_charts = Flask_config.r_serv_charts @@ -69,7 +69,7 @@ def progressionCharts(): @login_required @login_analyst def wordstrending(): - default_display = cfg.get("Flask", "default_display") + default_display = config_loader.get_config_str("Flask", "default_display") return render_template("Wordstrending.html", default_display = default_display) @@ -77,7 +77,7 @@ def wordstrending(): @login_required @login_analyst def protocolstrending(): - default_display = cfg.get("Flask", "default_display") + default_display = config_loader.get_config_str("Flask", "default_display") return render_template("Protocolstrending.html", default_display = default_display) @@ -85,7 +85,7 @@ def protocolstrending(): @login_required @login_analyst def trending(): - default_display = cfg.get("Flask", "default_display") + default_display = config_loader.get_config_str("Flask", "default_display") return render_template("Trending.html", default_display = default_display) diff --git a/var/www/modules/trendingmodules/Flask_trendingmodules.py b/var/www/modules/trendingmodules/Flask_trendingmodules.py index 80646ecb..816d8055 100644 --- a/var/www/modules/trendingmodules/Flask_trendingmodules.py +++ b/var/www/modules/trendingmodules/Flask_trendingmodules.py @@ -17,7 +17,6 @@ from flask_login import login_required import Flask_config app = Flask_config.app -cfg = Flask_config.cfg baseUrl = Flask_config.baseUrl r_serv_charts = Flask_config.r_serv_charts