chg: [lib ConfigLoader] add configLoader class

bin/Helper.py

@@ -58,7 +58,6 @@ class PubSub(object):
             for address in addresses.split(','):
                 new_sub = context.socket(zmq.SUB)
                 new_sub.connect(address)
-                # bytes64 encode bytes to ascii only bytes
                 new_sub.setsockopt_string(zmq.SUBSCRIBE, channel)
                 self.subscribers.append(new_sub)
 

bin/lib/ConfigLoader.py

@@ -0,0 +1,48 @@
+#!/usr/bin/python3
+
+"""
+The ``Domain``
+===================
+
+
+"""
+
+import os
+import sys
+import time
+import redis
+import configparser
+
+# Get Config file
+config_dir = os.path.join(os.environ['AIL_HOME'], 'configs')
+config_file = os.path.join(config_dir, 'core.cfg')
+if not os.path.exists(config_file):
+    raise Exception('Unable to find the configuration file. \
+                    Did you set environment variables? \
+                    Or activate the virtualenv.')
+
+ # # TODO: create sphinx doc
+
+ # # TODO: add config_field to reload
+
+class ConfigLoader(object):
+    """docstring for Config_Loader."""
+
+    def __init__(self):
+        self.cfg = configparser.ConfigParser()
+        self.cfg.read(config_file)
+
+    def get_redis_conn(self, redis_name, decode_responses=True): ## TODO: verify redis name
+        return redis.StrictRedis( host=self.cfg.get(redis_name, "host"),
+                                  port=self.cfg.getint(redis_name, "port"),
+                                  db=self.cfg.getint(redis_name, "db"),
+                                  decode_responses=decode_responses )
+
+    def get_config_str(self, section, key_name):
+        return self.cfg.get(section, key_name)
+
+    def get_config_int(self, section, key_name):
+        return self.cfg.getint(section, key_name)
+
+    def get_config_boolean(self, section, key_name):
+        return self.cfg.getboolean(section, key_name)

var/www/Flask_server.py

@@ -24,6 +24,10 @@ sys.path.append('./modules/')
 
 from User import User
 
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
+
 from pytaxonomies import Taxonomies
 
 # Import config
@@ -31,33 +35,21 @@ import Flask_config
 
 # Import Blueprint
 from blueprints.root import root
+from blueprints.crawler_splash import crawler_splash
 
 Flask_dir = os.environ['AIL_FLASK']
 
 # CONFIG #
-cfg = Flask_config.cfg
+config_loader = ConfigLoader.ConfigLoader()
-baseUrl = cfg.get("Flask", "baseurl")
+baseUrl = config_loader.get_config_str("Flask", "baseurl")
 baseUrl = baseUrl.replace('/', '')
 if baseUrl != '':
     baseUrl = '/'+baseUrl
 
 # ========= REDIS =========#
-r_serv_db = redis.StrictRedis(
+r_serv_db = config_loader.get_redis_conn("ARDB_DB")
-    host=cfg.get("ARDB_DB", "host"),
+r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
-    port=cfg.getint("ARDB_DB", "port"),
+r_cache = config_loader.get_redis_conn("Redis_Cache")
-    db=cfg.getint("ARDB_DB", "db"),
-    decode_responses=True)
-r_serv_tags = redis.StrictRedis(
-    host=cfg.get("ARDB_Tags", "host"),
-    port=cfg.getint("ARDB_Tags", "port"),
-    db=cfg.getint("ARDB_Tags", "db"),
-    decode_responses=True)
-
-r_cache = redis.StrictRedis(
-    host=cfg.get("Redis_Cache", "host"),
-    port=cfg.getint("Redis_Cache", "port"),
-    db=cfg.getint("Redis_Cache", "db"),
-    decode_responses=True)
 
 # logs
 log_dir = os.path.join(os.environ['AIL_HOME'], 'logs')
@@ -88,6 +80,7 @@ app.config['MAX_CONTENT_LENGTH'] = 900 * 1024 * 1024
 
 # =========  BLUEPRINT  =========#
 app.register_blueprint(root, url_prefix=baseUrl)
+app.register_blueprint(crawler_splash, url_prefix=baseUrl)
 # =========       =========#
 
 # ========= session ========
@@ -199,7 +192,7 @@ def add_header(response):
 
 @app.errorhandler(405)
 def _handle_client_error(e):
-    if request.path.startswith('/api/'): ## # TODO: add baseUrl 
+    if request.path.startswith('/api/'): ## # TODO: add baseUrl
         res_dict = {"status": "error", "reason": "Method Not Allowed: The method is not allowed for the requested URL"}
         anchor_id = request.path[8:]
         anchor_id = anchor_id.replace('/', '_')

var/www/modules/Flask_config.py

@@ -4,109 +4,34 @@
 '''
     Flask global variables shared accross modules
 '''
-import configparser
-import redis
 import os
 import re
 import sys
 
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
+import ConfigLoader
+
 # FLASK #
 app = None
 
 # CONFIG #
-configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
+config_loader = ConfigLoader.ConfigLoader()
-if not os.path.exists(configfile):
-    raise Exception('Unable to find the configuration file. \
-                    Did you set environment variables? \
-                    Or activate the virtualenv.')
-
-cfg = configparser.ConfigParser()
-cfg.read(configfile)
 
 # REDIS #
-r_serv = redis.StrictRedis(
+r_serv = config_loader.get_redis_conn("Redis_Queues")
-    host=cfg.get("Redis_Queues", "host"),
+r_cache = config_loader.get_redis_conn("Redis_Cache")
-    port=cfg.getint("Redis_Queues", "port"),
+r_serv_log = config_loader.get_redis_conn("Redis_Log")
-    db=cfg.getint("Redis_Queues", "db"),
+r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit")
-    decode_responses=True)
+r_serv_charts = config_loader.get_redis_conn("ARDB_Trending")
-
+r_serv_sentiment = config_loader.get_redis_conn("ARDB_Sentiment")
-r_cache = redis.StrictRedis(
+r_serv_term = config_loader.get_redis_conn("ARDB_Tracker")
-    host=cfg.get("Redis_Cache", "host"),
+r_serv_cred = config_loader.get_redis_conn("ARDB_TermCred")
-    port=cfg.getint("Redis_Cache", "port"),
+r_serv_pasteName = config_loader.get_redis_conn("Redis_Paste_Name")
-    db=cfg.getint("Redis_Cache", "db"),
+r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
-    decode_responses=True)
+r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
-
+r_serv_db = config_loader.get_redis_conn("ARDB_DB")
-r_serv_log = redis.StrictRedis(
+r_serv_statistics = config_loader.get_redis_conn("ARDB_Statistics")
-    host=cfg.get("Redis_Log", "host"),
+r_serv_onion = config_loader.get_redis_conn("ARDB_Onion")
-    port=cfg.getint("Redis_Log", "port"),
-    db=cfg.getint("Redis_Log", "db"),
-    decode_responses=True)
-
-r_serv_log_submit = redis.StrictRedis(
-    host=cfg.get("Redis_Log_submit", "host"),
-    port=cfg.getint("Redis_Log_submit", "port"),
-    db=cfg.getint("Redis_Log_submit", "db"),
-    decode_responses=True)
-
-r_serv_charts = redis.StrictRedis(
-    host=cfg.get("ARDB_Trending", "host"),
-    port=cfg.getint("ARDB_Trending", "port"),
-    db=cfg.getint("ARDB_Trending", "db"),
-    decode_responses=True)
-
-r_serv_sentiment = redis.StrictRedis(
-        host=cfg.get("ARDB_Sentiment", "host"),
-        port=cfg.getint("ARDB_Sentiment", "port"),
-        db=cfg.getint("ARDB_Sentiment", "db"),
-        decode_responses=True)
-
-r_serv_term = redis.StrictRedis(
-        host=cfg.get("ARDB_Tracker", "host"),
-        port=cfg.getint("ARDB_Tracker", "port"),
-        db=cfg.getint("ARDB_Tracker", "db"),
-        decode_responses=True)
-
-r_serv_cred = redis.StrictRedis(
-        host=cfg.get("ARDB_TermCred", "host"),
-        port=cfg.getint("ARDB_TermCred", "port"),
-        db=cfg.getint("ARDB_TermCred", "db"),
-        decode_responses=True)
-
-r_serv_pasteName = redis.StrictRedis(
-    host=cfg.get("Redis_Paste_Name", "host"),
-    port=cfg.getint("Redis_Paste_Name", "port"),
-    db=cfg.getint("Redis_Paste_Name", "db"),
-    decode_responses=True)
-
-r_serv_tags = redis.StrictRedis(
-    host=cfg.get("ARDB_Tags", "host"),
-    port=cfg.getint("ARDB_Tags", "port"),
-    db=cfg.getint("ARDB_Tags", "db"),
-    decode_responses=True)
-
-r_serv_metadata = redis.StrictRedis(
-    host=cfg.get("ARDB_Metadata", "host"),
-    port=cfg.getint("ARDB_Metadata", "port"),
-    db=cfg.getint("ARDB_Metadata", "db"),
-    decode_responses=True)
-
-r_serv_db = redis.StrictRedis(
-    host=cfg.get("ARDB_DB", "host"),
-    port=cfg.getint("ARDB_DB", "port"),
-    db=cfg.getint("ARDB_DB", "db"),
-    decode_responses=True)
-
-r_serv_statistics = redis.StrictRedis(
-    host=cfg.get("ARDB_Statistics", "host"),
-    port=cfg.getint("ARDB_Statistics", "port"),
-    db=cfg.getint("ARDB_Statistics", "db"),
-    decode_responses=True)
-
-r_serv_onion = redis.StrictRedis(
-    host=cfg.get("ARDB_Onion", "host"),
-    port=cfg.getint("ARDB_Onion", "port"),
-    db=cfg.getint("ARDB_Onion", "db"),
-    decode_responses=True)
 
 sys.path.append('../../configs/keys')
 # MISP #
@@ -146,17 +71,17 @@ if HiveApi != False:
         print('The Hive not connected')
 
 #### VARIABLES ####
-baseUrl = cfg.get("Flask", "baseurl")
+baseUrl = config_loader.get_config_str("Flask", "baseurl")
 baseUrl = baseUrl.replace('/', '')
 if baseUrl != '':
     baseUrl = '/'+baseUrl
 
-max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
+max_preview_char = int(config_loader.get_config_str("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
-max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
+max_preview_modal = int(config_loader.get_config_str("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
 
 max_tags_result = 50
 
-DiffMaxLineLength =  int(cfg.get("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value
+DiffMaxLineLength =  int(config_loader.get_config_str("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value
 
 bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
 
@@ -166,14 +91,14 @@ dict_update_description = {'v1.5':{'nb_background_update': 5, 'update_warning_me
 
 UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')
 
-PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + '/'
+PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
-SCREENSHOT_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot"), 'screenshot')
+SCREENSHOT_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "crawled_screenshot"), 'screenshot')
 
 REPO_ORIGIN = 'https://github.com/CIRCL/AIL-framework.git'
 
-max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs"))
+max_dashboard_logs = int(config_loader.get_config_str("Flask", "max_dashboard_logs"))
 
-crawler_enabled = cfg.getboolean("Crawler", "activate_crawler")
+crawler_enabled = config_loader.get_config_boolean("Crawler", "activate_crawler")
 
 email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
 email_regex = re.compile(email_regex)
@@ -191,6 +116,6 @@ try:
         vt_enabled = False
         print('VT submission is disabled')
 except:
-    vt_auth = {'apikey': cfg.get("Flask", "max_preview_char")}
+    vt_auth = {'apikey': config_loader.get_config_str("Flask", "max_preview_char")}
     vt_enabled = False
     print('VT submission is disabled')

var/www/modules/PasteSubmit/Flask_PasteSubmit.py

@@ -44,7 +44,6 @@ except:
 import Flask_config
 
 app = Flask_config.app
-cfg = Flask_config.cfg
 baseUrl = Flask_config.baseUrl
 r_serv_tags = Flask_config.r_serv_tags
 r_serv_metadata = Flask_config.r_serv_metadata

var/www/modules/Tags/Flask_Tags.py

@@ -23,7 +23,6 @@ import Flask_config
 import Tag
 
 app = Flask_config.app
-cfg = Flask_config.cfg
 baseUrl = Flask_config.baseUrl
 r_serv_tags = Flask_config.r_serv_tags
 r_serv_metadata = Flask_config.r_serv_metadata

var/www/modules/dashboard/Flask_dashboard.py

@@ -21,7 +21,7 @@ from flask_login import login_required
 import Flask_config
 
 app = Flask_config.app
-cfg = Flask_config.cfg
+config_loader = Flask_config.config_loader
 baseUrl = Flask_config.baseUrl
 r_serv = Flask_config.r_serv
 r_serv_log = Flask_config.r_serv_log
@@ -171,8 +171,8 @@ def stuff():
 @login_required
 @login_analyst
 def index():
-    default_minute = cfg.get("Flask", "minute_processed_paste")
+    default_minute = config_loader.get_config_str("Flask", "minute_processed_paste")
-    threshold_stucked_module = cfg.getint("Module_ModuleInformation", "threshold_stucked_module")
+    threshold_stucked_module = config_loader.get_config_int("Module_ModuleInformation", "threshold_stucked_module")
     log_select = {10, 25, 50, 100}
     log_select.add(max_dashboard_logs)
     log_select = list(log_select)

var/www/modules/hashDecoded/Flask_hashDecoded.py

@@ -24,7 +24,6 @@ from flask_login import login_required
 import Flask_config
 
 app = Flask_config.app
-cfg = Flask_config.cfg
 baseUrl = Flask_config.baseUrl
 r_serv_metadata = Flask_config.r_serv_metadata
 vt_enabled = Flask_config.vt_enabled

var/www/modules/hiddenServices/Flask_hiddenServices.py

@@ -23,7 +23,6 @@ from HiddenServices import HiddenServices
 import Flask_config
 
 app = Flask_config.app
-cfg = Flask_config.cfg
 baseUrl = Flask_config.baseUrl
 r_cache = Flask_config.r_cache
 r_serv_onion = Flask_config.r_serv_onion

var/www/modules/hunter/Flask_hunter.py

@@ -27,7 +27,6 @@ import Term
 import Flask_config
 
 app = Flask_config.app
-cfg = Flask_config.cfg
 baseUrl = Flask_config.baseUrl
 r_serv_term = Flask_config.r_serv_term
 r_serv_cred = Flask_config.r_serv_cred

var/www/modules/rawSkeleton/Flask_rawSkeleton.py

@@ -14,7 +14,6 @@ from flask_login import login_required
 import Flask_config
 
 app = Flask_config.app
-cfg = Flask_config.cfg
 
 rawSkeleton = Blueprint('rawSkeleton', __name__, template_folder='templates')
 

var/www/modules/restApi/Flask_restApi.py

@@ -31,7 +31,6 @@ import Flask_config
 
 
 app = Flask_config.app
-cfg = Flask_config.cfg
 baseUrl = Flask_config.baseUrl
 r_cache = Flask_config.r_cache
 r_serv_db = Flask_config.r_serv_db

var/www/modules/search/Flask_search.py

@@ -25,7 +25,7 @@ import time
 import Flask_config
 
 app = Flask_config.app
-cfg = Flask_config.cfg
+config_loader = Flask_config.config_loader
 baseUrl = Flask_config.baseUrl
 r_serv_pasteName = Flask_config.r_serv_pasteName
 r_serv_metadata = Flask_config.r_serv_metadata
@@ -34,9 +34,8 @@ max_preview_modal = Flask_config.max_preview_modal
 bootstrap_label = Flask_config.bootstrap_label
 PASTES_FOLDER = Flask_config.PASTES_FOLDER
 
-baseindexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path"))
+baseindexpath = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Indexer", "path"))
-indexRegister_path = os.path.join(os.environ['AIL_HOME'],
+indexRegister_path = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Indexer", "register"))
-                         cfg.get("Indexer", "register"))
 
 searches = Blueprint('searches', __name__, template_folder='templates')
 

var/www/modules/sentiment/Flask_sentiment.py

@@ -20,7 +20,6 @@ import Paste
 import Flask_config
 
 app = Flask_config.app
-cfg = Flask_config.cfg
 baseUrl = Flask_config.baseUrl
 r_serv_charts = Flask_config.r_serv_charts
 r_serv_sentiment = Flask_config.r_serv_sentiment

var/www/modules/settings/Flask_settings.py

@@ -19,7 +19,6 @@ import git_status
 import Flask_config
 
 app = Flask_config.app
-cfg = Flask_config.cfg
 baseUrl = Flask_config.baseUrl
 r_serv_db = Flask_config.r_serv_db
 max_preview_char = Flask_config.max_preview_char

var/www/modules/showpaste/Flask_showpaste.py

@@ -23,7 +23,6 @@ import requests
 import Flask_config
 
 app = Flask_config.app
-cfg = Flask_config.cfg
 baseUrl = Flask_config.baseUrl
 r_serv_pasteName = Flask_config.r_serv_pasteName
 r_serv_metadata = Flask_config.r_serv_metadata

var/www/modules/terms/Flask_terms.py

@@ -29,7 +29,6 @@ import Term
 import Flask_config
 
 app = Flask_config.app
-cfg = Flask_config.cfg
 baseUrl = Flask_config.baseUrl
 r_serv_term = Flask_config.r_serv_term
 r_serv_cred = Flask_config.r_serv_cred

var/www/modules/trendingcharts/Flask_trendingcharts.py

@@ -17,7 +17,7 @@ from flask_login import login_required
 import Flask_config
 
 app = Flask_config.app
-cfg = Flask_config.cfg
+config_loader = Flask_config.config_loader
 baseUrl = Flask_config.baseUrl
 r_serv_charts = Flask_config.r_serv_charts
 
@@ -69,7 +69,7 @@ def progressionCharts():
 @login_required
 @login_analyst
 def wordstrending():
-    default_display = cfg.get("Flask", "default_display")
+    default_display = config_loader.get_config_str("Flask", "default_display")
     return render_template("Wordstrending.html", default_display = default_display)
 
 
@@ -77,7 +77,7 @@ def wordstrending():
 @login_required
 @login_analyst
 def protocolstrending():
-    default_display = cfg.get("Flask", "default_display")
+    default_display = config_loader.get_config_str("Flask", "default_display")
     return render_template("Protocolstrending.html", default_display = default_display)
 
 
@@ -85,7 +85,7 @@ def protocolstrending():
 @login_required
 @login_analyst
 def trending():
-    default_display = cfg.get("Flask", "default_display")
+    default_display = config_loader.get_config_str("Flask", "default_display")
     return render_template("Trending.html", default_display = default_display)
 
 

var/www/modules/trendingmodules/Flask_trendingmodules.py

@@ -17,7 +17,6 @@ from flask_login import login_required
 import Flask_config
 
 app = Flask_config.app
-cfg = Flask_config.cfg
 baseUrl = Flask_config.baseUrl
 r_serv_charts = Flask_config.r_serv_charts