fix: [UI crawler endpoints] display crawler status + fix #353

var/www/Flask_server.py

@@ -48,10 +48,6 @@ except IOError:
     f = open('templates/ignored_modules.txt', 'w')
     f.close()
 
-activate_crawler = cfg.get("Crawler", "activate_crawler")
-if activate_crawler != 'True':
-    toIgnoreModule.add('hiddenServices')
-
 # Dynamically import routes and functions from modules
 # Also, prepare header.html
 to_add_to_header_dico = {}

var/www/modules/Flask_config.py

@@ -173,6 +173,8 @@ REPO_ORIGIN = 'https://github.com/CIRCL/AIL-framework.git'
 
 max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs"))
 
+crawler_enabled = cfg.getboolean("Crawler", "activate_crawler")
+
 # VT
 try:
     from virusTotalKEYS import vt_key

var/www/modules/hiddenServices/Flask_hiddenServices.py

@@ -25,6 +25,7 @@ baseUrl = Flask_config.baseUrl
 r_cache = Flask_config.r_cache
 r_serv_onion = Flask_config.r_serv_onion
 r_serv_metadata = Flask_config.r_serv_metadata
+crawler_enabled = Flask_config.crawler_enabled
 bootstrap_label = Flask_config.bootstrap_label
 
 hiddenServices = Blueprint('hiddenServices', __name__, template_folder='templates')
@@ -247,6 +248,7 @@ def dashboard():
     statDomains_regular = get_stats_last_crawled_domains('regular', date)
 
     return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion,
+                                crawler_enabled=crawler_enabled,
                                 crawler_metadata_regular=crawler_metadata_regular,
                                 statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular)
 
@@ -256,7 +258,7 @@ def hiddenServices_page_test():
 
 @hiddenServices.route("/crawlers/manual", methods=['GET'])
 def manual():
-    return render_template("Crawler_Splash_manual.html")
+    return render_template("Crawler_Splash_manual.html", crawler_enabled=crawler_enabled)
 
 @hiddenServices.route("/crawlers/crawler_splash_onion", methods=['GET'])
 def crawler_splash_onion():
@@ -294,6 +296,7 @@ def Crawler_Splash_last_by_type():
     crawler_metadata = get_crawler_splash_status(type)
 
     return render_template("Crawler_Splash_last_by_type.html", type=type, type_name=type_name,
+                            crawler_enabled=crawler_enabled,
                             last_domains=list_domains, statDomains=statDomains,
                             crawler_metadata=crawler_metadata, date_from=date_string, date_to=date_string)
 
@@ -507,6 +510,7 @@ def auto_crawler():
 
     return render_template("Crawler_auto.html", page=page, nb_page_max=nb_page_max,
                                 last_domains=last_domains,
+                                crawler_enabled=crawler_enabled,
                                 auto_crawler_domain_onions_metadata=auto_crawler_domain_onions_metadata,
                                 auto_crawler_domain_regular_metadata=auto_crawler_domain_regular_metadata)
 

var/www/modules/hiddenServices/templates/Crawler_Splash_last_by_type.html

@@ -46,6 +46,8 @@
 
 			<div class="col-12 col-lg-10" id="core_content">
 
+				{% include 'crawler/crawler_disabled.html' %}
+
 				<div class="row">
 					<div class="col-12 col-xl-6">
 

var/www/modules/hiddenServices/templates/Crawler_Splash_manual.html

@@ -30,6 +30,7 @@
 
 			<div class="col-12 col-lg-10" id="core_content">
 
+				{% include 'crawler/crawler_disabled.html' %}
 
 						<div class="card text-white bg-dark mb-3 mt-1">
 						  <div class="card-header">

var/www/modules/hiddenServices/templates/Crawler_auto.html

@@ -29,6 +29,8 @@
 
 			<div class="col-12 col-lg-10" id="core_content">
 
+				{% include 'crawler/crawler_disabled.html' %}
+
 				{%if last_domains%}
 					<div class="table-responsive mt-1 mb-3 table-hover table-borderless table-striped">
 						<table class="table">

var/www/modules/hiddenServices/templates/Crawler_dashboard.html

@@ -26,6 +26,8 @@
 
 			<div class="col-12 col-lg-10" id="core_content">
 
+				{% include 'crawler/crawler_disabled.html' %}
+
 				<div class="row">
 					<div class="col-xl-6">
 

var/www/templates/crawler/crawler_disabled.html

@@ -0,0 +1,6 @@
+{% if not crawler_enabled %}
+  <div class="alert alert-secondary text-center my-2" role="alert">
+    <h1><i class="fas fa-times-circle text-danger"></i> Crawler Disabled</h1>
+    <p>...</p>
+  </div>
+{% endif %}