From 9cb6312279020024a6a3d00c0c0829bc30aaa711 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 4 Jun 2019 14:22:46 +0200 Subject: [PATCH] fix: [UI crawler endpoints] display crawler status + fix #353 --- var/www/Flask_server.py | 4 ---- var/www/modules/Flask_config.py | 2 ++ var/www/modules/hiddenServices/Flask_hiddenServices.py | 6 +++++- .../templates/Crawler_Splash_last_by_type.html | 2 ++ .../hiddenServices/templates/Crawler_Splash_manual.html | 1 + var/www/modules/hiddenServices/templates/Crawler_auto.html | 2 ++ .../modules/hiddenServices/templates/Crawler_dashboard.html | 2 ++ var/www/templates/crawler/crawler_disabled.html | 6 ++++++ 8 files changed, 20 insertions(+), 5 deletions(-) create mode 100644 var/www/templates/crawler/crawler_disabled.html diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 95433757..ff6d5bac 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -48,10 +48,6 @@ except IOError: f = open('templates/ignored_modules.txt', 'w') f.close() -activate_crawler = cfg.get("Crawler", "activate_crawler") -if activate_crawler != 'True': - toIgnoreModule.add('hiddenServices') - # Dynamically import routes and functions from modules # Also, prepare header.html to_add_to_header_dico = {} diff --git a/var/www/modules/Flask_config.py b/var/www/modules/Flask_config.py index d4251e5f..899a26b5 100644 --- a/var/www/modules/Flask_config.py +++ b/var/www/modules/Flask_config.py @@ -173,6 +173,8 @@ REPO_ORIGIN = 'https://github.com/CIRCL/AIL-framework.git' max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs")) +crawler_enabled = cfg.getboolean("Crawler", "activate_crawler") + # VT try: from virusTotalKEYS import vt_key diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py index 0ee44974..ed5cbcd4 100644 --- a/var/www/modules/hiddenServices/Flask_hiddenServices.py +++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py @@ -25,6 +25,7 @@ baseUrl = Flask_config.baseUrl r_cache = Flask_config.r_cache r_serv_onion = Flask_config.r_serv_onion r_serv_metadata = Flask_config.r_serv_metadata +crawler_enabled = Flask_config.crawler_enabled bootstrap_label = Flask_config.bootstrap_label hiddenServices = Blueprint('hiddenServices', __name__, template_folder='templates') @@ -247,6 +248,7 @@ def dashboard(): statDomains_regular = get_stats_last_crawled_domains('regular', date) return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion, + crawler_enabled=crawler_enabled, crawler_metadata_regular=crawler_metadata_regular, statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular) @@ -256,7 +258,7 @@ def hiddenServices_page_test(): @hiddenServices.route("/crawlers/manual", methods=['GET']) def manual(): - return render_template("Crawler_Splash_manual.html") + return render_template("Crawler_Splash_manual.html", crawler_enabled=crawler_enabled) @hiddenServices.route("/crawlers/crawler_splash_onion", methods=['GET']) def crawler_splash_onion(): @@ -294,6 +296,7 @@ def Crawler_Splash_last_by_type(): crawler_metadata = get_crawler_splash_status(type) return render_template("Crawler_Splash_last_by_type.html", type=type, type_name=type_name, + crawler_enabled=crawler_enabled, last_domains=list_domains, statDomains=statDomains, crawler_metadata=crawler_metadata, date_from=date_string, date_to=date_string) @@ -507,6 +510,7 @@ def auto_crawler(): return render_template("Crawler_auto.html", page=page, nb_page_max=nb_page_max, last_domains=last_domains, + crawler_enabled=crawler_enabled, auto_crawler_domain_onions_metadata=auto_crawler_domain_onions_metadata, auto_crawler_domain_regular_metadata=auto_crawler_domain_regular_metadata) diff --git a/var/www/modules/hiddenServices/templates/Crawler_Splash_last_by_type.html b/var/www/modules/hiddenServices/templates/Crawler_Splash_last_by_type.html index a42e3880..bafe3ecf 100644 --- a/var/www/modules/hiddenServices/templates/Crawler_Splash_last_by_type.html +++ b/var/www/modules/hiddenServices/templates/Crawler_Splash_last_by_type.html @@ -46,6 +46,8 @@
+ {% include 'crawler/crawler_disabled.html' %} +
diff --git a/var/www/modules/hiddenServices/templates/Crawler_Splash_manual.html b/var/www/modules/hiddenServices/templates/Crawler_Splash_manual.html index bc2e6024..11df7730 100644 --- a/var/www/modules/hiddenServices/templates/Crawler_Splash_manual.html +++ b/var/www/modules/hiddenServices/templates/Crawler_Splash_manual.html @@ -30,6 +30,7 @@
+ {% include 'crawler/crawler_disabled.html' %}
diff --git a/var/www/modules/hiddenServices/templates/Crawler_auto.html b/var/www/modules/hiddenServices/templates/Crawler_auto.html index 977aed1e..87dd7569 100644 --- a/var/www/modules/hiddenServices/templates/Crawler_auto.html +++ b/var/www/modules/hiddenServices/templates/Crawler_auto.html @@ -29,6 +29,8 @@
+ {% include 'crawler/crawler_disabled.html' %} + {%if last_domains%}
diff --git a/var/www/modules/hiddenServices/templates/Crawler_dashboard.html b/var/www/modules/hiddenServices/templates/Crawler_dashboard.html index 91d300af..b746b05c 100644 --- a/var/www/modules/hiddenServices/templates/Crawler_dashboard.html +++ b/var/www/modules/hiddenServices/templates/Crawler_dashboard.html @@ -26,6 +26,8 @@
+ {% include 'crawler/crawler_disabled.html' %} +
diff --git a/var/www/templates/crawler/crawler_disabled.html b/var/www/templates/crawler/crawler_disabled.html new file mode 100644 index 00000000..455f350e --- /dev/null +++ b/var/www/templates/crawler/crawler_disabled.html @@ -0,0 +1,6 @@ +{% if not crawler_enabled %} + +{% endif %}