chg: [crawler] show all crawlers type on dashboard

bin/Crawler.py

@@ -349,7 +349,7 @@ if __name__ == '__main__':
                               'user_agent': p.config.get("Crawler", "default_crawler_user_agent")}
 
     # Track launched crawler
-    r_cache.sadd('all_crawler', splash_url)
+    r_cache.sadd('all_splash_crawlers', splash_url)
     r_cache.hset('metadata_crawler:{}'.format(splash_url), 'status', 'Waiting')
     r_cache.hset('metadata_crawler:{}'.format(splash_url), 'started_time', datetime.datetime.now().strftime("%Y/%m/%d  -  %H:%M.%S"))
 
@@ -385,7 +385,7 @@ if __name__ == '__main__':
                         'epoch': int(time.time())}
 
                 # Update crawler status type
-                r_cache.sadd('{}_crawlers'.format(to_crawl['type_service']), splash_url)
+                r_cache.hset('metadata_crawler:{}'.format(splash_url), 'type', to_crawl['type_service'])
 
                 crawler_config = load_crawler_config(to_crawl['type_service'], url_data['domain'], to_crawl['paste'],  to_crawl['url'], date)
                 # check if default crawler
@@ -437,7 +437,7 @@ if __name__ == '__main__':
                 r_cache.hdel('metadata_crawler:{}'.format(splash_url), 'crawling_domain')
 
                 # Update crawler status type
-                r_cache.srem('{}_crawlers'.format(to_crawl['type_service']), splash_url)
+                r_cache.hdel('metadata_crawler:{}'.format(splash_url), 'type', to_crawl['type_service'])
 
                 # add next auto Crawling in queue:
                 if to_crawl['paste'] == 'auto':

bin/core/Crawler_manager.py

@@ -35,6 +35,8 @@ def launch_crawlers():
             print('Please launch {} additional {} Dockers'.format( nb_crawlers - len(all_crawler_urls), splash_name))
             nb_crawlers = len(all_crawler_urls)
 
+        crawlers.reset_all_spash_crawler_status()
+
         for i in range(0, int(nb_crawlers)):
             splash_url = all_crawler_urls[i]
             print(all_crawler_urls[i])
@@ -59,6 +61,8 @@ if __name__ == '__main__':
 
     while True:
 
+        # # TODO: avoid multiple ping
+
         # check if manager is connected
         if int(time.time()) - last_check > 60:
             is_manager_connected = crawlers.is_splash_manager_connected()

bin/lib/crawlers.py

@@ -46,9 +46,20 @@ config_loader = None
 
 faup = Faup()
 
+# # # # # # # #
+#             #
+#   COMMON    #
+#             #
+# # # # # # # #
+
 def generate_uuid():
     return str(uuid.uuid4()).replace('-', '')
 
+def get_current_date():
+    return datetime.now().strftime("%Y%m%d")
+
+##-- COMMON --#
+
 ################################################################################
 
 # # TODO: handle prefix cookies
@@ -377,6 +388,55 @@ def api_create_cookie(user_id, cookiejar_uuid, cookie_dict):
 
 #### ####
 
+# # # # # # # #
+#             #
+#   CRAWLER   #
+#             #
+# # # # # # # #
+
+#### CRAWLER GLOBAL ####
+
+def get_all_spash_crawler_status():
+    crawler_metadata = []
+    all_crawlers = r_cache.smembers('all_splash_crawlers')
+    for crawler in all_crawlers:
+        crawler_metadata.append(get_splash_crawler_status(crawler))
+    return crawler_metadata
+
+def reset_all_spash_crawler_status():
+    r_cache.delete('all_splash_crawlers')
+
+def get_splash_crawler_status(spash_url):
+    crawler_type = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'type')
+    crawling_domain = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'crawling_domain')
+    started_time = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'started_time')
+    status_info = r_cache.hget('metadata_crawler:{}'.format(spash_url), 'status')
+    crawler_info = '{}  - {}'.format(spash_url, started_time)
+    if status_info=='Waiting' or status_info=='Crawling':
+        status=True
+    else:
+        status=False
+    return {'crawler_info': crawler_info, 'crawling_domain': crawling_domain, 'status_info': status_info, 'status': status, 'type': crawler_type}
+
+def get_stats_last_crawled_domains(crawler_types, date):
+    statDomains = {}
+    for crawler_type in crawler_types:
+        stat_type = {}
+        stat_type['domains_up'] = r_serv_onion.scard('{}_up:{}'.format(crawler_type, date))
+        stat_type['domains_down'] = r_serv_onion.scard('{}_down:{}'.format(crawler_type, date))
+        stat_type['total'] = stat_type['domains_up'] + stat_type['domains_down']
+        stat_type['domains_queue'] = get_nb_elem_to_crawl_by_type(crawler_type)
+        statDomains[crawler_type] = stat_type
+    return statDomains
+
+# # TODO: handle custom proxy
+def get_splash_crawler_latest_stats():
+    now = datetime.now()
+    date = now.strftime("%Y%m%d")
+    return get_stats_last_crawled_domains(['onion', 'regular'], date)
+
+##-- CRAWLER GLOBAL --##
+
 #### CRAWLER TASK ####
 def create_crawler_task(url, screenshot=True, har=True, depth_limit=1, max_pages=100, auto_crawler=False, crawler_delta=3600, cookiejar_uuid=None, user_agent=None):
 
@@ -587,10 +647,20 @@ def get_elem_to_crawl_by_queue_type(l_queue_type):
                 return {'url': url, 'paste': item_id, 'type_service': queue_type, 'original_message': message}
     return None
 
+def get_nb_elem_to_crawl_by_type(queue_type):
+    nb = r_serv_onion.scard('{}_crawler_priority_queue'.format(queue_type))
+    nb += r_serv_onion.scard('{}_crawler_discovery_queue'.format(queue_type))
+    nb += r_serv_onion.scard('{}_crawler_queue'.format(queue_type))
+    return nb
+
 #### ---- ####
 
+# # # # # # # # # # # #
+#                     #
+#   SPLASH MANAGER    #
+#                     #
+# # # # # # # # # # # #
 
-#### SPLASH MANAGER ####
 def get_splash_manager_url(reload=False): # TODO: add in db config
     return splash_manager_url
 
@@ -636,6 +706,8 @@ def ping_splash_manager():
             return True
         else:
             print(req.json())
+            update_splash_manager_connection_status(False)
+            return False
     except requests.exceptions.ConnectionError:
         pass
     # splash manager unreachable

var/www/blueprints/crawler_splash.py

@@ -48,6 +48,31 @@ def create_json_response(data, status_code):
     return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
 
 # ============= ROUTES ==============
+@crawler_splash.route("/crawlers/dashboard", methods=['GET'])
+@login_required
+@login_read_only
+def crawlers_dashboard():
+    # # TODO:  get splash manager status
+    crawler_enabled = crawlers.ping_splash_manager()
+    all_splash_crawler_status = crawlers.get_all_spash_crawler_status()
+    splash_crawlers_latest_stats = crawlers.get_splash_crawler_latest_stats()
+    date = crawlers.get_current_date()
+
+    return render_template("dashboard_splash_crawler.html", all_splash_crawler_status = all_splash_crawler_status,
+                                crawler_enabled=crawler_enabled, date=date,
+                                splash_crawlers_latest_stats=splash_crawlers_latest_stats)
+
+@crawler_splash.route("/crawlers/crawler_dashboard_json", methods=['GET'])
+@login_required
+@login_read_only
+def crawler_dashboard_json():
+
+    all_splash_crawler_status = crawlers.get_all_spash_crawler_status()
+    splash_crawlers_latest_stats = crawlers.get_splash_crawler_latest_stats()
+
+    return jsonify({'all_splash_crawler_status': all_splash_crawler_status,
+                        'splash_crawlers_latest_stats':splash_crawlers_latest_stats})
+
 @crawler_splash.route("/crawlers/manual", methods=['GET'])
 @login_required
 @login_read_only
@@ -403,4 +428,11 @@ def crawler_cookiejar_cookie_json_add_post():
 
     return redirect(url_for('crawler_splash.crawler_cookiejar_cookie_add', cookiejar_uuid=cookiejar_uuid))
 
+@crawler_splash.route('/crawler/cookiejar/cookie/json_add_post', methods=['GET'])
+@login_required
+@login_analyst
+def crawler_splash_setings():
+    
+    return render_template("settings_splash_crawler.html", cookiejar_uuid=True, cookie_uuid=False)
+
 ##  - -  ##

var/www/modules/hiddenServices/Flask_hiddenServices.py

@@ -231,22 +231,22 @@ def delete_auto_crawler(url):
 
 # ============= ROUTES ==============
 
-@hiddenServices.route("/crawlers/", methods=['GET'])
+# @hiddenServices.route("/crawlers/", methods=['GET'])
-@login_required
+# @login_required
-@login_read_only
+# @login_read_only
-def dashboard():
+# def dashboard():
-    crawler_metadata_onion = get_crawler_splash_status('onion')
+#     crawler_metadata_onion = get_crawler_splash_status('onion')
-    crawler_metadata_regular = get_crawler_splash_status('regular')
+#     crawler_metadata_regular = get_crawler_splash_status('regular')
-
+#
-    now = datetime.datetime.now()
+#     now = datetime.datetime.now()
-    date = now.strftime("%Y%m%d")
+#     date = now.strftime("%Y%m%d")
-    statDomains_onion = get_stats_last_crawled_domains('onion', date)
+#     statDomains_onion = get_stats_last_crawled_domains('onion', date)
-    statDomains_regular = get_stats_last_crawled_domains('regular', date)
+#     statDomains_regular = get_stats_last_crawled_domains('regular', date)
-
+#
-    return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion,
+#     return render_template("Crawler_dashboard.html", crawler_metadata_onion = crawler_metadata_onion,
-                                crawler_enabled=crawler_enabled, date=date,
+#                                 crawler_enabled=crawler_enabled, date=date,
-                                crawler_metadata_regular=crawler_metadata_regular,
+#                                 crawler_metadata_regular=crawler_metadata_regular,
-                                statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular)
+#                                 statDomains_onion=statDomains_onion, statDomains_regular=statDomains_regular)
 
 @hiddenServices.route("/crawlers/crawler_splash_onion", methods=['GET'])
 @login_required
@@ -439,23 +439,6 @@ def remove_auto_crawler():
         delete_auto_crawler(url)
     return redirect(url_for('hiddenServices.auto_crawler', page=page))
 
-@hiddenServices.route("/crawlers/crawler_dashboard_json", methods=['GET'])
-@login_required
-@login_read_only
-def crawler_dashboard_json():
-
-    crawler_metadata_onion = get_crawler_splash_status('onion')
-    crawler_metadata_regular = get_crawler_splash_status('regular')
-
-    now = datetime.datetime.now()
-    date = now.strftime("%Y%m%d")
-
-    statDomains_onion = get_stats_last_crawled_domains('onion', date)
-    statDomains_regular = get_stats_last_crawled_domains('regular', date)
-
-    return jsonify({'statDomains_onion': statDomains_onion, 'statDomains_regular': statDomains_regular,
-                        'crawler_metadata_onion':crawler_metadata_onion, 'crawler_metadata_regular':crawler_metadata_regular})
-
 # # TODO: refractor
 @hiddenServices.route("/hiddenServices/last_crawled_domains_with_stats_json", methods=['GET'])
 @login_required

var/www/modules/hiddenServices/templates/Crawler_dashboard.html

@@ -1,267 +0,0 @@
-<!DOCTYPE html>
-
-<html>
-<head>
-	<title>AIL-Framework</title>
-	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
-	<!-- Core CSS -->
-	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
-	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
-
-	<!-- JS -->
-	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
-	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
-
-
-</head>
-
-<body>
-
-	{% include 'nav_bar.html' %}
-
-	<div class="container-fluid">
-		<div class="row">
-
-			{% include 'crawler/menu_sidebar.html' %}
-
-			<div class="col-12 col-lg-10" id="core_content">
-
-				{% include 'crawler/crawler_disabled.html' %}
-
-				<div class="row">
-					<div class="col-xl-6">
-
-						<div class="card mt-1 mb-1">
-						  <div class="card-header text-white bg-dark">
-								<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=onion"><i class="fas fa-user-secret"></i> Onions Crawlers</a></h5>
-								<div class="row">
-									<div class="col-6">
-										<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_up=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_domain_up">{{ statDomains_onion['domains_up'] }}</a> UP
-										<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_onion_domain_down">{{ statDomains_onion['domains_down'] }}</a> DOWN
-									</div>
-									<div class="col-6">
-										<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_up=True&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_total">{{ statDomains_onion['total'] }}</a> Crawled
-										<span class="badge badge-warning ml-md-3" id="stat_onion_queue">{{ statDomains_onion['domains_queue'] }}</span> Queue
-									</div>
-								</div>
-							</div>
-						  <div class="card-body px-0 py-0 ">
-								<table class="table">
-									<tbody id="tbody_crawler_onion_info">
-										{% for crawler in crawler_metadata_onion %}
-											<tr>
-						      			<td>
-													<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
-												</td>
-												<td>
-													{{crawler['crawling_domain']}}
-												</td>
-												<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
-													{{crawler['status_info']}}
-												</td>
-											</tr>
-										{% endfor %}
-									</tbody>
-								</table>
-							</div>
-						</div>
-
-					</div>
-					<div class="col-xl-6">
-						<div class="card mt-1 mb-1">
-							<div class="card-header text-white bg-dark">
-								<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=regular"><i class="fab fa-html5"></i> Regular Crawlers</a></h5>
-								<div class="row">
-									<div class="col-6">
-										<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_up=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_regular_domain_up">{{ statDomains_regular['domains_up'] }}</a> UP
-										<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_regular_domain_down">{{ statDomains_regular['domains_down'] }}</a> DOWN
-									</div>
-									<div class="col-6">
-										<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_up=True&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_regular_total">{{ statDomains_regular['total'] }}</a> Crawled
-										<span class="badge badge-warning ml-md-3" id="stat_regular_queue">{{ statDomains_regular['domains_queue'] }}</span> Queue
-									</div>
-								</div>
-							</div>
-							<div class="card-body px-0 py-0 ">
-								<table class="table">
-									<tbody id="tbody_crawler_regular_info">
-										{% for crawler in crawler_metadata_regular %}
-											<tr>
-												<td>
-													<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
-												</td>
-												<td>
-													{{crawler['crawling_domain']}}
-												</td>
-												<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
-													{{crawler['status_info']}}
-												</td>
-											</tr>
-										{% endfor %}
-									</tbody>
-								</table>
-							</div>
-						</div>
-					</div>
-				</div>
-
-			<div class="d-flex justify-content-center my-4">
-				<div class="card border-secondary" style="max-width: 40rem;">
-					<div class="card-body text-dark">
-						<h5 class="card-title">Show Domain:</h5>
-						<form class="" action="{{url_for('crawler_splash.showDomain')}}" method="post">
-							<div class="input-group mb-3">
-							  <input type="text" class="form-control" style="min-width: 30rem;" placeholder="Domain name" aria-label="Domain name" aria-describedby="btn_show_domain" id="in_show_domain" , name="in_show_domain">
-							  <div class="input-group-append">
-							    <button class="btn btn-info" type="submit" id="btn_show_domain">
-										<i class="fas fa-search"></i>
-									</button>
-							  </div>
-							</div>
-						</form>
-					</div>
-				</div>
-			</div>
-
-
-			<hr>
-			<div class="row mb-3">
-				<div class="col-xl-6">
-					<div class="text-center">
-						<a class="btn btn-secondary" href="{{url_for('crawler_splash.domains_explorer_onion')}}" role="button">
-							<i class="fas fa-user-secret"></i>  Onion Domain Explorer
-						</a>
-					</div>
-				</div>
-				<div class="col-xl-6">
-					<div class="text-center">
-						<a class="btn btn-secondary" href="{{url_for('crawler_splash.domains_explorer_web')}}" role="button">
-							<i class="fab fa-html5"></i> Web Domain Explorer
-						</a>
-					</div>
-				</div>
-			</div>
-
-
-
-				{% with object_type='domain' %}
-					{% include 'tags/block_obj_tags_search.html' %}
-				{% endwith %}
-
-			</div>
-		</div>
-	</div>
-
-
-</body>
-
-<script>
-var to_refresh = false
-$(document).ready(function(){
-	$("#page-Crawler").addClass("active");
-	$("#nav_dashboard").addClass("active");
-	$( window ).on("focus", function() {
-			to_refresh = true
-			refresh_crawler_status();
-	});
-	$( window ).on("blur", function() {
-			to_refresh = false
-	});
-
-	to_refresh = true
-	refresh_crawler_status();
-});
-
-function toggle_sidebar(){
-	if($('#nav_menu').is(':visible')){
-		$('#nav_menu').hide();
-		$('#side_menu').removeClass('border-right')
-		$('#side_menu').removeClass('col-lg-2')
-		$('#core_content').removeClass('col-lg-10')
-	}else{
-		$('#nav_menu').show();
-		$('#side_menu').addClass('border-right')
-		$('#side_menu').addClass('col-lg-2')
-		$('#core_content').addClass('col-lg-10')
-	}
-}
-
-function refresh_crawler_status(){
-
-	$.getJSON("{{ url_for('hiddenServices.crawler_dashboard_json') }}",
-		function(data) {
-
-			$('#stat_onion_domain_up').text(data.statDomains_onion['domains_up']);
-			$('#stat_onion_domain_down').text(data.statDomains_onion['domains_down']);
-			$('#stat_onion_total').text(data.statDomains_onion['total']);
-			$('#stat_onion_queue').text(data.statDomains_onion['domains_queue']);
-
-			$('#stat_regular_domain_up').text(data.statDomains_regular['domains_up']);
-			$('#stat_regular_domain_down').text(data.statDomains_regular['domains_down']);
-			$('#stat_regular_total').text(data.statDomains_regular['total']);
-			$('#stat_regular_queue').text(data.statDomains_regular['domains_queue']);
-
-			if(data.crawler_metadata_onion.length!=0){
-				$("#tbody_crawler_onion_info").empty();
-				var tableRef = document.getElementById('tbody_crawler_onion_info');
-				for (var i = 0; i < data.crawler_metadata_onion.length; i++) {
-						var crawler = data.crawler_metadata_onion[i];
-						var newRow = tableRef.insertRow(tableRef.rows.length);
-						var text_color;
-						var icon;
-						if(crawler['status']){
-							text_color = 'Green';
-							icon = 'check';
-						} else {
-							text_color = 'Red';
-							icon = 'times';
-						}
-
-				    var newCell  = newRow.insertCell(0);
-						newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
-
-						newCell  = newRow.insertCell(1);
-						newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
-
-						newCell  = newRow.insertCell(2);
-						newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
-
-						//$("#panel_crawler").show();
-				}
-			}
-			if(data.crawler_metadata_regular.length!=0){
-				$("#tbody_crawler_regular_info").empty();
-				var tableRef = document.getElementById('tbody_crawler_regular_info');
-				for (var i = 0; i < data.crawler_metadata_regular.length; i++) {
-						var crawler = data.crawler_metadata_regular[i];
-						var newRow = tableRef.insertRow(tableRef.rows.length);
-						var text_color;
-						var icon;
-						if(crawler['status']){
-							text_color = 'Green';
-							icon = 'check';
-						} else {
-							text_color = 'Red';
-							icon = 'times';
-						}
-
-				    var newCell  = newRow.insertCell(0);
-						newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
-
-						newCell  = newRow.insertCell(1);
-						newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
-
-						newCell  = newRow.insertCell(2);
-						newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
-
-						//$("#panel_crawler").show();
-				}
-			}
-		}
-	);
-
-	if (to_refresh) {
-    setTimeout("refresh_crawler_status()", 10000);
-	}
-}
-</script>

var/www/templates/crawler/crawler_splash/dashboard_splash_crawler.html

@@ -0,0 +1,235 @@
+<!DOCTYPE html>
+
+<html>
+<head>
+	<title>AIL-Framework</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
+	<!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+
+	<!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+
+
+</head>
+
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'crawler/menu_sidebar.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+				{% include 'crawler/crawler_disabled.html' %}
+
+				<div class="row">
+					<div class="col-xl-6">
+
+						<div class="card mt-1 mb-1">
+						  <div class="card-header text-white bg-dark">
+								<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=onion"><i class="fas fa-user-secret"></i> Onions Crawlers</a></h5>
+								<div class="row">
+									<div class="col-6">
+										<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_up=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_domain_up">{{ splash_crawlers_latest_stats['onion']['domains_up'] }}</a> UP
+										<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_onion_domain_down">{{ splash_crawlers_latest_stats['onion']['domains_down'] }}</a> DOWN
+									</div>
+									<div class="col-6">
+										<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=onion&domains_up=True&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_onion_total">{{ splash_crawlers_latest_stats['onion']['total'] }}</a> Crawled
+										<span class="badge badge-warning ml-md-3" id="stat_onion_queue">{{ splash_crawlers_latest_stats['onion']['domains_queue'] }}</span> Queue
+									</div>
+								</div>
+							</div>
+						</div>
+
+					</div>
+					<div class="col-xl-6">
+						<div class="card mt-1 mb-1">
+							<div class="card-header text-white bg-dark">
+								<h5><a class="text-info" href="{{ url_for('hiddenServices.Crawler_Splash_last_by_type')}}?type=regular"><i class="fab fa-html5"></i> Regular Crawlers</a></h5>
+								<div class="row">
+									<div class="col-6">
+										<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_up=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_regular_domain_up">{{ splash_crawlers_latest_stats['regular']['domains_up'] }}</a> UP
+										<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-danger ml-md-3" id="stat_regular_domain_down">{{ splash_crawlers_latest_stats['regular']['domains_down'] }}</a> DOWN
+									</div>
+									<div class="col-6">
+										<a href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?service_type=regular&domains_up=True&domains_down=True&date_from={{date}}&date_to={{date}}" class="badge badge-success" id="stat_regular_total">{{ splash_crawlers_latest_stats['regular']['total'] }}</a> Crawled
+										<span class="badge badge-warning ml-md-3" id="stat_regular_queue">{{ splash_crawlers_latest_stats['regular']['domains_queue'] }}</span> Queue
+									</div>
+								</div>
+							</div>
+						</div>
+					</div>
+				</div>
+
+				<table class="table">
+					<tbody id="tbody_crawler_onion_info">
+						{% for splash_crawler in all_splash_crawler_status %}
+							<tr>
+								<td>
+									<i class="fas fa-{%if splash_crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if splash_crawler['status']%}Green{%else%}Red{%endif%};"></i> {{splash_crawler['crawler_info']}}
+								</td>
+								<td>
+									{%if splash_crawler['type']=='onion'%}
+										<i class="fas fa-user-secret"></i>
+									{%else%}
+										<i class="fab fa-html5">
+									{%endif%}
+								</td>
+								<td>
+									{{splash_crawler['crawling_domain']}}
+								</td>
+								<td style="color:{%if splash_crawler['status']%}Green{%else%}Red{%endif%};">
+									{{splash_crawler['status_info']}}
+								</td>
+							</tr>
+						{% endfor %}
+					</tbody>
+				</table>
+
+				<div class="d-flex justify-content-center my-4">
+					<div class="card border-secondary" style="max-width: 40rem;">
+						<div class="card-body text-dark">
+							<h5 class="card-title">Show Domain:</h5>
+							<form class="" action="{{url_for('crawler_splash.showDomain')}}" method="post">
+								<div class="input-group mb-3">
+								  <input type="text" class="form-control" style="min-width: 30rem;" placeholder="Domain name" aria-label="Domain name" aria-describedby="btn_show_domain" id="in_show_domain" , name="in_show_domain">
+								  <div class="input-group-append">
+								    <button class="btn btn-info" type="submit" id="btn_show_domain">
+											<i class="fas fa-search"></i>
+										</button>
+								  </div>
+								</div>
+							</form>
+						</div>
+					</div>
+				</div>
+
+
+				<hr>
+				<div class="row mb-3">
+					<div class="col-xl-6">
+						<div class="text-center">
+							<a class="btn btn-secondary" href="{{url_for('crawler_splash.domains_explorer_onion')}}" role="button">
+								<i class="fas fa-user-secret"></i> Onion Domain Explorer
+							</a>
+						</div>
+					</div>
+					<div class="col-xl-6">
+						<div class="text-center">
+							<a class="btn btn-secondary" href="{{url_for('crawler_splash.domains_explorer_web')}}" role="button">
+								<i class="fab fa-html5"></i> Web Domain Explorer
+							</a>
+						</div>
+					</div>
+				</div>
+
+
+
+				{% with object_type='domain' %}
+					{% include 'tags/block_obj_tags_search.html' %}
+				{% endwith %}
+
+			</div>
+		</div>
+	</div>
+
+
+</body>
+
+<script>
+var to_refresh = false
+$(document).ready(function(){
+	$("#page-Crawler").addClass("active");
+	$("#nav_dashboard").addClass("active");
+	$( window ).on("focus", function() {
+			to_refresh = true
+			refresh_crawler_status();
+	});
+	$( window ).on("blur", function() {
+			to_refresh = false
+	});
+
+	to_refresh = true
+	refresh_crawler_status();
+});
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+
+function refresh_crawler_status(){
+
+	$.getJSON("{{ url_for('crawler_splash.crawler_dashboard_json') }}",
+		function(data) {
+
+			$('#stat_onion_domain_up').text(data.splash_crawlers_latest_stats['onion']['domains_up']);
+			$('#stat_onion_domain_down').text(data.splash_crawlers_latest_stats['onion']['domains_down']);
+			$('#stat_onion_total').text(data.splash_crawlers_latest_stats['onion']['total']);
+			$('#stat_onion_queue').text(data.splash_crawlers_latest_stats['onion']['domains_queue']);
+
+			$('#stat_regular_domain_up').text(data.splash_crawlers_latest_stats['regular']['domains_up']);
+			$('#stat_regular_domain_down').text(data.splash_crawlers_latest_stats['regular']['domains_down']);
+			$('#stat_regular_total').text(data.splash_crawlers_latest_stats['regular']['total']);
+			$('#stat_regular_queue').text(data.splash_crawlers_latest_stats['regular']['domains_queue']);
+
+			if(data.all_splash_crawler_status.length!=0){
+				$("#tbody_crawler_onion_info").empty();
+				var tableRef = document.getElementById('tbody_crawler_onion_info');
+				for (var i = 0; i < data.all_splash_crawler_status.length; i++) {
+						var crawler = data.all_splash_crawler_status[i];
+						var newRow = tableRef.insertRow(tableRef.rows.length);
+						var text_color;
+						var icon;
+						if(crawler['status']){
+							text_color = 'Green';
+							icon = 'check';
+						} else {
+							text_color = 'Red';
+							icon = 'times';
+						}
+
+						if(crawler['type'] === 'onion'){
+							icon_t = 'fas fa-user-secret';
+						} else {
+							icon_t = 'fab fa-html5';
+						}
+
+				    var newCell  = newRow.insertCell(0);
+						newCell.innerHTML = "<td><i class=\"fas fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i> "+crawler['crawler_info']+"</td>";
+
+						var newCell  = newRow.insertCell(1);
+						newCell.innerHTML = "<td><i class=\""+icon_t+"\"></i></td>";
+
+						newCell  = newRow.insertCell(2);
+						newCell.innerHTML = "<td>"+crawler['crawling_domain']+"</td>";
+
+						newCell  = newRow.insertCell(3);
+						newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
+
+						//$("#panel_crawler").show();
+				}
+			}
+		}
+	);
+
+	if (to_refresh) {
+    setTimeout("refresh_crawler_status()", 10000);
+	}
+}
+</script>

var/www/templates/crawler/menu_sidebar.html

@@ -14,7 +14,7 @@
       		</h5>
       	  <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
       	    <li class="nav-item">
-      	      <a class="nav-link" href="{{url_for('hiddenServices.dashboard')}}" id="nav_dashboard">
+      	      <a class="nav-link" href="{{url_for('crawler_splash.crawlers_dashboard')}}" id="nav_dashboard">
       	        <i class="fas fa-search"></i>
       	        <span>Dashboard</span>
       	      </a>
@@ -43,6 +43,12 @@
                 Automatic Crawler
               </a>
             </li>
+            <li class="nav-item">
+              <a class="nav-link" href="{{url_for('hiddenServices.auto_crawler')}}" id="nav_settings">
+      					<i class="fas fa-cog"></i>
+                Settings
+              </a>
+            </li>
           </ul>
 
           <h5 class="d-flex text-muted w-100" id="nav_title_domains_explorer">

var/www/templates/nav_bar.html

@@ -22,7 +22,7 @@
         <a class="nav-link" id="page-Tracker" href="{{ url_for('hunter.tracked_menu') }}" aria-disabled="true"><i class="fas fa-crosshairs"></i> Leaks Hunter</a>
       </li>
       <li class="nav-item mr-3">
-        <a class="nav-link" id="page-Crawler" href="{{ url_for('hiddenServices.dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>
+        <a class="nav-link" id="page-Crawler" href="{{ url_for('crawler_splash.crawlers_dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>
       </li>
       <li class="nav-item mr-3">
         <a class="nav-link" id="page-Decoded" href="{{ url_for('hashDecoded.hashDecoded_page') }}" aria-disabled="true"><i class="fas fa-cube"></i> Objects</a>