chg: [Crawler UI Flask] refractor functions, templates

var/www/modules/hiddenServices/Flask_hiddenServices.py

@@ -30,6 +30,8 @@ PASTES_FOLDER = Flask_config.PASTES_FOLDER
 hiddenServices = Blueprint('hiddenServices', __name__, template_folder='templates')
 
 faup = Faup()
+list_types=['onion', 'regular']
+dic_type_name={'onion':'Onion', 'regular':'Website'}
 
 # ============ FUNCTIONS ============
 def one():
@@ -72,10 +74,10 @@ def unpack_paste_tags(p_tags):
         l_tags.append( (tag, complete_tag) )
     return l_tags
 
-def is_valid_onion_domain(onion_domain):
+def is_valid_domain(domain):
-    faup.decode(onion_domain)
+    faup.decode(domain)
     domain_unpack = faup.get()
-    if domain_unpack['tld']==b'onion' and domain_unpack['scheme'] is None and domain_unpack['port'] is None and domain_unpack['query_string'] is None:
+    if domain_unpack['tld'] is not None and domain_unpack['scheme'] is None and domain_unpack['port'] is None and domain_unpack['query_string'] is None:
         return True
     else:
         return False
@@ -93,6 +95,18 @@ def get_domain_type(domain):
     else:
         return 'regular'
 
+def get_last_domains_crawled(type):
+    return r_serv_onion.lrange('last_{}'.format(type), 0 ,-1)
+
+def get_stats_last_crawled_domains(type, date):
+    statDomains = {}
+    statDomains['domains_up'] = r_serv_onion.scard('{}_up:{}'.format(type, date))
+    statDomains['domains_down'] = r_serv_onion.scard('{}_down:{}'.format(type, date))
+    statDomains['total'] = statDomains['domains_up'] + statDomains['domains_down']
+    statDomains['domains_queue'] = r_serv_onion.scard('{}_crawler_queue'.format(type))
+    statDomains['domains_queue'] += r_serv_onion.scard('{}_crawler_priority_queue'.format(type))
+    return statDomains
+
 def get_last_crawled_domains_metadata(list_domains_crawled, date, type=None):
     list_crawled_metadata = []
     for domain_epoch in list_domains_crawled:
@@ -153,6 +167,10 @@ def send_url_to_crawl_in_queue(mode, service_type, url):
 
 # ============= ROUTES ==============
 
+@hiddenServices.route("/crawlers/", methods=['GET'])
+def dashboard():
+    return render_template("Crawler_dashboard.html")
+
 @hiddenServices.route("/hiddenServices/2", methods=['GET'])
 def hiddenServices_page_test():
     return render_template("Crawler_index.html")
@@ -163,76 +181,58 @@ def manual():
 
 @hiddenServices.route("/crawlers/crawler_splash_onion", methods=['GET'])
 def crawler_splash_onion():
-    last_onions = r_serv_onion.lrange('last_onion', 0 ,-1)
+    type = 'onion'
+    last_onions = get_last_domains_crawled(type)
     list_onion = []
 
     now = datetime.datetime.now()
     date = now.strftime("%Y%m%d")
-    statDomains = {}
+    statDomains = get_stats_last_crawled_domains(type, date)
-    statDomains['domains_up'] = r_serv_onion.scard('onion_up:{}'.format(date))
-    statDomains['domains_down'] = r_serv_onion.scard('onion_down:{}'.format(date))
-    statDomains['total'] = statDomains['domains_up'] + statDomains['domains_down']
-    statDomains['domains_queue'] = r_serv_onion.scard('onion_domain_crawler_queue')
 
-    list_onion = get_last_crawled_domains_metadata(last_onions, date, type='onion')
+    list_onion = get_last_crawled_domains_metadata(last_onions, date, type=type)
-    crawler_metadata = get_crawler_splash_status('onion')
+    crawler_metadata = get_crawler_splash_status(type)
 
     date_string = '{}-{}-{}'.format(date[0:4], date[4:6], date[6:8])
     return render_template("Crawler_Splash_onion.html", last_onions=list_onion, statDomains=statDomains,
                             crawler_metadata=crawler_metadata, date_from=date_string, date_to=date_string)
 
-@hiddenServices.route("/crawlers/manual_splash_crawler", methods=['GET'])
+@hiddenServices.route("/crawlers/crawler_splash_regular", methods=['GET'])
-def manual_splash_crawler():
+def crawler_splash_regular():
+    type = 'regular'
+    type_name = dic_type_name[type]
+    list_domains = []
 
     now = datetime.datetime.now()
-    date = '{}{}{}'.format(now.strftime("%Y"), now.strftime("%m"), now.strftime("%d"))
+    date = now.strftime("%Y%m%d")
-
-    # Stats
-    # user request == CHECK
-    # preconf crawlers == ?????
-    #################################################################################
-    statDomains = {}
-    #statDomains['domains_up'] = r_serv_onion.scard('onion_up:{}'.format(date))
-    #statDomains['domains_down'] = r_serv_onion.scard('onion_down:{}'.format(date))
-    #statDomains['total'] = statDomains['domains_up'] + statDomains['domains_down']
-    #statDomains['domains_queue'] = r_serv_onion.scard('onion_domain_crawler_queue')
-    ####################################################################################
-
-    last_crawled = r_serv_onion.lrange('last_crawled_manual', 0 ,-1)
-    list_crawled = get_last_crawled_domains_metadata(last_crawled)
-
-    crawler_metadata=[]
-    all_onion_crawler = r_cache.smembers('all_crawler:onion')
-    for crawler in all_onion_crawler:
-        crawling_domain = r_cache.hget('metadata_crawler:{}'.format(crawler), 'crawling_domain')
-        started_time = r_cache.hget('metadata_crawler:{}'.format(crawler), 'started_time')
-        status_info = r_cache.hget('metadata_crawler:{}'.format(crawler), 'status')
-        crawler_info = '{}  - {}'.format(crawler, started_time)
-        if status_info=='Waiting' or status_info=='Crawling':
-            status=True
-        else:
-            status=False
-        crawler_metadata.append({'crawler_info': crawler_info, 'crawling_domain': crawling_domain, 'status_info': status_info, 'status': status})
-
     date_string = '{}-{}-{}'.format(date[0:4], date[4:6], date[6:8])
-    return render_template("Crawler_Splash_onion.html", last_crawled=list_crawled, statDomains=statDomains,
+
+    statDomains = get_stats_last_crawled_domains(type, date)
+
+    list_domains = get_last_crawled_domains_metadata(get_last_domains_crawled(type), date, type=type)
+    crawler_metadata = get_crawler_splash_status(type)
+
+    return render_template("Crawler_Splash_last_by_type.html", type=type, type_name=type_name,
+                            last_domains=list_domains, statDomains=statDomains,
                             crawler_metadata=crawler_metadata, date_from=date_string, date_to=date_string)
 
-@hiddenServices.route("/crawlers/blacklisted_onion", methods=['GET'])
+@hiddenServices.route("/crawlers/blacklisted_domains", methods=['GET'])
-def blacklisted_onion():
+def blacklisted_domains():
-    blacklist_onion = request.args.get('blacklist_onion')
+    blacklist_domain = request.args.get('blacklist_domain')
-    unblacklist_onion = request.args.get('unblacklist_onion')
+    unblacklist_domain = request.args.get('unblacklist_domain')
-    if blacklist_onion is not None:
+    type = request.args.get('type')
-        blacklist_onion = int(blacklist_onion)
+    if type in list_types:
-    if unblacklist_onion is not None:
+        type_name = dic_type_name[type]
-        unblacklist_onion = int(unblacklist_onion)
+        if blacklist_domain is not None:
+            blacklist_domain = int(blacklist_domain)
+        if unblacklist_domain is not None:
+            unblacklist_domain = int(unblacklist_domain)
         try:
             page = int(request.args.get('page'))
         except:
             page = 1
         if page <= 0:
             page = 1
-    nb_page_max = r_serv_onion.scard('blacklist_onion')/(1000)
+        nb_page_max = r_serv_onion.scard('blacklist_{}'.format(type))/(1000)
         if isinstance(nb_page_max, float):
             nb_page_max = int(nb_page_max)+1
         if page > nb_page_max:
@@ -240,47 +240,56 @@ def blacklisted_onion():
         start = 1000*(page -1)
         stop = 1000*page
 
-    list_blacklisted = list(r_serv_onion.smembers('blacklist_onion'))
+        list_blacklisted = list(r_serv_onion.smembers('blacklist_{}'.format(type)))
         list_blacklisted_1 = list_blacklisted[start:stop]
         list_blacklisted_2 = list_blacklisted[stop:stop+1000]
-    return render_template("blacklisted_onion.html", list_blacklisted_1=list_blacklisted_1, list_blacklisted_2=list_blacklisted_2,
+        return render_template("blacklisted_domains.html", list_blacklisted_1=list_blacklisted_1, list_blacklisted_2=list_blacklisted_2,
-                            page=page, nb_page_max=nb_page_max,
+                                type=type, type_name=type_name, page=page, nb_page_max=nb_page_max,
-                            blacklist_onion=blacklist_onion, unblacklist_onion=unblacklist_onion)
+                                blacklist_domain=blacklist_domain, unblacklist_domain=unblacklist_domain)
+    else:
+        return 'Incorrect Type'
 
-@hiddenServices.route("/crawler/blacklist_onion", methods=['GET'])
+@hiddenServices.route("/crawler/blacklist_domain", methods=['GET'])
-def blacklist_onion():
+def blacklist_domain():
-    onion = request.args.get('onion')
+    domain = request.args.get('domain')
+    type = request.args.get('type')
     try:
         page = int(request.args.get('page'))
     except:
         page = 1
-    if is_valid_onion_domain(onion):
+    if type in list_types:
-        res = r_serv_onion.sadd('blacklist_onion', onion)
+        if is_valid_domain(domain):
-        print(res)
+            res = r_serv_onion.sadd('blacklist_{}'.format(type), domain)
             if page:
                 if res == 0:
-                return redirect(url_for('hiddenServices.blacklisted_onion', page=page, blacklist_onion=2))
+                    return redirect(url_for('hiddenServices.blacklisted_domains', page=page, type=type, blacklist_domain=2))
                 else:
-                return redirect(url_for('hiddenServices.blacklisted_onion', page=page, blacklist_onion=1))
+                    return redirect(url_for('hiddenServices.blacklisted_domains', page=page, type=type, blacklist_domain=1))
         else:
-        return redirect(url_for('hiddenServices.blacklisted_onion', page=page, blacklist_onion=0))
+            return redirect(url_for('hiddenServices.blacklisted_domains', page=page, type=type, blacklist_domain=0))
+    else:
+        return 'Incorrect type'
 
-@hiddenServices.route("/crawler/unblacklist_onion", methods=['GET'])
+@hiddenServices.route("/crawler/unblacklist_domain", methods=['GET'])
-def unblacklist_onion():
+def unblacklist_domain():
-    onion = request.args.get('onion')
+    domain = request.args.get('domain')
+    type = request.args.get('type')
     try:
         page = int(request.args.get('page'))
     except:
         page = 1
-    if is_valid_onion_domain(onion):
+    if type in list_types:
-        res = r_serv_onion.srem('blacklist_onion', onion)
+        if is_valid_domain(domain):
+            res = r_serv_onion.srem('blacklist_{}'.format(type), domain)
             if page:
                 if res == 0:
-                return redirect(url_for('hiddenServices.blacklisted_onion', page=page, unblacklist_onion=2))
+                    return redirect(url_for('hiddenServices.blacklisted_domains', page=page, type=type, unblacklist_domain=2))
                 else:
-                return redirect(url_for('hiddenServices.blacklisted_onion', page=page, unblacklist_onion=1))
+                    return redirect(url_for('hiddenServices.blacklisted_domains', page=page, type=type, unblacklist_domain=1))
         else:
-        return redirect(url_for('hiddenServices.blacklisted_onion', page=page, unblacklist_onion=0))
+            return redirect(url_for('hiddenServices.blacklisted_domains', page=page, type=type, unblacklist_domain=0))
+    else:
+        return 'Incorrect type'
 
 @hiddenServices.route("/crawlers/create_spider_splash", methods=['POST'])
 def create_spider_splash():
@@ -619,10 +628,11 @@ def domain_crawled_7days_json():
 
     return jsonify(json_domain_stats)
 
-@hiddenServices.route('/hiddenServices/automatic_onion_crawler_json')
+@hiddenServices.route('/hiddenServices/domain_crawled_by_type_json')
-def automatic_onion_crawler_json():
+def domain_crawled_by_type_json():
     current_date = request.args.get('date')
-    type = 'onion'
+    type = request.args.get('type')
+    if type in list_types:
 
         num_day_type = 7
         date_range = get_date_range(num_day_type)
@@ -636,6 +646,8 @@ def automatic_onion_crawler_json():
 
         return jsonify(range_decoder)
 
+    else:
+        return jsonify('Incorrect Type')
 
 # ========= REGISTRATION =========
 app.register_blueprint(hiddenServices, url_prefix=baseUrl)

var/www/modules/hiddenServices/templates/Crawler_Splash_last_by_type.html

@@ -0,0 +1,476 @@
+<!DOCTYPE html>
+
+<html>
+<head>
+	<title>AIL-Framework</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
+	<!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
+
+	<!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
+	<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
+	<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
+
+	<style>
+		.bar {
+				fill: steelblue;
+			}
+		.bar:hover{
+			fill: brown;
+			cursor: pointer;
+		}
+		.bar_stack:hover{
+			cursor: pointer;
+		}
+		div.tooltip {
+			position: absolute;
+			text-align: center;
+			padding: 2px;
+			font: 12px sans-serif;
+			background: #ebf4fb;
+			border: 2px solid #b7ddf2;
+			border-radius: 8px;
+			pointer-events: none;
+			color: #000000;
+		}
+	</style>
+
+</head>
+
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'crawler/menu_sidebar.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
+				<div class="row">
+					<div class="col-12 col-xl-6">
+
+						<div class="table-responsive mt-1 table-hover table-borderless table-striped">
+						  <table class="table">
+								<thead class="thead-dark">
+							      <tr>
+							        <th>Domain</th>
+							        <th>First Seen</th>
+											<th>Last Check</th>
+							        <th>Status</th>
+							      </tr>
+							  </thead>
+							  <tbody id="tbody_last_crawled">
+										{% for metadata_domain in last_domains %}
+								      <tr>
+								        <td><a target="_blank" href="{{ url_for('hiddenServices.onion_domain') }}?onion_domain={{ metadata_domain['domain'] }}">{{ metadata_domain['domain'] }}</a></td>
+								        <td>{{'{}/{}/{}'.format(metadata_domain['first_seen'][0:4], metadata_domain['first_seen'][4:6], metadata_domain['first_seen'][6:8])}}</td>
+												<td>{{'{}/{}/{}'.format(metadata_domain['last_check'][0:4], metadata_domain['last_check'][4:6], metadata_domain['last_check'][6:8])}}</td>
+								        <td><div style="color:{{metadata_domain['status_color']}}; display:inline-block">
+															<i class="fas {{metadata_domain['status_icon']}} "></i>
+															{{metadata_domain['status_text']}}
+														</div>
+												</td>
+								      </tr>
+										{% endfor %}
+							  </tbody>
+						  </table>
+						</div>
+
+						<a href="{{ url_for('hiddenServices.blacklisted_domains') }}?type={{type}}">
+							<button type="button" class="btn btn-outline-danger">Show Blacklisted {{type_name}}s</button>
+						</a>
+
+					</div>
+					<div class="col-12 col-xl-6">
+
+						<div class="card text-white bg-dark mb-3 mt-1">
+						  <div class="card-header">
+								<div class="row">
+									<div class="col-6">
+										<span class="badge badge-success">{{ statDomains['domains_up'] }}</span> UP
+										<span class="badge badge-danger ml-md-3">{{ statDomains['domains_down'] }}</span> DOWN
+									</div>
+									<div class="col-6">
+										<span class="badge badge-success">{{ statDomains['total'] }}</span> Crawled
+										<span class="badge badge-warning ml-md-3">{{ statDomains['domains_queue'] }}</span> Queue
+									</div>
+								</div>
+							</div>
+						  <div class="card-body">
+						    <h5 class="card-title">Select domains by date range :</h5>
+						    <p class="card-text">Some quick example text to build on the card title and make up the bulk of the card's content.</p>
+								<form action="{{ url_for('hiddenServices.get_onions_by_daterange') }}" id="hash_selector_form" method='post'>
+									<div class="row">
+										<div class="col-6">
+											<div class="input-group" id="date-range-from">
+												<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
+												<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from">
+											</div>
+											<div class="input-group" id="date-range-to">
+												<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
+												<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to">
+											</div>
+										</div>
+										<div class="col-6">
+											<div class="custom-control custom-switch">
+											  <input class="custom-control-input" type="checkbox" name="domains_up" value="True" id="domains_up_id" checked>
+												<label class="custom-control-label" for="domains_up_id">
+													<span class="badge badge-success"><i class="fas fa-check-circle"></i> Domains UP </span>
+											  </label>
+											</div>
+											<div class="custom-control custom-switch">
+											  <input class="custom-control-input" type="checkbox" name="domains_down" value="True" id="domains_down_id">
+												<label class="custom-control-label" for="domains_down_id">
+													<span class="badge badge-danger"><i class="fas fa-times-circle"></i> Domains DOWN</span>
+												</label>
+											</div>
+											<div class="custom-control custom-switch mt-2">
+											  <input class="custom-control-input" type="checkbox" name="domains_tags" value="True" id="domains_tags_id">
+												<label class="custom-control-label" for="domains_tags_id">
+													<span class="badge badge-dark"><i class="fas fa-tags"></i> Domains Tags</span>
+												</label>
+											</div>
+										</div>
+									</div>
+									<button class="btn btn-primary">
+										<i class="fas fa-eye"></i> Show {{type_name}}s
+									</button>
+								<form>
+						  </div>
+						</div>
+
+						<div id="barchart_type">
+		        </div>
+
+						<div class="card mt-1 mb-1">
+						  <div class="card-header text-white bg-dark">
+								Crawlers Status
+							</div>
+						  <div class="card-body px-0 py-0 ">
+								<table class="table">
+									<tbody id="tbody_crawler_info">
+										{% for crawler in crawler_metadata %}
+											<tr>
+												 <td>
+													<i class="fas fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
+												</td>
+												<td>
+													{{crawler['crawling_domain']}}
+												</td>
+												<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
+													{{crawler['status_info']}}
+												</td>
+											</tr>
+										{% endfor %}
+									</tbody>
+								</table>
+							</div>
+						</div>
+					</div>
+				</div>
+
+			</div>
+
+		</div>
+	</div>
+
+
+</body>
+
+<script>
+var chart = {};
+$(document).ready(function(){
+	$("#page-Crawler").addClass("active");
+	$("#nav_{{type}}_crawler").addClass("active");
+
+	$('#date-range-from').dateRangePicker({
+			separator : ' to ',
+			getValue: function(){
+					if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
+							return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
+					else
+							return '';
+					},
+					setValue: function(s,s1,s2){
+							$('#date-range-from-input').val(s1);
+							$('#date-range-to-input').val(s2);
+					}
+	});
+	$('#date-range-to').dateRangePicker({
+			separator : ' to ',
+			getValue: function(){
+					if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
+							return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
+					else
+							return '';
+			},
+			setValue: function(s,s1,s2){
+					$('#date-range-from-input').val(s1);
+					$('#date-range-to-input').val(s2);
+			}
+	});
+	chart.stackBarChart =barchart_type_stack("{{ url_for('hiddenServices.domain_crawled_by_type_json') }}?type={{type}}", 'id');
+
+	chart.onResize();
+	$(window).on("resize", function() {
+		chart.onResize();
+	});
+
+});
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+</script>
+
+
+
+<script>/*
+function refresh_list_crawled(){
+
+	$.getJSON("{{ url_for('hiddenServices.last_crawled_domains_with_stats_json') }}",
+		function(data) {
+
+			var tableRef = document.getElementById('tbody_last_crawled');
+			$("#tbody_last_crawled").empty()
+
+			for (var i = 0; i < data.last_domains.length; i++) {
+					var data_domain = data.last_domains[i]
+					var newRow   = tableRef.insertRow(tableRef.rows.length);
+
+			    var newCell  = newRow.insertCell(0);
+					newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.onion_domain') }}?onion_domain="+data_domain['domain']+"\">"+data_domain['domain']+"</a></td>";
+
+					newCell  = newRow.insertCell(1);
+					newCell.innerHTML = "<td>"+data_domain['first_seen'].substr(0, 4)+"/"+data_domain['first_seen'].substr(4, 2)+"/"+data_domain['first_seen'].substr(6, 2)+"</td>"
+
+					newCell  = newRow.insertCell(2);
+					newCell.innerHTML = "<td>"+data_domain['last_check'].substr(0, 4)+"/"+data_domain['last_check'].substr(4, 2)+"/"+data_domain['last_check'].substr(6, 2)+"</td>"
+
+					newCell  = newRow.insertCell(3);
+					newCell.innerHTML = "<td><div style=\"color:"+data_domain['status_color']+"; display:inline-block\"><i class=\"fa "+data_domain['status_icon']+" fa-2x\"></i>"+data_domain['status_text']+"</div></td>"
+
+			}
+			var statDomains = data.statDomains
+			document.getElementById('text_domain_up').innerHTML = statDomains['domains_up']
+			document.getElementById('text_domain_down').innerHTML = statDomains['domains_down']
+			document.getElementById('text_domain_queue').innerHTML = statDomains['domains_queue']
+			document.getElementById('text_total_domains').innerHTML = statDomains['total']
+
+			if(data.crawler_metadata.length!=0){
+				$("#tbody_crawler_info").empty();
+				var tableRef = document.getElementById('tbody_crawler_info');
+				for (var i = 0; i < data.crawler_metadata.length; i++) {
+						var crawler = data.crawler_metadata[i];
+						var newRow = tableRef.insertRow(tableRef.rows.length);
+						var text_color;
+						var icon;
+						if(crawler['status']){
+							text_color = 'Green';
+							icon = 'check';
+						} else {
+							text_color = 'Red';
+							icon = 'times';
+						}
+
+				    var newCell  = newRow.insertCell(0);
+						newCell.innerHTML = "<td><i class=\"fa fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i>"+crawler['crawler_info']+"</td>";
+
+						newCell  = newRow.insertCell(1);
+						newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.onion_domain') }}?onion_domain="+crawler['crawling_domain']+"\">"+crawler['crawling_domain']+"</a></td>";
+
+						newCell  = newRow.insertCell(2);
+						newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
+
+						$("#panel_crawler").show();
+				}
+			} else {
+				$("#panel_crawler").hide();
+			}
+		}
+	);
+
+	if (to_refresh) {
+    setTimeout("refresh_list_crawled()", 10000);
+	}
+}*/
+</script>
+
+<script>
+var margin = {top: 20, right: 90, bottom: 55, left: 0},
+				width = parseInt(d3.select('#barchart_type').style('width'), 10);
+				width  = 1000 - margin.left - margin.right,
+				height = 500  - margin.top  - margin.bottom;
+var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
+
+var y = d3.scaleLinear().rangeRound([height, 0]);
+
+var xAxis = d3.axisBottom(x);
+
+var yAxis = d3.axisLeft(y);
+
+var color = d3.scaleOrdinal(d3.schemeSet3);
+
+var svg = d3.select("#barchart_type").append("svg")
+				.attr("id", "thesvg")
+				.attr("viewBox", "0 0 "+width+" 500")
+				.attr("width",  width  + margin.left + margin.right)
+				.attr("height", height + margin.top  + margin.bottom)
+			.append("g")
+				.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
+
+
+function barchart_type_stack(url, id) {
+
+  d3.json(url)
+    .then(function(data){
+
+			var labelVar = 'date';  //A
+		  var varNames = d3.keys(data[0])
+		      .filter(function (key) { return key !== labelVar;}); //B
+
+		  data.forEach(function (d) { //D
+		    var y0 = 0;
+		    d.mapping = varNames.map(function (name) {
+		      return {
+		        name: name,
+		        label: d[labelVar],
+		        y0: y0,
+		        y1: y0 += +d[name]
+		      };
+		    });
+		    d.total = d.mapping[d.mapping.length - 1].y1;
+		  });
+
+		  x.domain(data.map(function (d) { return (d.date); })); //E
+		  y.domain([0, d3.max(data, function (d) { return d.total; })]);
+
+			svg.append("g")
+        .attr("class", "x axis")
+        .attr("transform", "translate(0," + height + ")")
+        .call(xAxis)
+				.selectAll("text")
+					.attr("class", "bar")
+					.on("click", function (d) { window.location.href = "#" })
+					.attr("transform", "rotate(-18)" )
+					//.attr("transform", "rotate(-40)" )
+					.style("text-anchor", "end");
+
+    	svg.append("g")
+        .attr("class", "y axis")
+        .call(yAxis)
+      	.append("text")
+        	.attr("transform", "rotate(-90)")
+        	.attr("y", 6)
+        	.attr("dy", ".71em")
+        	.style("text-anchor", "end");
+
+			var selection = svg.selectAll(".series")
+		    .data(data)
+		    .enter().append("g")
+		      .attr("class", "series")
+		      .attr("transform", function (d) { return "translate(" + x((d.date)) + ",0)"; });
+
+			selection.selectAll("rect")
+	      .data(function (d) { return d.mapping; })
+	    .enter().append("rect")
+				.attr("class", "bar_stack")
+	      .attr("width", x.bandwidth())
+	      .attr("y", function (d) { return y(d.y1); })
+	      .attr("height", function (d) { return y(d.y0) - y(d.y1); })
+	      .style("fill", function (d) { return color(d.name); })
+	      .style("stroke", "grey")
+	      .on("mouseover", function (d) { showPopover.call(this, d); })
+				.on("mouseout", function (d) { removePopovers(); })
+				.on("click", function(d){ window.location.href = "#" });
+
+
+				data.forEach(function(d) {
+					if(d.total != 0){
+						svg.append("text")
+								.attr("class", "bar")
+								.attr("dy", "-.35em")
+								.attr('x', x(d.date) + x.bandwidth()/2)
+								.attr('y', y(d.total))
+								.on("click", function () {window.location.href = "#" })
+								.style("text-anchor", "middle")
+								.text(d.total);
+					}
+				});
+
+			drawLegend(varNames);
+    });
+
+}
+
+function drawLegend (varNames) {
+    var legend = svg.selectAll(".legend")
+        .data(varNames.slice().reverse())
+      .enter().append("g")
+        .attr("class", "legend")
+        .attr("transform", function (d, i) { return "translate(0," + i * 20 + ")"; });
+
+    legend.append("rect")
+        .attr("x", 943)
+        .attr("width", 10)
+        .attr("height", 10)
+        .style("fill", color)
+        .style("stroke", "grey");
+
+    legend.append("text")
+				.attr("class", "svgText")
+        .attr("x", 941)
+        .attr("y", 6)
+        .attr("dy", ".35em")
+        .style("text-anchor", "end")
+        .text(function (d) { return d; });
+}
+
+function removePopovers () {
+    $('.popover').each(function() {
+      $(this).remove();
+    });
+  }
+
+function showPopover (d) {
+  $(this).popover({
+    title: d.name,
+    placement: 'top',
+    container: 'body',
+    trigger: 'manual',
+    html : true,
+    content: function() {
+      return d.label +
+            "<br/>num: " + d3.format(",")(d.value ? d.value: d.y1 - d.y0); }
+  });
+  $(this).popover('show')
+}
+
+chart.onResize = function () {
+    var aspect = width / height, chart = $("#thesvg");
+    var targetWidth = chart.parent().width();
+    chart.attr("width", targetWidth);
+    chart.attr("height", targetWidth / 2);
+}
+
+window.chart = chart;
+
+</script>

var/www/modules/hiddenServices/templates/Crawler_Splash_manual.html

@@ -26,40 +26,9 @@
 	<div class="container-fluid">
 		<div class="row">
 
-			<div class="col-12 col-lg-2 p-0 bg-light border-right">
+			{% include 'crawler/menu_sidebar.html' %}
 
-
+			<div class="col-12 col-lg-10" id="core_content">
-				<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2">
-					<h5 class="d-flex text-muted w-100">
-						<span>Splash Crawlers </span>
-						<a class="ml-auto" href="#">
-							<i class="fas fa-plus-circle ml-auto"></i>
-						</a>
-					</h5>
-	        <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
-	          <li class="nav-item">
-	            <a class="nav-link" href="#">
-	              <i class="fas fa-search"></i>
-	              <span>Dashboard</span>
-	            </a>
-	          </li>
-	          <li class="nav-item">
-	            <a class="nav-link active" href="#">
-								<i class="fas fa-sync"></i>
-	             	Automatic Onion Crawler
-	            </a>
-	          </li>
-            <li class="nav-item">
-              <a class="nav-link" href="#">
-								<i class="fas fa-clock"></i>
-                Manual Splash Crawler
-              </a>
-            </li>
-          </ul>
-	      </nav>
-			</div>
-
-			<div class="col-12 col-lg-10">
 
 
 						<div class="card text-white bg-dark mb-3 mt-1">
@@ -162,6 +131,7 @@
 var chart = {};
 $(document).ready(function(){
 	$("#page-Crawler").addClass("active");
+	$("#nav_manual_crawler").addClass("active");
 	manual_crawler_input_controler();
 
 	$('#crawler_type').change(function () {
@@ -169,6 +139,20 @@ $(document).ready(function(){
 	});
 });
 
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
+
 function manual_crawler_input_controler() {
 	if($('#crawler_type').is(':checked')){
 		$("#crawler_epoch_input").show();

var/www/modules/hiddenServices/templates/Crawler_Splash_onion.html

@@ -50,42 +50,7 @@
 	<div class="container-fluid">
 		<div class="row">
 
-			<div class="col-12 col-lg-2 p-0 bg-light border-right" id="side_menu">
+			{% include 'crawler/menu_sidebar.html' %}
-
-				<button type="button" class="btn btn-outline-secondary mt-1 ml-3" onclick="toggle_sidebar()">
-							 <i class="fas fa-align-left"></i>
-							 <span>Toggle Sidebar</span>
-				</button>
-
-				<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
-					<h5 class="d-flex text-muted w-100">
-						<span>Splash Crawlers </span>
-						<a class="ml-auto" href="#">
-							<i class="fas fa-plus-circle ml-auto"></i>
-						</a>
-					</h5>
-	        <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
-	          <li class="nav-item">
-	            <a class="nav-link" href="#">
-	              <i class="fas fa-search"></i>
-	              <span>Dashboard</span>
-	            </a>
-	          </li>
-	          <li class="nav-item">
-	            <a class="nav-link active" href="#">
-								<i class="fas fa-sync"></i>
-	             	Automatic Onion Crawler
-	            </a>
-	          </li>
-            <li class="nav-item">
-              <a class="nav-link" href="#">
-								<i class="fas fa-clock"></i>
-                Manual Splash Crawler
-              </a>
-            </li>
-          </ul>
-	      </nav>
-			</div>
 
 			<div class="col-12 col-lg-10" id="core_content">
 
@@ -224,6 +189,7 @@
 var chart = {};
 $(document).ready(function(){
 	$("#page-Crawler").addClass("active");
+	$("#nav_onion_crawler").addClass("active");
 
 	$('#date-range-from').dateRangePicker({
 			separator : ' to ',

var/www/modules/hiddenServices/templates/Crawler_dashboard.html

@@ -0,0 +1,82 @@
+<!DOCTYPE html>
+
+<html>
+<head>
+	<title>AIL-Framework</title>
+	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
+	<!-- Core CSS -->
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
+
+	<!-- JS -->
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+
+
+</head>
+
+<body>
+
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			<div class="col-12 col-md-2 p-0 bg-light border-right">
+
+
+				<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2">
+					<h5 class="d-flex text-muted w-100">
+						<span>Splash Crawlers </span>
+						<a class="ml-auto" href="#">
+							<i class="fas fa-plus-circle ml-auto"></i>
+						</a>
+					</h5>
+	            <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
+	              <li class="nav-item">
+	                <a class="nav-link active" href="#">
+	                  <i class="fas fa-search"></i>
+	                  <span>Dashboard</span>
+	                </a>
+	              </li>
+	              <li class="nav-item">
+	                <a class="nav-link" href="#">
+										<i class="fas fa-search"></i>
+	                	Automatic Splash Crawler
+	                </a>
+	              </li>
+
+	              <li class="nav-item">
+	                <a class="nav-link" href="#">
+										<i class="fas fa-search"></i>
+	                  Manual Splash Crawler
+	                </a>
+	              </li>
+	            </ul>
+	        </nav>
+			</div>
+
+
+
+			<div >
+				<pre>
+					--------------
+
+
+
+					--------------
+
+				</pre>
+			</div>
+
+		</div>
+	</div>
+
+
+</body>
+
+<script>
+$(document).ready(function(){
+	$("#page-Crawler").addClass("active");
+});
+</script>

var/www/modules/hiddenServices/templates/blacklisted_domains.html

@@ -24,45 +24,14 @@
 	<div class="container-fluid">
 		<div class="row">
 
-			<div class="col-12 col-lg-2 p-0 bg-light border-right">
+			{% include 'crawler/menu_sidebar.html' %}
 
-
+			<div class="col-12 col-lg-10" id="core_content">
-				<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2">
-					<h5 class="d-flex text-muted w-100">
-						<span>Splash Crawlers </span>
-						<a class="ml-auto" href="#">
-							<i class="fas fa-plus-circle ml-auto"></i>
-						</a>
-					</h5>
-	        <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100">
-	          <li class="nav-item">
-	            <a class="nav-link" href="#">
-	              <i class="fas fa-search"></i>
-	              <span>Dashboard</span>
-	            </a>
-	          </li>
-	          <li class="nav-item">
-	            <a class="nav-link" href="{{ url_for('hiddenServices.crawler_splash_onion') }}">
-								<i class="fas fa-sync"></i>
-	             	Automatic Onion Crawler
-	            </a>
-	          </li>
-            <li class="nav-item">
-              <a class="nav-link" href="#">
-								<i class="fas fa-clock"></i>
-                Manual Splash Crawler
-              </a>
-            </li>
-          </ul>
-	      </nav>
-			</div>
-
-			<div class="col-12 col-lg-10">
 
 				<div class="card-deck justify-content-center mx-0">
 					<div class="card border-dark mt-2">
 						<div class="card-header bg-dark text-white">
-							Blacklisted Onions
+							Blacklisted {{type_name}}s
 						</div>
 					  <div class="card-body text-dark">
 
@@ -70,38 +39,38 @@
 						    <div class="col-12 col-md-6">
 									<div class="card text-center border-danger">
 										<div class="card-body text-danger">
-											<h5 class="card-title">Blacklist Onion</h5>
+											<h5 class="card-title">Blacklist {{type_name}}</h5>
-												<input type="text" class="form-control {%if blacklist_onion is not none %}{%if blacklist_onion==1 %}is-valid{% else %}is-invalid{%endif%}{%endif%}" id="blacklist_onion_input" placeholder="Onion Address">
+												<input type="text" class="form-control {%if blacklist_domain is not none %}{%if blacklist_domain==1 %}is-valid{% else %}is-invalid{%endif%}{%endif%}" id="blacklist_domain_input" placeholder="{{type_name}} Address">
 												<div class="invalid-feedback">
-													{%if blacklist_onion==2 %}
+													{%if blacklist_domain==2 %}
-														This Onion is already blacklisted
+														This {{type_name}} is already blacklisted
 													{% else %}
-														Incorrect Onion address
+														Incorrect {{type_name}} address
 													{% endif %}
 												</div>
 												<div class="valid-feedback">
-													Onion Blacklisted
+													{{type_name}} Blacklisted
 												</div>
-												<button type="button" class="btn btn-danger mt-2" onclick="window.location.href ='{{ url_for('hiddenServices.blacklist_onion') }}?redirect=0&onion='+$('#blacklist_onion_input').val();">Blacklist Onion</button>
+												<button type="button" class="btn btn-danger mt-2" onclick="window.location.href ='{{ url_for('hiddenServices.blacklist_domain') }}?redirect=0&type={{type}}&domain='+$('#blacklist_domain_input').val();">Blacklist {{type_name}}</button>
 										</div>
 									</div>
 								</div>
 								<div class="col-12 col-md-6 mt-4 mt-md-0">
 									<div class="card text-center border-success">
 										<div class="card-body">
-											<h5 class="card-title">Unblacklist Onion</h5>
+											<h5 class="card-title">Unblacklist {{type_name}}</h5>
-												<input type="text" class="form-control {%if unblacklist_onion is not none %}{%if unblacklist_onion==1 %}is-valid{% else %}is-invalid{%endif%}{%endif%}" id="unblacklist_onion_input" placeholder="Onion Address">
+												<input type="text" class="form-control {%if unblacklist_domain is not none %}{%if unblacklist_domain==1 %}is-valid{% else %}is-invalid{%endif%}{%endif%}" id="unblacklist_domain_input" placeholder="{{type_name}} Address">
 												<div class="invalid-feedback">
-													{%if unblacklist_onion==2 %}
+													{%if unblacklist_domain==2 %}
-														This Onion is not blacklisted
+														This {{type_name}} is not blacklisted
 													{% else %}
-														Incorrect Onion address
+														Incorrect {{type_name}} address
 													{% endif %}
 												</div>
 												<div class="valid-feedback">
-													Onion Unblacklisted
+													{{type_name}} Unblacklisted
 												</div>
-												<button type="button" class="btn btn-outline-secondary mt-2" onclick="window.location.href ='{{ url_for('hiddenServices.unblacklist_onion') }}?redirect=0&onion='+$('#unblacklist_onion_input').val();">Unblacklist Onion</button>
+												<button type="button" class="btn btn-outline-secondary mt-2" onclick="window.location.href ='{{ url_for('hiddenServices.unblacklist_domain') }}?redirect=0&type={{type}}&domain='+$('#unblacklist_domain_input').val();">Unblacklist {{type_name}}</button>
 										</div>
 									</div>
 								</div>
@@ -112,17 +81,17 @@
 									<table class="table table-striped table-bordered table-hover" id="myTable_1">
 										<thead class="thead-dark">
 											<tr>
-												<th style="max-width: 800px;">Onion</th>
+												<th style="max-width: 800px;">{{type_name}}</th>
-												<th style="max-width: 800px;">Unblacklist Onion</th>
+												<th style="max-width: 800px;">Unblacklist {{type_name}}</th>
 											</tr>
 										</thead>
 										<tbody>
-											{% for onion in list_blacklisted_1 %}
+											{% for domain in list_blacklisted_1 %}
 											<tr>
-												<td>{{onion}}</td>
+												<td>{{domain}}</td>
 												<td>
-													<a href="{{ url_for('hiddenServices.unblacklist_onion') }}?page={{page}}&onion={{onion}}">
+													<a href="{{ url_for('hiddenServices.unblacklist_domain') }}?page={{page}}&domain={{domain}}&type={{type}}">
-														<button type="button" class="btn btn-outline-danger">UnBlacklist Onion</button>
+														<button type="button" class="btn btn-outline-danger">UnBlacklist {{type_name}}</button>
 													</a>
 												</td>
 											</tr>
@@ -134,17 +103,17 @@
 									<table class="table table-striped table-bordered table-hover" id="myTable_2">
 										<thead class="thead-dark">
 											<tr>
-												<th style="max-width: 800px;">Onion</th>
+												<th style="max-width: 800px;">{{type_name}}</th>
-												<th style="max-width: 800px;">Unblacklist Onion</th>
+												<th style="max-width: 800px;">Unblacklist {{type_name}}</th>
 											</tr>
 										</thead>
 										<tbody>
-											{% for onion in list_blacklisted_2 %}
+											{% for domain in list_blacklisted_2 %}
 											<tr>
-												<td>{{onion}}</td>
+												<td>{{domain}}</td>
 												<td>
-													<a href="{{ url_for('hiddenServices.unblacklist_onion') }}?page={{page}}&onion={{onion}}">
+													<a href="{{ url_for('hiddenServices.unblacklist_domain') }}?page={{page}}&domain={{domain}}&type={{type}}">
-														<button type="button" class="btn btn-outline-danger">UnBlacklist Onion</button>
+														<button type="button" class="btn btn-outline-danger">UnBlacklist {{type_name}}</button>
 													</a>
 												</td>
 											</tr>
@@ -162,34 +131,34 @@
 					<nav class="mt-4" aria-label="...">
 						<ul class="pagination">
 							<li class="page-item {%if page==1%}disabled{%endif%}">
-								<a class="page-link" href="{{ url_for('hiddenServices.blacklisted_onion') }}?page={{page-1}}">Previous</a>
+								<a class="page-link" href="{{ url_for('hiddenServices.blacklisted_domains') }}?page={{page-1}}">Previous</a>
 							</li>
 
 							{%if page>3%}
-								<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_onion') }}?page=1">1</a></li>
+								<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_domains') }}?page=1">1</a></li>
 								<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
-								<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_onion') }}?page={{page-1}}">{{page-1}}</a></li>
+								<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_domains') }}?page={{page-1}}">{{page-1}}</a></li>
-								<li class="page-item active"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_onion') }}?page={{page}}">{{page}}</a></li>
+								<li class="page-item active"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_domains') }}?page={{page}}">{{page}}</a></li>
 							{%else%}
-							{%if page>2%}<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_onion') }}?page={{page-2}}">{{page-2}}</a></li>{%endif%}
+							{%if page>2%}<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_domains') }}?page={{page-2}}">{{page-2}}</a></li>{%endif%}
-							{%if page>1%}<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_onion') }}?page={{page-1}}">{{page-1}}</a></li>{%endif%}
+							{%if page>1%}<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_domains') }}?page={{page-1}}">{{page-1}}</a></li>{%endif%}
-							<li class="page-item active"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_onion') }}?page={{page}}">{{page}}</a></li>
+							<li class="page-item active"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_domains') }}?page={{page}}">{{page}}</a></li>
 							{%endif%}
 
 							{%if nb_page_max-page>3%}
-								<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_onion') }}?page={{page+1}}">{{page+1}}</a></li>
+								<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_domains') }}?page={{page+1}}">{{page+1}}</a></li>
 								<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
-								<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_onion') }}?page={{nb_page_max}}">{{nb_page_max}}</a></li>
+								<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_domains') }}?page={{nb_page_max}}">{{nb_page_max}}</a></li>
 							{%else%}
-								{%if nb_page_max-page>2%}<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_onion') }}?page={{nb_page_max-2}}">{{nb_page_max-2}}</a></li>{%endif%}
+								{%if nb_page_max-page>2%}<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_domains') }}?page={{nb_page_max-2}}">{{nb_page_max-2}}</a></li>{%endif%}
-								{%if nb_page_max-page>1%}<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_onion') }}?page={{nb_page_max-1}}">{{nb_page_max-1}}</a></li>{%endif%}
+								{%if nb_page_max-page>1%}<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_domains') }}?page={{nb_page_max-1}}">{{nb_page_max-1}}</a></li>{%endif%}
-								{%if nb_page_max-page>0%}<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_onion') }}?page={{nb_page_max}}">{{nb_page_max}}</a></li>{%endif%}
+								{%if nb_page_max-page>0%}<li class="page-item"><a class="page-link" href="{{ url_for('hiddenServices.blacklisted_domains') }}?page={{nb_page_max}}">{{nb_page_max}}</a></li>{%endif%}
 							{%endif%}
 
 
 
 							<li class="page-item {%if page==nb_page_max%}disabled{%endif%}">
-								<a class="page-link" href="{{ url_for('hiddenServices.blacklisted_onion') }}?page={{page+1}}" aria-disabled="true">Next</a>
+								<a class="page-link" href="{{ url_for('hiddenServices.blacklisted_domains') }}?page={{page+1}}" aria-disabled="true">Next</a>
 							</li>
 						</ul>
 					</nav>
@@ -220,6 +189,20 @@ $(document).ready(function(){
 				"order": [[ 0, "asc" ]]
 			}
 		);
-
+		$("#page-Crawler").addClass("active");
 });
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
 </script>

var/www/templates/crawler/menu_sidebar.html

@@ -0,0 +1,48 @@
+      <div class="col-12 col-lg-2 p-0 bg-light border-right" id="side_menu">
+
+      	<button type="button" class="btn btn-outline-secondary mt-1 ml-3" onclick="toggle_sidebar()">
+          <i class="fas fa-align-left"></i>
+      	  <span>Toggle Sidebar</span>
+      	</button>
+
+      	<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
+      		<h5 class="d-flex text-muted w-100">
+      		  <span>Splash Crawlers </span>
+      			<a class="ml-auto" href="#">
+      				<i class="fas fa-plus-circle ml-auto"></i>
+      			</a>
+      		</h5>
+      	  <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
+      	    <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('hiddenServices.dashboard')}}" id="nav_dashboard">
+      	        <i class="fas fa-search"></i>
+      	        <span>Dashboard</span>
+      	      </a>
+      	    </li>
+      	    <li class="nav-item">
+      	      <a class="nav-link" href="{{url_for('hiddenServices.crawler_splash_onion')}}" id="nav_onion_crawler">
+      					<i class="fas fa-user-secret"></i>
+      	       	Onion Crawler
+      	      </a>
+      	    </li>
+            <li class="nav-item">
+              <a class="nav-link" href="{{url_for('hiddenServices.crawler_splash_regular')}}" id="nav_regular_crawler">
+      					<i class="fab fa-html5"></i>
+                Regular Crawler
+              </a>
+            </li>
+      			<li class="nav-item">
+              <a class="nav-link" href="{{url_for('hiddenServices.manual')}}" id="nav_manual_crawler">
+      					<i class="fas fa-spider"></i>
+                Manual Crawler
+              </a>
+            </li>
+      			<li class="nav-item">
+              <a class="nav-link" href="#" id="nav_auto_crawler">
+      					<i class="fas fa-sync"></i>
+                Automatic Crawler
+              </a>
+            </li>
+          </ul>
+      	</nav>
+      </div>