chg: [Flask UI] refractor crawler frontend + show crawled domain

bin/packages/HiddenServices.py

@@ -63,7 +63,7 @@ class HiddenServices(object):
         self.type = type
         self.tags = {}
 
-        if type == 'onion':
+        if type == 'onion' or type == 'regular':
             self.paste_directory = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes"))
             self.paste_crawled_directory = os.path.join(self.paste_directory, cfg.get("Directories", "crawled"))
             self.paste_crawled_directory_name = cfg.get("Directories", "crawled")
@@ -75,6 +75,10 @@ class HiddenServices(object):
             ## TODO: # FIXME: add error
             pass
 
+    def remove_absolute_path_link(self, key, value):
+        print(key)
+        print(value)
+
     def get_origin_paste_name(self):
         origin_item = self.r_serv_onion.hget('onion_metadata:{}'.format(self.domain), 'paste_parent')
         if origin_item is None:
@@ -105,10 +109,35 @@ class HiddenServices(object):
         for tag in p_tags:
             self.tags[tag] = self.tags.get(tag, 0) + 1
 
+    def get_first_crawled(self):
+        res = self.r_serv_onion.zrange('crawler_history_{}:{}'.format(self.type, self.domain), 0, 0, withscores=True)
+        if res:
+            res = res[0]
+            return {'root_item':res[0], 'epoch':res[1]}
+        else:
+            return {}
+
+    def get_last_crawled(self):
+        res = self.r_serv_onion.zrevrange('crawler_history_{}:{}'.format(self.type, self.domain), 0, 0, withscores=True)
+        if res:
+            res = res[0]
+            return {'root_item':res[0], 'epoch':res[1]}
+        else:
+            return {}
+
     #todo use the right paste
-    def get_last_crawled_pastes(self):
+    def get_last_crawled_pastes(self, epoch=None):
-        paste_root = self.r_serv_onion.zrevrange('crawler_history_{}:{}'.format(self.type, self.domain), 0, 0)[0]
+        if epoch is None:
-        return self.get_all_pastes_domain(paste_root)
+            list_root = self.r_serv_onion.zrevrange('crawler_history_{}:{}'.format(self.type, self.domain), 0, 0)
+        else:
+            list_root = self.r_serv_onion.zrevrangebyscore('crawler_history_{}:{}'.format(self.type, self.domain), int(epoch), int(epoch))
+        if list_root:
+            return self.get_all_pastes_domain(list_root[0])
+        else:
+            if epoch:
+                return self.get_last_crawled_pastes()
+            else:
+                return list_root
 
     def get_all_pastes_domain(self, root_item):
         if root_item is None:
@@ -135,6 +164,27 @@ class HiddenServices(object):
                 l_crawled_pastes.extend(self.get_item_crawled_children(children))
         return l_crawled_pastes
 
+    def get_item_link(self, item):
+        link = self.r_serv_metadata.hget('paste_metadata:{}'.format(item), 'real_link')
+        if link is None:
+            if self.paste_directory in item:
+                self.r_serv_metadata.hget('paste_metadata:{}'.format(item.replace(self.paste_directory+'/', '')), 'real_link')
+            else:
+                key = os.path.join(self.paste_directory, item)
+                link = self.r_serv_metadata.hget('paste_metadata:{}'.format(key), 'real_link')
+                if link:
+                    self.remove_absolute_path_link(key, link)
+
+        return link
+
+    def get_all_links(self, l_items):
+        dict_links = {}
+        for item in l_items:
+            link = self.get_item_link(item)
+            if link:
+                dict_links[item] = link
+        return dict_links
+
     # experimental
     def get_domain_son(self, l_paste):
         if l_paste is None:
@@ -177,11 +227,12 @@ class HiddenServices(object):
         l_screenshot_paste = []
         for paste in l_crawled_pastes:
             ## FIXME: # TODO: remove me
+            origin_paste = paste
             paste= paste.replace(self.paste_directory+'/', '')
 
             paste = paste.replace(self.paste_crawled_directory_name, '')
             if os.path.isfile( '{}{}.png'.format(self.screenshot_directory, paste) ):
-                l_screenshot_paste.append(paste[1:])
+                l_screenshot_paste.append({'screenshot': paste[1:], 'item': origin_paste})
 
         if len(l_screenshot_paste) > num_screenshot:
             l_random_screenshot = []

var/www/modules/hiddenServices/Flask_hiddenServices.py

@@ -8,6 +8,7 @@ import redis
 import datetime
 import sys
 import os
+import time
 import json
 from pyfaup.faup import Faup
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
@@ -95,6 +96,16 @@ def get_domain_type(domain):
     else:
         return 'regular'
 
+def get_type_domain(domain):
+    if domain is None:
+        type = 'regular'
+    else:
+        if domain.rsplit('.', 1)[1] == 'onion':
+            type = 'onion'
+        else:
+            type = 'regular'
+    return type
+
 def get_last_domains_crawled(type):
     return r_serv_onion.lrange('last_{}'.format(type), 0 ,-1)
 
@@ -560,25 +571,31 @@ def show_domains_by_daterange():
                                 date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down,
                                 domains_tags=domains_tags, bootstrap_label=bootstrap_label)
 
-@hiddenServices.route("/hiddenServices/onion_domain", methods=['GET'])
+@hiddenServices.route("/hiddenServices/show_domain", methods=['GET'])
-def onion_domain():
+def show_domain():
-    onion_domain = request.args.get('onion_domain')
+    domain = request.args.get('domain')
-    if onion_domain is None or not r_serv_onion.exists('onion_metadata:{}'.format(onion_domain)):
+    epoch = request.args.get('epoch')
+    type = get_type_domain(domain)
+    if domain is None or not r_serv_onion.exists('{}_metadata:{}'.format(type, domain)):
         return '404'
         # # TODO: FIXME return 404
 
-    last_check = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'last_check')
+    last_check = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'last_check')
     if last_check is None:
         last_check = '********'
     last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
-    first_seen = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'first_seen')
+    first_seen = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'first_seen')
     if first_seen is None:
         first_seen = '********'
     first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8])
-    origin_paste = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'paste_parent')
+    origin_paste = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'paste_parent')
 
-    h = HiddenServices(onion_domain, 'onion')
+    h = HiddenServices(domain, type)
-    l_pastes = h.get_last_crawled_pastes()
+    last_crawled_time = h.get_last_crawled()
+    if epoch in last_crawled_time:
+        last_check = '{} - {}'.format(last_check, time.strftime('%H:%M.%S', time.gmtime(last_crawled_time['epoch'])))
+    l_pastes = h.get_last_crawled_pastes(epoch=epoch)
+    dict_links = h.get_all_links(l_pastes)
     if l_pastes:
         status = True
     else:
@@ -600,8 +617,9 @@ def onion_domain():
         p_tags = r_serv_metadata.smembers('tag:'+path)
         paste_tags.append(unpack_paste_tags(p_tags))
 
-    return render_template("showDomain.html", domain=onion_domain, last_check=last_check, first_seen=first_seen,
+    return render_template("showDomain.html", domain=domain, last_check=last_check, first_seen=first_seen,
                             l_pastes=l_pastes, paste_tags=paste_tags, bootstrap_label=bootstrap_label,
+                            dict_links=dict_links,
                             path_name=path_name, origin_paste_tags=origin_paste_tags, status=status,
                             origin_paste=origin_paste, origin_paste_name=origin_paste_name,
                             domain_tags=domain_tags, screenshot=screenshot)

var/www/modules/hiddenServices/templates/Crawler_Splash_last_by_type.html

@@ -62,7 +62,7 @@
 							  <tbody id="tbody_last_crawled">
 										{% for metadata_domain in last_domains %}
 								      <tr data-toggle="popover" data-trigger="hover" title="<span class='badge badge-dark'>{{metadata_domain['domain']}}</span>"  data-content="{{metadata_domain['epoch']}}">
-								        <td><a target="_blank" href="{{ url_for('hiddenServices.onion_domain') }}?onion_domain={{ metadata_domain['domain'] }}">{{ metadata_domain['domain_name'] }}</a></td>
+								        <td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ metadata_domain['domain'] }}&epoch={{metadata_domain['epoch']}}">{{ metadata_domain['domain_name'] }}</a></td>
 								        <td>{{'{}/{}/{}'.format(metadata_domain['first_seen'][0:4], metadata_domain['first_seen'][4:6], metadata_domain['first_seen'][6:8])}}</td>
 												<td>{{'{}/{}/{}'.format(metadata_domain['last_check'][0:4], metadata_domain['last_check'][4:6], metadata_domain['last_check'][6:8])}}</td>
 								        <td><div style="color:{{metadata_domain['status_color']}}; display:inline-block">
@@ -255,7 +255,7 @@ function refresh_list_crawled(){
 					var newRow   = tableRef.insertRow(tableRef.rows.length);
 
 			    var newCell  = newRow.insertCell(0);
-					newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.onion_domain') }}?onion_domain="+data_domain['domain']+"\">"+data_domain['domain']+"</a></td>";
+					newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.show_domain') }}?onion_domain="+data_domain['domain']+"\">"+data_domain['domain']+"</a></td>";
 
 					newCell  = newRow.insertCell(1);
 					newCell.innerHTML = "<td>"+data_domain['first_seen'].substr(0, 4)+"/"+data_domain['first_seen'].substr(4, 2)+"/"+data_domain['first_seen'].substr(6, 2)+"</td>"
@@ -293,7 +293,7 @@ function refresh_list_crawled(){
 						newCell.innerHTML = "<td><i class=\"fa fa-"+icon+"-circle\" style=\"color:"+text_color+";\"></i>"+crawler['crawler_info']+"</td>";
 
 						newCell  = newRow.insertCell(1);
-						newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.onion_domain') }}?onion_domain="+crawler['crawling_domain']+"\">"+crawler['crawling_domain']+"</a></td>";
+						newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.show_domain') }}?onion_domain="+crawler['crawling_domain']+"\">"+crawler['crawling_domain']+"</a></td>";
 
 						newCell  = newRow.insertCell(2);
 						newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";

var/www/modules/hiddenServices/templates/showDomain.html

@@ -1,60 +1,53 @@
 <!DOCTYPE html>
 <html>
-
 <head>
-	  <meta charset="utf-8">
-	  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
   <title>Show Domain - AIL</title>
 	<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
-
 	<!-- Core CSS -->
-	  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
-	  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
-	  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
-	  <link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
 	<!-- JS -->
-	  <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
-	  <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
 	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
-		<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
-
-		<style>
-	      .test thead{
-	      	background: #d91f2d;
-		      color: #fff;
-	      }
-	  </style>
 
 </head>
 <body>
 
-            {% include 'navbar.html' %}
+	{% include 'nav_bar.html' %}
+
+	<div class="container-fluid">
+		<div class="row">
+
+			{% include 'crawler/menu_sidebar.html' %}
+
+			<div class="col-12 col-lg-10" id="core_content">
+
 
-            <div id="page-wrapper">
 
 				<div class="row">
 
-					<div class="col-md-6">
+					<div class="col-12 col-xl-6">
-						<div class="row">
+						<div class="card mt-2">
-							<div class="panel panel-info">
+							<div class="card-header bg-dark">
-								<div class="panel-heading">
+								<span class="badge badge-pill badge-light flex-row-reverse float-right">
 									{% if status %}
-										<div class="pull-right" style="color:Green;">
+										<div style="color:Green;">
-											<i class="fa fa-check-circle fa-2x"></i>
+											<i class="fas fa-check-circle fa-2x"></i>
 											UP
 										</div>
 									{% else %}
-										<div class="pull-right" style="color:Red;">
+										<div style="color:Red;">
-											<i class="fa fa-times-circle fa-2x"></i>
+											<i class="fas fa-times-circle fa-2x"></i>
 											DOWN
 										</div>
 									{% endif %}
-									<h3>{{ domain }} :</h3>
+								</span>
-									<ul class="list-group">
+								<h3 class="card-title text-white">{{ domain }} :</h3>
-										<li class="list-group-item">
+							</div>
-
+							<div class="card-body">
-											<table class="table table-condensed">
+								<table class="table table-responsive table-condensed">
 									<thead>
 										<tr>
 											<th>First Seen</th>
@@ -63,76 +56,80 @@
 									</thead>
 									<tbody>
 										<tr>
-								            <td class="panelText"><a href="#">{{ first_seen }}</a></td>
+											<td class="panelText">{{ first_seen }}</td>
-								            <td class="panelText"><a href="#">{{ last_check }}</a></td>
+											<td class="panelText">{{ last_check }}</td>
 										</tr>
 									</tbody>
 								</table>
 
-										</li>
+								Origin Paste:
-										<li class="list-group-item">
+								{% if origin_paste_name=='manual' or origin_paste_name=='auto' %}
-											Origin Paste: <a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste', paste=origin_paste) }}" />{{ origin_paste_name }}</a>
+									<span class="badge badge-dark">{{ origin_paste_name }}</span>
+								{%else%}
+								<a class="badge badge-dark" target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste', paste=origin_paste) }}" />{{ origin_paste_name }}</a>
+								{%endif%}
 								<div>
 									{% for tag in origin_paste_tags %}
 									<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag[1] }}">
-													<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
+										<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
 									</a>
 									{%  endfor %}
 									<br>
 								</div>
-										</li>
-									</ul>
 
 							</div>
 						</div>
+
 						<div>
 							{% for tag in domain_tags %}
 							<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
-									<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }} <i>{{ domain_tags[tag] }}</i></span>
+								<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }} <i>{{ domain_tags[tag] }}</i></span>
 							</a>
 							{%  endfor %}
 							<br>
 							<br>
 						</div>
 
-								<table class="test table table-striped table-bordered table-hover table-responsive " id="myTable_">
+						{% if l_pastes %}
-						      <thead>
+							<table class="table table-striped table-bordered table-hover" id="myTable_1">
+						    <thead class="thead-dark">
 						      <tr>
-						          <th style="max-width: 800px;">Crawled Pastes</th>
+						        <th>Crawled Pastes</th>
 						      </tr>
 						    </thead>
 						    <tbody>
-
 					        {% for path in l_pastes %}
 					          <tr>
-						              <td><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{path}}">{{ path_name[loop.index0] }}</a>
+				              <td>
+												<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{path}}" class="text-secondary">
+													<div style="line-height:0.9;">{{ dict_links[path] }}</div>
+												</a>
 				                <div>
 				                  {% for tag in paste_tags[loop.index0] %}
 					                  <a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag[1] }}">
-						                    <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
+					                    <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
 					                  </a>
 				                  {%  endfor %}
 				                </div>
 			              	</td>
 						        </tr>
 						      {%  endfor %}
-
 						    </tbody>
 						  </table>
+						{%endif%}
 
 					</div>
-					</div>
 
-					<div class="col-md-6">
+					<div class="col-12 col-xl-6">
-						<div class="panel panel-info" style="text-align:center;">
+						<div class="card my-2" style="background-color:#ecf0f1;">
-							<div class="panel-heading">
+							<div class="card-body py-2">
 								<div class="row">
 									<div class="col-md-8">
-											<input class="center" id="blocks" type="range" min="1" max="50" value="13">
+										<input class="custom-range mt-2" id="blocks" type="range" min="1" max="50" value="13">
 									</div>
 									<div class="col-md-4">
-											<button class="btn btn-primary btn-tags" onclick="blocks.value=50;pixelate();">
+										<button class="btn btn-primary" onclick="blocks.value=50;pixelate();">
-													<span class="glyphicon glyphicon-zoom-in"></span>
+												<i class="fas fa-search-plu"></i>
 												<span class="label-icon">Full resolution</span>
 										</button>
 									</div>
@@ -140,25 +137,47 @@
 							</div>
 						</div>
 						<canvas id="canvas" style="width:100%;"></canvas>
+						<div class="text-center">
+							<small>
+								<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{screenshot['item']}}" class="text-info">
+									<div style="line-height:0.9;">{{dict_links[screenshot['item']]}}</div>
+								</a>
+							<small>
+						</div>
 					</div>
 
 				</div>
-
 			</div>
-            <!-- /#page-wrapper -->
+		</div>
+	</div>
+
+</body>
+
 
 <script>
+	var table;
   $(document).ready(function(){
-                    activePage = "page-hiddenServices"
+	table = $('#myTable_1').DataTable(
-                    $("#"+activePage).addClass("active");
-										table = $('#myTable_').DataTable(
 	{
-												"aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
+		//"aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
-												"iDisplayLength": 5,
+		//"iDisplayLength": 5,
-												"order": [[ 0, "desc" ]]
+		//"order": [[ 0, "desc" ]]
-											}
-										);
 	});
+});
+
+function toggle_sidebar(){
+	if($('#nav_menu').is(':visible')){
+		$('#nav_menu').hide();
+		$('#side_menu').removeClass('border-right')
+		$('#side_menu').removeClass('col-lg-2')
+		$('#core_content').removeClass('col-lg-10')
+	}else{
+		$('#nav_menu').show();
+		$('#side_menu').addClass('border-right')
+		$('#side_menu').addClass('col-lg-2')
+		$('#core_content').addClass('col-lg-10')
+	}
+}
 </script>
 
 <script>
@@ -172,10 +191,9 @@
 img.addEventListener("error", img_error);
 var draw_img = false;
 
-							img.src = "{{ url_for('showsavedpastes.screenshot', filename=screenshot) }}";
+img.src = "{{ url_for('showsavedpastes.screenshot', filename=screenshot['screenshot']) }}";
 
 function pixelate() {
-
   /// use slider value
   if( blocks.value == 50 ){
     size = 1;
@@ -208,6 +226,4 @@
 }
 </script>
 
-        </body>
-
 </html>

var/www/modules/showpaste/templates/show_saved_paste.html

@@ -433,7 +433,7 @@
                         <tbody>
                           <tr>
                             <td>Domain</td>
-                            <td><a target="_blank" href="{{ url_for('hiddenServices.onion_domain') }}?onion_domain={{ crawler_metadata['domain'] }}" id='onion_domain'>{{ crawler_metadata['domain'] }}</a></td>
+                            <td><a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td>
                           </tr>
                           <tr>
                             <td>Father</td>