chg: [UI crawler, show_domain] add domain history list + navigation

bin/packages/HiddenServices.py

@@ -17,6 +17,7 @@ Conditions to fulfill to be able to use this class correctly:
 """
 
 import os
+import time
 import gzip
 import redis
 import random
@@ -124,6 +125,22 @@ class HiddenServices(object):
             for tag in p_tags:
                 self.tags[tag] = self.tags.get(tag, 0) + 1
 
+    def extract_epoch_from_history(self, crawled_history):
+        epoch_list = []
+        for res, epoch_val in crawled_history:
+            # domain down
+            if res == epoch_val:
+                status = False
+            # domain up
+            else:
+                status = True
+            epoch_val = int(epoch_val) # force int
+            epoch_list.append((epoch_val, time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(epoch_val)), status))
+        return epoch_list
+
+    def get_domain_crawled_history(self):
+        return self.r_serv_onion.zrange('crawler_history_{}:{}:{}'.format(self.type, self.domain, self.port), 0, -1, withscores=True)
+
     def get_first_crawled(self):
         res = self.r_serv_onion.zrange('crawler_history_{}:{}:{}'.format(self.type, self.domain, self.port), 0, 0, withscores=True)
         if res:

var/www/modules/hiddenServices/Flask_hiddenServices.py

@@ -747,6 +747,7 @@ def show_domain():
     if first_seen is None:
         first_seen = '********'
     first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8])
+    ports = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'ports')
     origin_paste = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'paste_parent')
 
     h = HiddenServices(domain, type, port=port)
@@ -776,9 +777,12 @@ def show_domain():
         p_tags = r_serv_metadata.smembers('tag:'+path)
         paste_tags.append(unpack_paste_tags(p_tags))
 
+    domain_history = h.extract_epoch_from_history(h.get_domain_crawled_history())
+
     return render_template("showDomain.html", domain=domain, last_check=last_check, first_seen=first_seen,
                             l_pastes=l_pastes, paste_tags=paste_tags, bootstrap_label=bootstrap_label,
                             dict_links=dict_links, port=port, epoch=epoch,
+                            ports=ports, domain_history=domain_history,
                             origin_paste_tags=origin_paste_tags, status=status,
                             origin_paste=origin_paste, origin_paste_name=origin_paste_name,
                             domain_tags=domain_tags, screenshot=screenshot)

var/www/modules/hiddenServices/templates/showDomain.html

@@ -53,12 +53,14 @@
 										<tr>
 											<th>First Seen</th>
 											<th>Last Check</th>
+                      <th>Ports</th>
 										</tr>
 									</thead>
 									<tbody>
 										<tr>
 											<td class="panelText">{{ first_seen }}</td>
 											<td class="panelText">{{ last_check }}</td>
+                      <td class="panelText">{{ ports }}</td>
 										</tr>
 									</tbody>
 								</table>
@@ -127,6 +129,34 @@
   						</a>
             </div>
 
+            {% if domain_history %}
+							<table class="table table-hover mt-2" id="myTable_2">
+						    <thead>
+						      <tr>
+						        <th><i class="fas fa-history"></i> Domain History</th>
+						      </tr>
+						    </thead>
+						    <tbody>
+					        {% for epoch_item in domain_history %}
+					          <tr>
+				              <td class="{% if epoch_item[0]==epoch %}text-dark table-info{% endif %}">
+												<a target="_blank" href="{{ url_for('hiddenServices.show_domain') }}?domain={{domain}}&port={{port}}&epoch={{epoch_item[0]}}" class="text-secondary">
+													<div class="d-flex justify-content-around" style="line-height:0.9;">
+                            <div>{{domain}}</div>
+                            {% if epoch_item[2] %}
+                              <div style="color:Green;"><i class="fas fa-check-circle"></i> UP</div>
+                            {% else %}
+                              <div style="color:Red;"><i class="fas fa-times-circle"></i>	DOWN</div>
+                            {% endif %}
+                            <div>{{ epoch_item[1] }}</div>
+                          </div>
+												</a>
+						        </tr>
+						      {%  endfor %}
+						    </tbody>
+						  </table>
+						{%endif%}
+
 					</div>
 
 					<div class="col-12 col-xl-6">