Merge branch 'master' into user_management

bin/MISP_The_Hive_feeder.py

@@ -55,21 +55,22 @@ from thehive4py.models import Case, CaseTask, CustomFieldHelper
 
 
 def create_the_hive_alert(source, path, tag):
-    tags = list(r_serv_metadata.smembers('tag:'+path))
+    # # TODO: check items status (processed by all modules)
+    # # TODO: add item metadata: decoded content, link to auto crawled content, pgp correlation, cryptocurrency correlation...
+    # # # TODO: description, add AIL link:show items ?
+    tags = list( r_serv_metadata.smembers('tag:{}'.format(path)) )
 
     artifacts = [
         AlertArtifact( dataType='uuid-ail', data=r_serv_db.get('ail:uuid') ),
         AlertArtifact( dataType='file', data=path, tags=tags )
     ]
 
-    l_tags = tag.split(',')
-
     # Prepare the sample Alert
     sourceRef = str(uuid.uuid4())[0:6]
     alert = Alert(title='AIL Leak',
                   tlp=3,
-                  tags=l_tags,
+                  tags=tags,
-                  description='infoleak',
+                  description='AIL Leak, triggered by {}'.format(tag),
                   type='ail',
                   source=source,
                   sourceRef=sourceRef,

bin/PgpDump.py

@@ -82,7 +82,12 @@ def get_pgp_packet(message, save_path):
         process1 = subprocess.Popen([ 'echo', '-e', save_path], stdout=subprocess.PIPE)
         process2 = subprocess.Popen([ 'pgpdump'], stdin=process1.stdout, stdout=subprocess.PIPE)
         process1.stdout.close()
-        output = process2.communicate()[0].decode()
+        output = process2.communicate()[0]
+        try:
+            output = output.decode()
+        except UnicodeDecodeError:
+            publisher.error('Error PgpDump UnicodeDecodeError: {}'.format(message))
+            output = ''
         return output
 
 def get_pgp_packet_file(file):

bin/packages/HiddenServices.py

@@ -17,10 +17,14 @@ Conditions to fulfill to be able to use this class correctly:
 """
 
 import os
+import time
 import gzip
 import redis
 import random
 
+from io import BytesIO
+import zipfile
+
 import configparser
 import sys
 sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
@@ -71,6 +75,7 @@ class HiddenServices(object):
             self.paste_crawled_directory = os.path.join(self.paste_directory, cfg.get("Directories", "crawled"))
             self.paste_crawled_directory_name = cfg.get("Directories", "crawled")
             self.screenshot_directory = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot"))
+            self.screenshot_directory_screenshot = os.path.join(self.screenshot_directory, 'screenshot')
         elif type == 'i2p':
             self.paste_directory = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot"))
             self.screenshot_directory = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot"))
@@ -120,6 +125,26 @@ class HiddenServices(object):
             for tag in p_tags:
                 self.tags[tag] = self.tags.get(tag, 0) + 1
 
+    def extract_epoch_from_history(self, crawled_history):
+        epoch_list = []
+        for res, epoch_val in crawled_history:
+            epoch_val = int(epoch_val) # force int
+            try:
+                # domain down
+                if int(res) == epoch_val:
+                    status = False
+                # domain up
+                else:
+                    status = True
+            except ValueError:
+                status = True
+            epoch_val = int(epoch_val) # force int
+            epoch_list.append((epoch_val, time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(epoch_val)), status))
+        return epoch_list
+
+    def get_domain_crawled_history(self):
+        return self.r_serv_onion.zrange('crawler_history_{}:{}:{}'.format(self.type, self.domain, self.port), 0, -1, withscores=True)
+
     def get_first_crawled(self):
         res = self.r_serv_onion.zrange('crawler_history_{}:{}:{}'.format(self.type, self.domain, self.port), 0, 0, withscores=True)
         if res:
@@ -230,6 +255,13 @@ class HiddenServices(object):
         return l_crawled_pastes
     '''
 
+    def get_item_screenshot(self, item):
+        screenshot = self.r_serv_metadata.hget('paste_metadata:{}'.format(item), 'screenshot')
+        if screenshot:
+            screenshot =  os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:])
+            return screenshot
+        return ''
+
     def get_domain_random_screenshot(self, l_crawled_pastes, num_screenshot = 1):
         l_screenshot_paste = []
         for paste in l_crawled_pastes:
@@ -237,9 +269,8 @@ class HiddenServices(object):
             origin_paste = paste
             paste= paste.replace(self.paste_directory+'/', '')
 
-            screenshot = self.r_serv_metadata.hget('paste_metadata:{}'.format(paste), 'screenshot')
+            screenshot = self.get_item_screenshot(paste)
             if screenshot:
-                screenshot =  os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:])
                 l_screenshot_paste.append({'screenshot': screenshot, 'item': origin_paste})
 
         if len(l_screenshot_paste) > num_screenshot:
@@ -250,6 +281,35 @@ class HiddenServices(object):
         else:
             return l_screenshot_paste
 
+    def get_all_domain_screenshot(self, l_crawled_pastes, filename=False):
+        l_screenshot_paste = []
+        for paste in l_crawled_pastes:
+            ## FIXME: # TODO: remove me
+            origin_paste = paste
+            paste= paste.replace(self.paste_directory+'/', '')
+
+            screenshot = self.get_item_screenshot(paste)
+            if screenshot:
+                screenshot = screenshot + '.png'
+                screenshot_full_path = os.path.join(self.screenshot_directory_screenshot, screenshot)
+                if filename:
+                    screen_file_name = os.path.basename(paste) + '.png'
+                    l_screenshot_paste.append( (screenshot_full_path, screen_file_name) )
+                else:
+                    l_screenshot_paste.append(screenshot_full_path)
+        return l_screenshot_paste
+
+    def get_all_item_full_path(self, l_items, filename=False):
+        l_full_items = []
+        for item in l_items:
+            item = os.path.join(self.PASTES_FOLDER, item)
+            if filename:
+                file_name = os.path.basename(item) + '.gz'
+                l_full_items.append( (item, file_name) )
+            else:
+                l_full_items.append(item)
+        return l_full_items
+
     def get_crawled_pastes_by_date(self, date):
 
         pastes_path = os.path.join(self.paste_crawled_directory, date[0:4], date[4:6], date[6:8])
@@ -258,6 +318,63 @@ class HiddenServices(object):
         l_crawled_pastes = []
         return l_crawled_pastes
 
+    def get_all_har(self, l_pastes, filename=False):
+        all_har = []
+        for item in l_pastes:
+            if filename:
+                all_har.append( (self.get_item_har(item), os.path.basename(item) + '.json') )
+            else:
+                all_har.append(self.get_item_har(item))
+        return all_har
+
+
+    def get_item_har(self, item_path):
+        item_path = item_path.replace('{}/'.format(self.paste_crawled_directory_name), '', 1)
+        har_path = os.path.join(self.screenshot_directory, item_path) + '.json'
+        return har_path
+
+    def create_domain_basic_archive(self, l_pastes):
+        all_har = self.get_all_har(l_pastes, filename=True)
+        all_screenshot = self.get_all_domain_screenshot(l_pastes, filename=True)
+        all_items = self.get_all_item_full_path(l_pastes, filename=True)
+
+        # try:
+
+        # zip buffer
+        zip_buffer = BytesIO()
+
+        with zipfile.ZipFile(zip_buffer, "a") as zf:
+
+            #print(all_har)
+            self.write_in_zip_buffer(zf, all_har)
+            self.write_in_zip_buffer(zf, all_screenshot)
+            self.write_in_zip_buffer(zf, all_items)
+
+            # write map url
+            map_file_content = self.get_metadata_file(l_pastes).encode()
+            zf.writestr( '_URL_MAP_', BytesIO(map_file_content).getvalue())
+
+        zip_buffer.seek(0)
+        return zip_buffer
+
+        # except Exception as e:
+        #     print(e)
+        #     return 'Server Error'
+
+    def write_in_zip_buffer(self, zf, list_file):
+        for file_path, file_name in list_file:
+            with open(file_path, "rb") as f:
+                har_content = f.read()
+                zf.writestr( file_name, BytesIO(har_content).getvalue())
+
+    def get_metadata_file(self, list_items):
+        file_content = ''
+        dict_url = self.get_all_links(list_items)
+        for key in dict_url:
+            file_content = '{}\n{}    :    {}'.format(file_content, os.path.basename(key), dict_url[key])
+        return file_content
+
+
     '''
     def get_last_crawled_pastes_fileSearch(self):
 

var/www/modules/dashboard/templates/index.html

@@ -2,23 +2,40 @@
 <html>
 
 <head>
-  <meta charset="utf-8">
-  <meta name="viewport" content="width=device-width, initial-scale=1.0">
-
   <title>Analysis Information Leak framework Dashboard</title>
   <link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
 
   <!-- Core CSS -->
-  <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
   <link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
-  <link href="{{ url_for('static', filename='css/switch_checkbox.css') }}" rel="stylesheet" type="text/css" />
+
   <!-- JS -->
   <script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
   <script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
   <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
   <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
+  <script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+
+  <script type="text/javascript" src="{{ url_for('static', filename='js/indexjavascript.js')}}"
+      data-urlstuff="{{ url_for('dashboard.stuff') }}" data-urllog="{{ url_for('dashboard.logs') }}">
+  </script>
+
+  <style>
+			.tableQueue tbody tr td,
+      .tableQueue tbody tr th,
+      .tableQueue thead tr td,
+      .tableQueue thead tr th{
+				padding: 1px;
+			}
+      tr.table-disabled {
+        background-color: #f2f2f2;
+      }
+      tr.table-log-warning {
+        background-color: #ccfae3;
+      }
+	</style>
+
   <script>
       window.default_minute = {{ default_minute }};
       window.glob_tabvar = []; // Avoid undefined
@@ -31,184 +48,127 @@
       };
       update_values();
   </script>
-  <style>
-			.tableQueue tbody tr td,
-      .tableQueue tbody tr th,
-      .tableQueue thead tr td,
-      .tableQueue thead tr th{
-				padding: 1px;
-			}
-	</style>
 
 </head>
 <body>
 
-    <div id="wrapper">
+
-      <nav class="navbar navbar-inverse navbar-static-top nav">
+{% include 'nav_bar.html' %}
-          {% include 'header.html' %}
+
-          <!-- /.navbar-top-links -->
+	<div class="container-fluid">
-          <div class="navbar-default sidebar" role="navigation">
+		<div class="row">
-              <div class="sidebar-collapse">
+
-                  <ul class="nav" id="side-menu">
+			{% include 'dashboard/menu_sidebar.html' %}
-                      <li class="sidebar-search">
+
-                          <a href="{{ url_for('dashboard.index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
+			<div class="col-12 col-lg-10" id="core_content">
-                          {% include 'searchbox.html' %}
-                          <!-- /input-group -->
-                      </li>
-                 </ul>
-                  <!-- /#side-menu -->
-              </div>
-              <!-- /.sidebar-collapse -->
-              <div class="panel panel-default">
-                  <div class="panel-heading">
-                      <i class="fa fa-dashboard  fa-fw"></i> Total pastes since {{ default_minute }} min
-                  </div>
-                  <div class="panel-body">
-                       <div id="global" style="height: 90px; padding: 0px; position: relative;"></div>
-                  </div>
-              </div>
-                  <!-- <div id="Graph_paste_num" style="height:90px; width:100%;"></div> -->
-              <div class='pull_right'>
-                  <label class="switch">
-                      <input id="button-toggle-queues" class="switch-input" type="checkbox" checked>
-                      <span class="switch-label" data-on="On" data-off="Off"></span>
-                      <span class="switch-handle"></span>
-                  </label>
-                  <strong style="top: 3px; position: relative;">Display queues</strong>
-              <div>
-              <div>
-                  <table id="queue-color-legend" class="table">
-                  <thead>
-                  </thead>
-                  <tbody>
-                      <tr><td class="legendColorBox" style="vertical-align: ; "><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #d0e9c6;overflow:hidden"></div></div></td><td> Working queues</td></tr>
-                      <tr><td class="legendColorBox" style="vertical-align: ;"><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #faf2cc;overflow:hidden"></div></div></td><td> Idling queues</td></tr>
-                      <tr><td class="legendColorBox" style="vertical-align: ;"><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #ebcccc;overflow:hidden"></div></div></td><td> Stuck queues</td></tr>
-                  </tbody>
-                  </table>
-              </div>
-              <div class="table-responsive", id="queueing" style="margin-top:10px; font-size: small;"></div>
-             <a  href="{{ url_for('dashboard.index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
-          </div>
-          <!-- /.navbar-static-side -->
-      </nav>
-        <div id="page-wrapper">
-        <div class="row">
-                <div class="col-lg-12">
-		</br>
-                </div>
-        </div>
 
         {%if update_in_progress%}
-          <div class="alert alert-warning alert-dismissible fade in">
+          <div class="alert alert-warning alert-dismissible fade show my-2" role="alert">
-            <a href="#" class="close" data-dismiss="alert" aria-label="close">&times;</a>
             <strong>Warning!</strong> {{update_warning_message}} <strong>{{update_warning_message_notice_me}}</strong>
             (<a href="{{ url_for('settings.settings_page') }}">Check Update Status</a>)
+            <button type="button" class="close" data-dismiss="alert" aria-label="Close">
+              <span aria-hidden="true">&times;</span>
+            </button>
           </div>
         {%endif%}
 
-        <div class="alert alert-info alert-dismissible fade in">
+        <div class="alert alert-info alert-dismissible fade show mt-1" role="alert">
-          <a href="#" class="close" data-dismiss="alert" aria-label="close">&times;</a>
+            <strong>Bootstrap 4 migration!</strong> Some pages are still in bootstrap 3. You can check the migration progress <strong><a href="https://github.com/CIRCL/AIL-framework/issues/330" target="_blank">Here</a></strong>.
-          <strong>Bootstrap 4 migration!</strong> Some pages are still in bootstrap 3. You can check the migration progress <strong><a href="https://github.com/CIRCL/AIL-framework/issues/330" target="_blank">Here</a></strong>.
+          <button type="button" class="close" data-dismiss="alert" aria-label="Close">
+            <span aria-hidden="true">&times;</span>
+          </button>
         </div>
 
-            <div class="row">
+        <div class="row my-2">
-                 <div class="col-lg-6">
-                     <div class="panel panel-default">
-                        <div class="panel-heading">
-                           <i class="fa fa-bar-chart-o fa-fw"></i> Feeder(s) Monitor:
-                        </div>
-                        <div id="panelbody" class="panel-body" style="height:420px;">
-
-                       <strong>Processed pastes</strong>
-                       <div id="Proc_feeder" style="height: 230px; padding: 0px; position: relative;"></div>
-                       <hr style="border-top: 2px solid #eee; margin-top: 7px; margin-bottom: 7px;">
-                       <strong>Filtered duplicates</strong>
-                       <div id="Dup_feeder" style="height: 100px; padding: 0px; position: relative;"></div>
-
-                        </div>
-                     <!-- /.panel-body -->
-                     </div>
-                      <!-- /.panel -->
-                 </div>
-                 <!-- /.col-lg-6 -->
-                 <div class="col-lg-6">
-                     <div class="panel panel-default">
-                        <div class="panel-heading">
-                           <i class="fa fa-bar-chart-o fa-fw"></i> Queues Monitor
-                        </div>
-                        <div class="panel-body">
-                           <div class="row">
-                               <div class="col-lg-6" id="Graph" style="height:195px; width:88%;"></div>
-                               <div style="height:10px;"></div>
-                               <div class="col-lg-6" id="Graph2" style="height:195px; width:88%;"></div>
-                           </div>
-                        <!-- /.row -->
-                        </div>
-                     <!-- /.panel-body -->
-                     </div>
-                      <!-- /.panel -->
-                 </div>
-                 <!-- /.col-lg-6 -->
-        <div class="col-lg-12">
-        <div class="panel panel-default">
-        <div class="panel-heading">
-            <i class="fa fa-tasks fa-fw"></i> Logs
-            <div class="pull-right">
-                <label style="padding-bottom:2px;">
-		<select class="form-control input-sm" id="log_select">
-      {% for log_selection in log_select %}
-        {% if log_selection == selected %}
-          <option value="{{ log_selection }}" selected>{{ log_selection }}</option>
-        {% else %}
-    		  <option value="{{ log_selection }}">{{ log_selection }}</option>
-        {% endif %}
-      {% endfor %}
-		</select>
-                </label>
-		<input id="checkbox_log_info" type="checkbox" value="info"> INFO
-		<input id="checkbox_log_warning" type="checkbox" value="warning" checked="true"> WARNING
-		<input id="checkbox_log_critical" type="checkbox" value="critical" checked="true"> CRITICAL
 
+          <div class="col-xl-6">
+            <div class="card">
+              <div class="card-header">
+                <i class="far fa-chart-bar"></i> Feeder(s) Monitor:
+              </div>
+              <div class="card-body" id="panelbody" style="height:420px;">
+                <strong>Processed pastes</strong>
+                <div id="Proc_feeder" style="height: 230px; padding: 0px; position: relative;"></div>
+                <hr style="border-top: 2px solid #eee; margin-top: 7px; margin-bottom: 7px;">
+                <strong>Filtered duplicates</strong>
+                <div id="Dup_feeder" style="height: 100px; padding: 0px; position: relative;"></div>
+              </div>
             </div>
+          </div>
+
+          <div class="col-xl-6">
+            <div class="card">
+              <div class="card-header">
+                <i class="far fa-chart-bar"></i> Queues Monitor
+              </div>
+              <div class="card-body">
+                <div class="" id="Graph" style="height:195px; width:88%;"></div>
+                <div style="height:10px;"></div>
+                <div class="" id="Graph2" style="height:195px; width:88%;"></div>
+              </div>
+
+              </div>
+            </div>
+          </div>
+
+
+          <div class="card">
+            <div class="card-header">
+              <i class="fas fa-clipboard-list"></i> Logs
+              <div class="float-right">
+                <label style="padding-bottom:2px;">
+              		<select class="form-control input-sm" id="log_select">
+                    {% for log_selection in log_select %}
+                      {% if log_selection == selected %}
+                        <option value="{{ log_selection }}" selected>{{ log_selection }}</option>
+                      {% else %}
+                  		  <option value="{{ log_selection }}">{{ log_selection }}</option>
+                      {% endif %}
+                    {% endfor %}
+              		</select>
+                </label>
+              	<input id="checkbox_log_info" type="checkbox" value="INFO"> INFO
+              	<input id="checkbox_log_warning" type="checkbox" value="WARNING" checked="true"> WARNING
+              	<input id="checkbox_log_critical" type="checkbox" value="CRITICAL" checked="true"> CRITICAL
+
+              </div>
+            </div>
+
+            <div class="card-body">
+        		  <table class="table table-hover table-sm" id="table_log">
+            		<thead>
+            			<tr>
+              		  <th>Time</th>
+              			<th>Channel</th>
+              			<th>Level</th>
+              			<th>Script Name</th>
+              			<th>Source</th>
+              			<th>Date</th>
+              			<th>Paste name</th>
+              			<th>Message</th>
+              			<th>Actions</th>
+            			</tr>
+            		</thead>
+            		<tbody id="tab_body">
+            		</tbody>
+              </table>
+            </div>
+          </div>
+
+
         </div>
-        <div class="panel-body">
+
-                    <div class="row">
+
-			    <div class="col-lg-12">
-				    <table class="table table-bordered table-hover table-striped" id="table_log">
-					<thead>
-						<tr>
-						<th>Time</th>
-						<th>Channel</th>
-						<th>Level</th>
-						<th>Script Name</th>
-						<th>Source</th>
-						<th>Date</th>
-						<th>Paste name</th>
-						<th>Message</th>
-						<th>Actions</th>
-						</tr>
-					</thead>
-					<tbody id="tab_body">
-					</tbody>
-			            </table>
-			    </div>
-                   </div>
-        </div>
-            <!-- /.row -->
-        </div>
-        <!-- /#page-wrapper -->
 
     </div>
+  </div>
 
-    <script> var url_showSavedPath = "{{ url_for('showsavedpastes.showsavedpaste') }}"; </script>
+</body>
-    <script type="text/javascript" src="{{ url_for('static', filename='js/indexjavascript.js')}}"
+
-      data-urlstuff="{{ url_for('dashboard.stuff') }}" data-urllog="{{ url_for('dashboard.logs') }}">
+<script> var url_showSavedPath = "{{ url_for('showsavedpastes.showsavedpaste') }}"; </script>
-    </script>
 
     <script>
-        activePage = "page-index";
+        $("#page-Dashboard").addClass("active");
-        $("#"+activePage).addClass("active");
 
         var tableBody = document.getElementById('tab_body')
         $.getJSON("{{ url_for('dashboard.get_last_logs_json') }}", function(data) {
@@ -224,7 +184,7 @@
             var msage = document.createElement('TD')
             var inspect = document.createElement('TD')
 
-            tr.className = "warning";
+            tr.className = "table-log-warning";
             time.appendChild(document.createTextNode(d.time))
             chan.appendChild(document.createTextNode('Script'))
             level.appendChild(document.createTextNode('WARNING'))
@@ -236,10 +196,10 @@
             var iconspan = document.createElement('SPAN');
             var message = d.message.split(" ")
             if (message[0] == "Detected"){
-                iconspan.className = "glyphicon glyphicon-eye-open";
+                iconspan.className = "fas fa-eye";
             }
             else if (message[0] == "Checked"){
-                iconspan.className = "glyphicon glyphicon-thumbs-up";
+                iconspan.className = "far fa-thumbs-up";
             }
             iconspan.innerHTML = "&nbsp;";
             msage.appendChild(iconspan);
@@ -249,7 +209,7 @@
             action_icon_a.setAttribute("TARGET", "_blank");
             action_icon_a.setAttribute("HREF", d.path);
             var action_icon_span = document.createElement('SPAN');
-            action_icon_span.className = "fa fa-search-plus";
+            action_icon_span.className = "fas fa-search-plus";
             action_icon_a.appendChild(action_icon_span);
             inspect.appendChild(action_icon_a)
             inspect.setAttribute("style", "text-align:center;");
@@ -269,14 +229,22 @@
 
 		      });
         });
-    </script>
 
-    <script type="text/javascript">
+function toggle_sidebar(){
-        initfunc( "{{ url_for('static', filename='csv/wordstrendingdata.csv') }}", {{ request.script_root|tojson|safe }} );
+  if($('#nav_menu').is(':visible')){
-    </script>
+    $('#nav_menu').hide();
-
+    $('#side-bard-dashboard-content').hide();
-    <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
+    $('#side_menu').removeClass('border-right')
-
+    $('#side_menu').removeClass('col-lg-2')
-</body>
+    $('#core_content').removeClass('col-lg-10')
+  }else{
+    $('#nav_menu').show();
+    $('#side-bard-dashboard-content').show();
+    $('#side_menu').addClass('border-right')
+    $('#side_menu').addClass('col-lg-2')
+    $('#core_content').addClass('col-lg-10')
+  }
+}
+</script>
 
 </html>

var/www/modules/hiddenServices/Flask_hiddenServices.py

@@ -11,7 +11,7 @@ import os
 import time
 import json
 from pyfaup.faup import Faup
-from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
+from flask import Flask, render_template, jsonify, request, send_file, Blueprint, redirect, url_for
 from flask_login import login_required
 
 from Date import Date
@@ -764,6 +764,7 @@ def show_domain():
     if first_seen is None:
         first_seen = '********'
     first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8])
+    ports = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'ports')
     origin_paste = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'paste_parent')
 
     h = HiddenServices(domain, type, port=port)
@@ -793,13 +794,67 @@ def show_domain():
         p_tags = r_serv_metadata.smembers('tag:'+path)
         paste_tags.append(unpack_paste_tags(p_tags))
 
+    domain_history = h.extract_epoch_from_history(h.get_domain_crawled_history())
+
     return render_template("showDomain.html", domain=domain, last_check=last_check, first_seen=first_seen,
                             l_pastes=l_pastes, paste_tags=paste_tags, bootstrap_label=bootstrap_label,
-                            dict_links=dict_links,
+                            dict_links=dict_links, port=port, epoch=epoch,
+                            ports=ports, domain_history=domain_history,
                             origin_paste_tags=origin_paste_tags, status=status,
                             origin_paste=origin_paste, origin_paste_name=origin_paste_name,
                             domain_tags=domain_tags, screenshot=screenshot)
 
+@hiddenServices.route("/crawlers/download_domain", methods=['GET'])
+def download_domain():
+    domain = request.args.get('domain')
+    epoch = request.args.get('epoch')
+    try:
+        epoch = int(epoch)
+    except:
+        epoch = None
+    port = request.args.get('port')
+    faup.decode(domain)
+    unpack_url = faup.get()
+
+    ## TODO: # FIXME: remove me
+    try:
+        domain = unpack_url['domain'].decode()
+    except:
+        domain = unpack_url['domain']
+
+    if not port:
+        if unpack_url['port']:
+            try:
+                port = unpack_url['port'].decode()
+            except:
+                port = unpack_url['port']
+        else:
+            port = 80
+    try:
+        port = int(port)
+    except:
+        port = 80
+    type = get_type_domain(domain)
+    if domain is None or not r_serv_onion.exists('{}_metadata:{}'.format(type, domain)):
+        return '404'
+        # # TODO: FIXME return 404
+
+    origin_paste = r_serv_onion.hget('{}_metadata:{}'.format(type, domain), 'paste_parent')
+
+    h = HiddenServices(domain, type, port=port)
+    item_core = h.get_domain_crawled_core_item(epoch=epoch)
+    if item_core:
+        l_pastes = h.get_last_crawled_pastes(item_root=item_core['root_item'])
+    else:
+        l_pastes = []
+    #dict_links = h.get_all_links(l_pastes)
+
+    zip_file = h.create_domain_basic_archive(l_pastes)
+    filename = domain + '.zip'
+
+    return send_file(zip_file, attachment_filename=filename, as_attachment=True)
+
+
 @hiddenServices.route("/hiddenServices/onion_son", methods=['GET'])
 @login_required
 def onion_son():

var/www/modules/hiddenServices/templates/showDomain.html

@@ -53,12 +53,14 @@
 										<tr>
 											<th>First Seen</th>
 											<th>Last Check</th>
+                      <th>Ports</th>
 										</tr>
 									</thead>
 									<tbody>
 										<tr>
 											<td class="panelText">{{ first_seen }}</td>
 											<td class="panelText">{{ last_check }}</td>
+                      <td class="panelText">{{ ports }}</td>
 										</tr>
 									</tbody>
 								</table>
@@ -119,6 +121,42 @@
 						  </table>
 						{%endif%}
 
+
+            <div class="text-center my-3">
+              <a href="{{ url_for('hiddenServices.download_domain') }}?domain={{domain}}&port={{port}}{% if epoch %}&epoch={{epoch}}{% endif %}" target="blank" style="font-size: 15px">
+  								<button class="btn btn-info"><i class="fas fa-download"></i> Download Domain Content
+  								</button>
+  						</a>
+            </div>
+
+            {% if domain_history %}
+							<table class="table table-hover mt-2" id="myTable_2">
+						    <thead>
+						      <tr>
+						        <th><i class="fas fa-history"></i> Domain History</th>
+						      </tr>
+						    </thead>
+						    <tbody>
+					        {% for epoch_item in domain_history %}
+					          <tr>
+				              <td class="{% if epoch_item[0]==epoch %}text-dark table-info{% endif %}">
+												<a href="{{ url_for('hiddenServices.show_domain') }}?domain={{domain}}&port={{port}}&epoch={{epoch_item[0]}}" class="text-secondary">
+													<div class="d-flex justify-content-around" style="line-height:0.9;">
+                            <div>{{domain}}</div>
+                            {% if epoch_item[2] %}
+                              <div style="color:Green;"><i class="fas fa-check-circle"></i> UP</div>
+                            {% else %}
+                              <div style="color:Red;"><i class="fas fa-times-circle"></i>	DOWN</div>
+                            {% endif %}
+                            <div>{{ epoch_item[1] }}</div>
+                          </div>
+												</a>
+						        </tr>
+						      {%  endfor %}
+						    </tbody>
+						  </table>
+						{%endif%}
+
 					</div>
 
 					<div class="col-12 col-xl-6">

var/www/static/js/indexjavascript.js

@@ -201,92 +201,94 @@ function create_log_table(obj_json) {
         return;
     }
 
-    if( chansplit[1] == "INFO" ){
+    if( parsedmess.length>2 ){
-        tr.className = "info";
+      if( chansplit[1] == "INFO" ){
-    }
+          tr.className = "table-disabled";
-    else if ( chansplit[1] == "WARNING" ){
+      }
-        tr.className = "warning";
+      else if ( chansplit[1] == "WARNING" ){
-    }
+          tr.className = "table-log-warning";
-    else if ( chansplit[1] == "CRITICAL"){
+      }
-        tr.className = "danger"
+      else if ( chansplit[1] == "CRITICAL"){
-    }
+          tr.className = "table-danger"
+      }
 
-    source_link = document.createElement("A");
+      source_link = document.createElement("A");
-    if (parsedmess[1] == "slexy.org"){
+      if (parsedmess[1] == "slexy.org"){
-        source_url = "http://"+parsedmess[1]+"/view/"+parsedmess[3].split(".")[0];
+          source_url = "http://"+parsedmess[1]+"/view/"+parsedmess[3].split(".")[0];
-    }
+      }
-    else{
+      else{
-        source_url = "http://"+parsedmess[1]+"/"+parsedmess[3].split(".")[0];
+          source_url = "http://"+parsedmess[1]+"/"+parsedmess[3].split(".")[0];
-    }
+      }
-    source_link.setAttribute("HREF",source_url);
+      source_link.setAttribute("HREF",source_url);
-    source_link.setAttribute("TARGET", "_blank");
+      source_link.setAttribute("TARGET", "_blank");
-    source_link.appendChild(document.createTextNode(parsedmess[1]));
+      source_link.appendChild(document.createTextNode(parsedmess[1]));
 
-    src.appendChild(source_link);
+      src.appendChild(source_link);
 
-    var now = new Date();
+      var now = new Date();
-    var timepaste = pad_2(now.getHours()) + ":" + pad_2(now.getMinutes()) + ":" + pad_2(now.getSeconds());
+      var timepaste = pad_2(now.getHours()) + ":" + pad_2(now.getMinutes()) + ":" + pad_2(now.getSeconds());
 
-    time.appendChild(document.createTextNode(timepaste));
+      time.appendChild(document.createTextNode(timepaste));
-    chan.appendChild(document.createTextNode(chansplit[0]));
+      chan.appendChild(document.createTextNode(chansplit[0]));
-    level.appendChild(document.createTextNode(chansplit[1]));
+      level.appendChild(document.createTextNode(chansplit[1]));
 
-    scrpt.appendChild(document.createTextNode(parsedmess[0]));
+      scrpt.appendChild(document.createTextNode(parsedmess[0]));
-    pdate.appendChild(document.createTextNode(parsedmess[2]));
+      pdate.appendChild(document.createTextNode(parsedmess[2]));
-    nam.appendChild(document.createTextNode(parsedmess[3]));
+      nam.appendChild(document.createTextNode(parsedmess[3]));
 
-    var iconspan = document.createElement('SPAN');
+      var iconspan = document.createElement('SPAN');
-    if (parsedmess[4].split(" ")[0] == "Detected"){
+      if (parsedmess[4].split(" ")[0] == "Detected"){
-        iconspan.className = "glyphicon glyphicon-eye-open";
+          iconspan.className = "fas fa-eye";
-    }
+      }
-    else if (parsedmess[4].split(" ")[0] == "Checked"){
+      else if (parsedmess[4].split(" ")[0] == "Checked"){
-        iconspan.className = "glyphicon glyphicon-thumbs-up";
+          iconspan.className = "far fa-thumbs-up";
-    }
+      }
-    iconspan.innerHTML = " ";
+      iconspan.innerHTML = " ";
-    msage.appendChild(iconspan);
+      msage.appendChild(iconspan);
-    var message = parsedmess[4].split(" ");
+      var message = parsedmess[4].split(" ");
-    message.shift();
+      message.shift();
 
-    msage.appendChild(document.createTextNode(message.join(" ")));
+      msage.appendChild(document.createTextNode(message.join(" ")));
 
-    var paste_path = parsedmess[5];
+      var paste_path = parsedmess[5];
-    var url_to_saved_paste = url_showSavedPath+"?paste="+paste_path+"&num="+parsedmess[0];
+      var url_to_saved_paste = url_showSavedPath+"?paste="+paste_path+"&num="+parsedmess[0];
 
-    var action_icon_a = document.createElement("A");
+      var action_icon_a = document.createElement("A");
-    action_icon_a.setAttribute("TARGET", "_blank");
+      action_icon_a.setAttribute("TARGET", "_blank");
-    action_icon_a.setAttribute("HREF", url_to_saved_paste);
+      action_icon_a.setAttribute("HREF", url_to_saved_paste);
-    var action_icon_span = document.createElement('SPAN');
+      var action_icon_span = document.createElement('SPAN');
-    action_icon_span.className = "fa fa-search-plus";
+      action_icon_span.className = "fas fa-search-plus";
-    action_icon_a.appendChild(action_icon_span);
+      action_icon_a.appendChild(action_icon_span);
 
-    inspect.appendChild(action_icon_a);
+      inspect.appendChild(action_icon_a);
-    inspect.setAttribute("style", "text-align:center;");
+      inspect.setAttribute("style", "text-align:center;");
 
-    tr.appendChild(time)
+      tr.appendChild(time)
-    tr.appendChild(chan);
+      tr.appendChild(chan);
-    tr.appendChild(level);
+      tr.appendChild(level);
-    tr.appendChild(scrpt);
+      tr.appendChild(scrpt);
-    tr.appendChild(src);
+      tr.appendChild(src);
-    tr.appendChild(pdate);
+      tr.appendChild(pdate);
-    tr.appendChild(nam);
+      tr.appendChild(nam);
-    tr.appendChild(msage);
+      tr.appendChild(msage);
-    tr.appendChild(inspect);
+      tr.appendChild(inspect);
 
-    if (tr.className == document.getElementById("checkbox_log_info").value && document.getElementById("checkbox_log_info").checked  == true) {
+      if (chansplit[1] == document.getElementById("checkbox_log_info").value && document.getElementById("checkbox_log_info").checked  == true) {
-           tableBody.appendChild(tr);
+             tableBody.appendChild(tr);
-       }
+         }
-    if (tr.className == document.getElementById("checkbox_log_warning").value && document.getElementById("checkbox_log_warning").checked == true) {
+      if (chansplit[1] == document.getElementById("checkbox_log_warning").value && document.getElementById("checkbox_log_warning").checked == true) {
-        tableBody.appendChild(tr);
+          tableBody.appendChild(tr);
-    }
+      }
-    if (tr.className == document.getElementById("checkbox_log_critical").value && document.getElementById("checkbox_log_critical").checked == true) {
+      if (chansplit[1] == document.getElementById("checkbox_log_critical").value && document.getElementById("checkbox_log_critical").checked == true) {
-        tableBody.appendChild(tr);
+          tableBody.appendChild(tr);
-    };
+      };
 
-    var sel = document.getElementById("log_select")
+      var sel = document.getElementById("log_select")
-    if (tableBody.rows.length > sel.options[sel.options.selectedIndex].value) {
+      if (tableBody.rows.length > sel.options[sel.options.selectedIndex].value) {
-        while (tableBody.rows.length != sel.options[sel.options.selectedIndex].value){
+          while (tableBody.rows.length != sel.options[sel.options.selectedIndex].value){
-            tableBody.deleteRow(0);
+              tableBody.deleteRow(0);
-        }
+          }
+      }
     }
 }
 
@@ -294,7 +296,7 @@ function create_queue_table() {
     document.getElementById("queueing").innerHTML = "";
     var Tablediv = document.getElementById("queueing")
     var table = document.createElement('TABLE')
-    table.className = "table table-bordered table-hover table-striped tableQueue";
+    table.className = "table table-bordered table-hover tableQueue";
     var tableHead = document.createElement('THEAD')
     var tableBody = document.createElement('TBODY')
 
@@ -323,8 +325,8 @@ function create_queue_table() {
         var td2 = document.createElement('TD');
         td.appendChild(document.createTextNode("No running queues"));
         td2.appendChild(document.createTextNode("Or no feed"));
-        td.className += " danger";
+        td.className += " table-danger";
-        td2.className += " danger";
+        td2.className += " table-danger";
         tr.appendChild(td);
         tr.appendChild(td2);
         tableBody.appendChild(tr);
@@ -345,11 +347,11 @@ function create_queue_table() {
             // - j=2: LastProcessedPasteTime
             // - j=3: Number of the module belonging in the same category
             if (parseInt(glob_tabvar.row1[i][2]) > window.threshold_stucked_module && parseInt(glob_tabvar.row1[i][1]) > 2)
-                tr.className += " danger";
+                tr.className += " table-danger";
             else if (parseInt(glob_tabvar.row1[i][1]) == 0)
-                tr.className += " warning";
+                tr.className += " table-disabled";
             else
-                tr.className += " success";
+                tr.className += " table-success";
             tableBody.appendChild(tr);
         }
     }
@@ -474,29 +476,12 @@ function load_queues() {
                     g.updateOptions( { 'file': data } );
                     g2.updateOptions( { 'file': data2 } );
 
-
-                   // TagCanvas.Reload('myCanvas');
-
                 }, interval);
             }
         });
     };
 
     refresh();
-
-    try {
-      var options = {
-      weight:true,
-      weightMode:"both",
-      noMouse:true,
-      textColour: '#2E9AFE'
-      }
-      TagCanvas.Start('myCanvas','',options);
-      TagCanvas.SetSpeed('myCanvas', [0.05, -0.15]);
-    } catch(e) {
-      // something went wrong, hide the canvas container
-      document.getElementById('myCanvasContainer').style.display = 'none';
-    }
 }
 
 function manage_undefined() {

var/www/templates/dashboard/menu_sidebar.html

@@ -0,0 +1,41 @@
+      <div class="col-12 col-lg-2 p-0 bg-light border-right" id="side_menu">
+
+      	<button type="button" class="btn btn-outline-secondary mt-1 ml-3" onclick="toggle_sidebar()">
+          <i class="fas fa-align-left"></i>
+      	  <span>Toggle Sidebar</span>
+      	</button>
+
+      	<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_menu">
+          
+      	</nav>
+
+        <span id="side-bard-dashboard-content">
+          <div class="card">
+            <div class="card-header">
+              <i class="fas fa-tachometer-alt"></i> Total pastes since {{ default_minute }} min
+            </div>
+            <div class="card-body">
+              <div id="global" style="height: 90px; padding: 0px; position: relative;"></div>
+            </div>
+          </div>
+
+          <div class="custom-control custom-switch ml-5 mt-2">
+            <input class="custom-control-input" id="button-toggle-queues" type="checkbox" checked>
+            <label class="custom-control-label" for="button-toggle-queues"><strong style="top: 3px; position: relative;">Display queues</strong></label>
+          </div>
+          <hr>
+
+          <div>
+            <div>
+              <table id="queue-color-legend" class="table table-borderless table-sm">
+                <tbody>
+                  <tr><td class="legendColorBox" style="vertical-align: ;"><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #e8e8e8;overflow:hidden"></div></div></td><td> Idling queues</td></tr>
+                  <tr><td class="legendColorBox" style="vertical-align: ; "><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #d0e9c6;overflow:hidden"></div></div></td><td> Working queues</td></tr>
+                  <tr><td class="legendColorBox" style="vertical-align: ;"><div style="border:1px solid #ccc;padding:1px"><div style="width:100%;height:0;border:5px solid #ebcccc;overflow:hidden"></div></div></td><td> Stuck queues</td></tr>
+                </tbody>
+              </table>
+            </div>
+            <div class="table-responsive", id="queueing" style="margin-top:10px; font-size: small;"></div>
+          </div>
+        </span>
+      </div>

var/www/templates/nav_bar.html

@@ -10,7 +10,7 @@
   <div class="collapse navbar-collapse" id="navbarSupportedContent">
     <ul class="navbar-nav">
       <li class="nav-item">
-        <a class="nav-link mr-3" href="{{ url_for('dashboard.index') }}">Home</a>
+        <a class="nav-link mr-3" id="page-Dashboard" href="{{ url_for('dashboard.index') }}">Home</a>
       </li>
       <li class="nav-item mr-3">
         <a class="nav-link" href="{{ url_for('PasteSubmit.PasteSubmit_page') }}" aria-disabled="true"><i class="fas fa-external-link-alt"></i> Submit</a>
@@ -38,10 +38,11 @@
       </li>
     </ul>
 
-    <form class="form-inline my-2 my-lg-0 ml-auto justify-content-center">
+    <form class="form-inline my-2 my-lg-0 ml-auto justify-content-center" action="{{ url_for('searches.search') }}" method=POST>
       <div class="d-flex flex-column">
         <div>
-        <input class="form-control mr-sm-2" type="search" id="global_search" placeholder="Search" aria-label="Search" aria-describedby="advanced_search">
+        <input class="form-control mr-sm-2" type="search" id="global_search" name="query" placeholder="Search" aria-label="Search" aria-describedby="advanced_search">
+        <input type="hidden" name="index_name" class="form-control" value="0" placeholder="Index Name">
         <button class="btn btn-outline-info my-2 my-sm-0" type="submit"><i class="fas fa-search"></i></button>
         </div>
         <small id="advanced_search" class="form-text"><a class="nav text-muted" href="#" aria-disabled="true">Advanced Search</a></small>