chg: [UI Tags search] bootstrap 4 migration

OVERVIEW.md

@@ -14,7 +14,6 @@ Redis and ARDB overview
 * ARDB on TCP port 6382
 
 
-
      DB 1 - Curve
      DB 2 - TermFreq
      DB 3 - Trending
@@ -24,7 +23,7 @@ Redis and ARDB overview
      DB 7 - Metadata
      DB 8 - Statistics
      DB 9 - Crawler
-     
+
 * ARDB on TCP port <year>
     - DB 0 - Lines duplicate
     - DB 1 - Hashes
@@ -36,32 +35,58 @@ Redis and ARDB overview
 ##### Hset:
 | Key | Field | Value |
 | ------ | ------ | ------ |
-| daily_tags:**<date>** | **<tag>** | **<nb tagged this day>** |
+| daily_tags:**date** | **tag** | **nb tagged this day** |
 | | |
-| tag_metadata:**<tag>** | first_seen | **<date>** |
-| tag_metadata:**<tag>** | last_seen | **<date>** |
+| tag_metadata:**tag** | first_seen | **date** |
+| tag_metadata:**tag** | last_seen | **date** |
 
 ##### Set:
 | Key | Value |
 | ------ | ------ |
-| list_tags | **<tag>** |
-| active_taxonomies | **<taxonomie>** |
-| active_galaxies | **<galaxie>** |
-| active_tag_**<taxonomie or galaxy>** | **<tag>** |
-| synonym_tag_misp-galaxy:**<galaxy>** | **<tag synonym>** |
-| list_export_tags | **<user_tag>** |
-| **<tag>**:**<date>** | **<paste>** |
+| list_tags | **tag** |
+| active_taxonomies | **taxonomie** |
+| active_galaxies | **galaxie** |
+| active_tag_**taxonomie or galaxy** | **tag** |
+| synonym_tag_misp-galaxy:**galaxy** | **tag synonym** |
+| list_export_tags | **user_tag** |
+| **tag**:**date** | **paste** |
 
 
 ##### old:
 | Key | Value |
 | ------ | ------ |
-| **<tag>** | **<paste>** |
+| *tag* | *paste* |
 
 
+## DB9 - Crawler:
 
+##### Hset:
+| Key | Field | Value |
+| ------ | ------ | ------ |
+| **service type**:**domain** | first_seen | **date** |
+| | last_check | **date** |
+| | paste_parent | **parent last crawling (can be auto or manual)** |
 
+##### Zset:
+| Key | Field | Value |
+| ------ | ------ | ------ |
+| crawler_history_**service type**:**domain** | **item root (first crawled item)** | **epoch (seconds)** |
 
+##### Regular key:
+| Key | Value |
+| ------ | ------ |
+| crawler_history_**service type**:**domain** | **json config** |
+
+##### exemple json config:
+```json
+{
+  "closespider_pagecount": 1,
+  "time": 3600,
+  "depth_limit": 0,
+  "har": 0,
+  "png": 0
+}
+```
 
 ARDB overview
 ---------------------------
@@ -130,5 +155,3 @@ ARDB_DB
 
 	GET  - 'base64_decoded:'+date	nd_decoded
 	GET  - 'binary_decoded:'+date	nd_decoded
-
-

bin/Tags.py

@@ -75,12 +75,12 @@ if __name__ == '__main__':
             if res == 1:
                 print("new tags added : {}".format(tag))
             # add the path to the tag set
-            date = get_paste_date(path)
-            res = server.sadd('{}:{}'.format(tag, date), path)
+            curr_date = datetime.date.today().strftime("%Y%m%d")
+            res = server.sadd('{}:{}'.format(tag, curr_date), path)
             if res == 1:
                 print("new paste: {}".format(path))
                 print("   tagged: {}".format(tag))
-                set_tag_metadata(tag, date)
+                set_tag_metadata(tag, curr_date)
             server_metadata.sadd('tag:{}'.format(path), tag)
 
             curr_date = datetime.date.today().strftime("%Y%m%d")

var/www/modules/Tags/Flask_Tags.py

@@ -8,7 +8,7 @@ import redis
 from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
 
 import json
-from datetime import datetime
+import datetime
 import ssdeep
 
 import Paste
@@ -71,9 +71,98 @@ def get_tags_with_synonyms(tag):
 
 # ============= ROUTES ==============
 
-@Tags.route("/Tags/", methods=['GET'])
+@Tags.route("/tags/", methods=['GET'])
 def Tags_page():
-    return render_template("Tags.html")
+    current_date = datetime.date.today().strftime("%Y-%m-%d")
+
+    tags = request.args.get('ltags')
+    if tags is None:
+        return render_template("Tags.html", date_from=current_date, date_to=current_date)
+
+    else:
+
+
+        tags = request.args.get('ltags')
+
+        list_tags = tags.split(',')
+        list_tag = []
+        for tag in list_tags:
+            list_tag.append(tag.replace('"','\"'))
+
+        # TODO verify input
+
+        if(type(list_tags) is list):
+            # no tag
+            if list_tags is False:
+                print('empty')
+            # 1 tag
+            elif len(list_tags) < 2:
+                tagged_pastes = r_serv_tags.smembers(list_tags[0])
+
+            # 2 tags or more
+            else:
+                tagged_pastes = r_serv_tags.sinter(list_tags[0], *list_tags[1:])
+
+        else :
+            return 'INCORRECT INPUT'
+
+        all_content = []
+        paste_date = []
+        paste_linenum = []
+        all_path = []
+        allPastes = list(tagged_pastes)
+        paste_tags = []
+
+        for path in allPastes[0:50]: ######################moduleName
+            all_path.append(path)
+            paste = Paste.Paste(path)
+            content = paste.get_p_content()
+            content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
+            all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " "))
+            curr_date = str(paste._get_p_date())
+            curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
+            paste_date.append(curr_date)
+            paste_linenum.append(paste.get_lines_info()[0])
+            p_tags = r_serv_metadata.smembers('tag:'+path)
+            complete_tags = []
+            l_tags = []
+            for tag in p_tags:
+                complete_tag = tag
+
+                tag = tag.split('=')
+                if len(tag) > 1:
+                    if tag[1] != '':
+                        tag = tag[1][1:-1]
+                    # no value
+                    else:
+                        tag = tag[0][1:-1]
+                # use for custom tags
+                else:
+                    tag = tag[0]
+
+                l_tags.append( (tag,complete_tag) )
+
+            paste_tags.append(l_tags)
+
+        if len(allPastes) > 10:
+            finished = False
+        else:
+            finished = True
+
+        return render_template("Tags.html",
+                all_path=all_path,
+                tags=tags,
+                list_tag = list_tag,
+                date_from=current_date,
+                date_to=current_date,
+                paste_tags=paste_tags,
+                bootstrap_label=bootstrap_label,
+                content=all_content,
+                paste_date=paste_date,
+                paste_linenum=paste_linenum,
+                char_to_display=max_preview_modal,
+                finished=finished)
+
 
 @Tags.route("/Tags/get_all_tags")
 def get_all_tags():
@@ -173,94 +262,6 @@ def get_tags_galaxy():
     else:
         return 'this galaxy is disable'
 
-
-@Tags.route("/Tags/get_tagged_paste")
-def get_tagged_paste():
-
-    tags = request.args.get('ltags')
-
-    list_tags = tags.split(',')
-    list_tag = []
-    for tag in list_tags:
-        list_tag.append(tag.replace('"','\"'))
-
-    # TODO verify input
-
-    if(type(list_tags) is list):
-        # no tag
-        if list_tags is False:
-            print('empty')
-        # 1 tag
-        elif len(list_tags) < 2:
-            tagged_pastes = r_serv_tags.smembers(list_tags[0])
-
-        # 2 tags or more
-        else:
-            tagged_pastes = r_serv_tags.sinter(list_tags[0], *list_tags[1:])
-
-    else :
-        return 'INCORRECT INPUT'
-
-    #TODO FIXME
-    currentSelectYear = int(datetime.now().year)
-
-    all_content = []
-    paste_date = []
-    paste_linenum = []
-    all_path = []
-    allPastes = list(tagged_pastes)
-    paste_tags = []
-
-    for path in allPastes[0:50]: ######################moduleName
-        all_path.append(path)
-        paste = Paste.Paste(path)
-        content = paste.get_p_content()
-        content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
-        all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " "))
-        curr_date = str(paste._get_p_date())
-        curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
-        paste_date.append(curr_date)
-        paste_linenum.append(paste.get_lines_info()[0])
-        p_tags = r_serv_metadata.smembers('tag:'+path)
-        complete_tags = []
-        l_tags = []
-        for tag in p_tags:
-            complete_tag = tag
-
-            tag = tag.split('=')
-            if len(tag) > 1:
-                if tag[1] != '':
-                    tag = tag[1][1:-1]
-                # no value
-                else:
-                    tag = tag[0][1:-1]
-            # use for custom tags
-            else:
-                tag = tag[0]
-
-            l_tags.append( (tag,complete_tag) )
-
-        paste_tags.append(l_tags)
-
-    if len(allPastes) > 10:
-        finished = False
-    else:
-        finished = True
-
-    return render_template("tagged.html",
-            year=currentSelectYear,
-            all_path=all_path,
-            tags=tags,
-            list_tag = list_tag,
-            paste_tags=paste_tags,
-            bootstrap_label=bootstrap_label,
-            content=all_content,
-            paste_date=paste_date,
-            paste_linenum=paste_linenum,
-            char_to_display=max_preview_modal,
-            finished=finished)
-
-
 @Tags.route("/Tags/remove_tag")
 def remove_tag():
 

var/www/modules/Tags/templates/Tags.html

@@ -9,12 +9,15 @@
 	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
 	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
 	<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
+	<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
 	<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
 
   <!-- JS -->
 	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
 	<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
 	<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
+	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
 	<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
 	<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
 	<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
@@ -68,6 +71,48 @@
 					</div>
 				</div>
 
+				<div>
+					{%if all_path%}
+						<table class="table table-bordered table-hover " id="myTable_">
+				      <thead class="thead-dark">
+				        <tr>
+									<th>Date</th>
+				          <th style="max-width: 800px;">Path</th>
+				          <th># of lines</th>
+				          <th>Action</th>
+				        </tr>
+				      </thead>
+				      <tbody>
+
+				        {% for path in all_path %}
+				          <tr>
+										<td class="pb-0">{{ paste_date[loop.index0] }}</td>
+				            <td class="pb-0"><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{path}}" class="text-secondary">
+													<div style="line-height:0.9;">{{ path }}</div>
+												</a>
+				              <div>
+				                {% for tag in paste_tags[loop.index0] %}
+				                <a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag[1] }}">
+				                  <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
+				                </a>
+				                {%  endfor %}
+				              </div>
+				            </td class="pb-0">
+				            <td class="pb-0">{{ paste_linenum[loop.index0] }}</td>
+				            <td class="pb-0"><p>
+											<span class="fas fa-info-circle" data-toggle="tooltip" data-placement="left" title="{{ content[loop.index0] }} "></span>
+											<button type="button" class="btn btn-light" data-num="{{ loop.index0 + 1 }}" data-toggle="modal" data-target="#mymodal" data-url="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{ path }}&num={{ loop.index0+1 }}" data-path="{{ path }}">
+												<span class="fas fa-search-plus"></span>
+											</button></p>
+										</td>
+				          </tr>
+				        {%  endfor %}
+
+				      </tbody>
+				    </table>
+					{%endif%}
+				</div>
+
 				<div>
 					<br></br>
 					<a class="btn btn-light text-secondary" href="{{ url_for('Tags.taxonomies') }}" target="_blank">
@@ -98,14 +143,38 @@
 </body>
 
 <script>
-var ltags
+var ltags;
+var search_table;
+var last_clicked_paste;
+var can_change_modal_content = true;
+$("#myTable_").attr('data-numElem', "{{ all_path|length }}");
+
 $(document).ready(function(){
 	$("#page-Browse-Items").addClass("active");
+	search_table = $('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
+
+	// Use to bind the button with the new displayed data
+  // (The bind do not happens if the dataTable is in tabs and the clicked data is in another page)
+  search_table.on( 'draw.dt', function () {
+    // Bind tooltip each time we draw a new page
+    $('[data-toggle="tooltip"]').tooltip();
+    // On click, get html content from url and update the corresponding modal
+    $("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
+        get_html_and_update_modal(event, $(this));
+    });
+  } );
+
+	var valueData = [
+    {% for tag in list_tag %}
+      '{{tag|safe}}',
+    {%  endfor %}
+  ];
 
 	$.getJSON("{{ url_for('Tags.get_all_tags') }}",
   function(data) {
     ltags = $('#ltags').tagSuggest({
       data: data,
+			value: valueData,
       sortOrder: 'name',
       maxDropHeight: 200,
       name: 'ltags'
@@ -144,7 +213,7 @@ $(document).ready(function(){
 <script>
 function searchTags() {
 	var data = ltags.getValue();
-	window.location.replace("{{ url_for('Tags.get_tagged_paste') }}?ltags=" + data);
+	window.location.replace("{{ url_for('Tags.Tags_page') }}?ltags=" + data);
 }
 function emptyTags() {
 	ltags.clear();
@@ -165,4 +234,99 @@ function toggle_sidebar(){
 }
 </script>
 
+<!-- Dynamically update the modal -->
+<script>
+    // static data
+    var alert_message = '<div class="alert alert-info alert-dismissable"><button type="button" class="close" data-dismiss="alert" aria-hidden="true">×</button><strong>No more data.</strong> Full paste displayed.</div>';
+    var complete_paste = null;
+    var char_to_display = {{ char_to_display }};
+    var start_index = 0;
+
+    // When the modal goes out, refresh it to normal content
+    $("#mymodal").on('hidden.bs.modal', function () {
+        can_change_modal_content = true;
+        $("#mymodalbody").html("<p>Loading paste information...</p>");
+        var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
+        $("#mymodalbody").append(loading_gif); // Show the loading GIF
+        $("#button_show_path").attr('href', '');
+        $("#button_show_path").hide();
+        complete_paste = null;
+        start_index = 0;
+    });
+
+   // Update the paste preview in the modal
+   function update_preview() {
+       if (start_index + char_to_display > complete_paste.length-1){ // end of paste reached
+           var final_index = complete_paste.length-1;
+           var flag_stop = true;
+       } else {
+           var final_index = start_index + char_to_display;
+       }
+
+       if (final_index != start_index){ // still have data to display
+           // Append the new content using text() and not append (XSS)
+           $("#mymodalbody").find("#paste-holder").text($("#mymodalbody").find("#paste-holder").text()+complete_paste.substring(start_index+1, final_index+1));
+           start_index = final_index;
+           if (flag_stop)
+               nothing_to_display();
+       } else {
+          nothing_to_display();
+      }
+   }
+   // Update the modal when there is no more data
+   function nothing_to_display() {
+       var new_content = $(alert_message).hide();
+       $("#mymodalbody").find("#panel-body").append(new_content);
+       new_content.show('fast');
+       $("#load-more-button").hide();
+   }
+
+   function get_html_and_update_modal(event, truemodal) {
+       event.preventDefault();
+
+       var modal=truemodal;
+       var url = " {{ url_for('showsavedpastes.showpreviewpaste')  }}?paste=" + modal.attr('data-path') + "&num=" + modal.attr('data-num');
+       last_clicked_paste =  modal.attr('data-num');
+       $.get(url, function (data) {
+
+           // verify that the reveived data is really the current clicked paste. Otherwise, ignore it.
+           var received_num = parseInt(data.split("|num|")[1]);
+           if (received_num == last_clicked_paste && can_change_modal_content) {
+               can_change_modal_content = false;
+
+               // clear data by removing html, body, head tags. prevent dark modal background stack bug.
+               var cleared_data = data.split("<body>")[1].split("</body>")[0];
+               $("#mymodalbody").html(cleared_data);
+
+               var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
+               button.tooltip();
+               $("#mymodalbody").children(".panel-default").append(button);
+
+               $("#button_show_path").attr('href', $(modal).attr('data-url'));
+               $("#button_show_path").show('fast');
+               $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
+               if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
+                   nothing_to_display();
+               }
+               // On click, donwload all paste's content
+               $("#load-more-button").on("click", function (event) {
+                   if (complete_paste == null) { //Donwload only once
+                       $.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
+                           complete_paste = data;
+                           update_preview();
+                       });
+                   } else {
+                       update_preview();
+                   }
+               });
+           } else if (can_change_modal_content) {
+               $("#mymodalbody").html("Ignoring previous not finished query of paste #" + received_num);
+           }
+       });
+   }
+
+
+</script>
+
+
 </html>

var/www/modules/hiddenServices/templates/header_hiddenServices.html

@@ -1 +1 @@
-<li id='page-hiddenServices'><a href="{{ url_for('hiddenServices.hiddenServices_page') }}"><i class="fa fa-user-secret"></i> hidden Services </a></li>
+<li id='page-hiddenServices'><a href="{{ url_for('hiddenServices.dashboard') }}"><i class="fa fa-user-secret"></i> hidden Services </a></li>

var/www/modules/hiddenServices/templates/showDomain.html

@@ -7,6 +7,7 @@
 	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
 	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
 	<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
 	<!-- JS -->
 	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
 	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
@@ -70,7 +71,7 @@
 								{%endif%}
 								<div>
 									{% for tag in origin_paste_tags %}
-									<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag[1] }}">
+									<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag[1] }}">
 										<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
 									</a>
 									{%  endfor %}
@@ -82,7 +83,7 @@
 
 						<div>
 							{% for tag in domain_tags %}
-							<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
+							<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
 								<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }} <i>{{ domain_tags[tag] }}</i></span>
 							</a>
 							{%  endfor %}
@@ -106,7 +107,7 @@
 												</a>
 				                <div>
 				                  {% for tag in paste_tags[loop.index0] %}
-					                  <a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag[1] }}">
+					                  <a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag[1] }}">
 					                    <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
 					                  </a>
 				                  {%  endfor %}

var/www/modules/search/templates/search.html

@@ -101,7 +101,7 @@
                                   <td><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{path}}">{{ path }}</a>
                                     <div>
                                       {% for tag in paste_tags[loop.index0] %}
-                                      <a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag[1] }}">
+                                      <a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag[1] }}">
                                         <span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
                                       </a>
                                       {%  endfor %}
@@ -201,7 +201,7 @@
                 var curr_preview = data.preview_array[i].replace(/\"/g, "\'");
                 var tag = ""
                 for(j=0; j<data.list_tags[i].length; j++) {
-                  tag = tag + "<a href=\"{{ url_for('Tags.get_tagged_paste') }}?ltags=" + data.list_tags[i][j][1] + ">"
+                  tag = tag + "<a href=\"{{ url_for('Tags.Tags_page') }}?ltags=" + data.list_tags[i][j][1] + ">"
                         + "<span class=\"label label-" + data.bootstrap_label[j % 5] + " pull-left\">" + data.list_tags[i][j][0]
                         + "</span>" + "</a>"
                 }

var/www/templates/nav_bar.html

@@ -22,7 +22,7 @@
         <a class="nav-link" href="#" aria-disabled="true"><i class="fas fa-crosshairs"></i> Leaks Hunter</a>
       </li>
       <li class="nav-item mr-3">
-        <a class="nav-link" id="page-Crawler" href="#" tabindex="-1" href="#" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>
+        <a class="nav-link" id="page-Crawler" href="{{ url_for('hiddenServices.dashboard') }}" tabindex="-1" aria-disabled="true"><i class="fas fa-spider"></i> Crawlers</a>
       </li>
       <li class="nav-item mr-3">
         <a class="nav-link" href="#" aria-disabled="true"><i class="fas fa-lock-open"></i> Decoded</a>