chg: [Domain + UI Crawler] refractor show domain

bin/lib/Domain.py

@@ -56,6 +56,7 @@ def get_link_tree():
     pass
 
 
+
 def get_domain_tags(domain):
     '''
     Retun all tags of a given domain.
@@ -103,11 +104,119 @@ def get_domain_all_correlation(domain, correlation_type=None):
         domain_correl['pgp'] = res
     return domain_correl
 
+ # TODO: handle port
+def get_domain_history(domain, domain_type, port): # TODO: add date_range: from to + nb_elem
+    '''
+    Retun .
+
+    :param domain: crawled domain
+    :type domain: str
+
+    :return:
+    :rtype: list of tuple (item_core, epoch)
+    '''
+    return r_serv_onion.zrange('crawler_history_{}:{}:{}'.format(domain_type, domain, port), 0, -1, withscores=True)
+
+def get_domain_history_with_status(domain, domain_type, port): # TODO: add date_range: from to + nb_elem
+    '''
+    Retun .
+
+    :param domain: crawled domain
+    :type domain: str
+
+    :return:
+    :rtype: list of dict (epoch, date: %Y/%m/%d - %H:%M.%S, boolean status)
+    '''
+    l_history = []
+    history = get_domain_history(domain, domain_type, port)
+    for root_item, epoch_val in history:
+        epoch_val = int(epoch_val) # force int
+        # domain down, root_item==epoch_val
+        try:
+            int(root_item)
+            status = False
+        # domain up, root_item=str
+        except ValueError:
+            status = True
+        l_history.append({"epoch": epoch_val, "date": time.strftime('%Y/%m/%d - %H:%M.%S', time.gmtime(epoch_val)), "status": status})
+    return l_history
+
 
 class Domain(object):
     """docstring for Domain."""
 
     def __init__(self, domain, port=80):
         self.domain = str(domain)
-        ## TODO: handle none port
         self.type = get_domain_type(domain)
+
+    def get_domain_first_seen(self):
+        '''
+        Get domain first seen date
+
+        :return: domain first seen date
+        :rtype: str
+        '''
+        first_seen = r_serv_onion.hget('{}_metadata:{}'.format(self.type, self.domain), 'first_seen')
+        if first_seen is not None:
+            first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8])
+        return first_seen
+
+    def get_domain_last_check(self):# # TODO: add epoch ???
+        '''
+        Get domain last check date
+
+        :return: domain last check date
+        :rtype: str
+        '''
+        last_check = r_serv_onion.hget('{}_metadata:{}'.format(self.type, self.domain), 'last_check')
+        if last_check is not None:
+            last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
+        return last_check
+
+    #def get_domain_all_ports(self):
+    #    pass
+
+    def get_domain_metadata(self, first_seen=True, last_ckeck=True, ports=True):
+        '''
+        Get Domain basic metadata
+
+        :param first_seen: get domain first_seen
+        :type first_seen: boolean
+        :param last_ckeck: get domain last_check
+        :type last_ckeck: boolean
+        :param ports: get all domain ports
+        :type ports: boolean
+
+        :return: a dict of all metadata for a given domain
+        :rtype: dict
+        '''
+        dict_metadata = {}
+        if first_seen:
+            res = self.get_domain_first_seen()
+            if res is not None:
+                dict_metadata['first_seen'] = res
+        if last_ckeck:
+            res = self.get_domain_last_check()
+            if res is not None:
+                dict_metadata['last_check'] = res
+        return dict_metadata
+
+    def get_domain_tags(self):
+        '''
+        Retun all tags of a given domain.
+
+        :param domain: crawled domain
+        '''
+        return get_domain_tags(self.domain)
+
+    def get_domain_correlation(self):
+        '''
+        Retun all cryptocurrencies of a given domain.
+        '''
+        return get_domain_all_correlation(self.domain)
+
+    def get_domain_history_with_status(self):
+        '''
+        Retun the full history of a given domain and port.
+        '''
+        return get_domain_history_with_status(self.domain, self.type, 80)

bin/packages/Correlation.py

@@ -108,6 +108,24 @@ class Correlation(object):
         else:
             return []
 
+    def _get_correlation_obj_domain(self, field_name, correlation_type):
+        '''
+        Return all domains that contain this correlation.
+
+        :param domain: field name
+        :type domain: str
+        :param correlation_type: correlation type
+        :type correlation_type: str
+
+        :return: a list of correlation
+        :rtype: list
+        '''
+        res = r_serv_metadata.smembers('set_domain_{}_{}:{}'.format(self.correlation_name, correlation_type, field_name))
+        if res:
+            return list(res)
+        else:
+            return []
+
     def get_domain_correlation_dict(self, domain, correlation_type=None):
         '''
         Return all correlation of a given domain.

var/www/blueprints/crawler_splash.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+# -*-coding:UTF-8 -*
+
+'''
+    Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
+'''
+
+import os
+import sys
+
+from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response
+from flask_login import login_required, current_user, login_user, logout_user
+
+sys.path.append('modules')
+import Flask_config
+
+# Import Role_Manager
+from Role_Manager import create_user_db, check_password_strength, check_user_role_integrity
+from Role_Manager import login_admin, login_analyst
+
+sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
+import Domain
+
+r_cache = Flask_config.r_cache
+r_serv_db = Flask_config.r_serv_db
+r_serv_tags = Flask_config.r_serv_tags
+bootstrap_label = Flask_config.bootstrap_label
+
+# ============ BLUEPRINT ============
+crawler_splash = Blueprint('crawler_splash', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/crawler/crawler_splash'))
+
+# ============ VARIABLES ============
+
+
+
+# ============ FUNCTIONS ============
+
+
+
+# ============= ROUTES ==============
+@crawler_splash.route('/crawlers/showDomain')
+#@login_required
+#@login_analyst
+def showDomain():
+    domain_name = request.args.get('domain')
+    epoch = request.args.get('epoch')
+    port = request.args.get('port')
+
+    domain = Domain.Domain(domain_name)
+
+    dict_domain = domain.get_domain_metadata()
+    dict_domain = {**dict_domain, **domain.get_domain_correlation()}
+    dict_domain['domain'] = domain_name
+    dict_domain['tags'] = domain.get_domain_tags()
+    dict_domain['history'] = domain.get_domain_history_with_status()
+
+    print(dict_domain)
+
+    return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label, screenshot={'item': None, '':None}, dict_links={})

var/www/templates/crawler/crawler_splash/showDomain.html

@@ -6,10 +6,10 @@
 	<!-- Core CSS -->
 	<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
 	<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
-	<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
+  <link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
-  <link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
 	<!-- JS -->
 	<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+  <script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
 	<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
 	<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
 
@@ -45,7 +45,7 @@
 										</div>
 									{% endif %}
 								</span>
-								<h3 class="card-title text-white">{{ domain }} :</h3>
+								<h3 class="card-title text-white">{{ dict_domain['domain'] }} :</h3>
 							</div>
 							<div class="card-body">
 								<table class="table table-responsive table-condensed">
@@ -58,23 +58,22 @@
 									</thead>
 									<tbody>
 										<tr>
-											<td class="panelText">{{ first_seen }}</td>
+											<td class="panelText">{%if "first_seen" in dict_domain%}{{ dict_domain['first_seen'] }}{%endif%}</td>
-											<td class="panelText">{{ last_check }}</td>
+											<td class="panelText">{%if "last_check" in dict_domain%}{{ dict_domain['last_check'] }}{%endif%}</td>
                       <td class="panelText">{{ ports }}</td>
 										</tr>
 									</tbody>
 								</table>
 
-								Origin Paste:
 								{% if origin_paste_name=='manual' or origin_paste_name=='auto' %}
 									<span class="badge badge-dark">{{ origin_paste_name }}</span>
 								{%else%}
 								<a class="badge badge-dark" target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste', paste=origin_paste) }}" />{{ origin_paste_name }}</a>
 								{%endif%}
 								<div>
-									{% for tag in origin_paste_tags %}
+									{% for tag in dict_domain['tags'] %}
-									<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag[1] }}">
+									<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
-										<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
+										<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
 									</a>
 									{%  endfor %}
 									<br>
@@ -83,17 +82,122 @@
 							</div>
 						</div>
 
-						<div>
+
-							{% for tag in domain_tags %}
+            {% if 'pgp' in dict_domain%}
-							<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag }}">
+              <div id="accordionpgp" class="mt-3">
-								<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }} <i>{{ domain_tags[tag] }}</i></span>
+                <div class="card">
-							</a>
+                  <div class="card-header" id="headingPgp">
-							{%  endfor %}
+                    <div class="row">
-							<br>
+                      <div class="col-11">
-							<br>
+                        <div class="mt-2">
-						</div>
+                          <i class="fas fa-key"></i> PGP Dumps&nbsp;&nbsp;
+                          <div class="badge badge-warning">{{l_64|length}}</div>
+                        </div>
+                      </div>
+                      <div class="col-1">
+                        <button class="btn btn-link collapsed rotate" data-toggle="collapse" data-target="#collapsePgp" aria-expanded="false" aria-controls="collapsePgp">
+                          <i class="fas fa-chevron-circle-down"></i>
+                        </button>
+                      </div>
+                    </div>
+                  </div>
+                  <div id="collapsePgp" class="collapse" aria-labelledby="headingPgp" data-parent="#accordionpgp">
+                    <div class="card-body">
+                      <table id="tablepgp" class="table table-striped">
+                        <thead class="thead-dark">
+                          <tr>
+                           <th>PGP Type</th>
+                           <th>Key ID</th>
+                          </tr>
+                         </thead>
+                        <tbody>
+                          {% for dict_key in dict_domain['pgp']%}
+                            {% if dict_key=="mail" %}
+                              {% set var_icon = "fas fa-at" %}
+                            {% elif dict_key=="name" %}
+                              {% set var_icon = "fas fa-user-tag" %}
+                            {% else %}
+                              {% set var_icon = "fas fa-key" %}
+                            {% endif %}
+                            {% for key_id in dict_domain['pgp'][dict_key]%}
+                              <tr>
+                                <td>
+                                  <i class="{{ var_icon }}"></i>
+                                  &nbsp;&nbsp;{{ dict_key }}
+                                </td>
+                                <td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ key_id }}">{{ key_id }}</a></td>
+                              </tr>
+                            {% endfor %}
+                          {% endfor %}
+                        </tbody>
+                      </table>
+                    </div>
+                  </div>
+                </div>
+              </div>
+            {% endif %}
+
+
+            {% if 'cryptocurrency' in dict_domain%}
+              <div id="accordioncurrency" class="mt-3">
+                <div class="card">
+                  <div class="card-header" id="headingcurrency">
+                    <div class="row">
+                      <div class="col-11">
+                        <div class="mt-2">
+                          <i class="fas fa-coins"></i> Cryptocurrencies&nbsp;&nbsp;
+                          <div class="badge badge-warning">{{l_64|length}}</div>
+                        </div>
+                      </div>
+                      <div class="col-1">
+                        <button class="btn btn-link collapsed rotate" data-toggle="collapse" data-target="#collapsecurrency" aria-expanded="false" aria-controls="collapsecurrency">
+                          <i class="fas fa-chevron-circle-down"></i>
+                        </button>
+                      </div>
+                    </div>
+                  </div>
+                  <div id="collapsecurrency" class="collapse" aria-labelledby="headingcurrency" data-parent="#accordioncurrency">
+                    <div class="card-body">
+                      <table id="tablecurrency" class="table table-striped">
+                        <thead class="thead-dark">
+                          <tr>
+                           <th>Currency</th>
+                           <th>address</th>
+                          </tr>
+                         </thead>
+                        <tbody>
+                          {% for dict_key in dict_domain['cryptocurrency']%}
+                            {% if dict_key=="bitcoin" %}
+                              {% set var_icon = "fab fa-bitcoin" %}
+                            {% elif dict_key=="monero" %}
+                              {% set var_icon = "fab fa-monero" %}
+                            {% else %}
+                              {% set var_icon = "fas fa-coins" %}
+                            {% endif %}
+                            {% for key_id in dict_domain['cryptocurrency'][dict_key]%}
+                              <tr>
+                                <td>
+                                  <i class="{{ var_icon }}"></i>
+                                  &nbsp;&nbsp;{{ dict_key }}
+                                </td>
+                                <td><a target="_blank" href="{{ url_for('hashDecoded.showHash') }}?hash={{ key_id }}">{{ key_id }}</a></td>
+                              </tr>
+                            {% endfor %}
+                          {% endfor %}
+                        </tbody>
+                      </table>
+                    </div>
+                  </div>
+                </div>
+              </div>
+            {% endif %}
+
+
+            <!-- TODO: ADD HASH DECODED -->
 
 						{% if l_pastes %}
+              <hr>
+
 							<table class="table table-striped table-bordered table-hover" id="myTable_1">
 						    <thead class="thead-dark">
 						      <tr>
@@ -129,7 +233,7 @@
   						</a>
             </div>
 
-            {% if domain_history %}
+            {% if dict_domain["domain_history"] %}
 							<table class="table table-hover mt-2" id="myTable_2">
 						    <thead>
 						      <tr>
@@ -142,7 +246,7 @@
 				              <td class="{% if epoch_item[0]==epoch %}text-dark table-info{% endif %}">
 												<a href="{{ url_for('hiddenServices.show_domain') }}?domain={{domain}}&port={{port}}&epoch={{epoch_item[0]}}" class="text-secondary">
 													<div class="d-flex justify-content-around" style="line-height:0.9;">
-                            <div>{{domain}}</div>
+                            <div>{{dict_domain["domain"]}}</div>
                             {% if epoch_item[2] %}
                               <div style="color:Green;"><i class="fas fa-check-circle"></i> UP</div>
                             {% else %}
@@ -177,11 +281,13 @@
 						</div>
 						<canvas id="canvas" style="width:100%;"></canvas>
 						<div class="text-center">
+              <!--
 							<small>
 								<a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{screenshot['item']}}" class="text-info">
 									<div style="line-height:0.9;">{{dict_links[screenshot['item']]}}</div>
 								</a>
 							<small>
+              -->
 						</div>
 					</div>
 
@@ -196,13 +302,19 @@
 <script>
 	var table;
   $(document).ready(function(){
-	table = $('#myTable_1').DataTable(
+    {% if 'pgp' in dict_domain%}
-	{
+      $('#tablepgp').DataTable();
-		//"aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
+    {% endif %}
-		//"iDisplayLength": 5,
+    {% if 'cryptocurrency' in dict_domain%}
-		//"order": [[ 0, "desc" ]]
+      $('#tablecurrency').DataTable({});
-	});
+    {% endif %}
-});
+  	table = $('#myTable_1').DataTable(
+  	{
+  		//"aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
+  		//"iDisplayLength": 5,
+  		//"order": [[ 0, "desc" ]]
+  	});
+  });
 
 function toggle_sidebar(){
 	if($('#nav_menu').is(':visible')){
@@ -230,7 +342,7 @@ img.onload = pixelate;
 img.addEventListener("error", img_error);
 var draw_img = false;
 
-img.src = "{{ url_for('showsavedpastes.screenshot', filename=screenshot['screenshot']) }}";
+//img.src = "{{ url_for('showsavedpastes.screenshot', filename=screenshot['screenshot']) }}";
 
 function pixelate() {
   /// use slider value