mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 00:28:22 +00:00
chg: [new title object] add new title object + correlation on page title
This commit is contained in:
Terrtia
parent
f7e0a357ea
commit
c008366f02
18 changed files with 1205 additions and 27 deletions
|
|
@ -19,6 +19,7 @@ from lib.ConfigLoader import ConfigLoader
|
|||
from lib.objects.Domains import Domain
|
||||
from lib.objects.Items import Item
|
||||
from lib.objects import Screenshots
|
||||
from lib.objects import Titles
|
||||
|
||||
logging.config.dictConfig(ail_logger.get_config(name='crawlers'))
|
||||
|
||||
|
|
@ -252,6 +253,13 @@ class Crawler(AbstractModule):
|
|||
self.root_item = item_id
|
||||
parent_id = item_id
|
||||
|
||||
item = Item(item_id)
|
||||
|
||||
title_content = crawlers.extract_title_from_html(entries['html'])
|
||||
if title_content:
|
||||
title = Titles.create_title(title_content)
|
||||
title.add(item.get_date(), item_id)
|
||||
|
||||
# SCREENSHOT
|
||||
if self.screenshot:
|
||||
if 'png' in entries and entries['png']:
|
||||
|
|
@ -260,7 +268,6 @@ class Crawler(AbstractModule):
|
|||
if not screenshot.is_tags_safe():
|
||||
unsafe_tag = 'dark-web:topic="pornography-child-exploitation"'
|
||||
self.domain.add_tag(unsafe_tag)
|
||||
item = Item(item_id)
|
||||
item.add_tag(unsafe_tag)
|
||||
# Remove Placeholder pages # TODO Replace with warning list ???
|
||||
if screenshot.id not in self.placeholder_screenshots:
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ config_loader = ConfigLoader()
|
|||
r_serv_db = config_loader.get_db_conn("Kvrocks_DB")
|
||||
config_loader = None
|
||||
|
||||
AIL_OBJECTS = sorted({'cve', 'cryptocurrency', 'decoded', 'domain', 'item', 'pgp', 'screenshot', 'username'})
|
||||
AIL_OBJECTS = sorted({'cve', 'cryptocurrency', 'decoded', 'domain', 'item', 'pgp', 'screenshot', 'title', 'username'})
|
||||
|
||||
def get_ail_uuid():
|
||||
ail_uuid = r_serv_db.get('ail:uuid')
|
||||
|
|
|
|||
|
|
@ -44,11 +44,12 @@ CORRELATION_TYPES_BY_OBJ = {
|
|||
"cryptocurrency": ["domain", "item"],
|
||||
"cve": ["domain", "item"],
|
||||
"decoded": ["domain", "item"],
|
||||
"domain": ["cve", "cryptocurrency", "decoded", "item", "pgp", "username", "screenshot"],
|
||||
"item": ["cve", "cryptocurrency", "decoded", "domain", "pgp", "username", "screenshot"],
|
||||
"domain": ["cve", "cryptocurrency", "decoded", "item", "pgp", "title", "screenshot", "username"],
|
||||
"item": ["cve", "cryptocurrency", "decoded", "domain", "pgp", "screenshot", "title", "username"],
|
||||
"pgp": ["domain", "item"],
|
||||
"username": ["domain", "item"],
|
||||
"screenshot": ["domain", "item"],
|
||||
"title": ["domain", "item"],
|
||||
"username": ["domain", "item"],
|
||||
}
|
||||
|
||||
def get_obj_correl_types(obj_type):
|
||||
|
|
|
|||
|
|
@ -183,6 +183,47 @@ def extract_favicon_from_html(html, url):
|
|||
|
||||
# # # - - # # #
|
||||
|
||||
# # # # # # # #
|
||||
# #
|
||||
# TITLE #
|
||||
# #
|
||||
# # # # # # # #
|
||||
|
||||
def extract_title_from_html(html):
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
title = soup.title
|
||||
if title:
|
||||
return str(title.string)
|
||||
return ''
|
||||
|
||||
def extract_description_from_html(html):
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
description = soup.find('meta', attrs={'name': 'description'})
|
||||
if description:
|
||||
return description['content']
|
||||
return ''
|
||||
|
||||
def extract_description_from_html(html):
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
description = soup.find('meta', attrs={'name': 'description'})
|
||||
if description:
|
||||
return description['content']
|
||||
return ''
|
||||
|
||||
def extract_keywords_from_html(html):
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
keywords = soup.find('meta', attrs={'name': 'keywords'})
|
||||
if keywords:
|
||||
return keywords['content']
|
||||
return ''
|
||||
|
||||
def extract_author_from_html(html):
|
||||
soup = BeautifulSoup(html, 'html.parser')
|
||||
keywords = soup.find('meta', attrs={'name': 'author'})
|
||||
if keywords:
|
||||
return keywords['content']
|
||||
return ''
|
||||
# # # - - # # #
|
||||
|
||||
################################################################################
|
||||
|
||||
|
|
@ -1711,7 +1752,7 @@ def test_ail_crawlers():
|
|||
load_blacklist()
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# task = CrawlerTask('2dffcae9-8f66-4cfa-8e2c-de1df738a6cd')
|
||||
# print(task.get_meta())
|
||||
# _clear_captures()
|
||||
|
||||
# item = Item('crawled/2023/03/06/foo.bec50a87b5-0c21-4ed4-9cb2-2d717a7a6507')
|
||||
# content = item.get_content()
|
||||
# r = extract_author_from_html(content)
|
||||
# print(r)
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@
|
|||
import json
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
import yara
|
||||
|
||||
|
|
@ -15,6 +14,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
##################################
|
||||
from lib.objects import ail_objects
|
||||
from lib.objects.Items import Item
|
||||
from lib.objects.Titles import Title
|
||||
from lib import correlations_engine
|
||||
from lib import regex_helper
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
|
|
@ -58,18 +58,25 @@ def get_correl_match(extract_type, obj_id, content):
|
|||
correl = correlations_engine.get_correlation_by_correl_type('item', '', obj_id, extract_type)
|
||||
to_extract = []
|
||||
map_subtype = {}
|
||||
map_value_id = {}
|
||||
for c in correl:
|
||||
subtype, value = c.split(':', 1)
|
||||
map_subtype[value] = subtype
|
||||
to_extract.append(value)
|
||||
if extract_type == 'title':
|
||||
title = Title(value).get_content()
|
||||
to_extract.append(title)
|
||||
map_value_id[title] = value
|
||||
else:
|
||||
map_subtype[value] = subtype
|
||||
to_extract.append(value)
|
||||
map_value_id[value] = value
|
||||
if to_extract:
|
||||
objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj_id, content)
|
||||
for obj in objs:
|
||||
if map_subtype[obj[2]]:
|
||||
if map_subtype.get(obj[2]):
|
||||
subtype = map_subtype[obj[2]]
|
||||
else:
|
||||
subtype = ''
|
||||
extracted.append([obj[0], obj[1], obj[2], f'{extract_type}:{subtype}:{obj[2]}'])
|
||||
extracted.append([obj[0], obj[1], obj[2], f'{extract_type}:{subtype}:{map_value_id[obj[2]]}'])
|
||||
return extracted
|
||||
|
||||
def _get_yara_match(data):
|
||||
|
|
@ -173,7 +180,7 @@ def extract(obj_id, content=None):
|
|||
if matches:
|
||||
extracted = extracted + matches
|
||||
|
||||
for obj_t in ['cve', 'cryptocurrency', 'username']: # Decoded, PGP->extract bloc
|
||||
for obj_t in ['cve', 'cryptocurrency', 'title', 'username']: # Decoded, PGP->extract bloc
|
||||
matches = get_correl_match(obj_t, obj_id, content)
|
||||
if matches:
|
||||
extracted = extracted + matches
|
||||
|
|
|
|||
114
bin/lib/objects/Titles.py
Executable file
114
bin/lib/objects/Titles.py
Executable file
|
|
@ -0,0 +1,114 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from hashlib import sha256
|
||||
from flask import url_for
|
||||
|
||||
from pymisp import MISPObject
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
r_objects = config_loader.get_db_conn("Kvrocks_Objects")
|
||||
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
|
||||
config_loader = None
|
||||
|
||||
|
||||
class Title(AbstractDaterangeObject):
|
||||
"""
|
||||
AIL Title Object.
|
||||
"""
|
||||
|
||||
def __init__(self, id):
|
||||
super(Title, self).__init__('title', id)
|
||||
|
||||
# def get_ail_2_ail_payload(self):
|
||||
# payload = {'raw': self.get_gzip_content(b64=True),
|
||||
# 'compress': 'gzip'}
|
||||
# return payload
|
||||
|
||||
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
|
||||
def delete(self):
|
||||
# # TODO:
|
||||
pass
|
||||
|
||||
def get_content(self, r_type='str'):
|
||||
if r_type == 'str':
|
||||
return self._get_field('content')
|
||||
|
||||
def get_link(self, flask_context=False):
|
||||
if flask_context:
|
||||
url = url_for('correlation.show_correlation', type=self.type, id=self.id)
|
||||
else:
|
||||
url = f'{baseurl}/correlation/show?type={self.type}&id={self.id}'
|
||||
return url
|
||||
|
||||
# TODO # CHANGE COLOR
|
||||
def get_svg_icon(self):
|
||||
return {'style': 'fas', 'icon': '\uf1dc', 'color': '#1E88E5', 'radius': 5}
|
||||
|
||||
def get_misp_object(self):
|
||||
obj_attrs = []
|
||||
obj = MISPObject('tsk-web-history')
|
||||
obj.first_seen = self.get_first_seen()
|
||||
obj.last_seen = self.get_last_seen()
|
||||
|
||||
obj_attrs.append(obj.add_attribute('title', value=self.get_content()))
|
||||
for obj_attr in obj_attrs:
|
||||
for tag in self.get_tags():
|
||||
obj_attr.add_tag(tag)
|
||||
return obj
|
||||
|
||||
def get_meta(self, options=set()):
|
||||
meta = self._get_meta(options=options)
|
||||
meta['id'] = self.id
|
||||
meta['tags'] = self.get_tags(r_list=True)
|
||||
meta['content'] = self.get_content()
|
||||
return meta
|
||||
|
||||
def add(self, date, item_id):
|
||||
self._add(date, item_id)
|
||||
|
||||
def create(self, content, _first_seen=None, _last_seen=None):
|
||||
self._set_field('content', content)
|
||||
self._create()
|
||||
|
||||
|
||||
def create_title(content):
|
||||
title_id = sha256(content.encode()).hexdigest()
|
||||
title = Title(title_id)
|
||||
if not title.exists():
|
||||
title.create(content)
|
||||
return title
|
||||
|
||||
class Titles(AbstractDaterangeObjects):
|
||||
"""
|
||||
Titles Objects
|
||||
"""
|
||||
def __init__(self):
|
||||
super().__init__('title')
|
||||
|
||||
def get_metas(self, obj_ids, options=set()):
|
||||
return self._get_metas(Title, obj_ids, options=options)
|
||||
|
||||
def sanitize_name_to_search(self, name_to_search):
|
||||
return name_to_search
|
||||
|
||||
|
||||
# if __name__ == '__main__':
|
||||
# from lib import crawlers
|
||||
# from lib.objects import Items
|
||||
# for item in Items.get_all_items_objects(filters={'sources': ['crawled']}):
|
||||
# title_content = crawlers.extract_title_from_html(item.get_content())
|
||||
# if title_content:
|
||||
# print(item.id, title_content)
|
||||
# title = create_title(title_content)
|
||||
# title.add(item.get_date(), item.id)
|
||||
|
|
@ -7,6 +7,7 @@ Base Class for AIL Objects
|
|||
# Import External packages
|
||||
##################################
|
||||
import os
|
||||
import re
|
||||
import sys
|
||||
from abc import abstractmethod, ABC
|
||||
|
||||
|
|
@ -44,8 +45,14 @@ class AbstractDaterangeObject(AbstractObject, ABC):
|
|||
def exists(self):
|
||||
return r_object.exists(f'meta:{self.type}:{self.id}')
|
||||
|
||||
def _get_field(self, field):
|
||||
return r_object.hget(f'meta:{self.type}:{self.id}', field)
|
||||
|
||||
def _set_field(self, field, value):
|
||||
return r_object.hset(f'meta:{self.type}:{self.id}', field, value)
|
||||
|
||||
def get_first_seen(self, r_int=False):
|
||||
first_seen = r_object.hget(f'meta:{self.type}:{self.id}', 'first_seen')
|
||||
first_seen = self._get_field('first_seen')
|
||||
if r_int:
|
||||
if first_seen:
|
||||
return int(first_seen)
|
||||
|
|
@ -55,7 +62,7 @@ class AbstractDaterangeObject(AbstractObject, ABC):
|
|||
return first_seen
|
||||
|
||||
def get_last_seen(self, r_int=False):
|
||||
last_seen = r_object.hget(f'meta:{self.type}:{self.id}', 'last_seen')
|
||||
last_seen = self._get_field('last_seen')
|
||||
if r_int:
|
||||
if last_seen:
|
||||
return int(last_seen)
|
||||
|
|
@ -83,10 +90,10 @@ class AbstractDaterangeObject(AbstractObject, ABC):
|
|||
return meta_dict
|
||||
|
||||
def set_first_seen(self, first_seen):
|
||||
r_object.hset(f'meta:{self.type}:{self.id}', 'first_seen', first_seen)
|
||||
self._set_field('first_seen', first_seen)
|
||||
|
||||
def set_last_seen(self, last_seen):
|
||||
r_object.hset(f'meta:{self.type}:{self.id}', 'last_seen', last_seen)
|
||||
self._set_field('last_seen', last_seen)
|
||||
|
||||
def update_daterange(self, date):
|
||||
date = int(date)
|
||||
|
|
@ -139,11 +146,85 @@ class AbstractDaterangeObject(AbstractObject, ABC):
|
|||
self.add_correlation('domain', '', domain)
|
||||
|
||||
# TODO:ADD objects + Stats
|
||||
def _create(self, first_seen, last_seen):
|
||||
self.set_first_seen(first_seen)
|
||||
self.set_last_seen(last_seen)
|
||||
def _create(self, first_seen=None, last_seen=None):
|
||||
if first_seen:
|
||||
self.set_first_seen(first_seen)
|
||||
if last_seen:
|
||||
self.set_last_seen(last_seen)
|
||||
r_object.sadd(f'{self.type}:all', self.id)
|
||||
|
||||
# TODO
|
||||
def _delete(self):
|
||||
pass
|
||||
|
||||
|
||||
class AbstractDaterangeObjects(ABC):
|
||||
"""
|
||||
Abstract Daterange Objects
|
||||
"""
|
||||
|
||||
def __init__(self, obj_type):
|
||||
""" Abstract for Daterange Objects
|
||||
|
||||
:param obj_type: object type (item, ...)
|
||||
"""
|
||||
self.type = obj_type
|
||||
|
||||
def get_all(self):
|
||||
return r_object.smembers(f'{self.type}:all')
|
||||
|
||||
def get_by_date(self, date):
|
||||
return r_object.zrange(f'{self.type}:date:{date}', 0, -1)
|
||||
|
||||
def get_nb_by_date(self, date):
|
||||
return r_object.zcard(f'{self.type}:date:{date}')
|
||||
|
||||
def get_by_daterange(self, date_from, date_to):
|
||||
obj_ids = set()
|
||||
for date in Date.substract_date(date_from, date_to):
|
||||
obj_ids = obj_ids | set(self.get_by_date(date))
|
||||
return obj_ids
|
||||
|
||||
@abstractmethod
|
||||
def get_metas(self, obj_ids, options=set()):
|
||||
pass
|
||||
|
||||
def _get_metas(self, obj_class_ref, obj_ids, options=set()):
|
||||
dict_obj = {}
|
||||
for obj_id in obj_ids:
|
||||
obj = obj_class_ref(obj_id)
|
||||
dict_obj[obj_id] = obj.get_meta(options=options)
|
||||
return dict_obj
|
||||
|
||||
@abstractmethod
|
||||
def sanitize_name_to_search(self, name_to_search):
|
||||
return name_to_search
|
||||
|
||||
def search_by_name(self, name_to_search, r_pos=False):
|
||||
objs = {}
|
||||
# for subtype in subtypes:
|
||||
r_name = self.sanitize_name_to_search(name_to_search)
|
||||
if not name_to_search or isinstance(r_name, dict):
|
||||
return objs
|
||||
r_name = re.compile(r_name)
|
||||
for title_name in self.get_all():
|
||||
res = re.search(r_name, title_name)
|
||||
if res:
|
||||
objs[title_name] = {}
|
||||
if r_pos:
|
||||
objs[title_name]['hl-start'] = res.start()
|
||||
objs[title_name]['hl-end'] = res.end()
|
||||
return objs
|
||||
|
||||
def api_get_chart_nb_by_daterange(self, date_from, date_to):
|
||||
date_type = []
|
||||
for date in Date.substract_date(date_from, date_to):
|
||||
d = {'date': f'{date[0:4]}-{date[4:6]}-{date[6:8]}',
|
||||
self.type: self.get_nb_by_date(date)}
|
||||
date_type.append(d)
|
||||
return date_type
|
||||
|
||||
def api_get_meta_by_daterange(self, date_from, date_to):
|
||||
date = Date.sanitise_date_range(date_from, date_to)
|
||||
return self.get_metas(self.get_by_daterange(date['date_from'], date['date_to']), options={'sparkline'})
|
||||
|
||||
|
|
|
|||
|
|
@ -187,7 +187,7 @@ class AbstractObject(ABC):
|
|||
pass
|
||||
|
||||
@staticmethod
|
||||
def get_misp_object_first_last_seen(misp_obj):
|
||||
def get_misp_object_first_last_seen(misp_obj): # TODO REMOVE ME ????
|
||||
"""
|
||||
:type misp_obj: MISPObject
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -21,6 +21,7 @@ from lib.objects.Domains import Domain
|
|||
from lib.objects.Items import Item, get_all_items_objects, get_nb_items_objects
|
||||
from lib.objects import Pgps
|
||||
from lib.objects.Screenshots import Screenshot
|
||||
from lib.objects import Titles
|
||||
from lib.objects import Usernames
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
|
|
@ -59,6 +60,8 @@ def get_object(obj_type, subtype, id):
|
|||
return CryptoCurrencies.CryptoCurrency(id, subtype)
|
||||
elif obj_type == 'pgp':
|
||||
return Pgps.Pgp(id, subtype)
|
||||
elif obj_type == 'title':
|
||||
return Titles.Title(id)
|
||||
elif obj_type == 'username':
|
||||
return Usernames.Username(id, subtype)
|
||||
|
||||
|
|
@ -160,10 +163,12 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False):
|
|||
obj = get_object(obj_type, subtype, id)
|
||||
meta = obj.get_meta()
|
||||
meta['icon'] = obj.get_svg_icon()
|
||||
if subtype or obj_type == 'cve':
|
||||
if subtype or obj_type == 'cve' or obj_type == 'title':
|
||||
meta['sparkline'] = obj.get_sparkline()
|
||||
if obj_type == 'cve':
|
||||
meta['cve_search'] = obj.get_cve_search()
|
||||
# if obj_type == 'title':
|
||||
# meta['cve_search'] = obj.get_cve_search()
|
||||
if subtype == 'bitcoin' and related_btc:
|
||||
meta["related_btc"] = btc_ail.get_bitcoin_info(obj.id)
|
||||
if obj.get_type() == 'decoded':
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ class Phone(AbstractModule):
|
|||
|
||||
def extract(self, obj_id, content, tag):
|
||||
extracted = []
|
||||
phones = self.regex_phone_iter('US', obj_id, content)
|
||||
phones = self.regex_phone_iter('ZZ', obj_id, content)
|
||||
for phone in phones:
|
||||
extracted.append([phone[0], phone[1], phone[2], f'tag:{tag}'])
|
||||
return extracted
|
||||
|
|
|
|||
|
|
@ -49,6 +49,7 @@ from blueprints.settings_b import settings_b
|
|||
from blueprints.objects_cve import objects_cve
|
||||
from blueprints.objects_decoded import objects_decoded
|
||||
from blueprints.objects_subtypes import objects_subtypes
|
||||
from blueprints.objects_title import objects_title
|
||||
|
||||
Flask_dir = os.environ['AIL_FLASK']
|
||||
|
||||
|
|
@ -102,6 +103,7 @@ app.register_blueprint(settings_b, url_prefix=baseUrl)
|
|||
app.register_blueprint(objects_cve, url_prefix=baseUrl)
|
||||
app.register_blueprint(objects_decoded, url_prefix=baseUrl)
|
||||
app.register_blueprint(objects_subtypes, url_prefix=baseUrl)
|
||||
app.register_blueprint(objects_title, url_prefix=baseUrl)
|
||||
# ========= =========#
|
||||
|
||||
# ========= Cookie name ========
|
||||
|
|
|
|||
86
var/www/blueprints/objects_title.py
Normal file
86
var/www/blueprints/objects_title.py
Normal file
|
|
@ -0,0 +1,86 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
|
||||
'''
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file
|
||||
from flask_login import login_required, current_user
|
||||
|
||||
# Import Role_Manager
|
||||
from Role_Manager import login_admin, login_analyst, login_read_only
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib.objects import Titles
|
||||
from packages import Date
|
||||
|
||||
# ============ BLUEPRINT ============
|
||||
objects_title = Blueprint('objects_title', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/title'))
|
||||
|
||||
# ============ VARIABLES ============
|
||||
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
|
||||
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
@objects_title.route("/objects/title", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def objects_titles():
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
show_objects = request.args.get('show_objects')
|
||||
date = Date.sanitise_date_range(date_from, date_to)
|
||||
date_from = date['date_from']
|
||||
date_to = date['date_to']
|
||||
|
||||
if show_objects:
|
||||
dict_objects = Titles.Titles().api_get_meta_by_daterange(date_from, date_to)
|
||||
else:
|
||||
dict_objects = {}
|
||||
|
||||
return render_template("TitleDaterange.html", date_from=date_from, date_to=date_to,
|
||||
dict_objects=dict_objects, show_objects=show_objects)
|
||||
|
||||
@objects_title.route("/objects/title/post", methods=['POST'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def objects_titles_post():
|
||||
date_from = request.form.get('date_from')
|
||||
date_to = request.form.get('date_to')
|
||||
show_objects = request.form.get('show_objects')
|
||||
return redirect(url_for('objects_title.objects_titles', date_from=date_from, date_to=date_to, show_objects=show_objects))
|
||||
|
||||
@objects_title.route("/objects/title/range/json", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def objects_title_range_json():
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
date = Date.sanitise_date_range(date_from, date_to)
|
||||
date_from = date['date_from']
|
||||
date_to = date['date_to']
|
||||
return jsonify(Titles.Titles().api_get_chart_nb_by_daterange(date_from, date_to))
|
||||
|
||||
@objects_title.route("/objects/title/search", methods=['POST'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def objects_title_search():
|
||||
to_search = request.form.get('object_id')
|
||||
|
||||
# TODO SANITIZE ID
|
||||
# TODO Search all
|
||||
title = Titles.Title(to_search)
|
||||
if not title.exists():
|
||||
abort(404)
|
||||
else:
|
||||
return redirect(title.get_link(flask_context=True))
|
||||
|
||||
# ============= ROUTES ==============
|
||||
|
||||
|
|
@ -108,7 +108,7 @@
|
|||
Tags:
|
||||
{% for tag in dict_object["metadata"]['tags'] %}
|
||||
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
|
||||
data-tagid="{{ tag }}" data-objtype="decoded" data-objsubtype="" data-objid="{{ dict_object["correlation_id"] }}">
|
||||
data-tagid="{{ tag }}" data-objtype="cve" data-objsubtype="" data-objid="{{ dict_object["correlation_id"] }}">
|
||||
{{ tag }}
|
||||
</button>
|
||||
{% endfor %}
|
||||
|
|
|
|||
173
var/www/templates/correlation/metadata_card_title.html
Normal file
173
var/www/templates/correlation/metadata_card_title.html
Normal file
|
|
@ -0,0 +1,173 @@
|
|||
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
|
||||
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
|
||||
|
||||
{% with modal_add_tags=dict_object['metadata_card']['add_tags_modal']%}
|
||||
{% include 'modals/add_tags.html' %}
|
||||
{% endwith %}
|
||||
|
||||
{% include 'modals/edit_tag.html' %}
|
||||
|
||||
<div class="card my-3">
|
||||
<div class="card-header" style="background-color:#d9edf7;font-size: 15px">
|
||||
<h4>{{ dict_object["metadata"]["content"] }}</h4>
|
||||
<div class="text-secondary">{{ dict_object["correlation_id"] }}</div>
|
||||
<ul class="list-group mb-2">
|
||||
<li class="list-group-item py-0">
|
||||
<div class="row">
|
||||
<div class="col-md-10">
|
||||
<table class="table">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Object type</th>
|
||||
<th>First seen</th>
|
||||
<th>Last seen</th>
|
||||
<th>Nb seen</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<svg height="26" width="26">
|
||||
<g class="nodes">
|
||||
<circle cx="13" cy="13" r="13" fill="orange"></circle>
|
||||
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text>
|
||||
</g>
|
||||
</svg>
|
||||
{{ dict_object["object_type"] }}
|
||||
</td>
|
||||
<td>{{ dict_object["metadata"]['first_seen'] }}</td>
|
||||
<td>{{ dict_object["metadata"]['last_seen'] }}</td>
|
||||
<td>{{ dict_object["metadata"]['nb_seen'] }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
<div class="col-md-1">
|
||||
<div id="sparkline"></div>
|
||||
</div>
|
||||
</div>
|
||||
</li>
|
||||
|
||||
<li class="list-group-item py-0">
|
||||
<br>
|
||||
<div class="mb-3">
|
||||
Tags:
|
||||
{% for tag in dict_object["metadata"]['tags'] %}
|
||||
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
|
||||
data-tagid="{{ tag }}" data-objtype="title" data-objsubtype="" data-objid="{{ dict_object["correlation_id"] }}">
|
||||
{{ tag }}
|
||||
</button>
|
||||
{% endfor %}
|
||||
<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
|
||||
<i class="far fa-plus-square"></i>
|
||||
</button>
|
||||
</div>
|
||||
</li>
|
||||
</ul>
|
||||
|
||||
{% with obj_type='title', obj_id=dict_object['correlation_id'], obj_subtype='' %}
|
||||
{% include 'modals/investigations_register_obj.html' %}
|
||||
{% endwith %}
|
||||
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
|
||||
<i class="fas fa-microscope"></i> Investigations
|
||||
</button>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
|
||||
<script>
|
||||
sparkline("sparkline", {{ dict_object["metadata_card"]["sparkline"] }}, {});
|
||||
</script>
|
||||
|
||||
|
||||
<script>
|
||||
function create_line_chart(id, url){
|
||||
|
||||
var width = 900;
|
||||
var height = Math.round(width / 4);
|
||||
|
||||
var margin = {top: 20, right: 55, bottom: 50, left: 40};
|
||||
|
||||
var x = d3.scaleTime().range([0, width]);
|
||||
var y = d3.scaleLinear().rangeRound([height, 0]);
|
||||
|
||||
var xAxis = d3.axisBottom(x);
|
||||
var yAxis = d3.axisLeft(y);
|
||||
|
||||
var parseTime = d3.timeParse("%Y-%m-%d");
|
||||
|
||||
var line = d3.line()
|
||||
.x(function(d) {
|
||||
return x(d.date);
|
||||
}).y(function(d) {
|
||||
return y(d.value);
|
||||
});
|
||||
|
||||
var svg_line = d3.select('#'+id).append('svg')
|
||||
.attr("id", "graph_div")
|
||||
.attr("width", width + margin.left + margin.right)
|
||||
.attr("height", height + margin.top + margin.bottom)
|
||||
.append('g')
|
||||
.attr('transform', "translate("+ margin.left +","+ margin.top +")");
|
||||
|
||||
var div = d3.select('body').append('div')
|
||||
.attr('class', 'tooltip')
|
||||
.style('opacity', 0);
|
||||
|
||||
//add div tooltip
|
||||
|
||||
d3.json(url)
|
||||
.then(function(data){
|
||||
|
||||
data.forEach(function(d) {
|
||||
d.date_label = d.date;
|
||||
d.date = parseTime(d.date);
|
||||
d.value = +d.value;
|
||||
});
|
||||
|
||||
// fit the data
|
||||
x.domain(d3.extent(data, function(d) { return d.date; }));
|
||||
//x.domain(data.map(function (d) { return d.date; })); //E
|
||||
y.domain([0, d3.max(data, function(d){ return d.value ; })]);
|
||||
|
||||
//line
|
||||
svg_line.append("path")
|
||||
.data([data])
|
||||
.attr("class", "line_graph")
|
||||
.attr("d", line);
|
||||
|
||||
// add X axis
|
||||
svg_line.append("g")
|
||||
.attr("transform", "translate(0," + height + ")")
|
||||
.call(d3.axisBottom(x))
|
||||
.selectAll("text")
|
||||
.style("text-anchor", "end")
|
||||
.attr("transform", "rotate(-45)" );
|
||||
|
||||
// Add the Y Axis
|
||||
svg_line.append("g")
|
||||
.call(d3.axisLeft(y));
|
||||
|
||||
//add a dot circle
|
||||
svg_line.selectAll('dot')
|
||||
.data(data).enter()
|
||||
.append('circle')
|
||||
.attr('r', 2)
|
||||
.attr('cx', function(d) { return x(d.date); })
|
||||
.attr('cy', function(d) { return y(d.value); })
|
||||
|
||||
.on('mouseover', function(d) {
|
||||
div.transition().style('opacity', .9);
|
||||
div.html('' + d.date_label+ '<br/>' + d.value).style('left', (d3.event.pageX) + 'px')
|
||||
.style("left", (d3.event.pageX) + "px")
|
||||
.style("top", (d3.event.pageY - 28) + "px");
|
||||
})
|
||||
.on('mouseout', function(d)
|
||||
{
|
||||
div.transition().style('opacity', 0);
|
||||
});
|
||||
|
||||
});
|
||||
}
|
||||
</script>
|
||||
|
|
@ -113,6 +113,8 @@
|
|||
{% include 'correlation/metadata_card_domain.html' %}
|
||||
{% elif dict_object["object_type"] == "screenshot" %}
|
||||
{% include 'correlation/metadata_card_screenshot.html' %}
|
||||
{% elif dict_object["object_type"] == "title" %}
|
||||
{% include 'correlation/metadata_card_title.html' %}
|
||||
{% elif dict_object["object_type"] == "item" %}
|
||||
{% include 'correlation/metadata_card_item.html' %}
|
||||
{% endif %}
|
||||
|
|
|
|||
|
|
@ -347,6 +347,46 @@
|
|||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if 'title' in dict_domain%}
|
||||
<div id="accordiontitle" class="mt-3">
|
||||
<div class="card">
|
||||
<div class="card-header" id="headingtitle">
|
||||
<div class="row">
|
||||
<div class="col-11">
|
||||
<div class="mt-2">
|
||||
<i class="fas fa-heading"></i> Titles
|
||||
<div class="badge badge-warning">{{dict_domain['title']|length}}</div>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-1">
|
||||
<button class="btn btn-link collapsed rotate" data-toggle="collapse" data-target="#collapsetitle" aria-expanded="false" aria-controls="collapsetitle">
|
||||
<i class="fas fa-chevron-circle-down"></i>
|
||||
</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<div id="collapsetitle" class="collapse" aria-labelledby="headingtitle" data-parent="#accordiontitle">
|
||||
<div class="card-body">
|
||||
<table id="tabletitle" class="table table-striped">
|
||||
<thead class="thead-dark">
|
||||
<tr>
|
||||
<th>Tilte</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for title in dict_domain['title']%}
|
||||
<tr>
|
||||
<td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?type=title&id={{ title[1] }}">{{ title[1] }}</a></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
{% endif %}
|
||||
|
||||
{% if dict_domain["history"] %}
|
||||
<hr class="my-4">
|
||||
<div class="card mb-1">
|
||||
|
|
@ -489,6 +529,9 @@
|
|||
{% endif %}
|
||||
{% if 'cryptocurrency' in dict_domain%}
|
||||
$('#tablecurrency').DataTable({});
|
||||
{% endif %}
|
||||
{% if 'title' in dict_domain%}
|
||||
$('#tabletitle').DataTable({});
|
||||
{% endif %}
|
||||
table = $('#myTable_1').DataTable(
|
||||
{
|
||||
|
|
|
|||
611
var/www/templates/objects/title/TitleDaterange.html
Normal file
611
var/www/templates/objects/title/TitleDaterange.html
Normal file
|
|
@ -0,0 +1,611 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Titles - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/d3/sparklines.js')}}"></script>
|
||||
|
||||
<style>
|
||||
.input-group .form-control {
|
||||
position: unset;
|
||||
}
|
||||
.line {
|
||||
fill: none;
|
||||
stroke: #000;
|
||||
stroke-width: 2.0px;
|
||||
}
|
||||
.bar {
|
||||
fill: steelblue;
|
||||
}
|
||||
.bar:hover{
|
||||
fill: brown;
|
||||
cursor: pointer;
|
||||
}
|
||||
.bar_stack:hover{
|
||||
cursor: pointer;
|
||||
}
|
||||
.pie_path:hover{
|
||||
cursor: pointer;
|
||||
}
|
||||
.svgText {
|
||||
pointer-events: none;
|
||||
}
|
||||
div.tooltip {
|
||||
position: absolute;
|
||||
text-align: center;
|
||||
padding: 2px;
|
||||
font: 12px sans-serif;
|
||||
background: #ebf4fb;
|
||||
border: 2px solid #b7ddf2;
|
||||
border-radius: 8px;
|
||||
pointer-events: none;
|
||||
color: #000000;
|
||||
}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
<div class="row">
|
||||
|
||||
{% include 'sidebars/sidebar_objects.html' %}
|
||||
|
||||
<div class="col-12 col-lg-10" id="core_content">
|
||||
|
||||
<div class="row">
|
||||
<div class="col-xl-10">
|
||||
<div class="mt-1" id="barchart_type"></div>
|
||||
|
||||
<div class="card border-secondary my-2">
|
||||
<div class="card-body text-dark">
|
||||
<h5 class="card-title">Search Title by name:</h5>
|
||||
<form action="{{ url_for('objects_title.objects_title_search') }}" id="search_subtype_onj" method='post'>
|
||||
<div class="input-group mb-1">
|
||||
<input type="text" class="form-control col-8" name="object_id" value="" placeholder="Title ID" required>
|
||||
<button class="btn btn-primary input-group-addon search-obj col-2"><i class="fas fa-search"></i></button>
|
||||
</div>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="col-xl-2">
|
||||
|
||||
<div class="card mb-3 mt-2" style="background-color:#d9edf7;">
|
||||
<div class="card-body text-center py-2">
|
||||
<h6 class="card-title" style="color:#286090;">Select a date range :</h6>
|
||||
<form action="{{ url_for('objects_title.objects_titles_post') }}" id="hash_selector_form" method='post'>
|
||||
<div class="input-group" id="date-range-from">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
|
||||
</div>
|
||||
<div class="input-group" id="date-range-to">
|
||||
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off">
|
||||
</div>
|
||||
<div class="form-check my-1">
|
||||
<input class="form-check-input" type="checkbox" id="checkbox-input-show" name="show_objects" value="True" {% if show_objects %}checked{% endif %}>
|
||||
<label class="form-check-label" for="checkbox-input-show">
|
||||
<span style="color:#286090; font-size: 14px;">
|
||||
Show Title <i class="fas fa-key"></i>
|
||||
</span>
|
||||
</label>
|
||||
</div>
|
||||
<button class="btn btn-primary" style="text-align:center;">
|
||||
<i class="fas fa-copy"></i> Search
|
||||
</button>
|
||||
</form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div id="pie_chart_encoded">
|
||||
</div>
|
||||
<div id="pie_chart_top5_types">
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{% if dict_objects %}
|
||||
{% if date_from|string == date_to|string %}
|
||||
<h3> {{ date_from }} Title: </h3>
|
||||
{% else %}
|
||||
<h3> {{ date_from }} to {{ date_to }} Title: </h3>
|
||||
{% endif %}
|
||||
<table id="tableb64" class="table table-striped table-bordered">
|
||||
<thead class="bg-dark text-white">
|
||||
<tr>
|
||||
<th></th>
|
||||
<th>First Seen</th>
|
||||
<th>Last Seen</th>
|
||||
<th>Total</th>
|
||||
<th>Last days</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody style="font-size: 15px;">
|
||||
{% for obj_id in dict_objects %}
|
||||
<tr>
|
||||
<td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?type=title&id={{ obj_id }}">{{ dict_objects[obj_id]['content'] }}</a></td>
|
||||
<td>{{ dict_objects[obj_id]['first_seen'] }}</td>
|
||||
<td>{{ dict_objects[obj_id]['last_seen'] }}</td>
|
||||
<td>{{ dict_objects[obj_id]['nb_seen'] }}</td>
|
||||
<td id="sparklines_{{ obj_id }}" style="text-align:center;"></td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
|
||||
{% else %}
|
||||
{% if show_objects %}
|
||||
{% if date_from|string == date_to|string %}
|
||||
<h3> {{ date_from }}, No Title</h3>
|
||||
{% else %}
|
||||
<h3> {{ date_from }} to {{ date_to }}, No Title</h3>
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
{% endif %}
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var chart = {};
|
||||
$(document).ready(function(){
|
||||
$("#page-Decoded").addClass("active");
|
||||
$("#nav_title").addClass("active");
|
||||
|
||||
$('#date-range-from').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function()
|
||||
{
|
||||
if ($('#date-range-from-input').val() && $('#date-range-to').val() )
|
||||
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2)
|
||||
{
|
||||
$('#date-range-from-input').val(s1);
|
||||
$('#date-range-to-input').val(s2);
|
||||
},
|
||||
});
|
||||
$('#date-range-to').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function()
|
||||
{
|
||||
if ($('#date-range-from-input').val() && $('#date-range-to').val() )
|
||||
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2)
|
||||
{
|
||||
$('#date-range-from-input').val(s1);
|
||||
$('#date-range-to-input').val(s2);
|
||||
},
|
||||
});
|
||||
|
||||
$('#date-range-from').data('dateRangePicker').setDateRange('{{date_from}}','{{date_to}}');
|
||||
$('#date-range-to').data('dateRangePicker').setDateRange('{{date_from}}','{{date_to}}');
|
||||
|
||||
$('#tableb64').DataTable({
|
||||
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||
"iDisplayLength": 10,
|
||||
"order": [[ 3, "desc" ]]
|
||||
});
|
||||
|
||||
|
||||
chart.stackBarChart = barchart_type_stack("{{ url_for('objects_title.objects_title_range_json') }}?date_from={{date_from}}&date_to={{date_to}}", 'id');
|
||||
|
||||
|
||||
chart.onResize();
|
||||
$(window).on("resize", function() {
|
||||
chart.onResize();
|
||||
});
|
||||
});
|
||||
|
||||
function toggle_sidebar(){
|
||||
if($('#nav_menu').is(':visible')){
|
||||
$('#nav_menu').hide();
|
||||
$('#side_menu').removeClass('border-right')
|
||||
$('#side_menu').removeClass('col-lg-2')
|
||||
$('#core_content').removeClass('col-lg-10')
|
||||
}else{
|
||||
$('#nav_menu').show();
|
||||
$('#side_menu').addClass('border-right')
|
||||
$('#side_menu').addClass('col-lg-2')
|
||||
$('#core_content').addClass('col-lg-10')
|
||||
}
|
||||
}
|
||||
</script>
|
||||
|
||||
<script>
|
||||
{% for obj_id in dict_objects %}
|
||||
sparkline("sparklines_{{ obj_id }}", {{ dict_objects[obj_id]['sparkline'] }}, {});
|
||||
{% endfor %}
|
||||
</script>
|
||||
|
||||
<script>
|
||||
var margin = {top: 20, right: 100, bottom: 55, left: 45},
|
||||
width = 1000 - margin.left - margin.right,
|
||||
height = 500 - margin.top - margin.bottom;
|
||||
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
|
||||
|
||||
var y = d3.scaleLinear().rangeRound([height, 0]);
|
||||
|
||||
var xAxis = d3.axisBottom(x);
|
||||
|
||||
var yAxis = d3.axisLeft(y);
|
||||
|
||||
var color = d3.scaleOrdinal(d3.schemeSet3);
|
||||
|
||||
var svg = d3.select("#barchart_type").append("svg")
|
||||
.attr("id", "thesvg")
|
||||
.attr("viewBox", "0 0 1000 500")
|
||||
.attr("width", width + margin.left + margin.right)
|
||||
.attr("height", height + margin.top + margin.bottom)
|
||||
.append("g")
|
||||
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
|
||||
|
||||
function barchart_type_stack(url, id) {
|
||||
|
||||
d3.json(url)
|
||||
.then(function(data){
|
||||
|
||||
var labelVar = 'date'; //A
|
||||
var varNames = d3.keys(data[0])
|
||||
.filter(function (key) { return key !== labelVar;}); //B
|
||||
|
||||
data.forEach(function (d) { //D
|
||||
var y0 = 0;
|
||||
d.mapping = varNames.map(function (name) {
|
||||
return {
|
||||
name: name,
|
||||
label: d[labelVar],
|
||||
y0: y0,
|
||||
y1: y0 += +d[name]
|
||||
};
|
||||
});
|
||||
d.total = d.mapping[d.mapping.length - 1].y1;
|
||||
});
|
||||
|
||||
x.domain(data.map(function (d) { return (d.date); })); //E
|
||||
y.domain([0, d3.max(data, function (d) { return d.total; })]);
|
||||
|
||||
svg.append("g")
|
||||
.attr("class", "x axis")
|
||||
.attr("transform", "translate(0," + height + ")")
|
||||
.call(xAxis)
|
||||
.selectAll("text")
|
||||
.attr("class", "bar")
|
||||
{% if date_from|string == date_to|string and type is none %}
|
||||
.on("click", function (d) { window.location.href = "{{ url_for('objects_title.objects_titles') }}?date_from={{date_from}}&date_to={{date_to}}&type_id="+d })
|
||||
.attr("transform", "rotate(-18)" )
|
||||
{% elif date_from|string == date_to|string and type is not none %}
|
||||
.on("click", function (d) { window.location.href = "{{ url_for('objects_title.objects_titles') }}?date_from="+d+'&date_to='+d })
|
||||
.attr("transform", "rotate(-18)" )
|
||||
{% else %}
|
||||
.on("click", function (d) { window.location.href = "{{ url_for('objects_title.objects_titles') }}?date_from="+d+'&date_to='+d })
|
||||
.attr("transform", "rotate(-40)" )
|
||||
{% endif %}
|
||||
.style("text-anchor", "end");
|
||||
|
||||
svg.append("g")
|
||||
.attr("class", "y axis")
|
||||
.call(yAxis)
|
||||
.append("text")
|
||||
.attr("transform", "rotate(-90)")
|
||||
.attr("y", 6)
|
||||
.attr("dy", ".71em")
|
||||
.style("text-anchor", "end");
|
||||
|
||||
var selection = svg.selectAll(".series")
|
||||
.data(data)
|
||||
.enter().append("g")
|
||||
.attr("class", "series")
|
||||
.attr("transform", function (d) { return "translate(" + x((d.date)) + ",0)"; });
|
||||
|
||||
selection.selectAll("rect")
|
||||
.data(function (d) { return d.mapping; })
|
||||
.enter().append("rect")
|
||||
.attr("class", "bar_stack")
|
||||
.attr("width", x.bandwidth())
|
||||
.attr("y", function (d) { return y(d.y1); })
|
||||
.attr("height", function (d) { return y(d.y0) - y(d.y1); })
|
||||
.style("fill", function (d) { return color(d.name); })
|
||||
.style("stroke", "grey")
|
||||
.on("mouseover", function (d) { showPopover.call(this, d); })
|
||||
.on("mouseout", function (d) { removePopovers(); })
|
||||
{% if date_from|string == date_to|string and type is none %}
|
||||
.on("click", function(d){ window.location.href = "{{ url_for('objects_title.objects_titles') }}" +'?date_from={{date_from}}&date_to={{date_to}}&type_id='+d.label+'&encoding='+d.name; });
|
||||
{% elif date_from|string == date_to|string and type is not none %}
|
||||
.on("click", function(d){ window.location.href = "{{ url_for('objects_title.objects_titles') }}" +'?type_id={{type_id}}&date_from='+d.label+'&date_to='+d.label+'&encoding='+d.name; });
|
||||
{% else %}
|
||||
.on("click", function(d){ window.location.href = "{{ url_for('objects_title.objects_titles') }}" +'?type_id='+ d.name +'&date_from='+d.label+'&date_to='+d.label; });
|
||||
{% endif %}
|
||||
|
||||
data.forEach(function(d) {
|
||||
if(d.total !== 0){
|
||||
svg.append("text")
|
||||
.attr("class", "bar")
|
||||
.attr("dy", "-.35em")
|
||||
.attr('x', x(d.date) + x.bandwidth()/2)
|
||||
.attr('y', y(d.total))
|
||||
{% if date_from|string == date_to|string and type is none %}
|
||||
.on("click", function () {window.location.href = "{{ url_for('objects_title.objects_titles') }}"+'?date_from={{date_from}}&date_to={{date_to}}&type_id='+d.date })
|
||||
{% elif date_from|string == date_to|string and type is not none %}
|
||||
.on("click", function () {window.location.href = "{{ url_for('objects_title.objects_titles') }}?type_id={{type_id}}&date_from="+d.date+'&date_to='+d.date })
|
||||
{% else %}
|
||||
.on("click", function () {window.location.href = "{{ url_for('objects_title.objects_titles') }}"+'?date_from='+d.date+'&date_to='+d.date })
|
||||
{% endif %}
|
||||
.style("text-anchor", "middle")
|
||||
.text(d.total);
|
||||
}
|
||||
});
|
||||
|
||||
drawLegend(varNames);
|
||||
});
|
||||
|
||||
}
|
||||
|
||||
function drawLegend (varNames) {
|
||||
var legend = svg.selectAll(".legend")
|
||||
.data(varNames.slice().reverse())
|
||||
.enter().append("g")
|
||||
.attr("class", "legend")
|
||||
.attr("transform", function (d, i) { return "translate(0," + i * 20 + ")"; });
|
||||
|
||||
legend.append("rect")
|
||||
.attr("x", 943)
|
||||
.attr("width", 10)
|
||||
.attr("height", 10)
|
||||
.style("fill", color)
|
||||
.style("stroke", "grey");
|
||||
|
||||
legend.append("text")
|
||||
.attr("class", "svgText")
|
||||
.attr("x", 941)
|
||||
.attr("y", 6)
|
||||
.attr("dy", ".35em")
|
||||
.style("text-anchor", "end")
|
||||
.text(function (d) { return d; });
|
||||
}
|
||||
|
||||
function removePopovers () {
|
||||
$('.popover').each(function() {
|
||||
$(this).remove();
|
||||
});
|
||||
}
|
||||
|
||||
function showPopover (d) {
|
||||
$(this).popover({
|
||||
title: "<b><span id='tooltip-id-name-bar'></span></b>",
|
||||
placement: 'top',
|
||||
container: 'body',
|
||||
trigger: 'manual',
|
||||
html : true,
|
||||
content: function() {
|
||||
return "<span id='tooltip-id-label'></span>" +
|
||||
"<br/>num: <span id='tooltip-id-value-bar'></span>"; }
|
||||
});
|
||||
$(this).popover('show');
|
||||
$("#tooltip-id-name-bar").text(d.name);
|
||||
$("#tooltip-id-label").text(d.label);
|
||||
$("#tooltip-id-value-bar").text(d3.format(",")(d.value ? d.value: d.y1 - d.y0));
|
||||
}
|
||||
|
||||
chart.onResize = function () {
|
||||
var aspect = 1000 / 500, chart = $("#thesvg");
|
||||
var targetWidth = chart.parent().width();
|
||||
chart.attr("width", targetWidth);
|
||||
chart.attr("height", targetWidth / aspect);
|
||||
}
|
||||
|
||||
window.chart = chart;
|
||||
|
||||
</script>
|
||||
|
||||
<script>
|
||||
function draw_pie_chart(id, url_json, pie_on_click_url) {
|
||||
|
||||
var width_pie = 200;
|
||||
var height_pie = 200;
|
||||
var padding_pie = 10;
|
||||
var opacity_pie = .8;
|
||||
|
||||
var radius_pie = Math.min(width_pie - padding_pie, height_pie - padding_pie) / 2;
|
||||
//var color_pie = d3.scaleOrdinal(d3.schemeCategory10);
|
||||
var color_pie = d3.scaleOrdinal(d3.schemeSet3);
|
||||
|
||||
var div_pie = d3.select("body").append("div")
|
||||
.attr("class", "tooltip")
|
||||
.style("opacity", 0);
|
||||
|
||||
var svg_pie = d3.select("#"+id)
|
||||
.append('svg')
|
||||
.attr("width", '100%')
|
||||
.attr("height", '100%')
|
||||
.attr('viewBox','0 0 '+Math.min(width_pie,height_pie) +' '+Math.min(width_pie,height_pie) )
|
||||
.attr('preserveAspectRatio','xMinYMin')
|
||||
|
||||
|
||||
var g_pie = svg_pie.append('g')
|
||||
.attr('transform', 'translate(' + (width_pie/2) + ',' + (height_pie/2) + ')');
|
||||
|
||||
var arc_pie = d3.arc()
|
||||
.innerRadius(0)
|
||||
.outerRadius(radius_pie);
|
||||
|
||||
d3.json(url_json)
|
||||
.then(function(data){
|
||||
|
||||
var pie_pie = d3.pie()
|
||||
.value(function(d) { return d.value; })
|
||||
.sort(null);
|
||||
|
||||
var path_pie = g_pie.selectAll('path')
|
||||
.data(pie_pie(data))
|
||||
.enter()
|
||||
.append("g")
|
||||
.append('path')
|
||||
.attr('d', arc_pie)
|
||||
.attr('fill', (d,i) => color_pie(i))
|
||||
.attr('class', 'pie_path')
|
||||
.on("mouseover", mouseovered_pie)
|
||||
.on("mouseout", mouseouted_pie)
|
||||
.on("click", function (d) {window.location.href = pie_on_click_url+d.data.name })
|
||||
.style('opacity', opacity_pie)
|
||||
.style('stroke', 'white');
|
||||
});
|
||||
|
||||
|
||||
function mouseovered_pie(d) {
|
||||
//remove old content
|
||||
$("#tooltip-id-name").remove();
|
||||
$("#tooltip-id-value").remove();
|
||||
|
||||
// tooltip
|
||||
var content;
|
||||
|
||||
content = "<b><span id='tooltip-id-name'></span></b><br/>"+
|
||||
"<br/>"+
|
||||
"<i>Decoded</i>: <span id='tooltip-id-value'></span><br/>"
|
||||
|
||||
div_pie.transition()
|
||||
.duration(200)
|
||||
.style("opacity", .9);
|
||||
div_pie.html(content)
|
||||
.style("left", (d3.event.pageX) + "px")
|
||||
.style("top", (d3.event.pageY - 28) + "px");
|
||||
|
||||
$("#tooltip-id-name").text(d.data.name);
|
||||
$("#tooltip-id-value").text(d.data.value);
|
||||
}
|
||||
|
||||
function mouseouted_pie() {
|
||||
div_pie.transition()
|
||||
.duration(500)
|
||||
.style("opacity", 0);
|
||||
}
|
||||
}
|
||||
|
||||
</script>
|
||||
|
||||
|
||||
|
||||
<script>
|
||||
function barchart_type(url, id) {
|
||||
|
||||
|
||||
var margin = {top: 20, right: 20, bottom: 70, left: 40};
|
||||
|
||||
var width = 960 - margin.left - margin.right;
|
||||
var height = 500 - margin.top - margin.bottom;
|
||||
|
||||
var x = d3.scaleBand().rangeRound([0, width]).padding(0.1);
|
||||
var y = d3.scaleLinear().rangeRound([height, 0]);
|
||||
|
||||
var xAxis = d3.axisBottom(x)
|
||||
//.tickFormat(d3.time.format("%Y-%m"));
|
||||
|
||||
var yAxis = d3.axisLeft(y)
|
||||
.ticks(10);
|
||||
|
||||
/*var svg = d3.select(id).append("svg")
|
||||
.attr("width", width + margin.left + margin.right)
|
||||
.attr("height", height + margin.top + margin.bottom)
|
||||
.attr("id", "thesvg")
|
||||
.append("g")
|
||||
.attr("transform",
|
||||
"translate(" + margin.left + "," + margin.top + ")");*/
|
||||
|
||||
|
||||
d3.json(url)
|
||||
.then(function(data){
|
||||
|
||||
data.forEach(function(d) {
|
||||
d.value = +d.value;
|
||||
});
|
||||
|
||||
x.domain(data.map(function(d) { return d.date; }));
|
||||
y.domain([0, d3.max(data, function(d) { return d.value; })]);
|
||||
|
||||
var label = svg.append("g")
|
||||
.attr("class", "x axis")
|
||||
.attr("transform", "translate(0," + height + ")")
|
||||
.call(xAxis)
|
||||
.selectAll("text")
|
||||
.style("text-anchor", "end")
|
||||
.attr("dx", "-.8em")
|
||||
.attr("dy", "-.55em")
|
||||
{% if daily_type_chart %}
|
||||
.attr("transform", "rotate(-20)" );
|
||||
{% else %}
|
||||
.attr("transform", "rotate(-70)" )
|
||||
.attr("class", "bar")
|
||||
{% endif %}
|
||||
|
||||
svg.append("g")
|
||||
.attr("class", "y axis")
|
||||
.call(yAxis)
|
||||
.append("text")
|
||||
.attr("transform", "rotate(-90)")
|
||||
.attr("y", 6)
|
||||
.attr("dy", ".71em")
|
||||
.style("text-anchor", "end")
|
||||
.text("Value ($)");
|
||||
|
||||
var bar = svg.selectAll("bar")
|
||||
.data(data)
|
||||
.enter().append("rect")
|
||||
.attr("class", "bar")
|
||||
//.style("fill", "steelblue")
|
||||
.attr("x", function(d) { return x(d.date); })
|
||||
.attr("width", x.bandwidth())
|
||||
.attr("y", function(d) { return y(d.value); })
|
||||
.attr("height", function(d) { return height - y(d.value); })
|
||||
|
||||
|
||||
data.forEach(function(d) {
|
||||
if(d.value != 0){
|
||||
svg.append("text")
|
||||
.attr("class", "bar")
|
||||
.attr("dy", "-.35em")
|
||||
//.text(function(d) { return d.value; });
|
||||
.text(d.value)
|
||||
.style("text-anchor", "middle")
|
||||
.attr('x', x(d.date) + x.bandwidth()/2)
|
||||
.attr('y', y(d.value));
|
||||
}
|
||||
});
|
||||
|
||||
});
|
||||
|
||||
}
|
||||
</script>
|
||||
|
||||
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -34,6 +34,12 @@
|
|||
<span>CVE</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('objects_title.objects_titles')}}" id="nav_title">
|
||||
<i class="fas fa-heading"></i>
|
||||
<span>Title</span>
|
||||
</a>
|
||||
</li>
|
||||
<li class="nav-item">
|
||||
<a class="nav-link" href="{{url_for('objects_decoded.decodeds_dashboard')}}" id="nav_dashboard">
|
||||
<i class="fas fa-lock-open"></i>
|
||||
|
|
@ -61,7 +67,6 @@
|
|||
</ul>
|
||||
|
||||
|
||||
|
||||
<h5 class="d-flex text-muted w-100">
|
||||
<span>
|
||||
<img src="{{ url_for('static', filename='image/misp-logo.png')}}" alt="MISP" style="width:80px;">
|
||||
|
|
|
|||
Loading…
Reference in a new issue