mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-26 07:47:17 +00:00
chg: [dashboard] add echart feeder graph + cleanup olg graph libs + dashboard, show today nb objects
This commit is contained in:
terrtia
parent
456258c976
commit
73543ae5ad
25 changed files with 992 additions and 206 deletions
97
bin/lib/ail_stats.py
Executable file
97
bin/lib/ail_stats.py
Executable file
|
|
@ -0,0 +1,97 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
import datetime
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
|
||||
# from datetime import datetime
|
||||
from logging import lastResort
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects import ail_objects
|
||||
|
||||
|
||||
# Config
|
||||
config_loader = ConfigLoader()
|
||||
r_stats = config_loader.get_db_conn("Kvrocks_Stats")
|
||||
# r_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||
config_loader = None
|
||||
|
||||
|
||||
def get_feeders():
|
||||
return r_stats.smembers(f'feeders:name')
|
||||
|
||||
def get_current_feeder_timestamp(timestamp):
|
||||
return int(timestamp - (timestamp % 30))
|
||||
|
||||
def get_next_feeder_timestamp(timestamp):
|
||||
return int(timestamp + 30 - (timestamp % 30))
|
||||
|
||||
def get_feeders_by_time(timestamp): # TODO
|
||||
feeders = {}
|
||||
for row in r_stats.zrange(f'feeders:{timestamp}', 0, -1, withscores=True):
|
||||
feeders[row[0]] = int(row[1])
|
||||
return feeders
|
||||
|
||||
def get_feeders_dashboard_full():
|
||||
timestamp = get_current_feeder_timestamp(int(time.time()))
|
||||
print(timestamp)
|
||||
# timestamp = 1731491970
|
||||
f_dashboard = {}
|
||||
|
||||
feeders = get_feeders()
|
||||
d_time = []
|
||||
for i in range(timestamp - 30*20, timestamp +30, 30):
|
||||
t_feeders = get_feeders_by_time(i)
|
||||
for feeder in feeders:
|
||||
if feeder not in f_dashboard:
|
||||
f_dashboard[feeder] = []
|
||||
if feeder in t_feeders:
|
||||
f_dashboard[feeder].append(t_feeders[feeder])
|
||||
else:
|
||||
f_dashboard[feeder].append(0)
|
||||
d_time.append(datetime.datetime.utcfromtimestamp(i).strftime('%H:%M:%S'))
|
||||
return {'data': f_dashboard, 'dates': d_time}
|
||||
|
||||
def get_feeders_dashboard():
|
||||
timestamp = get_current_feeder_timestamp(int(time.time()))
|
||||
print(timestamp)
|
||||
|
||||
f_dashboard = {}
|
||||
t_feeders = get_feeders_by_time(timestamp)
|
||||
for feeder in get_feeders():
|
||||
if feeder in t_feeders:
|
||||
f_dashboard[feeder] = t_feeders[feeder]
|
||||
else:
|
||||
f_dashboard[feeder] = 0
|
||||
|
||||
date = datetime.datetime.utcfromtimestamp(timestamp).strftime('%H:%M:%S')
|
||||
return {'data': f_dashboard, 'date': date}
|
||||
|
||||
|
||||
def add_feeders(timestamp, feeders):
|
||||
if feeders:
|
||||
r = r_stats.zadd(f'feeders:{timestamp}', feeders)
|
||||
print(r)
|
||||
for feeder in feeders:
|
||||
r_stats.sadd(f'feeders:name', feeder)
|
||||
# cleanup keys
|
||||
r_stats.sadd(f'feeders:timestamps', timestamp)
|
||||
|
||||
def get_nb_objs_today():
|
||||
date = datetime.date.today().strftime("%Y%m%d")
|
||||
nb_objs = ail_objects.get_nb_objects_by_date(date)
|
||||
return nb_objs
|
||||
|
||||
def get_nb_objs_dashboard():
|
||||
date = datetime.date.today().strftime("%Y%m%d")
|
||||
return ail_objects.get_nb_objects_dashboard(date)
|
||||
|
||||
|
||||
|
||||
|
|
@ -150,6 +150,19 @@ class Barcodes(AbstractDaterangeObjects):
|
|||
def __init__(self):
|
||||
super().__init__('barcode', Barcode)
|
||||
|
||||
def get_name(self):
|
||||
return 'Barcodes'
|
||||
|
||||
def get_icon(self):
|
||||
return {'fa': 'fas', 'icon': 'barcode'}
|
||||
|
||||
def get_link(self, flask_context=False):
|
||||
if flask_context:
|
||||
url = url_for('objects_barcode.objects_barcodes')
|
||||
else:
|
||||
url = f'{baseurl}/objects/barcodes'
|
||||
return url
|
||||
|
||||
def sanitize_id_to_search(self, name_to_search):
|
||||
return name_to_search # TODO
|
||||
|
||||
|
|
|
|||
|
|
@ -109,6 +109,19 @@ class CookiesNames(AbstractDaterangeObjects):
|
|||
def __init__(self):
|
||||
super().__init__('cookie-name', CookieName)
|
||||
|
||||
def get_name(self):
|
||||
return 'Cookie-Names'
|
||||
|
||||
def get_icon(self):
|
||||
return {'fa': 'fas', 'icon': 'cookie-bite'}
|
||||
|
||||
def get_link(self, flask_context=False):
|
||||
if flask_context:
|
||||
url = url_for('objects_cookie_name.objects_cookies_names')
|
||||
else:
|
||||
url = f'{baseurl}/objects/cookie-name'
|
||||
return url
|
||||
|
||||
def sanitize_id_to_search(self, name_to_search):
|
||||
return name_to_search # TODO
|
||||
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ sys.path.append(os.environ['AIL_BIN'])
|
|||
# Import Project packages
|
||||
##################################
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib.objects.abstract_daterange_object import AbstractDaterangeObject
|
||||
from lib.objects.abstract_daterange_object import AbstractDaterangeObject, AbstractDaterangeObjects
|
||||
from packages import Date
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
|
|
@ -97,6 +97,29 @@ class Cve(AbstractDaterangeObject):
|
|||
except requests.exceptions.ReadTimeout:
|
||||
return {'error': f'Timeout Error'}
|
||||
|
||||
class Cves(AbstractDaterangeObjects):
|
||||
"""
|
||||
Barcodes Objects
|
||||
"""
|
||||
def __init__(self):
|
||||
super().__init__('cve', Cve)
|
||||
|
||||
def get_name(self):
|
||||
return 'Cves'
|
||||
|
||||
def get_icon(self):
|
||||
return {'fa': 'fas', 'icon': 'bug'}
|
||||
|
||||
def get_link(self, flask_context=False):
|
||||
if flask_context:
|
||||
url = url_for('objects_cve.objects_cves')
|
||||
else:
|
||||
url = f'{baseurl}/objects/cves'
|
||||
return url
|
||||
|
||||
def sanitize_id_to_search(self, name_to_search):
|
||||
return name_to_search # TODO
|
||||
|
||||
|
||||
# TODO ADD SEARCH FUNCTION
|
||||
|
||||
|
|
|
|||
|
|
@ -114,6 +114,19 @@ class DomHashs(AbstractDaterangeObjects):
|
|||
def __init__(self):
|
||||
super().__init__('dom-hash', DomHash)
|
||||
|
||||
def get_name(self):
|
||||
return 'DomHashs'
|
||||
|
||||
def get_icon(self):
|
||||
return {'fa': 'fas', 'icon': 'align-left'}
|
||||
|
||||
def get_link(self, flask_context=False):
|
||||
if flask_context:
|
||||
url = url_for('objects_dom_hash.objects_dom_hashs')
|
||||
else:
|
||||
url = f'{baseurl}/objects/dom-hashs'
|
||||
return url
|
||||
|
||||
def sanitize_id_to_search(self, name_to_search):
|
||||
return name_to_search
|
||||
|
||||
|
|
|
|||
|
|
@ -109,6 +109,19 @@ class Etags(AbstractDaterangeObjects):
|
|||
def __init__(self):
|
||||
super().__init__('etag', Etag)
|
||||
|
||||
def get_name(self):
|
||||
return 'Etags'
|
||||
|
||||
def get_icon(self):
|
||||
return {'fa': 'fas', 'icon': 'tag'}
|
||||
|
||||
def get_link(self, flask_context=False):
|
||||
if flask_context:
|
||||
url = url_for('objects_etag.objects_etags')
|
||||
else:
|
||||
url = f'{baseurl}/objects/etags'
|
||||
return url
|
||||
|
||||
def sanitize_id_to_search(self, name_to_search):
|
||||
return name_to_search # TODO
|
||||
|
||||
|
|
|
|||
|
|
@ -135,6 +135,19 @@ class Favicons(AbstractDaterangeObjects):
|
|||
def __init__(self):
|
||||
super().__init__('favicon', Favicon)
|
||||
|
||||
def get_name(self):
|
||||
return 'Favicons'
|
||||
|
||||
def get_icon(self):
|
||||
return {'fa': 'fas', 'icon': 'star-half'}
|
||||
|
||||
def get_link(self, flask_context=False):
|
||||
if flask_context:
|
||||
url = url_for('objects_favicon.objects_favicons')
|
||||
else:
|
||||
url = f'{baseurl}/objects/favicons'
|
||||
return url
|
||||
|
||||
def sanitize_id_to_search(self, name_to_search):
|
||||
return name_to_search # TODO
|
||||
|
||||
|
|
|
|||
|
|
@ -84,6 +84,20 @@ class FilesNames(AbstractDaterangeObjects):
|
|||
def __init__(self):
|
||||
super().__init__('file-name', FileName)
|
||||
|
||||
def get_name(self):
|
||||
return 'File-Names'
|
||||
|
||||
def get_icon(self):
|
||||
return {'fa': 'far', 'icon': 'file'}
|
||||
|
||||
def get_link(self, flask_context=False):
|
||||
pass
|
||||
# if flask_context:
|
||||
# url = url_for('objects_favicon.objects_favicons')
|
||||
# else:
|
||||
# url = f'{baseurl}/objects/favicons'
|
||||
# return url
|
||||
|
||||
def sanitize_id_to_search(self, name_to_search):
|
||||
return name_to_search
|
||||
|
||||
|
|
|
|||
|
|
@ -126,6 +126,19 @@ class HHHashs(AbstractDaterangeObjects):
|
|||
def __init__(self):
|
||||
super().__init__('hhhash', HHHash)
|
||||
|
||||
def get_name(self):
|
||||
return 'HHHashs'
|
||||
|
||||
def get_icon(self):
|
||||
return {'fas': 'far', 'icon': 'align-left'}
|
||||
|
||||
def get_link(self, flask_context=False):
|
||||
if flask_context:
|
||||
url = url_for('objects_hhhash.objects_hhhashs')
|
||||
else:
|
||||
url = f'{baseurl}/objects/hhhashs'
|
||||
return url
|
||||
|
||||
def sanitize_id_to_search(self, name_to_search):
|
||||
return name_to_search # TODO
|
||||
|
||||
|
|
|
|||
|
|
@ -152,6 +152,19 @@ class Images(AbstractDaterangeObjects):
|
|||
def __init__(self):
|
||||
super().__init__('image', Image)
|
||||
|
||||
def get_name(self):
|
||||
return 'Images'
|
||||
|
||||
def get_icon(self):
|
||||
return {'fas': 'fas', 'icon': 'image'}
|
||||
|
||||
def get_link(self, flask_context=False):
|
||||
if flask_context:
|
||||
url = url_for('objects_image.objects_images')
|
||||
else:
|
||||
url = f'{baseurl}/objects/images'
|
||||
return url
|
||||
|
||||
def sanitize_id_to_search(self, name_to_search):
|
||||
return name_to_search # TODO
|
||||
|
||||
|
|
|
|||
|
|
@ -323,6 +323,19 @@ class Ocrs(AbstractDaterangeObjects):
|
|||
def __init__(self):
|
||||
super().__init__('ocr', Ocr)
|
||||
|
||||
def get_name(self):
|
||||
return 'Ocrs'
|
||||
|
||||
def get_icon(self):
|
||||
return {'fas': 'far', 'icon': 'expand'}
|
||||
|
||||
def get_link(self, flask_context=False):
|
||||
if flask_context:
|
||||
url = url_for('objects_ocr.objects_ocrs')
|
||||
else:
|
||||
url = f'{baseurl}/objects/ocrs'
|
||||
return url
|
||||
|
||||
def sanitize_id_to_search(self, name_to_search):
|
||||
return name_to_search # TODO
|
||||
|
||||
|
|
|
|||
|
|
@ -150,6 +150,19 @@ class Qrcodes(AbstractDaterangeObjects):
|
|||
def __init__(self):
|
||||
super().__init__('qrcode', Qrcode)
|
||||
|
||||
def get_name(self):
|
||||
return 'Qrcodes'
|
||||
|
||||
def get_icon(self):
|
||||
return {'fas': 'far', 'icon': 'qrcode'}
|
||||
|
||||
def get_link(self, flask_context=False):
|
||||
if flask_context:
|
||||
url = url_for('objects_qrcode.objects_qrcodes')
|
||||
else:
|
||||
url = f'{baseurl}/objects/qrcodes'
|
||||
return url
|
||||
|
||||
def sanitize_id_to_search(self, name_to_search):
|
||||
return name_to_search # TODO
|
||||
|
||||
|
|
|
|||
|
|
@ -104,6 +104,19 @@ class Titles(AbstractDaterangeObjects):
|
|||
def __init__(self):
|
||||
super().__init__('title', Title)
|
||||
|
||||
def get_name(self):
|
||||
return 'Titles'
|
||||
|
||||
def get_icon(self):
|
||||
return {'fas': 'far', 'icon': 'heading'}
|
||||
|
||||
def get_link(self, flask_context=False):
|
||||
if flask_context:
|
||||
url = url_for('objects_title.objects_titles')
|
||||
else:
|
||||
url = f'{baseurl}/objects/titles'
|
||||
return url
|
||||
|
||||
def sanitize_id_to_search(self, name_to_search):
|
||||
return name_to_search
|
||||
|
||||
|
|
|
|||
|
|
@ -193,6 +193,18 @@ class AbstractDaterangeObjects(ABC):
|
|||
self.type = obj_type
|
||||
self.obj_class = obj_class
|
||||
|
||||
@abstractmethod
|
||||
def get_name(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_icon(self):
|
||||
pass
|
||||
|
||||
@abstractmethod
|
||||
def get_link(self, flask_context=False):
|
||||
pass
|
||||
|
||||
################################################
|
||||
################################################
|
||||
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ from lib.objects import ChatSubChannels
|
|||
from lib.objects import ChatThreads
|
||||
from lib.objects import CryptoCurrencies
|
||||
from lib.objects import CookiesNames
|
||||
from lib.objects.Cves import Cve
|
||||
from lib.objects import Cves
|
||||
from lib.objects.Decodeds import Decoded, get_all_decodeds_objects, get_nb_decodeds_objects
|
||||
from lib.objects.Domains import Domain
|
||||
from lib.objects import Etags
|
||||
|
|
@ -44,9 +44,36 @@ from lib.objects import Titles
|
|||
from lib.objects import UsersAccount
|
||||
from lib.objects import Usernames
|
||||
|
||||
config_loader = ConfigLoader()
|
||||
|
||||
config_loader = None
|
||||
# config_loader = ConfigLoader()
|
||||
#
|
||||
# config_loader = None
|
||||
# TODO INIT objs classes ????
|
||||
OBJECTS_CLASS = {
|
||||
'barcode': {'obj': BarCodes.Barcode, 'objs': BarCodes.Barcodes},
|
||||
'chat': {'obj': Chats.Chat, 'objs': None}, ## SUBTYPE #########################################
|
||||
'chat-subchannel': {'obj': ChatSubChannels.ChatSubChannel, 'objs': None}, ###### ######
|
||||
'chat-thread': {'obj': ChatThreads.ChatThread, 'objs': None}, ###### ######
|
||||
'cookie-name': {'obj': CookiesNames.CookieName, 'objs': CookiesNames.CookiesNames},
|
||||
'cve': {'obj': Cves.Cve, 'objs': Cves.Cves},
|
||||
'cryptocurrency': {'obj': CryptoCurrencies.CryptoCurrency, 'objs': None}, ## SUBTYPE #########################################
|
||||
'decoded': {'obj': Decoded, 'objs': None}, ###############################################################################################
|
||||
'domain': {'obj': Domain, 'objs': None}, ####################################################################################################
|
||||
'dom-hash': {'obj': DomHashs.DomHash, 'objs': DomHashs.DomHashs},
|
||||
'etag': {'obj': Etags.Etag, 'objs': Etags.Etags},
|
||||
'favicon': {'obj': Favicons.Favicon, 'objs': Favicons.Favicons},
|
||||
'file-name': {'obj': FilesNames.FileName, 'objs': FilesNames.FilesNames},
|
||||
'hhhash': {'obj': HHHashs.HHHash, 'objs': HHHashs.HHHashs},
|
||||
'item': {'obj': Item, 'objs': None}, ######
|
||||
'image': {'obj': Images.Image, 'objs': Images.Images},
|
||||
'message': {'obj': Messages.Message, 'objs': None}, ######
|
||||
'ocr': {'obj': Ocrs.Ocr, 'objs': Ocrs.Ocrs},
|
||||
'pgp': {'obj': Pgps.Pgp, 'objs': None}, ## SUBTYPE ###########################################################################
|
||||
'qrcode': {'obj': QrCodes.Qrcode, 'objs': QrCodes.Qrcodes},
|
||||
'screenshot': {'obj': Screenshots.Screenshot, 'objs': None}, ######
|
||||
'title': {'obj': Titles.Title, 'objs': Titles.Titles},
|
||||
'user-account': {'obj': UsersAccount.UserAccount, 'objs': None}, ## SUBTYPE ###########################################################################
|
||||
'username': {'obj': Usernames.Username, 'objs': None}, ## SUBTYPE ###########################################################################
|
||||
}
|
||||
|
||||
|
||||
def is_valid_object_type(obj_type):
|
||||
|
|
@ -70,67 +97,29 @@ def sanitize_objs_types(objs, default=False):
|
|||
l_types = get_all_objects()
|
||||
return l_types
|
||||
|
||||
|
||||
#### OBJECT ####
|
||||
|
||||
def get_obj_class(obj_type):
|
||||
if obj_type in OBJECTS_CLASS:
|
||||
return OBJECTS_CLASS[obj_type]['obj']
|
||||
|
||||
def get_objs_class(obj_type):
|
||||
if obj_type in OBJECTS_CLASS:
|
||||
return OBJECTS_CLASS[obj_type]['objs']
|
||||
|
||||
def get_object(obj_type, subtype, obj_id):
|
||||
if subtype == 'None':
|
||||
subtype = None
|
||||
obj_id = str(obj_id)
|
||||
if not subtype:
|
||||
if obj_type == 'item':
|
||||
return Item(obj_id)
|
||||
elif obj_type == 'domain':
|
||||
return Domain(obj_id)
|
||||
elif obj_type == 'decoded':
|
||||
return Decoded(obj_id)
|
||||
elif obj_type == 'cookie-name':
|
||||
return CookiesNames.CookieName(obj_id)
|
||||
elif obj_type == 'cve':
|
||||
return Cve(obj_id)
|
||||
elif obj_type == 'etag':
|
||||
return Etags.Etag(obj_id)
|
||||
elif obj_type == 'favicon':
|
||||
return Favicons.Favicon(obj_id)
|
||||
elif obj_type == 'file-name':
|
||||
return FilesNames.FileName(obj_id)
|
||||
elif obj_type == 'dom-hash':
|
||||
return DomHashs.DomHash(obj_id)
|
||||
elif obj_type == 'hhhash':
|
||||
return HHHashs.HHHash(obj_id)
|
||||
elif obj_type == 'image':
|
||||
return Images.Image(obj_id)
|
||||
elif obj_type == 'message':
|
||||
return Messages.Message(obj_id)
|
||||
elif obj_type == 'ocr':
|
||||
return Ocrs.Ocr(obj_id)
|
||||
elif obj_type == 'barcode':
|
||||
return BarCodes.Barcode(obj_id)
|
||||
elif obj_type == 'qrcode':
|
||||
return QrCodes.Qrcode(obj_id)
|
||||
elif obj_type == 'screenshot':
|
||||
return Screenshots.Screenshot(obj_id)
|
||||
elif obj_type == 'title':
|
||||
return Titles.Title(obj_id)
|
||||
else:
|
||||
obj_class = OBJECTS_CLASS[obj_type]['obj']
|
||||
if not obj_class:
|
||||
raise AILObjectUnknown(f'Unknown AIL object: {obj_type} {subtype} {obj_id}')
|
||||
if not subtype:
|
||||
return obj_class(obj_id)
|
||||
# SUBTYPES
|
||||
else:
|
||||
if obj_type == 'chat':
|
||||
return Chats.Chat(obj_id, subtype)
|
||||
elif obj_type == 'chat-subchannel':
|
||||
return ChatSubChannels.ChatSubChannel(obj_id, subtype)
|
||||
elif obj_type == 'chat-thread':
|
||||
return ChatThreads.ChatThread(obj_id, subtype)
|
||||
elif obj_type == 'cryptocurrency':
|
||||
return CryptoCurrencies.CryptoCurrency(obj_id, subtype)
|
||||
elif obj_type == 'pgp':
|
||||
return Pgps.Pgp(obj_id, subtype)
|
||||
elif obj_type == 'user-account':
|
||||
return UsersAccount.UserAccount(obj_id, subtype)
|
||||
elif obj_type == 'username':
|
||||
return Usernames.Username(obj_id, subtype)
|
||||
else:
|
||||
raise AILObjectUnknown(f'Unknown AIL object: {obj_type} {subtype} {obj_id}')
|
||||
obj_class(obj_id, subtype)
|
||||
|
||||
def exists_obj(obj_type, subtype, obj_id):
|
||||
obj = get_object(obj_type, subtype, obj_id)
|
||||
|
|
@ -172,6 +161,32 @@ def api_get_object_global_id(global_id):
|
|||
|
||||
#### --API-- ####
|
||||
|
||||
|
||||
#### OBJECTS ####
|
||||
|
||||
def get_nb_objects_by_date(date):
|
||||
objs = {}
|
||||
for obj_type in get_all_objects():
|
||||
objs_class = get_objs_class(obj_type)
|
||||
if objs_class:
|
||||
objs_class = objs_class()
|
||||
objs[obj_type] = objs_class.get_nb_by_date(date)
|
||||
return objs
|
||||
|
||||
def get_nb_objects_dashboard(date, flask_context=True):
|
||||
objs = {}
|
||||
for obj_type in get_all_objects():
|
||||
objs_class = get_objs_class(obj_type)
|
||||
if objs_class:
|
||||
objs_class = objs_class()
|
||||
objs[obj_type] = {}
|
||||
objs[obj_type]['nb'] = objs_class.get_nb_by_date(date)
|
||||
objs[obj_type]['name'] = objs_class.get_name()
|
||||
objs[obj_type]['icon'] = objs_class.get_icon()
|
||||
objs[obj_type]['link'] = objs_class.get_link(flask_context=flask_context)
|
||||
return objs
|
||||
|
||||
|
||||
#########################################################################################
|
||||
#########################################################################################
|
||||
#########################################################################################
|
||||
|
|
@ -241,6 +256,9 @@ def add_obj_tags(obj_type, subtype, id, tags):
|
|||
|
||||
# -TAGS- #
|
||||
|
||||
#### OBJ META ####
|
||||
|
||||
|
||||
def get_object_meta(obj_type, subtype, id, options=set(), flask_context=False):
|
||||
obj = get_object(obj_type, subtype, id)
|
||||
meta = obj.get_meta(options=options)
|
||||
|
|
|
|||
|
|
@ -176,4 +176,3 @@ class CodeReader(AbstractModule):
|
|||
if __name__ == '__main__':
|
||||
module = CodeReader()
|
||||
module.run()
|
||||
|
||||
|
|
@ -31,16 +31,17 @@ Note that the hash of the content is defined as the sha1(gzip64encoded).
|
|||
"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
import hashlib
|
||||
import time
|
||||
|
||||
# import hashlib
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from modules.abstract_module import AbstractModule
|
||||
from lib.ConfigLoader import ConfigLoader
|
||||
from lib import ail_stats
|
||||
|
||||
|
||||
class Mixer(AbstractModule):
|
||||
|
|
@ -51,12 +52,14 @@ class Mixer(AbstractModule):
|
|||
|
||||
config_loader = ConfigLoader()
|
||||
self.r_cache = config_loader.get_redis_conn("Redis_Mixer_Cache")
|
||||
# self.r_cache_s = config_loader.get_redis_conn("Redis_Log_submit")
|
||||
|
||||
self.pending_seconds = 5
|
||||
self.pending_seconds = 1
|
||||
|
||||
self.refresh_time = 30
|
||||
self.last_refresh = time.time()
|
||||
timestamp = int(time.time())
|
||||
self.last_refresh = int(timestamp - (timestamp % 30))
|
||||
if timestamp > self.last_refresh:
|
||||
self.last_refresh += 30
|
||||
|
||||
self.operation_mode = config_loader.get_config_int("Module_Mixer", "operation_mode")
|
||||
print(f'Operation mode {self.operation_mode}')
|
||||
|
|
@ -64,71 +67,25 @@ class Mixer(AbstractModule):
|
|||
self.ttl_key = config_loader.get_config_int("Module_Mixer", "ttl_duplicate")
|
||||
self.default_feeder_name = config_loader.get_config_str("Module_Mixer", "default_unnamed_feed_name")
|
||||
|
||||
self.nb_processed_items = 0
|
||||
self.feeders_processed = {}
|
||||
self.feeders_duplicate = {}
|
||||
|
||||
self.logger.info(f"Module: {self.module_name} Launched")
|
||||
|
||||
# TODO Save stats in cache
|
||||
# def get_feeders(self):
|
||||
# return self.r_cache_s.smembers("mixer_cache:feeders")
|
||||
#
|
||||
# def get_feeder_nb_last_processed(self, feeder):
|
||||
# nb = self.r_cache_s.hget("mixer_cache:feeders:last_processed", feeder)
|
||||
# if nb:
|
||||
# return int(nb)
|
||||
# else:
|
||||
# return 0
|
||||
#
|
||||
# def get_cache_feeders_nb_last_processed(self):
|
||||
# feeders = {}
|
||||
# for feeder in self.get_feeders():
|
||||
# feeders[feeder] = self.get_feeder_nb_last_processed(feeder)
|
||||
# return feeders
|
||||
|
||||
def clear_feeders_stat(self):
|
||||
pass
|
||||
# self.r_cache_s.delete("mixer_cache:feeders:last_processed")
|
||||
|
||||
def increase_stat_processed(self, feeder):
|
||||
self.nb_processed_items += 1
|
||||
try:
|
||||
self.feeders_processed[feeder] += 1
|
||||
except KeyError:
|
||||
self.feeders_processed[feeder] = 1
|
||||
|
||||
def increase_stat_duplicate(self, feeder):
|
||||
self.nb_processed_items += 1
|
||||
try:
|
||||
self.feeders_duplicate[feeder] += 1
|
||||
except KeyError:
|
||||
self.feeders_duplicate[feeder] = 1
|
||||
|
||||
# TODO Save stats in cache
|
||||
def refresh_stats(self):
|
||||
if int(time.time() - self.last_refresh) > self.refresh_time:
|
||||
# update internal feeder
|
||||
to_print = f'Mixer; ; ; ;mixer_all All_feeders Processed {self.nb_processed_items} item(s) in {self.refresh_time}sec'
|
||||
print(to_print)
|
||||
self.redis_logger.info(to_print)
|
||||
self.nb_processed_items = 0
|
||||
|
||||
for feeder in self.feeders_processed:
|
||||
to_print = f'Mixer; ; ; ;mixer_{feeder} {feeder} Processed {self.feeders_processed[feeder]} item(s) in {self.refresh_time}sec'
|
||||
print(to_print)
|
||||
self.redis_logger.info(to_print)
|
||||
self.feeders_processed[feeder] = 0
|
||||
|
||||
for feeder in self.feeders_duplicate:
|
||||
to_print = f'Mixer; ; ; ;mixer_{feeder} {feeder} Duplicated {self.feeders_duplicate[feeder]} item(s) in {self.refresh_time}sec'
|
||||
print(to_print)
|
||||
self.redis_logger.info(to_print)
|
||||
self.feeders_duplicate[feeder] = 0
|
||||
|
||||
self.last_refresh = time.time()
|
||||
self.clear_feeders_stat()
|
||||
time.sleep(0.5)
|
||||
timestamp = int(time.time())
|
||||
if timestamp >= self.last_refresh:
|
||||
timestamp = timestamp - timestamp % self.refresh_time
|
||||
print('update', timestamp)
|
||||
print(self.feeders_processed)
|
||||
ail_stats.add_feeders(timestamp, self.feeders_processed)
|
||||
self.feeders_processed = {}
|
||||
self.last_refresh = self.last_refresh + 30
|
||||
|
||||
def computeNone(self):
|
||||
self.refresh_stats()
|
||||
|
|
@ -163,22 +120,19 @@ class Mixer(AbstractModule):
|
|||
self.queue.rename_message_obj(self.obj.id, obj_id)
|
||||
|
||||
|
||||
relay_message = gzip64encoded
|
||||
# print(relay_message)
|
||||
|
||||
# TODO only work for item object
|
||||
# Avoid any duplicate coming from any sources
|
||||
if self.operation_mode == 1:
|
||||
digest = hashlib.sha1(gzip64encoded.encode('utf8')).hexdigest()
|
||||
if self.r_cache.exists(digest): # Content already exists
|
||||
# STATS
|
||||
self.increase_stat_duplicate(feeder_name)
|
||||
else: # New content
|
||||
self.r_cache.sadd(digest, feeder_name)
|
||||
self.r_cache.expire(digest, self.ttl_key)
|
||||
|
||||
self.increase_stat_processed(feeder_name)
|
||||
self.add_message_to_queue(message=relay_message)
|
||||
# # TODO only work for item object
|
||||
# # Avoid any duplicate coming from any sources
|
||||
# if self.operation_mode == 1:
|
||||
# digest = hashlib.sha1(gzip64encoded.encode('utf8')).hexdigest()
|
||||
# if self.r_cache.exists(digest): # Content already exists
|
||||
# # STATS
|
||||
# self.increase_stat_duplicate(feeder_name)
|
||||
# else: # New content
|
||||
# self.r_cache.sadd(digest, feeder_name)
|
||||
# self.r_cache.expire(digest, self.ttl_key)
|
||||
#
|
||||
# self.increase_stat_processed(feeder_name)
|
||||
# self.add_message_to_queue(message=relay_message)
|
||||
|
||||
# Need To Be Fixed, Currently doesn't check the source (-> same as operation 1)
|
||||
# # Keep duplicate coming from different sources
|
||||
|
|
@ -213,11 +167,9 @@ class Mixer(AbstractModule):
|
|||
# self.increase_stat_duplicate(feeder_name)
|
||||
|
||||
# No Filtering
|
||||
else:
|
||||
# else:
|
||||
|
||||
self.increase_stat_processed(feeder_name)
|
||||
if self.obj.type == 'item':
|
||||
self.add_message_to_queue(obj=self.obj, message=gzip64encoded)
|
||||
else:
|
||||
self.add_message_to_queue(obj=self.obj, message=gzip64encoded)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -77,6 +77,7 @@ phonenumbers>8.12.1
|
|||
# Web
|
||||
flask>=2.3.3
|
||||
flask-login
|
||||
flask-sock
|
||||
bcrypt>3.1.6
|
||||
pyotp
|
||||
segno
|
||||
|
|
|
|||
|
|
@ -6,13 +6,13 @@ import sys
|
|||
import ssl
|
||||
import json
|
||||
import time
|
||||
import uuid
|
||||
import random
|
||||
import logging
|
||||
import logging.config
|
||||
|
||||
from flask import Flask, render_template, jsonify, request, Request, Response, session, redirect, url_for
|
||||
from flask_login import LoginManager, current_user, login_user, logout_user, login_required
|
||||
from flask_sock import Sock
|
||||
|
||||
import importlib
|
||||
from os.path import join
|
||||
|
|
@ -28,6 +28,7 @@ from lib.ail_users import AILUser, get_session_user
|
|||
from lib import Tag
|
||||
from lib import ail_core
|
||||
from lib import ail_logger
|
||||
from lib import ail_stats
|
||||
|
||||
from packages.git_status import clear_git_meta_cache
|
||||
|
||||
|
|
@ -47,6 +48,7 @@ from blueprints.hunters import hunters
|
|||
from blueprints.old_endpoints import old_endpoints
|
||||
from blueprints.ail_2_ail_sync import ail_2_ail_sync
|
||||
from blueprints.settings_b import settings_b
|
||||
from blueprints.objects_objs import objects_objs
|
||||
from blueprints.objects_cve import objects_cve
|
||||
from blueprints.objects_decoded import objects_decoded
|
||||
from blueprints.objects_subtypes import objects_subtypes
|
||||
|
|
@ -134,6 +136,7 @@ app.register_blueprint(old_endpoints, url_prefix=baseUrl)
|
|||
app.register_blueprint(ail_2_ail_sync, url_prefix=baseUrl)
|
||||
app.register_blueprint(settings_b, url_prefix=baseUrl)
|
||||
app.register_blueprint(objects_cve, url_prefix=baseUrl)
|
||||
app.register_blueprint(objects_objs, url_prefix=baseUrl)
|
||||
app.register_blueprint(objects_decoded, url_prefix=baseUrl)
|
||||
app.register_blueprint(objects_subtypes, url_prefix=baseUrl)
|
||||
app.register_blueprint(objects_title, url_prefix=baseUrl)
|
||||
|
|
@ -163,7 +166,7 @@ login_manager.init_app(app)
|
|||
# ========= LOGIN MANAGER ========
|
||||
|
||||
@login_manager.user_loader
|
||||
def load_user(session_id): # TODO USE Alternative ID
|
||||
def load_user(session_id):
|
||||
# print(session)
|
||||
user_id = get_session_user(session_id)
|
||||
if user_id:
|
||||
|
|
@ -186,9 +189,7 @@ try:
|
|||
except IOError:
|
||||
pass
|
||||
|
||||
# Dynamically import routes and functions from modules
|
||||
# Also, prepare header.html
|
||||
to_add_to_header_dico = {}
|
||||
# Dynamically import routes and functions from modules # # # # TODO REMOVE ME ################################################
|
||||
for root, dirs, files in os.walk(os.path.join(Flask_dir, 'modules')):
|
||||
sys.path.append(join(root))
|
||||
|
||||
|
|
@ -204,36 +205,13 @@ for root, dirs, files in os.walk(os.path.join(Flask_dir, 'modules')):
|
|||
continue
|
||||
name = name.strip('.py')
|
||||
importlib.import_module(name)
|
||||
elif name == 'header_{}.html'.format(module_name):
|
||||
with open(join(root, name), 'r') as f:
|
||||
to_add_to_header_dico[module_name] = f.read()
|
||||
|
||||
# create header.html
|
||||
with open(os.path.join(Flask_dir, 'templates', 'header_base.html'), 'r') as f:
|
||||
complete_header = f.read()
|
||||
modified_header = complete_header
|
||||
|
||||
# Add the header in the supplied order
|
||||
for module_name, txt in list(to_add_to_header_dico.items()):
|
||||
to_replace = '<!--{}-->'.format(module_name)
|
||||
if to_replace in complete_header:
|
||||
modified_header = modified_header.replace(to_replace, txt)
|
||||
del to_add_to_header_dico[module_name]
|
||||
|
||||
# Add the header for no-supplied order
|
||||
to_add_to_header = []
|
||||
for module_name, txt in to_add_to_header_dico.items():
|
||||
to_add_to_header.append(txt)
|
||||
|
||||
modified_header = modified_header.replace('<!--insert here-->', '\n'.join(to_add_to_header))
|
||||
|
||||
# Write the header.html file
|
||||
with open(os.path.join(Flask_dir, 'templates', 'header.html'), 'w') as f:
|
||||
f.write(modified_header)
|
||||
|
||||
# ========= JINJA2 FUNCTIONS ========
|
||||
def list_len(s):
|
||||
return len(s)
|
||||
|
||||
|
||||
app.jinja_env.filters['list_len'] = list_len
|
||||
|
||||
|
||||
|
|
@ -316,6 +294,33 @@ def page_not_found(e):
|
|||
return render_template('error/404.html'), 404
|
||||
|
||||
|
||||
# ========== WEBSOCKET ============
|
||||
|
||||
app.config['SOCK_SERVER_OPTIONS'] = {'ping_interval': 25}
|
||||
sock = Sock(app)
|
||||
|
||||
@login_required
|
||||
@sock.route('/ws/dashboard')
|
||||
def ws_dashboard(ws):
|
||||
# TODO wait %30
|
||||
next_feeders = ail_stats.get_next_feeder_timestamp(int(time.time())) + 1
|
||||
try:
|
||||
while True:
|
||||
# TODO CHECK IF NEEDED
|
||||
# if ws.closed:
|
||||
# print('WebSocket connection closed')
|
||||
# break
|
||||
if int(time.time()) >= next_feeders:
|
||||
feeders = ail_stats.get_feeders_dashboard()
|
||||
# feeders['data']['telegram'] = 600
|
||||
# feeders['data']['test'] = 1300
|
||||
ws.send(json.dumps({'feeders': feeders}))
|
||||
next_feeders = next_feeders + 30
|
||||
time.sleep(1)
|
||||
except Exception as e: # ConnectionClosed ?
|
||||
print("WEBSOCKET", e)
|
||||
|
||||
|
||||
# ========== INITIAL taxonomies ============
|
||||
default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"]
|
||||
# enable default taxonomies
|
||||
|
|
|
|||
|
|
@ -30,7 +30,7 @@ bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
|
|||
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
@objects_cve.route("/objects/cve", methods=['GET'])
|
||||
@objects_cve.route("/objects/cves", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def objects_cves():
|
||||
|
|
|
|||
48
var/www/blueprints/objects_objs.py
Normal file
48
var/www/blueprints/objects_objs.py
Normal file
|
|
@ -0,0 +1,48 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
'''
|
||||
Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
|
||||
'''
|
||||
|
||||
import os
|
||||
import sys
|
||||
import json
|
||||
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file, stream_with_context
|
||||
from flask_login import login_required
|
||||
|
||||
# Import Role_Manager
|
||||
from Role_Manager import login_admin, login_read_only
|
||||
|
||||
sys.path.append(os.environ['AIL_BIN'])
|
||||
##################################
|
||||
# Import Project packages
|
||||
##################################
|
||||
from lib.objects import ail_objects
|
||||
from lib import ail_stats
|
||||
|
||||
|
||||
|
||||
# ============ BLUEPRINT ============
|
||||
objects_objs = Blueprint('objects_objs', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects'))
|
||||
|
||||
|
||||
# ============ VARIABLES ============
|
||||
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
|
||||
|
||||
|
||||
# ============ FUNCTIONS ============
|
||||
@objects_objs.route("/objects", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def objects():
|
||||
nb_objects = ail_stats.get_nb_objs_dashboard()
|
||||
print(nb_objects)
|
||||
feeders_dashboard = ail_stats.get_feeders_dashboard_full()
|
||||
return render_template("objs_dashboard.html", feeders_dashboard=feeders_dashboard, nb_objects=nb_objects)
|
||||
|
||||
|
||||
|
||||
# ============= ROUTES ==============
|
||||
|
||||
|
|
@ -34,7 +34,7 @@ def create_json_response(data, status_code):
|
|||
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
|
||||
|
||||
# ============= ROUTES ==============
|
||||
@objects_title.route("/objects/title", methods=['GET'])
|
||||
@objects_title.route("/objects/titles", methods=['GET'])
|
||||
@login_required
|
||||
@login_read_only
|
||||
def objects_titles():
|
||||
|
|
|
|||
9
var/www/templates/objects/block_obj_button.html
Normal file
9
var/www/templates/objects/block_obj_button.html
Normal file
|
|
@ -0,0 +1,9 @@
|
|||
<a class="icon-button btn-outline-dark px-2" href="{{ url }}">
|
||||
<div class="icon-wrapper text-center">
|
||||
<i class="fas fa-{{ icon }} fa-4x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
{{ nb }}
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>{{ name }}</b></div>
|
||||
</a>
|
||||
484
var/www/templates/objects/objs_dashboard.html
Normal file
484
var/www/templates/objects/objs_dashboard.html
Normal file
|
|
@ -0,0 +1,484 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<title>Objects - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||
|
||||
<!-- JS -->
|
||||
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/echarts.min.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||
|
||||
|
||||
{# <script src="https://cdn.jsdelivr.net/npm/echarts/dist/echarts.min.js"></script>#}
|
||||
|
||||
{# <script defer src="{{ url_for('static', filename='js/regular.js')}}"></script>#}
|
||||
{# <script defer src="{{ url_for('static', filename='js/solid.js')}}"></script>#}
|
||||
{# <script defer src="{{ url_for('static', filename='js/fontawesome.js')}}"></script>#}
|
||||
|
||||
|
||||
|
||||
<style>
|
||||
.icon-button {
|
||||
position: relative;
|
||||
display: inline-block;
|
||||
border: none;
|
||||
background: none;
|
||||
{#padding: 0;#}
|
||||
margin: 0;
|
||||
cursor: pointer;
|
||||
outline: inherit;
|
||||
transition: color .15s ease-in-out,background-color .15s ease-in-out,border-color .15s ease-in-out,box-shadow .15s ease-in-out;
|
||||
}
|
||||
}
|
||||
|
||||
.icon-wrapper {
|
||||
position: relative;
|
||||
display: inline-block;
|
||||
}
|
||||
|
||||
.notification-badge {
|
||||
position: absolute;
|
||||
top: -10px;
|
||||
right: -10px;
|
||||
font-size: 14px;
|
||||
background: forestgreen;
|
||||
}
|
||||
</style>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'nav_bar.html' %}
|
||||
|
||||
<div class="container-fluid">
|
||||
|
||||
|
||||
<div class="row">
|
||||
|
||||
<div class="col-12 col-xl-6">
|
||||
<div id="feeders_dashboard" style="width: 100%; height:600px;"></div>
|
||||
</div>
|
||||
|
||||
<div class="col-12 col-xl-6 mt-4">
|
||||
|
||||
<div class="row">
|
||||
|
||||
{% for obj_type in nb_objects %}
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
{% with name=nb_objects[obj_type]['name'], icon=nb_objects[obj_type]['icon']['icon'], nb=nb_objects[obj_type]['nb'], url=nb_objects[obj_type]['link'] %}
|
||||
{% include 'objects/block_obj_button.html' %}
|
||||
{% endwith %}
|
||||
</div>
|
||||
{% endfor %}
|
||||
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<a class="icon-button btn-outline-dark px-2" href="{{url_for('chats_explorer.chats_explorer_protocols')}}">
|
||||
<div class="icon-wrapper text-center">
|
||||
<i class="fas fa-comment fa-4x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
384556
|
||||
<span class="sr-only">Chats by days</span>
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>Chats</b></div>
|
||||
</a>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
<div class="col-12" id="core_content">
|
||||
|
||||
<div class="container mt-5">
|
||||
<div class="row">
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<a class="icon-button btn-outline-dark px-2" href="{{url_for('chats_explorer.chats_explorer_protocols')}}">
|
||||
<div class="icon-wrapper text-center">
|
||||
<i class="fas fa-comment fa-6x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
384556
|
||||
<span class="sr-only">Chats by days</span>
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>Chats</b></div>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<a class="btn" style="position:relative;display: inline-block;">
|
||||
<div class="icon-wrapper">
|
||||
<i class="fas fa-bug fa-6x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
5896
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>CVEs</b></div>
|
||||
</a>
|
||||
</div>
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<div class="icon-wrapper">
|
||||
<i class="fas fa-bug fa-6x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
5896
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>CVEs</b></div>
|
||||
</div>
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<div class="icon-wrapper">
|
||||
<i class="fas fa-bug fa-6x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
5896
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>CVEs</b></div>
|
||||
</div>
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<div class="icon-wrapper">
|
||||
<i class="fas fa-bug fa-6x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
5896
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>CVEs</b></div>
|
||||
</div>
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<div class="icon-wrapper">
|
||||
<i class="fas fa-bug fa-6x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
5896
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>CVEs</b></div>
|
||||
</div>
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<div class="icon-wrapper">
|
||||
<i class="fas fa-bug fa-6x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
5896
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>CVEs</b></div>
|
||||
</div>
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<div class="icon-wrapper">
|
||||
<i class="fas fa-bug fa-6x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
5896
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>CVEs</b></div>
|
||||
</div>
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<div class="icon-wrapper">
|
||||
<i class="fas fa-bug fa-6x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
5896
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>CVEs</b></div>
|
||||
</div>
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<div class="icon-wrapper">
|
||||
<i class="fas fa-bug fa-6x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
5896
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>CVEs</b></div>
|
||||
</div>
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<div class="icon-wrapper">
|
||||
<i class="fas fa-bug fa-6x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
5896
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>CVEs</b></div>
|
||||
</div>
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<div class="icon-wrapper">
|
||||
<i class="fas fa-bug fa-6x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
5896
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>CVEs</b></div>
|
||||
</div>
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<div class="icon-wrapper">
|
||||
<i class="fas fa-bug fa-6x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
5896
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>CVEs</b></div>
|
||||
</div>
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<div class="icon-wrapper">
|
||||
<i class="fas fa-bug fa-6x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
5896
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>CVEs</b></div>
|
||||
</div>
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<div class="icon-wrapper">
|
||||
<i class="fas fa-bug fa-6x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
5896
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>CVEs</b></div>
|
||||
</div>
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<div class="icon-wrapper">
|
||||
<i class="fas fa-bug fa-6x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
5896
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>CVEs</b></div>
|
||||
</div>
|
||||
|
||||
<div class="col-sm-6 col-md-4 col-lg-3 text-center mb-4">
|
||||
<div class="icon-wrapper">
|
||||
<i class="fas fa-bug fa-6x"></i>
|
||||
<span class="badge badge-pill badge-danger notification-badge">
|
||||
5896
|
||||
</span>
|
||||
</div>
|
||||
<div class="h4"><b>CVEs</b></div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
|
||||
|
||||
<span class="fa-6x">
|
||||
<span class="fa-layers fa-fw">
|
||||
<i class="fa-solid fa-comment"></i>
|
||||
<span class="fa-layers-counter" style="background:Tomato">1,419</span>
|
||||
</span>
|
||||
<span class="fa-layers fa-fw">
|
||||
<i class="fa-solid fa-bug"></i>
|
||||
<span class="fa-layers-counter" style="background:Tomato">1,419</span>
|
||||
</span>
|
||||
<span class="fa-layers fa-fw">
|
||||
<i class="fa-solid fa-cookie-bite"></i>
|
||||
<span class="fa-layers-counter" style="background:Tomato">1,419</span>
|
||||
</span>
|
||||
<span class="fa-layers fa-fw">
|
||||
<i class="fa-solid fa-tag"></i>
|
||||
<span class="fa-layers-counter" style="background:Tomato">1,419</span>
|
||||
</span>
|
||||
<span class="fa-layers fa-fw">
|
||||
<i class="fa-solid fa-align-left"></i>
|
||||
<span class="fa-layers-counter" style="background:Tomato">1,419</span>
|
||||
</span>
|
||||
<span class="fa-layers fa-fw">
|
||||
<i class="fa-solid fa-skull-crossbones"></i>
|
||||
<span class="fa-layers-counter" style="background:Tomato">1,419</span>
|
||||
</span>
|
||||
<span class="fa-layers fa-fw">
|
||||
<i class="fa-solid fa-star-half"></i>
|
||||
<span class="fa-layers-counter" style="background:Tomato">1,419</span>
|
||||
</span>
|
||||
<span class="fa-layers fa-fw">
|
||||
<i class="fa-solid fa-image"></i>
|
||||
<span class="fa-layers-counter" style="background:Tomato">1,419</span>
|
||||
</span>
|
||||
<span class="fa-layers fa-fw">
|
||||
<i class="fa-solid fa-expand"></i>
|
||||
<span class="fa-layers-counter" style="background:Tomato">1,419</span>
|
||||
</span>
|
||||
<span class="fa-layers fa-fw">
|
||||
<i class="fa-solid fa-barcode"></i>
|
||||
<span class="fa-layers-counter" style="background:Tomato">1,419</span>
|
||||
</span>
|
||||
<span class="fa-layers fa-fw">
|
||||
<i class="fa-solid fa-qrcode"></i>
|
||||
<span class="fa-layers-counter" style="background:Tomato">1,419</span>
|
||||
</span>
|
||||
<span class="fa-layers fa-fw">
|
||||
<i class="fa-solid fa-heading"></i>
|
||||
<span class="fa-layers-counter" style="background:Tomato">1,419</span>
|
||||
</span>
|
||||
<span class="fa-layers fa-fw">
|
||||
<i class="fa-solid fa-lock-open"></i>
|
||||
<span class="fa-layers-counter" style="background:Tomato">1,419</span>
|
||||
</span>
|
||||
<span class="fa-layers fa-fw">
|
||||
<i class="fa-solid fa-key"></i>
|
||||
<span class="fa-layers-counter" style="background:Tomato">1,419</span>
|
||||
</span>
|
||||
<span class="fa-layers fa-fw">
|
||||
<i class="fa-solid fa-coins"></i>
|
||||
<span class="fa-layers-counter" style="background:Tomato">1,419</span>
|
||||
</span>
|
||||
<span class="fa-layers fa-fw">
|
||||
<i class="fa-solid fa-user"></i>
|
||||
<span class="fa-layers-counter" style="background:Tomato">1,419</span>
|
||||
</span>
|
||||
<span class="fa-layers fa-fw">
|
||||
<i class="fa-solid fa-user-circle"></i>
|
||||
<span class="fa-layers-counter" style="background:Tomato">1,419</span>
|
||||
</span>
|
||||
</span>
|
||||
|
||||
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
var chart = {};
|
||||
$(document).ready(function(){
|
||||
$("#page-Decoded").addClass("active");
|
||||
$("#nav_dashboard_{{obj_type}}").addClass("active");
|
||||
});
|
||||
</script>
|
||||
|
||||
<script>
|
||||
// Init Dashboard
|
||||
var feederChart = echarts.init(document.getElementById('feeders_dashboard'));
|
||||
window.addEventListener('resize', function() {
|
||||
feederChart.resize();
|
||||
});
|
||||
var optionFeeder;
|
||||
var maxDataLength = 21;
|
||||
|
||||
var feeders_names = [{% for feeder in feeders_dashboard['data'] %}'{{ feeder }}', {% endfor %}]
|
||||
var feeders_xaxis = [{% for date in feeders_dashboard['dates'] %}'{{ date }}', {% endfor %}]
|
||||
var feeders_data = {
|
||||
{% for feeder in feeders_dashboard['data'] %}
|
||||
{{ feeder }}: {{ feeders_dashboard['data'][feeder] }},
|
||||
{% endfor %}
|
||||
}
|
||||
|
||||
|
||||
|
||||
optionFeeder = {
|
||||
title: {
|
||||
text: 'Feeders'
|
||||
},
|
||||
tooltip: {
|
||||
trigger: 'axis',
|
||||
axisPointer: {
|
||||
type: 'cross',
|
||||
label: {
|
||||
backgroundColor: '#6a7985'
|
||||
}
|
||||
}
|
||||
},
|
||||
legend: {
|
||||
data: feeders_names
|
||||
},
|
||||
grid: {
|
||||
left: '3%',
|
||||
right: '4%',
|
||||
bottom: '3%',
|
||||
containLabel: true
|
||||
},
|
||||
xAxis: [
|
||||
{
|
||||
type: 'category',
|
||||
boundaryGap: false,
|
||||
data: feeders_xaxis
|
||||
}
|
||||
],
|
||||
yAxis: [
|
||||
{
|
||||
type: 'value'
|
||||
}
|
||||
],
|
||||
series: [
|
||||
{% for feeder in feeders_dashboard['data'] %}
|
||||
{
|
||||
name: '{{ feeder }}',
|
||||
type: 'line',
|
||||
stack: 'Total',
|
||||
areaStyle: {},
|
||||
emphasis: {
|
||||
focus: 'series'
|
||||
},
|
||||
data: feeders_data['{{ feeder }}']
|
||||
},
|
||||
{% endfor %}
|
||||
]
|
||||
};
|
||||
|
||||
optionFeeder && feederChart.setOption(optionFeeder);
|
||||
|
||||
{# TODO UPDATE NEW FEEDER NAME#}
|
||||
function updateFeederChart(data) {
|
||||
let new_date = data['feeders']['date']
|
||||
feeders_xaxis.push(new_date)
|
||||
feeders_xaxis.shift()
|
||||
|
||||
for (const f_name in data['feeders']['data']) {
|
||||
if (f_name in feeders_data) {
|
||||
feeders_data[f_name].push(data['feeders']['data'][f_name])
|
||||
feeders_data[f_name].shift()
|
||||
} else {
|
||||
let fdata = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, data['feeders']['data'][f_name]]
|
||||
feeders_names.push(f_name)
|
||||
feeders_data[f_name] = fdata
|
||||
// add new feeder
|
||||
optionFeeder.legend.data.push(f_name)
|
||||
optionFeeder.series.push({name: f_name, type: 'line', stack: 'Total', areaStyle: {}, emphasis: {focus: 'series'}, data: feeders_data[f_name]})
|
||||
|
||||
}
|
||||
}
|
||||
feederChart.setOption(optionFeeder)
|
||||
}
|
||||
|
||||
// WebSocket
|
||||
var socket = new WebSocket("{{ url_for('ws_dashboard') }}");
|
||||
socket.wsocket = function(event) {
|
||||
console.log("WebSocket connection opened:", event);
|
||||
};
|
||||
socket.onmessage = function(event) {
|
||||
let data = JSON.parse(event.data);
|
||||
updateFeederChart(data);
|
||||
};
|
||||
socket.onerror = function(error) {
|
||||
console.error('WebSocket error:', error);
|
||||
};
|
||||
socket.onclose = function(event) {
|
||||
console.log('WebSocket connection closed:', event);
|
||||
// Optionally implement reconnection logic here
|
||||
};
|
||||
|
||||
|
||||
</script>
|
||||
|
||||
</body>
|
||||
</html>
|
||||
|
|
@ -5,28 +5,43 @@
|
|||
# submodules
|
||||
git submodule update
|
||||
|
||||
wget -q http://dygraphs.com/dygraph-combined.js -O ./static/js/dygraph-combined.js
|
||||
|
||||
BOOTSTRAP_VERSION='4.2.1'
|
||||
FONT_AWESOME_VERSION='6.6.0'
|
||||
|
||||
D3_JS_VERSION='5.16.0'
|
||||
wget https://d3js.org/d3.v7.min.js -O ./static/js/d3.v7.min.js
|
||||
|
||||
rm -rf temp
|
||||
mkdir temp
|
||||
|
||||
wget https://github.com/twbs/bootstrap/releases/download/v${BOOTSTRAP_VERSION}/bootstrap-${BOOTSTRAP_VERSION}-dist.zip -O temp/bootstrap${BOOTSTRAP_VERSION}.zip
|
||||
wget https://github.com/FortAwesome/Font-Awesome/archive/${FONT_AWESOME_VERSION}.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip
|
||||
#### D3JS ####
|
||||
#### TODO UPDATE ALL D3 JS to V7
|
||||
wget https://d3js.org/d3.v7.min.js -O ./static/js/d3.v7.min.js
|
||||
|
||||
# OLD
|
||||
D3_JS_VERSION='5.16.0'
|
||||
wget https://github.com/d3/d3/releases/download/v${D3_JS_VERSION}/d3.zip -O temp/d3_${D3_JS_VERSION}.zip
|
||||
unzip -qq temp/d3_${D3_JS_VERSION}.zip -d temp/
|
||||
|
||||
mv temp/d3.min.js ./static/js/
|
||||
#### ####
|
||||
|
||||
#### FONT_AWESOME ####
|
||||
FONT_AWESOME_VERSION='6.6.0'
|
||||
wget https://github.com/FortAwesome/Font-Awesome/archive/${FONT_AWESOME_VERSION}.zip -O temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip
|
||||
unzip temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/
|
||||
|
||||
rm -rf ./static/webfonts/
|
||||
mv temp/Font-Awesome-${FONT_AWESOME_VERSION}/css/all.min.css ./static/css/font-awesome.min.css
|
||||
mv temp/Font-Awesome-${FONT_AWESOME_VERSION}/webfonts ./static/webfonts
|
||||
|
||||
rm -rf ./static/fonts/ ./static/font-awesome/
|
||||
mv temp/font-awesome/ ./static
|
||||
#### ####
|
||||
|
||||
BOOTSTRAP_VERSION='4.2.1'
|
||||
|
||||
wget https://github.com/twbs/bootstrap/releases/download/v${BOOTSTRAP_VERSION}/bootstrap-${BOOTSTRAP_VERSION}-dist.zip -O temp/bootstrap${BOOTSTRAP_VERSION}.zip
|
||||
|
||||
# dateRangePicker
|
||||
wget https://github.com/moment/moment/archive/2.24.0.zip -O temp/moment.zip
|
||||
wget https://github.com/longbill/jquery-date-range-picker/archive/v0.20.0.zip -O temp/daterangepicker.zip
|
||||
|
||||
unzip -qq temp/bootstrap${BOOTSTRAP_VERSION}.zip -d temp/
|
||||
unzip temp/FONT_AWESOME_${FONT_AWESOME_VERSION}.zip -d temp/
|
||||
unzip -qq temp/d3_${D3_JS_VERSION}.zip -d temp/
|
||||
|
||||
unzip -qq temp/moment.zip -d temp/
|
||||
unzip -qq temp/daterangepicker.zip -d temp/
|
||||
|
|
@ -36,19 +51,10 @@ mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/js/bootstrap.min.js.map ./static/js/
|
|||
mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css ./static/css/bootstrap4.min.css
|
||||
mv temp/bootstrap-${BOOTSTRAP_VERSION}-dist/css/bootstrap.min.css.map ./static/css/bootstrap4.min.css.map
|
||||
|
||||
rm -rf ./static/webfonts/
|
||||
mv temp/Font-Awesome-${FONT_AWESOME_VERSION}/css/all.min.css ./static/css/font-awesome.min.css
|
||||
mv temp/Font-Awesome-${FONT_AWESOME_VERSION}/webfonts ./static/webfonts
|
||||
|
||||
rm -rf ./static/js/plugins
|
||||
|
||||
rm -rf ./static/fonts/ ./static/font-awesome/
|
||||
mv temp/font-awesome/ ./static/
|
||||
|
||||
rm -rf ./static/css/plugins/
|
||||
mv temp/jquery-date-range-picker-0.20.0/dist/daterangepicker.min.css ./static/css/
|
||||
|
||||
mv temp/d3.min.js ./static/js/
|
||||
mv temp/moment-2.24.0/min/moment.min.js ./static/js/
|
||||
mv temp/jquery-date-range-picker-0.20.0/dist/jquery.daterangepicker.min.js ./static/js/
|
||||
|
||||
|
|
@ -57,8 +63,6 @@ wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js
|
|||
|
||||
#Ressources for dataTable
|
||||
wget https://cdn.datatables.net/1.10.20/js/jquery.dataTables.min.js -O ./static/js/jquery.dataTables.min.js
|
||||
wget https://cdn.datatables.net/plug-ins/1.10.20/integration/bootstrap/3/dataTables.bootstrap.css -O ./static/css/dataTables.bootstrap.css
|
||||
wget https://cdn.datatables.net/plug-ins/1.10.20/integration/bootstrap/3/dataTables.bootstrap.js -O ./static/js/dataTables.bootstrap.js
|
||||
|
||||
wget https://cdn.datatables.net/1.10.20/css/dataTables.bootstrap4.min.css -O ./static/css/dataTables.bootstrap.min.css
|
||||
wget https://cdn.datatables.net/1.10.20/js/dataTables.bootstrap4.min.js -O ./static/js/dataTables.bootstrap.min.js
|
||||
|
|
@ -71,20 +75,10 @@ mv temp/floating-ui-${POPPER_VERSION}/dist/umd/popper.min.js ./static/js/
|
|||
mv temp/floating-ui-${POPPER_VERSION}/dist/umd/popper.min.js.map ./static/js/
|
||||
|
||||
#Ressource for graph
|
||||
# DASHBOARD # TODO REFACTOR DASHBOARD GRAPHS
|
||||
wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.js -O ./static/js/jquery.flot.js
|
||||
wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.pie.js -O ./static/js/jquery.flot.pie.js
|
||||
wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.time.js -O ./static/js/jquery.flot.time.js
|
||||
wget https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.stack.js -O ./static/js/jquery.flot.stack.js
|
||||
# DASHBOARD # TODO Extract from github
|
||||
wget https://cdn.jsdelivr.net/npm/echarts@5.5.1/dist/echarts.min.js - O ./static/js/echarts.min.js
|
||||
|
||||
#Ressources for sparkline and canvasJS and slider
|
||||
#wget http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js
|
||||
#wget https://canvasjs.com/assets/script/canvasjs.min.js -O ./static/js/jquery.canvasjs.min.js
|
||||
|
||||
wget https://jqueryui.com/resources/download/jquery-ui-1.12.1.zip -O temp/jquery-ui.zip
|
||||
unzip -qq temp/jquery-ui.zip -d temp/
|
||||
mv temp/jquery-ui-1.12.1/jquery-ui.min.js ./static/js/jquery-ui.min.js
|
||||
mv temp/jquery-ui-1.12.1/jquery-ui.min.css ./static/css/jquery-ui.min.css
|
||||
|
||||
# INSTALL YARA
|
||||
YARA_VERSION="4.3.0"
|
||||
|
|
|
|||
Loading…
Reference in a new issue