mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
chg: [v4.1] add Investigation with MISP Export + v4.1 update
This commit is contained in:
parent
303c8001e5
commit
f40d8f47c2
41 changed files with 3140 additions and 488 deletions
|
@ -22,6 +22,10 @@ import Correlate_object
|
||||||
import AILObjects
|
import AILObjects
|
||||||
import Export
|
import Export
|
||||||
|
|
||||||
|
|
||||||
|
from Investigations import Investigation
|
||||||
|
import Tag
|
||||||
|
|
||||||
# # TODO: # FIXME: REFRACTOR ME => use UI/Global config
|
# # TODO: # FIXME: REFRACTOR ME => use UI/Global config
|
||||||
sys.path.append('../../configs/keys')
|
sys.path.append('../../configs/keys')
|
||||||
try:
|
try:
|
||||||
|
@ -390,6 +394,7 @@ def create_misp_event(event, distribution=0, threat_level_id=4, publish=False, a
|
||||||
# # TODO: handle multiple MISP instance
|
# # TODO: handle multiple MISP instance
|
||||||
misp = PyMISP(misp_url, misp_key, misp_verifycert)
|
misp = PyMISP(misp_url, misp_key, misp_verifycert)
|
||||||
#print(event.to_json())
|
#print(event.to_json())
|
||||||
|
|
||||||
misp_event = misp.add_event(event)
|
misp_event = misp.add_event(event)
|
||||||
#print(misp_event)
|
#print(misp_event)
|
||||||
# # TODO: handle error
|
# # TODO: handle error
|
||||||
|
@ -414,9 +419,50 @@ def extract_event_metadata(event):
|
||||||
# LVL 1 => DETAILED Also add correlated_items correlation
|
# LVL 1 => DETAILED Also add correlated_items correlation
|
||||||
######
|
######
|
||||||
|
|
||||||
if __name__ == '__main__':
|
# # TODO: # create object relationships
|
||||||
|
def create_investigation_event(investigation_uuid):
|
||||||
|
investigation = Investigation(investigation_uuid)
|
||||||
|
|
||||||
l_obj = [{'id': 'bfd5f1d89e55b10a8b122a9d7ce31667ec1d086a', 'type': 'decoded', 'lvl': 2}]
|
event = MISPEvent()
|
||||||
create_list_of_objs_to_export(l_obj)
|
event.info = investigation.get_info()
|
||||||
|
event.uuid = investigation.get_uuid()
|
||||||
|
event.date = investigation.get_date()
|
||||||
|
event.analysis = investigation.get_analysis()
|
||||||
|
event.threat_level_id = investigation.get_threat_level()
|
||||||
|
|
||||||
|
taxonomies_tags, galaxies_tags = Tag.sort_tags_taxonomies_galaxies(investigation.get_tags())
|
||||||
|
event.Tag = taxonomies_tags
|
||||||
|
event.Galaxy = galaxies_tags
|
||||||
|
#event.add_galaxy(galaxies_tags)
|
||||||
|
|
||||||
|
investigation_objs = investigation.get_objects()
|
||||||
|
for obj in investigation_objs:
|
||||||
|
# if subtype -> obj_id = 'subtype:type'
|
||||||
|
if obj['subtype']:
|
||||||
|
obj_id = f"{obj['subtype']}:{obj['id']}"
|
||||||
|
else:
|
||||||
|
obj_id = obj['id']
|
||||||
|
misp_obj = create_misp_obj(obj['type'], obj_id)
|
||||||
|
if misp_obj:
|
||||||
|
event.add_object(misp_obj)
|
||||||
|
|
||||||
|
# if publish:
|
||||||
|
# event.publish()
|
||||||
|
|
||||||
|
# res = event.to_json()
|
||||||
|
# print(event.to_json())
|
||||||
|
|
||||||
|
misp = PyMISP(misp_url, misp_key, misp_verifycert)
|
||||||
|
misp_event = misp.add_event(event)
|
||||||
|
# print(misp_event)
|
||||||
|
|
||||||
|
# # TODO: handle error
|
||||||
|
event_metadata = extract_event_metadata(misp_event)
|
||||||
|
return event_metadata
|
||||||
|
|
||||||
|
# if __name__ == '__main__':
|
||||||
|
|
||||||
|
# l_obj = [{'id': 'bfd5f1d89e55b10a8b122a9d7ce31667ec1d086a', 'type': 'decoded', 'lvl': 2}]
|
||||||
|
# create_list_of_objs_to_export(l_obj)
|
||||||
|
|
||||||
#print(event.to_json())
|
#print(event.to_json())
|
||||||
|
|
|
@ -24,44 +24,116 @@ config_loader = None
|
||||||
## data retention
|
## data retention
|
||||||
#########################
|
#########################
|
||||||
|
|
||||||
default_config = {
|
|
||||||
|
ail_config = {
|
||||||
"crawler": {
|
"crawler": {
|
||||||
"enable_har_by_default": False,
|
"enable_har_by_default": {
|
||||||
"enable_screenshot_by_default": True,
|
"default": False,
|
||||||
"default_depth_limit": 1,
|
"type": bool,
|
||||||
"default_closespider_pagecount": 50,
|
"info": "Enable HAR by default"
|
||||||
"default_user_agent": "Mozilla/5.0 (Windows NT 10.0; rv:68.0) Gecko/20100101 Firefox/68.0",
|
},
|
||||||
"default_timeout": 30
|
"enable_screenshot_by_default": {
|
||||||
|
"default": True,
|
||||||
|
"type": bool,
|
||||||
|
"info": "Enable screenshot by default"
|
||||||
|
},
|
||||||
|
"depth_limit": {
|
||||||
|
"default": 1,
|
||||||
|
"type": int,
|
||||||
|
"info": "Maximum number of url depth"
|
||||||
|
},
|
||||||
|
"closespider_pagecount": {
|
||||||
|
"default": 50,
|
||||||
|
"type": int,
|
||||||
|
"info": "Maximum number of pages"
|
||||||
|
},
|
||||||
|
"user_agent": {
|
||||||
|
"default": 50,
|
||||||
|
"type": str,
|
||||||
|
"info": "User agent used by default"
|
||||||
|
},
|
||||||
|
"timeout": {
|
||||||
|
"default": 30,
|
||||||
|
"type": int,
|
||||||
|
"info": "Crawler connection timeout"
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"misp": {
|
||||||
|
"url": {
|
||||||
|
"default": "https://localhost:8443/",
|
||||||
|
"type": str,
|
||||||
|
"info": "Crawler connection timeout"
|
||||||
|
},
|
||||||
|
"key": {
|
||||||
|
"default": "",
|
||||||
|
"type": str,
|
||||||
|
"info": "Crawler connection timeout"
|
||||||
|
},
|
||||||
|
"verifycert": {
|
||||||
|
"default": True,
|
||||||
|
"type": bool,
|
||||||
|
"info": "Crawler connection timeout"
|
||||||
|
},
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
# The MISP auth key can be found on the MISP web interface under the automation section
|
||||||
|
|
||||||
|
def get_config_value(section, field, value):
|
||||||
|
return r_serv_db.hget(f'ail:config:global:{section}', field, value)
|
||||||
|
|
||||||
|
def get_config_default_value(section, field, value):
|
||||||
|
return ail_config[section][field]['default']
|
||||||
|
|
||||||
|
def get_config_type(section, field, value):
|
||||||
|
return ail_config[section][field]['type']
|
||||||
|
|
||||||
|
def get_config_info(section, field, value):
|
||||||
|
return ail_config[section][field]['info']
|
||||||
|
|
||||||
|
def save_config(section, field, value):
|
||||||
|
if section in ail_config:
|
||||||
|
if is_valid_type(value, section, field, value_type=value_type):
|
||||||
|
# if value_type in ['list', 'set', 'dict']:
|
||||||
|
# pass
|
||||||
|
# else:
|
||||||
|
r_serv_db.hset(f'ail:config:global:{section}', field, value)
|
||||||
|
|
||||||
|
|
||||||
|
config_documentation = {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
|
default_config = {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
def get_default_config():
|
def get_default_config():
|
||||||
return default_config
|
return default_config
|
||||||
|
|
||||||
def get_default_config_value(section, field):
|
def get_default_config_value(section, field):
|
||||||
return default_config[section][field]
|
return default_config[section][field]
|
||||||
|
|
||||||
config_type = {
|
|
||||||
# crawler config
|
|
||||||
"crawler": {
|
|
||||||
"enable_har_by_default": bool,
|
|
||||||
"enable_screenshot_by_default": bool,
|
|
||||||
"default_depth_limit": int,
|
|
||||||
"default_closespider_pagecount": int,
|
|
||||||
"default_user_agent": str,
|
|
||||||
"default_timeout": int
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def get_config_type(section, field):
|
|
||||||
return config_type[section][field]
|
|
||||||
|
#### DEFAULT CONFIG ####
|
||||||
|
|
||||||
|
#### CONFIG TYPE ####
|
||||||
|
# CONFIG DOC
|
||||||
|
config_type = {
|
||||||
|
|
||||||
|
}
|
||||||
|
|
||||||
# # TODO: add set, dict, list and select_(multiple_)value
|
# # TODO: add set, dict, list and select_(multiple_)value
|
||||||
def is_valid_type(obj, section, field, value_type=None):
|
def is_valid_type(obj, section, field, value_type=None):
|
||||||
res = isinstance(obj, get_config_type(section, field))
|
res = isinstance(obj, get_config_type(section, field))
|
||||||
return res
|
return res
|
||||||
|
|
||||||
|
# # TODO: ###########################################################
|
||||||
def reset_default_config():
|
def reset_default_config():
|
||||||
|
for section in config_type:
|
||||||
|
|
||||||
pass
|
pass
|
||||||
|
|
||||||
def set_default_config(section, field):
|
def set_default_config(section, field):
|
||||||
|
@ -92,45 +164,12 @@ def get_config_dict_by_section(section):
|
||||||
config_dict[field] = get_config(section, field)
|
config_dict[field] = get_config(section, field)
|
||||||
return config_dict
|
return config_dict
|
||||||
|
|
||||||
def save_config(section, field, value, value_type=None): ###########################################
|
|
||||||
if section in default_config:
|
|
||||||
if is_valid_type(value, section, field, value_type=value_type):
|
|
||||||
if value_type in ['list', 'set', 'dict']:
|
|
||||||
pass
|
|
||||||
else:
|
|
||||||
r_serv_db.hset(f'config:global:{section}', field, value)
|
|
||||||
# used by check_integrity
|
|
||||||
r_serv_db.sadd('config:all_global_section', field, value)
|
|
||||||
|
|
||||||
# check config value + type
|
# check config value + type
|
||||||
def check_integrity():
|
def check_integrity():
|
||||||
pass
|
pass
|
||||||
|
|
||||||
|
|
||||||
config_documentation = {
|
|
||||||
"crawler": {
|
|
||||||
"enable_har_by_default": 'Enable HAR by default',
|
|
||||||
"enable_screenshot_by_default": 'Enable screenshot by default',
|
|
||||||
"default_depth_limit": 'Maximum number of url depth',
|
|
||||||
"default_closespider_pagecount": 'Maximum number of pages',
|
|
||||||
"default_user_agent": "User agent used by default",
|
|
||||||
"default_timeout": "Crawler connection timeout"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
def get_config_documentation(section, field):
|
|
||||||
return config_documentation[section][field]
|
|
||||||
|
|
||||||
# def conf_view():
|
|
||||||
# class F(MyBaseForm):
|
|
||||||
# pass
|
|
||||||
#
|
|
||||||
# F.username = TextField('username')
|
|
||||||
# for name in iterate_some_model_dynamically():
|
|
||||||
# setattr(F, name, TextField(name.title()))
|
|
||||||
#
|
|
||||||
# form = F(request.POST, ...)
|
|
||||||
|
|
||||||
def get_field_full_config(section, field):
|
def get_field_full_config(section, field):
|
||||||
dict_config = {}
|
dict_config = {}
|
||||||
dict_config['value'] = get_config(section, field)
|
dict_config['value'] = get_config(section, field)
|
||||||
|
|
449
bin/lib/Investigations.py
Executable file
449
bin/lib/Investigations.py
Executable file
|
@ -0,0 +1,449 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
##################################################################
|
||||||
|
##################################################################
|
||||||
|
|
||||||
|
# TODO: /!\ MISP ORG UUID
|
||||||
|
|
||||||
|
##################################################################
|
||||||
|
##################################################################
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import datetime
|
||||||
|
import redis
|
||||||
|
import time
|
||||||
|
import uuid
|
||||||
|
|
||||||
|
from abc import ABC
|
||||||
|
from enum import Enum
|
||||||
|
from flask import escape
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
|
import ConfigLoader
|
||||||
|
from exceptions import UpdateInvestigationError
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||||
|
import Tag
|
||||||
|
|
||||||
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
r_tracking = config_loader.get_redis_conn("DB_Tracking")
|
||||||
|
config_loader = None
|
||||||
|
|
||||||
|
|
||||||
|
#### UUID ####
|
||||||
|
def is_valid_uuid_v4(UUID):
|
||||||
|
if not UUID:
|
||||||
|
return False
|
||||||
|
UUID = UUID.replace('-', '')
|
||||||
|
try:
|
||||||
|
uuid_test = uuid.UUID(hex=UUID, version=4)
|
||||||
|
return uuid_test.hex == UUID
|
||||||
|
except:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def sanityze_uuid(UUID):
|
||||||
|
sanityzed_uuid = uuid.UUID(hex=UUID, version=4)
|
||||||
|
return str(sanityzed_uuid).replace('-', '')
|
||||||
|
|
||||||
|
def exists_obj_type(obj_type):
|
||||||
|
return obj_type in ['domain', 'item', 'pgp', 'cryptocurrency', 'decoded', 'screenshot', 'username']
|
||||||
|
|
||||||
|
def generate_uuid():
|
||||||
|
return str(uuid.uuid4()).replace('-', '')
|
||||||
|
|
||||||
|
##-- UUID --##
|
||||||
|
|
||||||
|
# status
|
||||||
|
# created
|
||||||
|
# last change
|
||||||
|
# tags
|
||||||
|
# comment/info
|
||||||
|
# level
|
||||||
|
|
||||||
|
## threat_level:
|
||||||
|
# 1 = high
|
||||||
|
# 2 = medium
|
||||||
|
# 3 = low
|
||||||
|
# 4 = undefined
|
||||||
|
|
||||||
|
## analysis:
|
||||||
|
# 0 = Initial
|
||||||
|
# 1 = Ongoing
|
||||||
|
# 2 = Complete
|
||||||
|
|
||||||
|
# # TODO: Save correlation between investigations ?
|
||||||
|
|
||||||
|
class ThreatLevel(Enum):
|
||||||
|
high = 1
|
||||||
|
medium = 2
|
||||||
|
low = 3
|
||||||
|
undefined = 4
|
||||||
|
|
||||||
|
class Analysis(Enum):
|
||||||
|
initial = 0
|
||||||
|
ongoing = 1
|
||||||
|
completed = 2
|
||||||
|
|
||||||
|
class Investigation(object):
|
||||||
|
"""Investigation."""
|
||||||
|
|
||||||
|
def __init__(self, investigation_uuid):
|
||||||
|
self.uuid = investigation_uuid
|
||||||
|
|
||||||
|
def get_uuid(self):
|
||||||
|
return self.uuid
|
||||||
|
|
||||||
|
# # TODO: Replace by title ??????
|
||||||
|
def get_name(self):
|
||||||
|
return r_tracking.hget(f'investigations:data:{self.uuid}', 'name')
|
||||||
|
|
||||||
|
def get_threat_level(self):
|
||||||
|
try:
|
||||||
|
return int(r_tracking.hget(f'investigations:data:{self.uuid}', 'threat_level'))
|
||||||
|
except:
|
||||||
|
return 1
|
||||||
|
|
||||||
|
def get_threat_level_str(self):
|
||||||
|
return ThreatLevel(self.get_threat_level()).name
|
||||||
|
|
||||||
|
def get_analysis(self):
|
||||||
|
try:
|
||||||
|
return int(r_tracking.hget(f'investigations:data:{self.uuid}', 'analysis'))
|
||||||
|
except:
|
||||||
|
return 0
|
||||||
|
|
||||||
|
def get_analysis_str(self):
|
||||||
|
return Analysis(self.get_analysis()).name
|
||||||
|
|
||||||
|
def get_tags(self):
|
||||||
|
return r_tracking.smembers(f'investigations:tags:{self.uuid}')
|
||||||
|
|
||||||
|
# save all editor ??????
|
||||||
|
def get_creator_user(self):
|
||||||
|
return r_tracking.hget(f'investigations:data:{self.uuid}', 'creator_user')
|
||||||
|
|
||||||
|
def get_info(self):
|
||||||
|
return r_tracking.hget(f'investigations:data:{self.uuid}', 'info')
|
||||||
|
|
||||||
|
def get_date(self):
|
||||||
|
return r_tracking.hget(f'investigations:data:{self.uuid}', 'date')
|
||||||
|
|
||||||
|
def get_timestamp(self, r_str=False):
|
||||||
|
timestamp = r_tracking.hget(f'investigations:data:{self.uuid}', 'timestamp')
|
||||||
|
if r_str and timestamp:
|
||||||
|
timestamp = datetime.datetime.fromtimestamp(float(timestamp)).strftime('%Y-%m-%d %H:%M:%S')
|
||||||
|
return timestamp
|
||||||
|
|
||||||
|
def get_last_change(self, r_str=False):
|
||||||
|
last_change = r_tracking.hget(f'investigations:data:{self.uuid}', 'last_change')
|
||||||
|
if r_str and last_change:
|
||||||
|
last_change = datetime.datetime.fromtimestamp(float(last_change)).strftime('%Y-%m-%d %H:%M:%S')
|
||||||
|
return last_change
|
||||||
|
|
||||||
|
# # TODO: DATE FORMAT
|
||||||
|
def get_metadata(self, r_str=False):
|
||||||
|
if r_str:
|
||||||
|
analysis = self.get_analysis_str()
|
||||||
|
threat_level = self.get_threat_level_str()
|
||||||
|
else:
|
||||||
|
analysis = self.get_analysis()
|
||||||
|
threat_level = self.get_threat_level()
|
||||||
|
return {'uuid': self.uuid,
|
||||||
|
'name': self.get_name(),
|
||||||
|
'threat_level': threat_level,
|
||||||
|
'analysis': analysis,
|
||||||
|
'tags': self.get_tags(),
|
||||||
|
'user_creator': self.get_creator_user(),
|
||||||
|
'date': self.get_date(),
|
||||||
|
'timestamp': self.get_timestamp(r_str=r_str),
|
||||||
|
'last_change': self.get_last_change(r_str=r_str),
|
||||||
|
'info': self.get_info(),
|
||||||
|
'nb_objects': self.get_nb_objects()}
|
||||||
|
|
||||||
|
def set_name(self, name):
|
||||||
|
r_tracking.hset(f'investigations:data:{self.uuid}', 'name', name)
|
||||||
|
|
||||||
|
def set_info(self, info):
|
||||||
|
r_tracking.hset(f'investigations:data:{self.uuid}', 'info', info)
|
||||||
|
|
||||||
|
def set_date(self, date):
|
||||||
|
r_tracking.hset(f'investigations:data:{self.uuid}', 'date', date)
|
||||||
|
|
||||||
|
def set_last_change(self, last_change):
|
||||||
|
r_tracking.hset(f'investigations:data:{self.uuid}', 'last_change', last_change)
|
||||||
|
|
||||||
|
def set_threat_level(self, threat_level):
|
||||||
|
try:
|
||||||
|
threat_level = int(threat_level)
|
||||||
|
except:
|
||||||
|
raise UpdateInvestigationError('threat_level Not an integer')
|
||||||
|
if threat_level >= 1 and threat_level <= 4:
|
||||||
|
r_tracking.hset(f'investigations:data:{self.uuid}', 'threat_level', threat_level)
|
||||||
|
else:
|
||||||
|
raise UpdateInvestigationError(f'Invalid threat_level: {threat_level}')
|
||||||
|
|
||||||
|
def set_analysis(self, analysis):
|
||||||
|
try:
|
||||||
|
analysis = int(analysis)
|
||||||
|
except:
|
||||||
|
raise UpdateInvestigationError('analysis Not an integer')
|
||||||
|
if analysis >= 0 and analysis <= 2:
|
||||||
|
r_tracking.hset(f'investigations:data:{self.uuid}', 'analysis', analysis)
|
||||||
|
else:
|
||||||
|
raise UpdateInvestigationError(f'Invalid analysis: {analysis}')
|
||||||
|
|
||||||
|
def set_tags(self, tags):
|
||||||
|
# delete previous tags
|
||||||
|
r_tracking.delete(f'investigations:tags:{self.uuid}')
|
||||||
|
for tag in tags:
|
||||||
|
r_tracking.sadd(f'investigations:tags:{self.uuid}', tag)
|
||||||
|
|
||||||
|
def get_nb_objects(self):
|
||||||
|
return r_tracking.scard(f'investigations:objs:{self.uuid}')
|
||||||
|
|
||||||
|
def _get_objects(self):
|
||||||
|
return r_tracking.smembers(f'investigations:objs:{self.uuid}')
|
||||||
|
|
||||||
|
# # TODO: return Python object ???? ################################
|
||||||
|
# TODO: PAGINATE
|
||||||
|
def get_objects(self):
|
||||||
|
# obj_dict = {}
|
||||||
|
# for str_obj in self._get_objects():
|
||||||
|
# obj_type, subtype, id = str_obj.split(':', 2)
|
||||||
|
# if not obj_dict[obj_type]:
|
||||||
|
# obj_dict[obj_type] = []
|
||||||
|
# obj_dict[obj_type].append({'subtype': subtype, 'id': id})
|
||||||
|
objs = []
|
||||||
|
for str_obj in self._get_objects():
|
||||||
|
obj_type, subtype, obj_id = str_obj.split(':', 2)
|
||||||
|
dict_obj = {'type': obj_type, 'subtype': subtype, 'id': obj_id}
|
||||||
|
objs.append(dict_obj)
|
||||||
|
return objs
|
||||||
|
|
||||||
|
# # TODO: def register_object(self, Object): in OBJECT CLASS
|
||||||
|
|
||||||
|
def register_object(self, obj_id, obj_type, subtype):
|
||||||
|
r_tracking.sadd(f'investigations:objs:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}')
|
||||||
|
r_tracking.sadd(f'obj:investigations:{obj_type}:{subtype}:{obj_id}', self.uuid)
|
||||||
|
timestamp = int(time.time())
|
||||||
|
self.set_last_change(timestamp)
|
||||||
|
|
||||||
|
|
||||||
|
def unregister_object(self, obj_id, obj_type, subtype):
|
||||||
|
r_tracking.srem(f'investigations:objs:{self.uuid}', f'{obj_type}:{subtype}:{obj_id}')
|
||||||
|
r_tracking.srem(f'obj:investigations:{obj_type}:{subtype}:{obj_id}', self.uuid)
|
||||||
|
timestamp = int(time.time())
|
||||||
|
self.set_last_change(timestamp)
|
||||||
|
|
||||||
|
def delete(self):
|
||||||
|
for str_obj in self._get_objects():
|
||||||
|
obj_type, subtype, obj_id = str_obj.split(':', 2)
|
||||||
|
self.unregister_object(obj_id, obj_type, subtype=subtype)
|
||||||
|
|
||||||
|
r_tracking.srem('investigations:all', self.uuid)
|
||||||
|
# user map
|
||||||
|
r_tracking.srem(f'investigations:user:{self.get_creator_user()}', self.uuid)
|
||||||
|
# metadata
|
||||||
|
r_tracking.delete(f'investigations:data:{self.uuid}')
|
||||||
|
r_tracking.delete(f'investigations:tags:{self.uuid}')
|
||||||
|
|
||||||
|
##-- Class --##
|
||||||
|
|
||||||
|
def get_all_investigations():
|
||||||
|
return r_tracking.smembers('investigations:all')
|
||||||
|
|
||||||
|
def exists_investigation(investigation_uuid):
|
||||||
|
return r_tracking.sismember('investigations:all', investigation_uuid)
|
||||||
|
|
||||||
|
# created by user
|
||||||
|
def get_user_all_investigations(user_id):
|
||||||
|
return r_tracking.smembers('investigations:user:{user_id}')
|
||||||
|
|
||||||
|
def is_object_investigated(obj_id, obj_type, subtype=''):
|
||||||
|
return r_tracking.exists(f'obj:investigations:{obj_type}:{subtype}:{obj_id}')
|
||||||
|
|
||||||
|
def get_obj_investigations(obj_id, obj_type, subtype=''):
|
||||||
|
return r_tracking.smembers(f'obj:investigations:{obj_type}:{subtype}:{obj_id}')
|
||||||
|
|
||||||
|
# # TODO: fix default threat_level analysis
|
||||||
|
# # TODO: limit description + name
|
||||||
|
# # TODO: sanityze tags
|
||||||
|
# # TODO: sanityze date
|
||||||
|
def create_investigation(user_id, date, name, threat_level, analysis, info, tags=[]):
|
||||||
|
investigation_uuid = generate_uuid()
|
||||||
|
r_tracking.sadd('investigations:all', investigation_uuid)
|
||||||
|
# user map
|
||||||
|
r_tracking.sadd('investigations:user:{user_id}', investigation_uuid)
|
||||||
|
# metadata
|
||||||
|
r_tracking.hset(f'investigations:data:{investigation_uuid}', 'creator_user', user_id)
|
||||||
|
|
||||||
|
# TODO: limit info + name
|
||||||
|
investigation = Investigation(investigation_uuid)
|
||||||
|
investigation.set_info(info)
|
||||||
|
#investigation.set_name(name) ##############################################
|
||||||
|
investigation.set_date(date)
|
||||||
|
investigation.set_threat_level(threat_level)
|
||||||
|
investigation.set_analysis(analysis)
|
||||||
|
|
||||||
|
# # TODO: sanityze tags
|
||||||
|
if tags:
|
||||||
|
investigation.set_tags(tags)
|
||||||
|
|
||||||
|
timestamp = int(time.time())
|
||||||
|
r_tracking.hset(f'investigations:data:{investigation_uuid}', 'timestamp', timestamp)
|
||||||
|
investigation.set_last_change(timestamp)
|
||||||
|
|
||||||
|
return investigation_uuid
|
||||||
|
|
||||||
|
def get_all_investigations_meta(r_str=False):
|
||||||
|
investigations_meta = []
|
||||||
|
for investigation_uuid in get_all_investigations():
|
||||||
|
investigation = Investigation(investigation_uuid)
|
||||||
|
investigations_meta.append(investigation.get_metadata(r_str=r_str))
|
||||||
|
return investigations_meta
|
||||||
|
|
||||||
|
def get_investigations_selector():
|
||||||
|
l_investigations = []
|
||||||
|
for investigation_uuid in get_all_investigations():
|
||||||
|
investigation = Investigation(investigation_uuid)
|
||||||
|
name = investigation.get_info()
|
||||||
|
l_investigations.append({"id":investigation_uuid, "name": name})
|
||||||
|
return l_investigations
|
||||||
|
|
||||||
|
#{id:'8dc4b81aeff94a9799bd70ba556fa345',name:"Paris"}
|
||||||
|
|
||||||
|
|
||||||
|
#### API ####
|
||||||
|
|
||||||
|
# # TODO: CHECK Mandatory Fields
|
||||||
|
# # TODO: SANITYZE Fields
|
||||||
|
# # TODO: Name ?????
|
||||||
|
def api_add_investigation(json_dict):
|
||||||
|
user_id = json_dict.get('user_id')
|
||||||
|
name = json_dict.get('name') ##### mandatory ?
|
||||||
|
name = escape(name)
|
||||||
|
threat_level = json_dict.get('threat_level', 4)
|
||||||
|
analysis = json_dict.get('analysis', 0)
|
||||||
|
|
||||||
|
# # TODO: sanityze date
|
||||||
|
date = json_dict.get('date')
|
||||||
|
|
||||||
|
info = json_dict.get('info', '')
|
||||||
|
info = escape(info)
|
||||||
|
info = info[:1000]
|
||||||
|
tags = json_dict.get('tags', [])
|
||||||
|
if not Tag.are_enabled_tags(tags):
|
||||||
|
return {"status": "error", "reason": "Invalid/Disabled tags"}, 400
|
||||||
|
|
||||||
|
try:
|
||||||
|
res = create_investigation(user_id, date, name, threat_level, analysis, info, tags=tags)
|
||||||
|
except UpdateInvestigationError as e:
|
||||||
|
return e.message, 400
|
||||||
|
return res, 200
|
||||||
|
|
||||||
|
# # TODO: edit threat level / status
|
||||||
|
def api_edit_investigation(json_dict):
|
||||||
|
investigation_uuid = json_dict.get('uuid', '').replace(' ', '')
|
||||||
|
if not is_valid_uuid_v4(investigation_uuid):
|
||||||
|
return {"status": "error", "reason": "Invalid Investigation uuid"}, 400
|
||||||
|
investigation_uuid = sanityze_uuid(investigation_uuid)
|
||||||
|
if not exists_investigation(investigation_uuid):
|
||||||
|
return {"status": "error", "reason": "Investigation not found"}, 404
|
||||||
|
investigation = Investigation(investigation_uuid)
|
||||||
|
|
||||||
|
name = json_dict.get('name') ##### mandatory ?
|
||||||
|
name = escape(name)
|
||||||
|
threat_level = json_dict.get('threat_level', 4)
|
||||||
|
try:
|
||||||
|
investigation.set_threat_level(threat_level)
|
||||||
|
except UpdateInvestigationError:
|
||||||
|
return {"status": "error", "reason": "Invalid Investigation threat_level"}, 400
|
||||||
|
|
||||||
|
analysis = json_dict.get('analysis', 0)
|
||||||
|
try:
|
||||||
|
investigation.set_analysis(analysis)
|
||||||
|
except UpdateInvestigationError:
|
||||||
|
return {"status": "error", "reason": "Invalid Investigation analysis"}, 400
|
||||||
|
|
||||||
|
info = json_dict.get('info', '')
|
||||||
|
info = escape(info)
|
||||||
|
info = info[:1000]
|
||||||
|
tags = json_dict.get('tags', [])
|
||||||
|
if not Tag.are_enabled_tags(tags):
|
||||||
|
return {"status": "error", "reason": "Invalid/Disabled tags"}, 400
|
||||||
|
|
||||||
|
investigation.set_info(info)
|
||||||
|
investigation.set_name(name)
|
||||||
|
investigation.set_tags(tags)
|
||||||
|
|
||||||
|
timestamp = int(time.time())
|
||||||
|
investigation.set_last_change(timestamp)
|
||||||
|
|
||||||
|
return investigation_uuid, 200
|
||||||
|
|
||||||
|
def api_delete_investigation(json_dict):
|
||||||
|
investigation_uuid = json_dict.get('uuid', '').replace(' ', '')
|
||||||
|
if not is_valid_uuid_v4(investigation_uuid):
|
||||||
|
return {"status": "error", "reason": "Invalid Investigation uuid"}, 400
|
||||||
|
investigation_uuid = sanityze_uuid(investigation_uuid)
|
||||||
|
if not exists_investigation(investigation_uuid):
|
||||||
|
return {"status": "error", "reason": "Investigation not found"}, 404
|
||||||
|
investigation = Investigation(investigation_uuid)
|
||||||
|
res = investigation.delete()
|
||||||
|
return res, 200
|
||||||
|
|
||||||
|
def api_register_object(json_dict):
|
||||||
|
investigation_uuid = json_dict.get('uuid', '').replace(' ', '')
|
||||||
|
if not is_valid_uuid_v4(investigation_uuid):
|
||||||
|
return {"status": "error", "reason": f"Invalid Investigation uuid: {investigation_uuid}"}, 400
|
||||||
|
investigation_uuid = sanityze_uuid(investigation_uuid)
|
||||||
|
if not exists_investigation(investigation_uuid):
|
||||||
|
return {"status": "error", "reason": f"Investigation not found: {investigation_uuid}"}, 404
|
||||||
|
investigation = Investigation(investigation_uuid)
|
||||||
|
|
||||||
|
obj_type = json_dict.get('type', '').replace(' ', '')
|
||||||
|
if not exists_obj_type(obj_type):
|
||||||
|
return {"status": "error", "reason": f"Invalid Object Type: {obj_type}"}, 400
|
||||||
|
|
||||||
|
subtype = json_dict.get('subtype', '')
|
||||||
|
if subtype == 'None':
|
||||||
|
subtype = ''
|
||||||
|
obj_id = json_dict.get('id', '').replace(' ', '')
|
||||||
|
res = investigation.register_object(obj_id, obj_type, subtype)
|
||||||
|
return res, 200
|
||||||
|
|
||||||
|
def api_unregister_object(json_dict):
|
||||||
|
investigation_uuid = json_dict.get('uuid', '').replace(' ', '')
|
||||||
|
if not is_valid_uuid_v4(investigation_uuid):
|
||||||
|
return {"status": "error", "reason": f"Invalid Investigation uuid: {investigation_uuid}"}, 400
|
||||||
|
investigation_uuid = sanityze_uuid(investigation_uuid)
|
||||||
|
if not exists_investigation(investigation_uuid):
|
||||||
|
return {"status": "error", "reason": f"Investigation not found: {investigation_uuid}"}, 404
|
||||||
|
investigation = Investigation(investigation_uuid)
|
||||||
|
|
||||||
|
obj_type = json_dict.get('type', '').replace(' ', '')
|
||||||
|
subtype = json_dict.get('subtype', '')
|
||||||
|
if subtype == 'None':
|
||||||
|
subtype = ''
|
||||||
|
obj_id = json_dict.get('id', '').replace(' ', '')
|
||||||
|
res = investigation.unregister_object(obj_id, obj_type, subtype)
|
||||||
|
return res, 200
|
||||||
|
|
||||||
|
##-- API --##
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
# res = create_star_list(user_id, name, description)
|
||||||
|
# print(res)
|
||||||
|
|
||||||
|
# res = r_tracking.dbsize()
|
||||||
|
# print(res)
|
||||||
|
|
||||||
|
investigation_uuid = 'e4e1c8e3b0a349bf81482f2f823efc0f'
|
||||||
|
|
||||||
|
investigation = Investigation(investigation_uuid)
|
||||||
|
investigation.delete()
|
||||||
|
|
||||||
|
# # TODO: PAGINATION
|
|
@ -1,406 +0,0 @@
|
||||||
#!/usr/bin/env python3
|
|
||||||
# -*-coding:UTF-8 -*
|
|
||||||
|
|
||||||
import os
|
|
||||||
import sys
|
|
||||||
import uuid
|
|
||||||
import redis
|
|
||||||
|
|
||||||
from abc import ABC
|
|
||||||
from flask import url_for
|
|
||||||
|
|
||||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
|
||||||
import Tag
|
|
||||||
|
|
||||||
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
|
||||||
import ConfigLoader
|
|
||||||
|
|
||||||
config_loader = ConfigLoader.ConfigLoader()
|
|
||||||
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
|
||||||
config_loader = None
|
|
||||||
|
|
||||||
class AbstractObject(ABC):
|
|
||||||
"""
|
|
||||||
Abstract Object
|
|
||||||
"""
|
|
||||||
|
|
||||||
# first seen last/seen ??
|
|
||||||
# # TODO: - tags
|
|
||||||
# - handle + refactor coorelations
|
|
||||||
# - creates others objects
|
|
||||||
|
|
||||||
def __init__(self, obj_type, id, subtype=None):
|
|
||||||
""" Abstract for all the AIL object
|
|
||||||
|
|
||||||
:param obj_type: object type (item, ...)
|
|
||||||
:param id: Object ID
|
|
||||||
"""
|
|
||||||
self.id = id
|
|
||||||
self.type = obj_type
|
|
||||||
self.subtype = None
|
|
||||||
|
|
||||||
def get_id(self):
|
|
||||||
return self.id
|
|
||||||
|
|
||||||
def get_type(self):
|
|
||||||
return self.type
|
|
||||||
|
|
||||||
def get_subtype(self, r_str=False):
|
|
||||||
if not self.subtype:
|
|
||||||
if r_str:
|
|
||||||
return ''
|
|
||||||
return self.subtype
|
|
||||||
|
|
||||||
def get_default_meta(self):
|
|
||||||
dict_meta = {'id': self.get_id(),
|
|
||||||
'type': self.get_type()}
|
|
||||||
if self.subtype:
|
|
||||||
dict_meta['subtype'] = self.subtype
|
|
||||||
return dict_meta
|
|
||||||
|
|
||||||
def get_tags(self, r_set=False):
|
|
||||||
tags = Tag.get_obj_tag(self.id)
|
|
||||||
if r_set:
|
|
||||||
tags = set(tags)
|
|
||||||
return tags
|
|
||||||
|
|
||||||
## ADD TAGS ????
|
|
||||||
#def add_tags(self):
|
|
||||||
|
|
||||||
def _delete(self):
|
|
||||||
# DELETE TAGS
|
|
||||||
Tag.delete_obj_all_tags(self.id, self.type)
|
|
||||||
if self.type == 'item':
|
|
||||||
# delete tracker
|
|
||||||
pass
|
|
||||||
|
|
||||||
|
|
||||||
def is_valid_object_type(object_type):
|
|
||||||
if object_type in ['domain', 'item', 'image', 'decoded']:
|
|
||||||
return True
|
|
||||||
else:
|
|
||||||
return False
|
|
||||||
|
|
||||||
def get_all_objects():
|
|
||||||
return ['domain', 'paste', 'pgp', 'cryptocurrency', 'decoded', 'screenshot']
|
|
||||||
|
|
||||||
def get_all_correlation_names():
|
|
||||||
'''
|
|
||||||
Return a list of all available correlations
|
|
||||||
'''
|
|
||||||
return ['pgp', 'cryptocurrency', 'decoded', 'screenshot']
|
|
||||||
|
|
||||||
def get_all_correlation_objects():
|
|
||||||
'''
|
|
||||||
Return a list of all correllated objects
|
|
||||||
'''
|
|
||||||
return ['domain', 'paste']
|
|
||||||
|
|
||||||
def exist_object(object_type, correlation_id, type_id=None):
|
|
||||||
if object_type == 'domain':
|
|
||||||
return Domain.verify_if_domain_exist(correlation_id)
|
|
||||||
elif object_type == 'paste' or object_type == 'item':
|
|
||||||
return Item.exist_item(correlation_id)
|
|
||||||
elif object_type == 'decoded':
|
|
||||||
return Decoded.exist_decoded(correlation_id)
|
|
||||||
elif object_type == 'pgp':
|
|
||||||
return Pgp.pgp._exist_corelation_field(type_id, correlation_id)
|
|
||||||
elif object_type == 'cryptocurrency':
|
|
||||||
return Cryptocurrency.cryptocurrency._exist_corelation_field(type_id, correlation_id)
|
|
||||||
elif object_type == 'screenshot' or object_type == 'image':
|
|
||||||
return Screenshot.exist_screenshot(correlation_id)
|
|
||||||
else:
|
|
||||||
return False
|
|
||||||
|
|
||||||
def get_obj_date(object_type, object_id):
|
|
||||||
if object_type == "item":
|
|
||||||
return int(Item.get_item_date(object_id))
|
|
||||||
else:
|
|
||||||
return None
|
|
||||||
|
|
||||||
# request_type => api or ui
|
|
||||||
def get_object_metadata(object_type, correlation_id, type_id=None):
|
|
||||||
if object_type == 'domain':
|
|
||||||
return Domain.Domain(correlation_id).get_domain_metadata(tags=True)
|
|
||||||
elif object_type == 'paste' or object_type == 'item':
|
|
||||||
return Item.get_item({"id": correlation_id, "date": True, "date_separator": True, "tags": True})[0]
|
|
||||||
elif object_type == 'decoded':
|
|
||||||
return Decoded.get_decoded_metadata(correlation_id, nb_seen=True, size=True, file_type=True, tag=True)
|
|
||||||
elif object_type == 'pgp':
|
|
||||||
return Pgp.pgp.get_metadata(type_id, correlation_id)
|
|
||||||
elif object_type == 'cryptocurrency':
|
|
||||||
return Cryptocurrency.cryptocurrency.get_metadata(type_id, correlation_id)
|
|
||||||
elif object_type == 'screenshot' or object_type == 'image':
|
|
||||||
return Screenshot.get_metadata(correlation_id)
|
|
||||||
|
|
||||||
def get_object_correlation(object_type, value, correlation_names=None, correlation_objects=None, requested_correl_type=None):
|
|
||||||
if object_type == 'domain':
|
|
||||||
return Domain.get_domain_all_correlation(value, correlation_names=correlation_names)
|
|
||||||
elif object_type == 'paste' or object_type == 'item':
|
|
||||||
return Item.get_item_all_correlation(value, correlation_names=correlation_names)
|
|
||||||
elif object_type == 'decoded':
|
|
||||||
return Decoded.get_decoded_correlated_object(value, correlation_objects=correlation_objects)
|
|
||||||
elif object_type == 'pgp':
|
|
||||||
return Pgp.pgp.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects)
|
|
||||||
elif object_type == 'cryptocurrency':
|
|
||||||
return Cryptocurrency.cryptocurrency.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects)
|
|
||||||
elif object_type == 'screenshot' or object_type == 'image':
|
|
||||||
return Screenshot.get_screenshot_correlated_object(value, correlation_objects=correlation_objects)
|
|
||||||
return {}
|
|
||||||
|
|
||||||
def get_correlation_node_icon(correlation_name, correlation_type=None, value=None):
|
|
||||||
'''
|
|
||||||
Used in UI Graph.
|
|
||||||
Return a font awesome icon for a given correlation_name.
|
|
||||||
|
|
||||||
:param correlation_name: correlation name
|
|
||||||
:param correlation_name: str
|
|
||||||
:param correlation_type: correlation type
|
|
||||||
:type correlation_type: str, optional
|
|
||||||
|
|
||||||
:return: a dictionnary {font awesome class, icon_code}
|
|
||||||
:rtype: dict
|
|
||||||
'''
|
|
||||||
icon_class = 'fas'
|
|
||||||
icon_text = ''
|
|
||||||
node_color = "#332288"
|
|
||||||
node_radius = 6
|
|
||||||
if correlation_name == "pgp":
|
|
||||||
node_color = '#44AA99'
|
|
||||||
if correlation_type == 'key':
|
|
||||||
icon_text = '\uf084'
|
|
||||||
elif correlation_type == 'name':
|
|
||||||
icon_text = '\uf507'
|
|
||||||
elif correlation_type == 'mail':
|
|
||||||
icon_text = '\uf1fa'
|
|
||||||
else:
|
|
||||||
icon_text = 'times'
|
|
||||||
|
|
||||||
elif correlation_name == 'cryptocurrency':
|
|
||||||
node_color = '#DDCC77'
|
|
||||||
if correlation_type == 'bitcoin':
|
|
||||||
icon_class = 'fab'
|
|
||||||
icon_text = '\uf15a'
|
|
||||||
elif correlation_type == 'monero':
|
|
||||||
icon_class = 'fab'
|
|
||||||
icon_text = '\uf3d0'
|
|
||||||
elif correlation_type == 'ethereum':
|
|
||||||
icon_class = 'fab'
|
|
||||||
icon_text = '\uf42e'
|
|
||||||
else:
|
|
||||||
icon_text = '\uf51e'
|
|
||||||
|
|
||||||
elif correlation_name == 'decoded':
|
|
||||||
node_color = '#88CCEE'
|
|
||||||
correlation_type = Decoded.get_decoded_item_type(value).split('/')[0]
|
|
||||||
if correlation_type == 'application':
|
|
||||||
icon_text = '\uf15b'
|
|
||||||
elif correlation_type == 'audio':
|
|
||||||
icon_text = '\uf1c7'
|
|
||||||
elif correlation_type == 'image':
|
|
||||||
icon_text = '\uf1c5'
|
|
||||||
elif correlation_type == 'text':
|
|
||||||
icon_text = '\uf15c'
|
|
||||||
else:
|
|
||||||
icon_text = '\uf249'
|
|
||||||
|
|
||||||
elif correlation_name == 'screenshot' or correlation_name == 'image':
|
|
||||||
node_color = '#E1F5DF'
|
|
||||||
icon_text = '\uf03e'
|
|
||||||
|
|
||||||
elif correlation_name == 'domain':
|
|
||||||
node_radius = 5
|
|
||||||
node_color = '#3DA760'
|
|
||||||
if Domain.get_domain_type(value) == 'onion':
|
|
||||||
icon_text = '\uf06e'
|
|
||||||
else:
|
|
||||||
icon_class = 'fab'
|
|
||||||
icon_text = '\uf13b'
|
|
||||||
|
|
||||||
elif correlation_name == 'paste':
|
|
||||||
node_radius = 5
|
|
||||||
if Item.is_crawled(value):
|
|
||||||
node_color = 'red'
|
|
||||||
else:
|
|
||||||
node_color = '#332288'
|
|
||||||
|
|
||||||
return {"icon_class": icon_class, "icon_text": icon_text, "node_color": node_color, "node_radius": node_radius}
|
|
||||||
|
|
||||||
def get_item_url(correlation_name, value, correlation_type=None):
|
|
||||||
'''
|
|
||||||
Warning: use only in flask
|
|
||||||
'''
|
|
||||||
url = '#'
|
|
||||||
if correlation_name == "pgp":
|
|
||||||
endpoint = 'correlation.show_correlation'
|
|
||||||
url = url_for(endpoint, object_type="pgp", type_id=correlation_type, correlation_id=value)
|
|
||||||
elif correlation_name == 'cryptocurrency':
|
|
||||||
endpoint = 'correlation.show_correlation'
|
|
||||||
url = url_for(endpoint, object_type="cryptocurrency", type_id=correlation_type, correlation_id=value)
|
|
||||||
elif correlation_name == 'decoded':
|
|
||||||
endpoint = 'correlation.show_correlation'
|
|
||||||
url = url_for(endpoint, object_type="decoded", correlation_id=value)
|
|
||||||
elif correlation_name == 'screenshot' or correlation_name == 'image': ### # TODO: rename me
|
|
||||||
endpoint = 'correlation.show_correlation'
|
|
||||||
url = url_for(endpoint, object_type="screenshot", correlation_id=value)
|
|
||||||
elif correlation_name == 'domain':
|
|
||||||
endpoint = 'crawler_splash.showDomain'
|
|
||||||
url = url_for(endpoint, domain=value)
|
|
||||||
elif correlation_name == 'item':
|
|
||||||
endpoint = 'showsavedpastes.showsavedpaste'
|
|
||||||
url = url_for(endpoint, paste=value)
|
|
||||||
elif correlation_name == 'paste': ### # TODO: remove me
|
|
||||||
endpoint = 'showsavedpastes.showsavedpaste'
|
|
||||||
url = url_for(endpoint, paste=value)
|
|
||||||
return url
|
|
||||||
|
|
||||||
def get_obj_tag_table_keys(object_type):
|
|
||||||
'''
|
|
||||||
Warning: use only in flask (dynamic templates)
|
|
||||||
'''
|
|
||||||
if object_type=="domain":
|
|
||||||
return ['id', 'first_seen', 'last_check', 'status'] # # TODO: add root screenshot
|
|
||||||
|
|
||||||
|
|
||||||
def create_graph_links(links_set):
|
|
||||||
graph_links_list = []
|
|
||||||
for link in links_set:
|
|
||||||
graph_links_list.append({"source": link[0], "target": link[1]})
|
|
||||||
return graph_links_list
|
|
||||||
|
|
||||||
def create_graph_nodes(nodes_set, root_node_id):
|
|
||||||
graph_nodes_list = []
|
|
||||||
for node_id in nodes_set:
|
|
||||||
correlation_name, correlation_type, value = node_id.split(';', 3)
|
|
||||||
dict_node = {"id": node_id}
|
|
||||||
dict_node['style'] = get_correlation_node_icon(correlation_name, correlation_type, value)
|
|
||||||
dict_node['text'] = value
|
|
||||||
if node_id == root_node_id:
|
|
||||||
dict_node["style"]["node_color"] = 'orange'
|
|
||||||
dict_node["style"]["node_radius"] = 7
|
|
||||||
dict_node['url'] = get_item_url(correlation_name, value, correlation_type)
|
|
||||||
graph_nodes_list.append(dict_node)
|
|
||||||
return graph_nodes_list
|
|
||||||
|
|
||||||
def create_node_id(correlation_name, value, correlation_type=''):
|
|
||||||
if correlation_type is None:
|
|
||||||
correlation_type = ''
|
|
||||||
return '{};{};{}'.format(correlation_name, correlation_type, value)
|
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
# # TODO: filter by correlation type => bitcoin, mail, ...
|
|
||||||
def get_graph_node_object_correlation(object_type, root_value, mode, correlation_names, correlation_objects, max_nodes=300, requested_correl_type=None):
|
|
||||||
links = set()
|
|
||||||
nodes = set()
|
|
||||||
|
|
||||||
root_node_id = create_node_id(object_type, root_value, requested_correl_type)
|
|
||||||
nodes.add(root_node_id)
|
|
||||||
|
|
||||||
root_correlation = get_object_correlation(object_type, root_value, correlation_names, correlation_objects, requested_correl_type=requested_correl_type)
|
|
||||||
for correl in root_correlation:
|
|
||||||
if correl in ('pgp', 'cryptocurrency'):
|
|
||||||
for correl_type in root_correlation[correl]:
|
|
||||||
for correl_val in root_correlation[correl][correl_type]:
|
|
||||||
|
|
||||||
# add correlation
|
|
||||||
correl_node_id = create_node_id(correl, correl_val, correl_type)
|
|
||||||
|
|
||||||
if mode=="union":
|
|
||||||
if len(nodes) > max_nodes:
|
|
||||||
break
|
|
||||||
nodes.add(correl_node_id)
|
|
||||||
links.add((root_node_id, correl_node_id))
|
|
||||||
|
|
||||||
# get second correlation
|
|
||||||
res = get_object_correlation(correl, correl_val, correlation_names, correlation_objects, requested_correl_type=correl_type)
|
|
||||||
if res:
|
|
||||||
for corr_obj in res:
|
|
||||||
for correl_key_val in res[corr_obj]:
|
|
||||||
#filter root value
|
|
||||||
if correl_key_val == root_value:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if len(nodes) > max_nodes:
|
|
||||||
break
|
|
||||||
new_corel_1 = create_node_id(corr_obj, correl_key_val)
|
|
||||||
new_corel_2 = create_node_id(correl, correl_val, correl_type)
|
|
||||||
nodes.add(new_corel_1)
|
|
||||||
nodes.add(new_corel_2)
|
|
||||||
links.add((new_corel_1, new_corel_2))
|
|
||||||
|
|
||||||
if mode=="inter":
|
|
||||||
nodes.add(correl_node_id)
|
|
||||||
links.add((root_node_id, correl_node_id))
|
|
||||||
if correl in ('decoded', 'screenshot', 'domain', 'paste'):
|
|
||||||
for correl_val in root_correlation[correl]:
|
|
||||||
|
|
||||||
correl_node_id = create_node_id(correl, correl_val)
|
|
||||||
if mode=="union":
|
|
||||||
if len(nodes) > max_nodes:
|
|
||||||
break
|
|
||||||
nodes.add(correl_node_id)
|
|
||||||
links.add((root_node_id, correl_node_id))
|
|
||||||
|
|
||||||
res = get_object_correlation(correl, correl_val, correlation_names, correlation_objects)
|
|
||||||
if res:
|
|
||||||
for corr_obj in res:
|
|
||||||
if corr_obj in ('decoded', 'domain', 'paste', 'screenshot'):
|
|
||||||
for correl_key_val in res[corr_obj]:
|
|
||||||
#filter root value
|
|
||||||
if correl_key_val == root_value:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if len(nodes) > max_nodes:
|
|
||||||
break
|
|
||||||
new_corel_1 = create_node_id(corr_obj, correl_key_val)
|
|
||||||
new_corel_2 = create_node_id(correl, correl_val)
|
|
||||||
nodes.add(new_corel_1)
|
|
||||||
nodes.add(new_corel_2)
|
|
||||||
links.add((new_corel_1, new_corel_2))
|
|
||||||
|
|
||||||
if mode=="inter":
|
|
||||||
nodes.add(correl_node_id)
|
|
||||||
links.add((root_node_id, correl_node_id))
|
|
||||||
|
|
||||||
if corr_obj in ('pgp', 'cryptocurrency'):
|
|
||||||
for correl_key_type in res[corr_obj]:
|
|
||||||
for correl_key_val in res[corr_obj][correl_key_type]:
|
|
||||||
#filter root value
|
|
||||||
if correl_key_val == root_value:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if len(nodes) > max_nodes:
|
|
||||||
break
|
|
||||||
new_corel_1 = create_node_id(corr_obj, correl_key_val, correl_key_type)
|
|
||||||
new_corel_2 = create_node_id(correl, correl_val)
|
|
||||||
nodes.add(new_corel_1)
|
|
||||||
nodes.add(new_corel_2)
|
|
||||||
links.add((new_corel_1, new_corel_2))
|
|
||||||
|
|
||||||
if mode=="inter":
|
|
||||||
nodes.add(correl_node_id)
|
|
||||||
links.add((root_node_id, correl_node_id))
|
|
||||||
|
|
||||||
|
|
||||||
return {"nodes": create_graph_nodes(nodes, root_node_id), "links": create_graph_links(links)}
|
|
||||||
|
|
||||||
|
|
||||||
def get_obj_global_id(obj_type, obj_id, obj_sub_type=None):
|
|
||||||
if obj_sub_type:
|
|
||||||
return '{}:{}:{}'.format(obj_type, obj_sub_type, obj_id)
|
|
||||||
else:
|
|
||||||
# # TODO: remove me
|
|
||||||
if obj_type=='paste':
|
|
||||||
obj_type='item'
|
|
||||||
# # TODO: remove me
|
|
||||||
if obj_type=='screenshot':
|
|
||||||
obj_type='image'
|
|
||||||
|
|
||||||
return '{}:{}'.format(obj_type, obj_id)
|
|
||||||
|
|
||||||
######## API EXPOSED ########
|
|
||||||
def sanitize_object_type(object_type):
|
|
||||||
if not is_valid_object_type(object_type):
|
|
||||||
return ({'status': 'error', 'reason': 'Incorrect object_type'}, 400)
|
|
||||||
######## ########
|
|
30
bin/lib/ail_users.py
Executable file
30
bin/lib/ail_users.py
Executable file
|
@ -0,0 +1,30 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import uuid
|
||||||
|
import redis
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
|
import ConfigLoader
|
||||||
|
|
||||||
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
|
||||||
|
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||||
|
config_loader = None
|
||||||
|
|
||||||
|
class User(object):
|
||||||
|
"""AIL User."""
|
||||||
|
|
||||||
|
def __init__(self, id):
|
||||||
|
self.id = id
|
||||||
|
if self.id == '__anonymous__':
|
||||||
|
self.role = 'anonymous'
|
||||||
|
else:
|
||||||
|
self.role = None
|
||||||
|
|
||||||
|
def get_role(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
|
15
bin/lib/exceptions.py
Executable file
15
bin/lib/exceptions.py
Executable file
|
@ -0,0 +1,15 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
class AIL_ERROR(Exception):
|
||||||
|
"""docstring for AIL_ERROR."""
|
||||||
|
|
||||||
|
def __init__(self, message):
|
||||||
|
super(AIL_ERROR, self).__init__(message)
|
||||||
|
self.message = message
|
||||||
|
|
||||||
|
class UpdateInvestigationError(AIL_ERROR):
|
||||||
|
pass
|
||||||
|
|
||||||
|
class NewTagError(AIL_ERROR):
|
||||||
|
pass
|
76
bin/lib/objects/CryptoCurrencies.py
Executable file
76
bin/lib/objects/CryptoCurrencies.py
Executable file
|
@ -0,0 +1,76 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import redis
|
||||||
|
|
||||||
|
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
|
import ConfigLoader
|
||||||
|
|
||||||
|
from abstract_object import AbstractObject
|
||||||
|
|
||||||
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
|
||||||
|
config_loader = None
|
||||||
|
|
||||||
|
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
|
||||||
|
class CryptoCurrency(AbstractObject):
|
||||||
|
"""
|
||||||
|
AIL CryptoCurrency Object. (strings)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, id, subtype):
|
||||||
|
super(CryptoCurrency, self).__init__('cryptocurrency', id, subtype=subtype)
|
||||||
|
|
||||||
|
# def get_ail_2_ail_payload(self):
|
||||||
|
# payload = {'raw': self.get_gzip_content(b64=True),
|
||||||
|
# 'compress': 'gzip'}
|
||||||
|
# return payload
|
||||||
|
|
||||||
|
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
|
||||||
|
def delete(self):
|
||||||
|
# # TODO:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def get_link(self, flask_context=False):
|
||||||
|
if flask_context:
|
||||||
|
url = url_for('correlation.show_correlation', object_type=self.type, type_id=self.subtype, correlation_id=self.id)
|
||||||
|
else:
|
||||||
|
url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&type_id={self.subtype}&correlation_id={self.id}'
|
||||||
|
return url
|
||||||
|
|
||||||
|
def get_svg_icon(self):
|
||||||
|
if self.subtype == 'bitcoin':
|
||||||
|
style = 'fab'
|
||||||
|
icon = '\uf15a'
|
||||||
|
elif self.subtype == 'monero':
|
||||||
|
style = 'fab'
|
||||||
|
icon = '\uf3d0'
|
||||||
|
elif self.subtype == 'ethereum':
|
||||||
|
style = 'fab'
|
||||||
|
icon = '\uf42e'
|
||||||
|
else:
|
||||||
|
style = 'fas'
|
||||||
|
icon = '\uf51e'
|
||||||
|
return {'style': style, 'icon': icon, 'color': '#DDCC77', 'radius':5}
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
def exist_correlation(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#if __name__ == '__main__':
|
85
bin/lib/objects/Decodeds.py
Executable file
85
bin/lib/objects/Decodeds.py
Executable file
|
@ -0,0 +1,85 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import redis
|
||||||
|
|
||||||
|
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
|
import ConfigLoader
|
||||||
|
|
||||||
|
from abstract_object import AbstractObject
|
||||||
|
|
||||||
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||||
|
HASH_DIR = config_loader.get_config_str('Directories', 'hash')
|
||||||
|
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
|
||||||
|
config_loader = None
|
||||||
|
|
||||||
|
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
|
||||||
|
# # TODO: COMPLETE CLASS
|
||||||
|
|
||||||
|
class Decoded(AbstractObject):
|
||||||
|
"""
|
||||||
|
AIL Decoded Object. (strings)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, id):
|
||||||
|
super(Decoded, self).__init__('decoded', id)
|
||||||
|
|
||||||
|
# def get_ail_2_ail_payload(self):
|
||||||
|
# payload = {'raw': self.get_gzip_content(b64=True),
|
||||||
|
# 'compress': 'gzip'}
|
||||||
|
# return payload
|
||||||
|
|
||||||
|
def get_sha1(self):
|
||||||
|
return self.id.split('/')[0]
|
||||||
|
|
||||||
|
def get_file_type(self):
|
||||||
|
return r_serv_metadata.hget(f'metadata_hash:{self.get_sha1()}', 'estimated_type')
|
||||||
|
|
||||||
|
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
|
||||||
|
def delete(self):
|
||||||
|
# # TODO:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def get_link(self, flask_context=False):
|
||||||
|
if flask_context:
|
||||||
|
url = url_for('correlation.show_correlation', object_type="decoded", correlation_id=value)
|
||||||
|
else:
|
||||||
|
url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&correlation_id={self.id}'
|
||||||
|
return url
|
||||||
|
|
||||||
|
def get_svg_icon(self):
|
||||||
|
file_type = self.get_file_type()
|
||||||
|
if file_type == 'application':
|
||||||
|
icon = '\uf15b'
|
||||||
|
elif file_type == 'audio':
|
||||||
|
icon = '\uf1c7'
|
||||||
|
elif file_type == 'image':
|
||||||
|
icon = '\uf1c5'
|
||||||
|
elif file_type == 'text':
|
||||||
|
icon = '\uf15c'
|
||||||
|
else:
|
||||||
|
icon = '\uf249'
|
||||||
|
return {'style': 'fas', 'icon': icon, 'color': '#88CCEE', 'radius':5}
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
def exist_correlation(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#if __name__ == '__main__':
|
77
bin/lib/objects/Domains.py
Executable file
77
bin/lib/objects/Domains.py
Executable file
|
@ -0,0 +1,77 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import redis
|
||||||
|
|
||||||
|
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
|
import ConfigLoader
|
||||||
|
|
||||||
|
from abstract_object import AbstractObject
|
||||||
|
|
||||||
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
|
||||||
|
config_loader = None
|
||||||
|
|
||||||
|
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
|
||||||
|
class Domain(AbstractObject):
|
||||||
|
"""
|
||||||
|
AIL Decoded Object. (strings)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, id):
|
||||||
|
super(Domain, self).__init__('domain', id)
|
||||||
|
|
||||||
|
# def get_ail_2_ail_payload(self):
|
||||||
|
# payload = {'raw': self.get_gzip_content(b64=True),
|
||||||
|
# 'compress': 'gzip'}
|
||||||
|
# return payload
|
||||||
|
|
||||||
|
def get_domain_type(self):
|
||||||
|
if str(self.id).endswith('.onion'):
|
||||||
|
return 'onion'
|
||||||
|
else:
|
||||||
|
return 'regular'
|
||||||
|
|
||||||
|
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
|
||||||
|
def delete(self):
|
||||||
|
# # TODO:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def get_link(self, flask_context=False):
|
||||||
|
if flask_context:
|
||||||
|
url = url_for('crawler_splash.showDomain', domain=self.id)
|
||||||
|
else:
|
||||||
|
url = f'{baseurl}/crawlers/showDomain?domain={self.id}'
|
||||||
|
return url
|
||||||
|
|
||||||
|
def get_svg_icon(self):
|
||||||
|
color = '#3DA760'
|
||||||
|
if self.get_domain_type() == 'onion':
|
||||||
|
style = 'fas'
|
||||||
|
icon = '\uf06e'
|
||||||
|
else:
|
||||||
|
style = 'fab'
|
||||||
|
icon = '\uf13b'
|
||||||
|
return {'style': style, 'icon': icon, 'color':color, 'radius':5}
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
def exist_correlation(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#if __name__ == '__main__':
|
690
bin/lib/objects/Items.py
Executable file
690
bin/lib/objects/Items.py
Executable file
|
@ -0,0 +1,690 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import base64
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import redis
|
||||||
|
import cld3
|
||||||
|
import html2text
|
||||||
|
|
||||||
|
from io import BytesIO
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||||
|
import Tag
|
||||||
|
import Cryptocurrency
|
||||||
|
import Pgp
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
|
import item_basic
|
||||||
|
import domain_basic
|
||||||
|
import ConfigLoader
|
||||||
|
import Correlate_object
|
||||||
|
import Decoded
|
||||||
|
import Screenshot
|
||||||
|
import Username
|
||||||
|
|
||||||
|
from abstract_object import AbstractObject
|
||||||
|
from item_basic import *
|
||||||
|
|
||||||
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
# get and sanityze PASTE DIRECTORY
|
||||||
|
# # TODO: rename PASTES_FOLDER
|
||||||
|
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
|
||||||
|
PASTES_FOLDER = os.path.join(os.path.realpath(PASTES_FOLDER), '')
|
||||||
|
|
||||||
|
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||||
|
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||||
|
screenshot_directory = config_loader.get_files_directory('screenshot')
|
||||||
|
har_directory = config_loader.get_files_directory('har')
|
||||||
|
baseurl = config_loader.get_config_str("Notifications", "ail_domain")
|
||||||
|
config_loader = None
|
||||||
|
|
||||||
|
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
|
||||||
|
class Item(AbstractObject):
|
||||||
|
"""
|
||||||
|
AIL Item Object. (strings)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, id):
|
||||||
|
super(Item, self).__init__('item', id)
|
||||||
|
|
||||||
|
def get_date(self, separator=False):
|
||||||
|
"""
|
||||||
|
Returns Item date
|
||||||
|
"""
|
||||||
|
return item_basic.get_item_date(self.id, add_separator=separator)
|
||||||
|
|
||||||
|
def get_source(self):
|
||||||
|
"""
|
||||||
|
Returns Item source/feeder name
|
||||||
|
"""
|
||||||
|
#return self.id.split('/')[-5]
|
||||||
|
l_source = self.id.split('/')[:-4]
|
||||||
|
return os.path.join(*l_source)
|
||||||
|
|
||||||
|
def get_basename(self):
|
||||||
|
return os.path.basename(self.id)
|
||||||
|
|
||||||
|
def get_filename(self):
|
||||||
|
# Creating the full filepath
|
||||||
|
filename = os.path.join(PASTES_FOLDER, self.id)
|
||||||
|
filename = os.path.realpath(filename)
|
||||||
|
|
||||||
|
# incorrect filename
|
||||||
|
if not os.path.commonprefix([filename, PASTES_FOLDER]) == PASTES_FOLDER:
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
return filename
|
||||||
|
|
||||||
|
def get_content(self):
|
||||||
|
"""
|
||||||
|
Returns Item content
|
||||||
|
"""
|
||||||
|
return item_basic.get_item_content(self.id)
|
||||||
|
|
||||||
|
def get_gzip_content(self, b64=False):
|
||||||
|
with open(self.get_filename(), 'rb') as f:
|
||||||
|
content = f.read()
|
||||||
|
if b64:
|
||||||
|
content = base64.b64encode(content)
|
||||||
|
return content.decode()
|
||||||
|
|
||||||
|
def get_ail_2_ail_payload(self):
|
||||||
|
payload = {'raw': self.get_gzip_content(b64=True),
|
||||||
|
'compress': 'gzip'}
|
||||||
|
return payload
|
||||||
|
|
||||||
|
# # TODO:
|
||||||
|
def create(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
|
||||||
|
# TODO: DELETE ITEM CORRELATION + TAGS + METADATA + ...
|
||||||
|
def delete(self):
|
||||||
|
try:
|
||||||
|
os.remove(self.get_filename())
|
||||||
|
return True
|
||||||
|
except FileNotFoundError:
|
||||||
|
return False
|
||||||
|
|
||||||
|
def get_link(self, flask_context=False):
|
||||||
|
if flask_context:
|
||||||
|
url = url_for('objects_item.showItem', id=value)
|
||||||
|
else:
|
||||||
|
url = f'{baseurl}/object/item?id={self.id}'
|
||||||
|
return url
|
||||||
|
|
||||||
|
def get_svg_icon(self):
|
||||||
|
if is_crawled(self.id):
|
||||||
|
color = 'red'
|
||||||
|
else:
|
||||||
|
color = '#332288'
|
||||||
|
return {'style': '', 'icon': '', 'color': color, 'radius':5}
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
def exist_correlation(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
|
||||||
|
def exist_item(item_id):
|
||||||
|
return item_basic.exist_item(item_id)
|
||||||
|
|
||||||
|
def get_basename(item_id):
|
||||||
|
return os.path.basename(item_id)
|
||||||
|
|
||||||
|
def get_item_id(full_path):
|
||||||
|
return full_path.replace(PASTES_FOLDER, '', 1)
|
||||||
|
|
||||||
|
def get_item_filepath(item_id):
|
||||||
|
return item_basic.get_item_filepath(item_id)
|
||||||
|
|
||||||
|
def get_item_date(item_id, add_separator=False):
|
||||||
|
return item_basic.get_item_date(item_id, add_separator=add_separator)
|
||||||
|
|
||||||
|
def get_source(item_id):
|
||||||
|
return item_basic.get_source(item_id)
|
||||||
|
|
||||||
|
def get_all_sources():
|
||||||
|
return item_basic.get_all_items_sources(r_list=True)
|
||||||
|
|
||||||
|
def get_item_basename(item_id):
|
||||||
|
return os.path.basename(item_id)
|
||||||
|
|
||||||
|
def get_item_size(item_id):
|
||||||
|
return round(os.path.getsize(os.path.join(PASTES_FOLDER, item_id))/1024.0, 2)
|
||||||
|
|
||||||
|
def get_item_encoding(item_id):
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_lines_info(item_id, item_content=None):
|
||||||
|
if not item_content:
|
||||||
|
item_content = get_item_content(item_id)
|
||||||
|
max_length = 0
|
||||||
|
line_id = 0
|
||||||
|
nb_line = 0
|
||||||
|
for line in item_content.splitlines():
|
||||||
|
length = len(line)
|
||||||
|
if length > max_length:
|
||||||
|
max_length = length
|
||||||
|
nb_line += 1
|
||||||
|
return {'nb': nb_line, 'max_length': max_length}
|
||||||
|
|
||||||
|
|
||||||
|
def get_item_metadata(item_id, item_content=None):
|
||||||
|
## TODO: FIXME ##performance
|
||||||
|
# encoding
|
||||||
|
# language
|
||||||
|
# lines info
|
||||||
|
item_metadata = {'date': get_item_date(item_id, add_separator=True),
|
||||||
|
'source': get_source(item_id),
|
||||||
|
'size': get_item_size(item_id),
|
||||||
|
'encoding': get_item_encoding(item_id),
|
||||||
|
'lines': get_lines_info(item_id, item_content=item_content)
|
||||||
|
}
|
||||||
|
return item_metadata
|
||||||
|
|
||||||
|
def get_item_parent(item_id):
|
||||||
|
return item_basic.get_item_parent(item_id)
|
||||||
|
|
||||||
|
def add_item_parent(item_parent, item_id):
|
||||||
|
return item_basic.add_item_parent(item_parent, item_id)
|
||||||
|
|
||||||
|
def get_item_content(item_id):
|
||||||
|
return item_basic.get_item_content(item_id)
|
||||||
|
|
||||||
|
def get_item_content_html2text(item_id, item_content=None, ignore_links=False):
|
||||||
|
if not item_content:
|
||||||
|
item_content = get_item_content(item_id)
|
||||||
|
h = html2text.HTML2Text()
|
||||||
|
h.ignore_links = ignore_links
|
||||||
|
h.ignore_images = ignore_links
|
||||||
|
return h.handle(item_content)
|
||||||
|
|
||||||
|
def remove_all_urls_from_content(item_id, item_content=None):
|
||||||
|
if not item_content:
|
||||||
|
item_content = get_item_content(item_id)
|
||||||
|
regex = r'\b(?:http://|https://)?(?:[a-zA-Z\d-]{,63}(?:\.[a-zA-Z\d-]{,63})+)(?:\:[0-9]+)*(?:/(?:$|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*\b'
|
||||||
|
url_regex = re.compile(regex)
|
||||||
|
urls = url_regex.findall(item_content)
|
||||||
|
urls = sorted(urls, key=len, reverse=True)
|
||||||
|
for url in urls:
|
||||||
|
item_content = item_content.replace(url, '')
|
||||||
|
|
||||||
|
regex_pgp_public_blocs = r'-----BEGIN PGP PUBLIC KEY BLOCK-----[\s\S]+?-----END PGP PUBLIC KEY BLOCK-----'
|
||||||
|
regex_pgp_signature = r'-----BEGIN PGP SIGNATURE-----[\s\S]+?-----END PGP SIGNATURE-----'
|
||||||
|
regex_pgp_message = r'-----BEGIN PGP MESSAGE-----[\s\S]+?-----END PGP MESSAGE-----'
|
||||||
|
re.compile(regex_pgp_public_blocs)
|
||||||
|
re.compile(regex_pgp_signature)
|
||||||
|
re.compile(regex_pgp_message)
|
||||||
|
|
||||||
|
res = re.findall(regex_pgp_public_blocs, item_content)
|
||||||
|
for it in res:
|
||||||
|
item_content = item_content.replace(it, '')
|
||||||
|
res = re.findall(regex_pgp_signature, item_content)
|
||||||
|
for it in res:
|
||||||
|
item_content = item_content.replace(it, '')
|
||||||
|
res = re.findall(regex_pgp_message, item_content)
|
||||||
|
for it in res:
|
||||||
|
item_content = item_content.replace(it, '')
|
||||||
|
|
||||||
|
return item_content
|
||||||
|
|
||||||
|
def get_item_languages(item_id, min_len=600, num_langs=3, min_proportion=0.2, min_probability=0.7):
|
||||||
|
all_languages = []
|
||||||
|
|
||||||
|
## CLEAN CONTENT ##
|
||||||
|
content = get_item_content_html2text(item_id, ignore_links=True)
|
||||||
|
content = remove_all_urls_from_content(item_id, item_content=content)
|
||||||
|
|
||||||
|
# REMOVE USELESS SPACE
|
||||||
|
content = ' '.join(content.split())
|
||||||
|
#- CLEAN CONTENT -#
|
||||||
|
|
||||||
|
#print(content)
|
||||||
|
#print(len(content))
|
||||||
|
if len(content) >= min_len:
|
||||||
|
for lang in cld3.get_frequent_languages(content, num_langs=num_langs):
|
||||||
|
if lang.proportion >= min_proportion and lang.probability >= min_probability and lang.is_reliable:
|
||||||
|
all_languages.append(lang)
|
||||||
|
return all_languages
|
||||||
|
|
||||||
|
# API
|
||||||
|
def get_item(request_dict):
|
||||||
|
if not request_dict:
|
||||||
|
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
|
||||||
|
|
||||||
|
item_id = request_dict.get('id', None)
|
||||||
|
if not item_id:
|
||||||
|
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
|
||||||
|
if not exist_item(item_id):
|
||||||
|
return {'status': 'error', 'reason': 'Item not found'}, 404
|
||||||
|
|
||||||
|
dict_item = {}
|
||||||
|
dict_item['id'] = item_id
|
||||||
|
date = request_dict.get('date', True)
|
||||||
|
if date:
|
||||||
|
add_separator = False
|
||||||
|
if request_dict.get('date_separator', False):
|
||||||
|
add_separator = True
|
||||||
|
dict_item['date'] = get_item_date(item_id, add_separator=add_separator)
|
||||||
|
tags = request_dict.get('tags', True)
|
||||||
|
if tags:
|
||||||
|
dict_item['tags'] = Tag.get_obj_tag(item_id)
|
||||||
|
|
||||||
|
size = request_dict.get('size', False)
|
||||||
|
if size:
|
||||||
|
dict_item['size'] = get_item_size(item_id)
|
||||||
|
|
||||||
|
content = request_dict.get('content', False)
|
||||||
|
if content:
|
||||||
|
# UTF-8 outpout, # TODO: use base64
|
||||||
|
dict_item['content'] = get_item_content(item_id)
|
||||||
|
|
||||||
|
raw_content = request_dict.get('raw_content', False)
|
||||||
|
if raw_content:
|
||||||
|
dict_item['raw_content'] = get_raw_content(item_id)
|
||||||
|
|
||||||
|
lines_info = request_dict.get('lines', False)
|
||||||
|
if lines_info:
|
||||||
|
dict_item['lines'] = get_lines_info(item_id, dict_item.get('content', 'None'))
|
||||||
|
|
||||||
|
if request_dict.get('pgp'):
|
||||||
|
dict_item['pgp'] = {}
|
||||||
|
if request_dict['pgp'].get('key'):
|
||||||
|
dict_item['pgp']['key'] = get_item_pgp_key(item_id)
|
||||||
|
if request_dict['pgp'].get('mail'):
|
||||||
|
dict_item['pgp']['mail'] = get_item_pgp_mail(item_id)
|
||||||
|
if request_dict['pgp'].get('name'):
|
||||||
|
dict_item['pgp']['name'] = get_item_pgp_name(item_id)
|
||||||
|
|
||||||
|
if request_dict.get('cryptocurrency'):
|
||||||
|
dict_item['cryptocurrency'] = {}
|
||||||
|
if request_dict['cryptocurrency'].get('bitcoin'):
|
||||||
|
dict_item['cryptocurrency']['bitcoin'] = get_item_bitcoin(item_id)
|
||||||
|
|
||||||
|
return dict_item, 200
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
def api_get_item_content_base64_utf8(request_dict):
|
||||||
|
item_id = request_dict.get('id', None)
|
||||||
|
if not request_dict:
|
||||||
|
return {'status': 'error', 'reason': 'Malformed JSON'}, 400
|
||||||
|
if not item_id:
|
||||||
|
return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
|
||||||
|
if not exist_item(item_id):
|
||||||
|
return {'status': 'error', 'reason': 'Item not found'}, 404
|
||||||
|
|
||||||
|
item_content = get_item_content(item_id)
|
||||||
|
item_content = base64.b64encode((item_content.encode('utf-8'))).decode('UTF-8')
|
||||||
|
return {'status': 'success', 'content': item_content}, 200
|
||||||
|
|
||||||
|
|
||||||
|
def api_get_items_sources():
|
||||||
|
item_content = {'sources': get_all_sources()}
|
||||||
|
return item_content, 200
|
||||||
|
|
||||||
|
# def check_item_source(request_dict):
|
||||||
|
# source = request_dict.get('source', None)
|
||||||
|
# if not request_dict:
|
||||||
|
# return {'status': 'error', 'reason': 'Malformed JSON'}, 400
|
||||||
|
# if not source:
|
||||||
|
# return {'status': 'error', 'reason': 'Mandatory parameter(s) not provided'}, 400
|
||||||
|
#
|
||||||
|
# all_sources = item_basic.get_all_items_sources()
|
||||||
|
#
|
||||||
|
# if source not in all_sources:
|
||||||
|
# return {'status': 'error', 'reason': 'Invalid source', 'provide': source}, 400
|
||||||
|
# return {'status': 'success', 'reason': 'Valid source', 'provide': source}, 200
|
||||||
|
|
||||||
|
###
|
||||||
|
### correlation
|
||||||
|
###
|
||||||
|
def get_item_cryptocurrency(item_id, currencies_type=None, get_nb=False):
|
||||||
|
'''
|
||||||
|
Return all cryptocurrencies of a given item.
|
||||||
|
|
||||||
|
:param item_id: item id
|
||||||
|
:param currencies_type: list of cryptocurrencies type
|
||||||
|
:type currencies_type: list, optional
|
||||||
|
'''
|
||||||
|
return Cryptocurrency.cryptocurrency.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb)
|
||||||
|
|
||||||
|
def get_item_pgp(item_id, currencies_type=None, get_nb=False):
|
||||||
|
'''
|
||||||
|
Return all pgp of a given item.
|
||||||
|
|
||||||
|
:param item_id: item id
|
||||||
|
:param currencies_type: list of cryptocurrencies type
|
||||||
|
:type currencies_type: list, optional
|
||||||
|
'''
|
||||||
|
return Pgp.pgp.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb)
|
||||||
|
|
||||||
|
def get_item_username(item_id, sub_type=None, get_nb=False):
|
||||||
|
'''
|
||||||
|
Return all pgp of a given item.
|
||||||
|
|
||||||
|
:param item_id: item id
|
||||||
|
:param sub_type: list of username type
|
||||||
|
:type sub_type: list, optional
|
||||||
|
'''
|
||||||
|
return Username.correlation.get_item_correlation_dict(item_id, correlation_type=sub_type, get_nb=get_nb)
|
||||||
|
|
||||||
|
def get_item_decoded(item_id):
|
||||||
|
'''
|
||||||
|
Return all pgp of a given item.
|
||||||
|
|
||||||
|
:param item_id: item id
|
||||||
|
:param currencies_type: list of cryptocurrencies type
|
||||||
|
:type currencies_type: list, optional
|
||||||
|
'''
|
||||||
|
return Decoded.get_item_decoded(item_id)
|
||||||
|
|
||||||
|
def get_item_all_screenshot(item_id):
|
||||||
|
'''
|
||||||
|
Return all screenshot of a given item.
|
||||||
|
|
||||||
|
:param item_id: item id
|
||||||
|
'''
|
||||||
|
return Screenshot.get_item_screenshot_list(item_id)
|
||||||
|
|
||||||
|
def get_item_all_correlation(item_id, correlation_names=[], get_nb=False):
|
||||||
|
'''
|
||||||
|
Retun all correlation of a given item id.
|
||||||
|
|
||||||
|
:param item_id: item id
|
||||||
|
:type domain: str
|
||||||
|
|
||||||
|
:return: a dict of all correlation for a item id
|
||||||
|
:rtype: dict
|
||||||
|
'''
|
||||||
|
if not correlation_names:
|
||||||
|
correlation_names = Correlate_object.get_all_correlation_names()
|
||||||
|
item_correl = {}
|
||||||
|
for correlation_name in correlation_names:
|
||||||
|
if correlation_name=='cryptocurrency':
|
||||||
|
res = get_item_cryptocurrency(item_id, get_nb=get_nb)
|
||||||
|
elif correlation_name=='pgp':
|
||||||
|
res = get_item_pgp(item_id, get_nb=get_nb)
|
||||||
|
elif correlation_name=='username':
|
||||||
|
res = get_item_username(item_id, get_nb=get_nb)
|
||||||
|
elif correlation_name=='decoded':
|
||||||
|
res = get_item_decoded(item_id)
|
||||||
|
elif correlation_name=='screenshot':
|
||||||
|
res = get_item_all_screenshot(item_id)
|
||||||
|
else:
|
||||||
|
res = None
|
||||||
|
# add correllation to dict
|
||||||
|
if res:
|
||||||
|
item_correl[correlation_name] = res
|
||||||
|
return item_correl
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
## TODO: REFRACTOR
|
||||||
|
def _get_item_correlation(correlation_name, correlation_type, item_id):
|
||||||
|
res = r_serv_metadata.smembers('item_{}_{}:{}'.format(correlation_name, correlation_type, item_id))
|
||||||
|
if res:
|
||||||
|
return list(res)
|
||||||
|
else:
|
||||||
|
return []
|
||||||
|
|
||||||
|
## TODO: REFRACTOR
|
||||||
|
def get_item_bitcoin(item_id):
|
||||||
|
return _get_item_correlation('cryptocurrency', 'bitcoin', item_id)
|
||||||
|
|
||||||
|
## TODO: REFRACTOR
|
||||||
|
def get_item_pgp_key(item_id):
|
||||||
|
return _get_item_correlation('pgpdump', 'key', item_id)
|
||||||
|
|
||||||
|
## TODO: REFRACTOR
|
||||||
|
def get_item_pgp_name(item_id):
|
||||||
|
return _get_item_correlation('pgpdump', 'name', item_id)
|
||||||
|
|
||||||
|
## TODO: REFRACTOR
|
||||||
|
def get_item_pgp_mail(item_id):
|
||||||
|
return _get_item_correlation('pgpdump', 'mail', item_id)
|
||||||
|
|
||||||
|
## TODO: REFRACTOR
|
||||||
|
def get_item_pgp_correlation(item_id):
|
||||||
|
pass
|
||||||
|
|
||||||
|
###
|
||||||
|
### GET Internal Module DESC
|
||||||
|
###
|
||||||
|
def get_item_list_desc(list_item_id):
|
||||||
|
desc_list = []
|
||||||
|
for item_id in list_item_id:
|
||||||
|
desc_list.append( {'id': item_id, 'date': get_item_date(item_id), 'tags': Tag.get_obj_tag(item_id)} )
|
||||||
|
return desc_list
|
||||||
|
|
||||||
|
def is_crawled(item_id):
|
||||||
|
return item_basic.is_crawled(item_id)
|
||||||
|
|
||||||
|
def get_crawler_matadata(item_id, ltags=None):
|
||||||
|
dict_crawler = {}
|
||||||
|
if is_crawled(item_id):
|
||||||
|
dict_crawler['domain'] = get_item_domain(item_id)
|
||||||
|
if not ltags:
|
||||||
|
ltags = Tag.get_obj_tag(item_id)
|
||||||
|
dict_crawler['is_tags_safe'] = Tag.is_tags_safe(ltags)
|
||||||
|
dict_crawler['url'] = get_item_link(item_id)
|
||||||
|
dict_crawler['screenshot'] = get_item_screenshot(item_id)
|
||||||
|
dict_crawler['har'] = get_item_har_name(item_id)
|
||||||
|
return dict_crawler
|
||||||
|
|
||||||
|
def is_onion(item_id):
|
||||||
|
is_onion = False
|
||||||
|
if len(is_onion) > 62:
|
||||||
|
if is_crawled(item_id) and item_id[-42:-36] == '.onion':
|
||||||
|
is_onion = True
|
||||||
|
return is_onion
|
||||||
|
|
||||||
|
def is_item_in_domain(domain, item_id):
|
||||||
|
is_in_domain = False
|
||||||
|
domain_lenght = len(domain)
|
||||||
|
if len(item_id) > (domain_lenght+48):
|
||||||
|
if item_id[-36-domain_lenght:-36] == domain:
|
||||||
|
is_in_domain = True
|
||||||
|
return is_in_domain
|
||||||
|
|
||||||
|
def get_item_domain(item_id):
|
||||||
|
return item_basic.get_item_domain(item_id)
|
||||||
|
|
||||||
|
def get_domain(item_id):
|
||||||
|
item_id = item_id.split('/')
|
||||||
|
item_id = item_id[-1]
|
||||||
|
return item_id[:-36]
|
||||||
|
|
||||||
|
def get_item_domain_with_port(item_id):
|
||||||
|
return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'domain')
|
||||||
|
|
||||||
|
def get_item_link(item_id):
|
||||||
|
return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'real_link')
|
||||||
|
|
||||||
|
def get_item_screenshot(item_id):
|
||||||
|
screenshot = r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'screenshot')
|
||||||
|
if screenshot:
|
||||||
|
return os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:])
|
||||||
|
return ''
|
||||||
|
|
||||||
|
def get_item_har_name(item_id):
|
||||||
|
har_path = os.path.join(har_directory, item_id) + '.json'
|
||||||
|
if os.path.isfile(har_path):
|
||||||
|
return har_path
|
||||||
|
else:
|
||||||
|
return None
|
||||||
|
|
||||||
|
def get_item_har(har_path):
|
||||||
|
pass
|
||||||
|
|
||||||
|
def get_item_filename(item_id):
|
||||||
|
# Creating the full filepath
|
||||||
|
filename = os.path.join(PASTES_FOLDER, item_id)
|
||||||
|
filename = os.path.realpath(filename)
|
||||||
|
|
||||||
|
# incorrect filename
|
||||||
|
if not os.path.commonprefix([filename, PASTES_FOLDER]) == PASTES_FOLDER:
|
||||||
|
return None
|
||||||
|
else:
|
||||||
|
return filename
|
||||||
|
|
||||||
|
def get_item_duplicate(item_id, r_list=True):
|
||||||
|
res = r_serv_metadata.smembers('dup:{}'.format(item_id))
|
||||||
|
if r_list:
|
||||||
|
if res:
|
||||||
|
return list(res)
|
||||||
|
else:
|
||||||
|
return []
|
||||||
|
return res
|
||||||
|
|
||||||
|
def get_item_nb_duplicates(item_id):
|
||||||
|
return r_serv_metadata.scard('dup:{}'.format(item_id))
|
||||||
|
|
||||||
|
def get_item_duplicates_dict(item_id):
|
||||||
|
dict_duplicates = {}
|
||||||
|
for duplicate in get_item_duplicate(item_id):
|
||||||
|
duplicate = duplicate[1:-1].replace('\'', '').replace(' ', '').split(',')
|
||||||
|
duplicate_id = duplicate[1]
|
||||||
|
if not duplicate_id in dict_duplicates:
|
||||||
|
dict_duplicates[duplicate_id] = {'date': get_item_date(duplicate_id, add_separator=True), 'algo': {}}
|
||||||
|
algo = duplicate[0]
|
||||||
|
if algo == 'tlsh':
|
||||||
|
similarity = 100 - int(duplicate[2])
|
||||||
|
else:
|
||||||
|
similarity = int(duplicate[2])
|
||||||
|
dict_duplicates[duplicate_id]['algo'][algo] = similarity
|
||||||
|
return dict_duplicates
|
||||||
|
|
||||||
|
def add_item_duplicate(item_id, l_dup):
|
||||||
|
for item_dup in l_dup:
|
||||||
|
r_serv_metadata.sadd('dup:{}'.format(item_dup), item_id)
|
||||||
|
r_serv_metadata.sadd('dup:{}'.format(item_id), item_dup)
|
||||||
|
|
||||||
|
def delete_item_duplicate(item_id):
|
||||||
|
item_dup = get_item_duplicate(item_id)
|
||||||
|
for item_dup in get_item_duplicate(item_id):
|
||||||
|
r_serv_metadata.srem('dup:{}'.format(item_dup), item_id)
|
||||||
|
r_serv_metadata.delete('dup:{}'.format(item_id))
|
||||||
|
|
||||||
|
def get_raw_content(item_id):
|
||||||
|
filepath = get_item_filepath(item_id)
|
||||||
|
with open(filepath, 'rb') as f:
|
||||||
|
file_content = BytesIO(f.read())
|
||||||
|
return file_content
|
||||||
|
|
||||||
|
def save_raw_content(item_id, io_content):
|
||||||
|
filepath = get_item_filename(item_id)
|
||||||
|
if os.path.isfile(filepath):
|
||||||
|
#print('File already exist')
|
||||||
|
return False
|
||||||
|
# create subdir
|
||||||
|
dirname = os.path.dirname(filepath)
|
||||||
|
if not os.path.exists(dirname):
|
||||||
|
os.makedirs(dirname)
|
||||||
|
# # TODO: check if is IO file
|
||||||
|
with open(filepath, 'wb') as f:
|
||||||
|
f.write(io_content.getvalue())
|
||||||
|
return True
|
||||||
|
|
||||||
|
# IDEA: send item to duplicate ?
|
||||||
|
def create_item(obj_id, obj_metadata, io_content):
|
||||||
|
'''
|
||||||
|
Create a new Item (Import or Test only).
|
||||||
|
|
||||||
|
:param obj_id: item id
|
||||||
|
:type obj_metadata: dict - 'first_seen', 'tags'
|
||||||
|
|
||||||
|
:return: is item created
|
||||||
|
:rtype: boolean
|
||||||
|
'''
|
||||||
|
# check if datetime match ??
|
||||||
|
|
||||||
|
|
||||||
|
# # TODO: validate obj_id
|
||||||
|
|
||||||
|
res = save_raw_content(obj_id, io_content)
|
||||||
|
# item saved
|
||||||
|
if res:
|
||||||
|
# creata tags
|
||||||
|
if 'tags' in obj_metadata:
|
||||||
|
# # TODO: handle mixed tags: taxonomies and Galaxies
|
||||||
|
Tag.api_add_obj_tags(tags=obj_metadata['tags'], object_id=obj_id, object_type="item")
|
||||||
|
return True
|
||||||
|
|
||||||
|
# Item not created
|
||||||
|
return False
|
||||||
|
|
||||||
|
def delete_item(obj_id):
|
||||||
|
# check if item exists
|
||||||
|
if not exist_item(obj_id):
|
||||||
|
return False
|
||||||
|
else:
|
||||||
|
delete_item_duplicate(obj_id)
|
||||||
|
# delete MISP event
|
||||||
|
r_serv_metadata.delete('misp_events:{}'.format(obj_id))
|
||||||
|
r_serv_metadata.delete('hive_cases:{}'.format(obj_id))
|
||||||
|
|
||||||
|
os.remove(get_item_filename(obj_id))
|
||||||
|
|
||||||
|
# get all correlation
|
||||||
|
obj_correlations = get_item_all_correlation(obj_id)
|
||||||
|
for correlation in obj_correlations:
|
||||||
|
if correlation=='cryptocurrency' or correlation=='pgp':
|
||||||
|
for obj2_subtype in obj_correlations[correlation]:
|
||||||
|
for obj2_id in obj_correlations[correlation][obj2_subtype]:
|
||||||
|
Correlate_object.delete_obj_relationship(correlation, obj2_id, 'item', obj_id,
|
||||||
|
obj1_subtype=obj2_subtype)
|
||||||
|
else:
|
||||||
|
for obj2_id in obj_correlations[correlation]:
|
||||||
|
Correlate_object.delete_obj_relationship(correlation, obj2_id, 'item', obj_id)
|
||||||
|
|
||||||
|
# delete father/child
|
||||||
|
delete_node(obj_id)
|
||||||
|
|
||||||
|
# delete item metadata
|
||||||
|
r_serv_metadata.delete('paste_metadata:{}'.format(obj_id))
|
||||||
|
|
||||||
|
return True
|
||||||
|
|
||||||
|
### TODO in inport V2
|
||||||
|
# delete from tracked items
|
||||||
|
|
||||||
|
# # # TODO: # FIXME: LATER
|
||||||
|
# delete from queue
|
||||||
|
###
|
||||||
|
return False
|
||||||
|
|
||||||
|
#### ####
|
||||||
|
def delete_node(item_id):
|
||||||
|
if is_node(item_id):
|
||||||
|
if is_crawled(item_id):
|
||||||
|
delete_domain_node(item_id)
|
||||||
|
item_basic._delete_node(item_id)
|
||||||
|
|
||||||
|
def delete_domain_node(item_id):
|
||||||
|
if is_domain_root(item_id):
|
||||||
|
# remove from domain history
|
||||||
|
domain, port = get_item_domain_with_port(item_id).split(':')
|
||||||
|
domain_basic.delete_domain_item_core(item_id, domain, port)
|
||||||
|
for child_id in get_all_domain_node_by_item_id(item_id):
|
||||||
|
delete_item(child_id)
|
||||||
|
|
||||||
|
|
||||||
|
#if __name__ == '__main__':
|
72
bin/lib/objects/Pgps.py
Executable file
72
bin/lib/objects/Pgps.py
Executable file
|
@ -0,0 +1,72 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import redis
|
||||||
|
|
||||||
|
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
|
import ConfigLoader
|
||||||
|
|
||||||
|
from abstract_object import AbstractObject
|
||||||
|
|
||||||
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
|
||||||
|
config_loader = None
|
||||||
|
|
||||||
|
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
|
||||||
|
class Pgp(AbstractObject):
|
||||||
|
"""
|
||||||
|
AIL Pgp Object. (strings)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, id, subtype):
|
||||||
|
super(Pgp, self).__init__('pgp', id, subtype=subtype)
|
||||||
|
|
||||||
|
# def get_ail_2_ail_payload(self):
|
||||||
|
# payload = {'raw': self.get_gzip_content(b64=True),
|
||||||
|
# 'compress': 'gzip'}
|
||||||
|
# return payload
|
||||||
|
|
||||||
|
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
|
||||||
|
def delete(self):
|
||||||
|
# # TODO:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def get_link(self, flask_context=False):
|
||||||
|
if flask_context:
|
||||||
|
url = url_for('correlation.show_correlation', object_type=self.type, type_id=self.subtype, correlation_id=self.id)
|
||||||
|
else:
|
||||||
|
url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&type_id={self.subtype}&correlation_id={self.id}'
|
||||||
|
return url
|
||||||
|
|
||||||
|
def get_svg_icon(self):
|
||||||
|
if self.subtype == 'key':
|
||||||
|
icon = '\uf084'
|
||||||
|
elif self.subtype == 'name':
|
||||||
|
icon = '\uf507'
|
||||||
|
elif self.subtype == 'mail':
|
||||||
|
icon = '\uf1fa'
|
||||||
|
else:
|
||||||
|
icon = 'times'
|
||||||
|
return {'style': 'fas', 'icon': icon, 'color': '#44AA99', 'radius':5}
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
def exist_correlation(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#if __name__ == '__main__':
|
65
bin/lib/objects/Screenshots.py
Executable file
65
bin/lib/objects/Screenshots.py
Executable file
|
@ -0,0 +1,65 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import redis
|
||||||
|
|
||||||
|
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
|
import ConfigLoader
|
||||||
|
|
||||||
|
from abstract_object import AbstractObject
|
||||||
|
|
||||||
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||||
|
HASH_DIR = config_loader.get_config_str('Directories', 'hash')
|
||||||
|
config_loader = None
|
||||||
|
|
||||||
|
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
|
||||||
|
class Screenshot(AbstractObject):
|
||||||
|
"""
|
||||||
|
AIL Screenshot Object. (strings)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, id):
|
||||||
|
super(Screenshot, self).__init__('screenshot', id)
|
||||||
|
|
||||||
|
# def get_ail_2_ail_payload(self):
|
||||||
|
# payload = {'raw': self.get_gzip_content(b64=True),
|
||||||
|
# 'compress': 'gzip'}
|
||||||
|
# return payload
|
||||||
|
|
||||||
|
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
|
||||||
|
def delete(self):
|
||||||
|
# # TODO:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def get_link(self, flask_context=False):
|
||||||
|
if flask_context:
|
||||||
|
url = url_for('correlation.show_correlation', object_type=self.type, correlation_id=self.id)
|
||||||
|
else:
|
||||||
|
url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&correlation_id={self.id}'
|
||||||
|
return url
|
||||||
|
|
||||||
|
def get_svg_icon(self):
|
||||||
|
return {'style': 'fas', 'icon': '\uf03e', 'color': '#E1F5DF', 'radius':5}
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
def exist_correlation(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#if __name__ == '__main__':
|
73
bin/lib/objects/Usernames.py
Executable file
73
bin/lib/objects/Usernames.py
Executable file
|
@ -0,0 +1,73 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import redis
|
||||||
|
|
||||||
|
# sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
|
import ConfigLoader
|
||||||
|
|
||||||
|
from abstract_object import AbstractObject
|
||||||
|
|
||||||
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
|
||||||
|
config_loader = None
|
||||||
|
|
||||||
|
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
|
||||||
|
class Username(AbstractObject):
|
||||||
|
"""
|
||||||
|
AIL Username Object. (strings)
|
||||||
|
"""
|
||||||
|
|
||||||
|
def __init__(self, id, subtype):
|
||||||
|
super(Username, self).__init__('username', id, subtype=subtype)
|
||||||
|
|
||||||
|
# def get_ail_2_ail_payload(self):
|
||||||
|
# payload = {'raw': self.get_gzip_content(b64=True),
|
||||||
|
# 'compress': 'gzip'}
|
||||||
|
# return payload
|
||||||
|
|
||||||
|
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
|
||||||
|
def delete(self):
|
||||||
|
# # TODO:
|
||||||
|
pass
|
||||||
|
|
||||||
|
def get_link(self, flask_context=False):
|
||||||
|
if flask_context:
|
||||||
|
url = url_for('correlation.show_correlation', object_type=self.type, type_id=self.subtype, correlation_id=self.id)
|
||||||
|
else:
|
||||||
|
url = f'{baseurl}/correlation/show_correlation?object_type={self.type}&type_id={self.subtype}&correlation_id={self.id}'
|
||||||
|
return url
|
||||||
|
|
||||||
|
def get_svg_icon(self):
|
||||||
|
if self.subtype == 'telegram':
|
||||||
|
style = 'fab'
|
||||||
|
icon = '\uf2c6'
|
||||||
|
elif self.subtype == 'twitter':
|
||||||
|
style = 'fab'
|
||||||
|
icon = '\uf099'
|
||||||
|
else:
|
||||||
|
style = 'fas'
|
||||||
|
icon = '\uf007'
|
||||||
|
return {'style': style, 'icon': icon, 'color': '#4dffff', 'radius':5}
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
def exist_correlation(self):
|
||||||
|
pass
|
||||||
|
|
||||||
|
############################################################################
|
||||||
|
############################################################################
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
#if __name__ == '__main__':
|
135
bin/lib/objects/abstract_object.py
Executable file
135
bin/lib/objects/abstract_object.py
Executable file
|
@ -0,0 +1,135 @@
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
"""
|
||||||
|
Base Class for AIL Objects
|
||||||
|
"""
|
||||||
|
|
||||||
|
##################################
|
||||||
|
# Import External packages
|
||||||
|
##################################
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
from abc import ABC, abstractmethod
|
||||||
|
|
||||||
|
#from flask import url_for
|
||||||
|
|
||||||
|
sys.path.append(os.environ['AIL_BIN'])
|
||||||
|
##################################
|
||||||
|
# Import Project packages
|
||||||
|
##################################
|
||||||
|
from packages import Tag
|
||||||
|
from lib.Investigations import is_object_investigated, get_obj_investigations
|
||||||
|
|
||||||
|
# # TODO: ADD CORRELATION ENGINE
|
||||||
|
|
||||||
|
class AbstractObject(ABC):
|
||||||
|
"""
|
||||||
|
Abstract Object
|
||||||
|
"""
|
||||||
|
|
||||||
|
# first seen last/seen ??
|
||||||
|
# # TODO: - tags
|
||||||
|
# - handle + refactor coorelations
|
||||||
|
# - creates others objects
|
||||||
|
|
||||||
|
def __init__(self, obj_type, id, subtype=None):
|
||||||
|
""" Abstract for all the AIL object
|
||||||
|
|
||||||
|
:param obj_type: object type (item, ...)
|
||||||
|
:param id: Object ID
|
||||||
|
"""
|
||||||
|
self.id = id
|
||||||
|
self.type = obj_type
|
||||||
|
self.subtype = subtype
|
||||||
|
|
||||||
|
def get_id(self):
|
||||||
|
return self.id
|
||||||
|
|
||||||
|
def get_type(self):
|
||||||
|
return self.type
|
||||||
|
|
||||||
|
def get_subtype(self, r_str=False):
|
||||||
|
if not self.subtype:
|
||||||
|
if r_str:
|
||||||
|
return ''
|
||||||
|
return self.subtype
|
||||||
|
|
||||||
|
def get_default_meta(self, tags=False):
|
||||||
|
dict_meta = {'id': self.get_id(),
|
||||||
|
'type': self.get_type(),
|
||||||
|
'subtype': self.get_subtype()}
|
||||||
|
if tags:
|
||||||
|
dict_meta['tags'] = self.get_tags()
|
||||||
|
return dict_meta
|
||||||
|
|
||||||
|
## Tags ##
|
||||||
|
def get_tags(self, r_set=False):
|
||||||
|
tags = Tag.get_obj_tag(self.id)
|
||||||
|
if r_set:
|
||||||
|
tags = set(tags)
|
||||||
|
return tags
|
||||||
|
|
||||||
|
## ADD TAGS ????
|
||||||
|
#def add_tags(self):
|
||||||
|
|
||||||
|
#- Tags -#
|
||||||
|
|
||||||
|
## Investigations ##
|
||||||
|
# # TODO: unregister =====
|
||||||
|
|
||||||
|
def is_investigated(self):
|
||||||
|
if not self.subtype:
|
||||||
|
is_investigated = is_object_investigated(self.id, self.type)
|
||||||
|
else:
|
||||||
|
is_investigated = is_object_investigated(self.id, self.type, self.subtype)
|
||||||
|
return is_investigated
|
||||||
|
|
||||||
|
def get_investigations(self):
|
||||||
|
if not self.subtype:
|
||||||
|
investigations = get_obj_investigations(self.id, self.type)
|
||||||
|
else:
|
||||||
|
investigations = get_obj_investigations(self.id, self.type, self.subtype)
|
||||||
|
return investigations
|
||||||
|
#- Investigations -#
|
||||||
|
|
||||||
|
def _delete(self):
|
||||||
|
# DELETE TAGS
|
||||||
|
Tag.delete_obj_all_tags(self.id, self.type)
|
||||||
|
# # TODO: remove from investigations
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def delete(self):
|
||||||
|
"""
|
||||||
|
Delete Object: used for the Data Retention
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
# @abstractmethod
|
||||||
|
# def get_meta(self):
|
||||||
|
# """
|
||||||
|
# get Object metadata
|
||||||
|
# """
|
||||||
|
# pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_svg_icon(self):
|
||||||
|
"""
|
||||||
|
Get object svg icon
|
||||||
|
"""
|
||||||
|
pass
|
||||||
|
|
||||||
|
@abstractmethod
|
||||||
|
def get_link(self, flask_context=False):
|
||||||
|
pass
|
||||||
|
|
||||||
|
# # TODO:
|
||||||
|
# @abstractmethod
|
||||||
|
# def get_correlations(self, message):
|
||||||
|
# """
|
||||||
|
# Get object correlations
|
||||||
|
# """
|
||||||
|
# pass
|
||||||
|
|
||||||
|
|
||||||
|
# # TODO: get favicon
|
||||||
|
# # TODO: get url
|
||||||
|
# # TODO: get metadata
|
87
bin/lib/objects/ail_objects.py
Executable file
87
bin/lib/objects/ail_objects.py
Executable file
|
@ -0,0 +1,87 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import uuid
|
||||||
|
import redis
|
||||||
|
|
||||||
|
from abc import ABC
|
||||||
|
from flask import url_for
|
||||||
|
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
|
import ConfigLoader
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/objects'))
|
||||||
|
from Decodeds import Decoded
|
||||||
|
from Domains import Domain
|
||||||
|
from CryptoCurrencies import CryptoCurrency
|
||||||
|
from Items import Item
|
||||||
|
from Pgps import Pgp
|
||||||
|
from Screenshots import Screenshot
|
||||||
|
from Usernames import Username
|
||||||
|
|
||||||
|
##################################################################
|
||||||
|
##################################################################
|
||||||
|
#sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
|
|
||||||
|
#sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
|
||||||
|
##################################################################
|
||||||
|
##################################################################
|
||||||
|
|
||||||
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
|
||||||
|
config_loader = None
|
||||||
|
|
||||||
|
class AILObjects(object):
|
||||||
|
initial = 0
|
||||||
|
ongoing = 1
|
||||||
|
completed = 2
|
||||||
|
|
||||||
|
|
||||||
|
# # TODO: check change paste => item
|
||||||
|
def get_all_objects():
|
||||||
|
return ['domain', 'item', 'pgp', 'cryptocurrency', 'decoded', 'screenshot', 'username']
|
||||||
|
|
||||||
|
def get_object(obj_type, subtype, id):
|
||||||
|
if obj_type == 'item':
|
||||||
|
return Item(id)
|
||||||
|
elif obj_type == 'domain':
|
||||||
|
return Domain(id)
|
||||||
|
elif obj_type == 'decoded':
|
||||||
|
return Decoded(id)
|
||||||
|
elif obj_type == 'screenshot':
|
||||||
|
return Screenshot(id)
|
||||||
|
elif obj_type == 'cryptocurrency':
|
||||||
|
return CryptoCurrency(id, subtype)
|
||||||
|
elif obj_type == 'pgp':
|
||||||
|
return Pgp(id, subtype)
|
||||||
|
elif obj_type == 'username':
|
||||||
|
return Username(id, subtype)
|
||||||
|
|
||||||
|
def get_object_svg(obj_type, subtype, id):
|
||||||
|
object = get_object(obj_type, subtype, id)
|
||||||
|
return object.get_svg_icon()
|
||||||
|
|
||||||
|
def get_objects_meta(l_dict_objs, icon=False, url=False, flask_context=False):
|
||||||
|
l_meta = []
|
||||||
|
for dict_obj in l_dict_objs:
|
||||||
|
object = get_object(dict_obj['type'], dict_obj['subtype'], dict_obj['id'])
|
||||||
|
dict_meta = object.get_default_meta(tags=True)
|
||||||
|
if icon:
|
||||||
|
dict_meta['icon'] = object.get_svg_icon()
|
||||||
|
if url:
|
||||||
|
dict_meta['link'] = object.get_link(flask_context=flask_context)
|
||||||
|
l_meta.append(dict_meta)
|
||||||
|
return l_meta
|
||||||
|
|
||||||
|
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
||||||
|
################################################################################
|
|
@ -26,7 +26,7 @@ import Decoded
|
||||||
import Screenshot
|
import Screenshot
|
||||||
import Username
|
import Username
|
||||||
|
|
||||||
from ail_objects import AbstractObject
|
from objects.abstract_object import AbstractObject
|
||||||
from item_basic import *
|
from item_basic import *
|
||||||
|
|
||||||
config_loader = ConfigLoader.ConfigLoader()
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
|
|
@ -73,9 +73,6 @@ criticalNumberToAlert=8
|
||||||
#Will be considered as false positive if less that X matches from the top password list
|
#Will be considered as false positive if less that X matches from the top password list
|
||||||
minTopPassList=5
|
minTopPassList=5
|
||||||
|
|
||||||
[Curve]
|
|
||||||
max_execution_time = 90
|
|
||||||
|
|
||||||
[Onion]
|
[Onion]
|
||||||
save_i2p = False
|
save_i2p = False
|
||||||
max_execution_time = 180
|
max_execution_time = 180
|
||||||
|
@ -149,31 +146,27 @@ port = 6381
|
||||||
db = 1
|
db = 1
|
||||||
|
|
||||||
##### ARDB #####
|
##### ARDB #####
|
||||||
[ARDB_Curve]
|
|
||||||
|
[ARDB_DB]
|
||||||
host = localhost
|
host = localhost
|
||||||
port = 6382
|
port = 6382
|
||||||
db = 1
|
db = 0
|
||||||
|
|
||||||
|
[DB_Tracking]
|
||||||
|
host = localhost
|
||||||
|
port = 6382
|
||||||
|
db = 3
|
||||||
|
|
||||||
[ARDB_Sentiment]
|
[ARDB_Sentiment]
|
||||||
host = localhost
|
host = localhost
|
||||||
port = 6382
|
port = 6382
|
||||||
db = 4
|
db = 4
|
||||||
|
|
||||||
[ARDB_TermFreq]
|
|
||||||
host = localhost
|
|
||||||
port = 6382
|
|
||||||
db = 2
|
|
||||||
|
|
||||||
[ARDB_TermCred]
|
[ARDB_TermCred]
|
||||||
host = localhost
|
host = localhost
|
||||||
port = 6382
|
port = 6382
|
||||||
db = 5
|
db = 5
|
||||||
|
|
||||||
[ARDB_DB]
|
|
||||||
host = localhost
|
|
||||||
port = 6382
|
|
||||||
db = 0
|
|
||||||
|
|
||||||
[ARDB_Trending]
|
[ARDB_Trending]
|
||||||
host = localhost
|
host = localhost
|
||||||
port = 6382
|
port = 6382
|
||||||
|
|
36
update/v4.1/Update.py
Executable file
36
update/v4.1/Update.py
Executable file
|
@ -0,0 +1,36 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
import os
|
||||||
|
import re
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import redis
|
||||||
|
import datetime
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
|
||||||
|
import ConfigLoader
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
|
||||||
|
from ail_updater import AIL_Updater
|
||||||
|
|
||||||
|
class Updater(AIL_Updater):
|
||||||
|
"""default Updater."""
|
||||||
|
|
||||||
|
def __init__(self, version):
|
||||||
|
super(Updater, self).__init__(version)
|
||||||
|
|
||||||
|
def update(self):
|
||||||
|
config_loader = ConfigLoader.ConfigLoader()
|
||||||
|
r_tracking = config_loader.get_redis_conn("DB_Tracking")
|
||||||
|
config_loader = None
|
||||||
|
|
||||||
|
# FLUSH OLD DB
|
||||||
|
r_tracking.flushdb()
|
||||||
|
|
||||||
|
if __name__ == '__main__':
|
||||||
|
|
||||||
|
updater = Updater('v4.1')
|
||||||
|
|
||||||
|
|
||||||
|
updater.run_update()
|
25
update/v4.1/Update.sh
Executable file
25
update/v4.1/Update.sh
Executable file
|
@ -0,0 +1,25 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||||
|
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||||
|
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||||
|
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||||
|
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||||
|
|
||||||
|
export PATH=$AIL_HOME:$PATH
|
||||||
|
export PATH=$AIL_REDIS:$PATH
|
||||||
|
export PATH=$AIL_ARDB:$PATH
|
||||||
|
export PATH=$AIL_BIN:$PATH
|
||||||
|
export PATH=$AIL_FLASK:$PATH
|
||||||
|
|
||||||
|
GREEN="\\033[1;32m"
|
||||||
|
DEFAULT="\\033[0;39m"
|
||||||
|
|
||||||
|
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
|
||||||
|
bash ${AIL_BIN}/LAUNCH.sh -ks
|
||||||
|
wait
|
||||||
|
|
||||||
|
# SUBMODULES #
|
||||||
|
git submodule update
|
||||||
|
|
||||||
|
exit 0
|
|
@ -42,6 +42,7 @@ from blueprints.crawler_splash import crawler_splash
|
||||||
from blueprints.correlation import correlation
|
from blueprints.correlation import correlation
|
||||||
from blueprints.tags_ui import tags_ui
|
from blueprints.tags_ui import tags_ui
|
||||||
from blueprints.import_export import import_export
|
from blueprints.import_export import import_export
|
||||||
|
from blueprints.investigations_b import investigations_b
|
||||||
from blueprints.objects_item import objects_item
|
from blueprints.objects_item import objects_item
|
||||||
from blueprints.hunters import hunters
|
from blueprints.hunters import hunters
|
||||||
from blueprints.old_endpoints import old_endpoints
|
from blueprints.old_endpoints import old_endpoints
|
||||||
|
@ -101,6 +102,7 @@ app.register_blueprint(crawler_splash, url_prefix=baseUrl)
|
||||||
app.register_blueprint(correlation, url_prefix=baseUrl)
|
app.register_blueprint(correlation, url_prefix=baseUrl)
|
||||||
app.register_blueprint(tags_ui, url_prefix=baseUrl)
|
app.register_blueprint(tags_ui, url_prefix=baseUrl)
|
||||||
app.register_blueprint(import_export, url_prefix=baseUrl)
|
app.register_blueprint(import_export, url_prefix=baseUrl)
|
||||||
|
app.register_blueprint(investigations_b, url_prefix=baseUrl)
|
||||||
app.register_blueprint(objects_item, url_prefix=baseUrl)
|
app.register_blueprint(objects_item, url_prefix=baseUrl)
|
||||||
app.register_blueprint(hunters, url_prefix=baseUrl)
|
app.register_blueprint(hunters, url_prefix=baseUrl)
|
||||||
app.register_blueprint(old_endpoints, url_prefix=baseUrl)
|
app.register_blueprint(old_endpoints, url_prefix=baseUrl)
|
||||||
|
|
|
@ -28,7 +28,7 @@ import crawlers
|
||||||
import Domain
|
import Domain
|
||||||
import Language
|
import Language
|
||||||
|
|
||||||
import Config_DB
|
#import Config_DB
|
||||||
|
|
||||||
r_cache = Flask_config.r_cache
|
r_cache = Flask_config.r_cache
|
||||||
r_serv_db = Flask_config.r_serv_db
|
r_serv_db = Flask_config.r_serv_db
|
||||||
|
@ -295,6 +295,15 @@ def domains_search_name():
|
||||||
l_dict_domains=l_dict_domains, bootstrap_label=bootstrap_label,
|
l_dict_domains=l_dict_domains, bootstrap_label=bootstrap_label,
|
||||||
domains_types=domains_types)
|
domains_types=domains_types)
|
||||||
|
|
||||||
|
@crawler_splash.route('/domains/TODO', methods=['GET'])
|
||||||
|
@login_required
|
||||||
|
@login_analyst
|
||||||
|
def domains_todo():
|
||||||
|
domain_type = request.args.get('type')
|
||||||
|
last_domains = Domain.get_last_crawled_domains(domain_type)
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
##-- --##
|
##-- --##
|
||||||
|
|
||||||
|
|
||||||
|
@ -349,8 +358,8 @@ def crawler_cookiejar_add_post():
|
||||||
return redirect(url_for('crawler_splash.crawler_cookiejar_show', cookiejar_uuid=cookiejar_uuid))
|
return redirect(url_for('crawler_splash.crawler_cookiejar_show', cookiejar_uuid=cookiejar_uuid))
|
||||||
|
|
||||||
@crawler_splash.route('/crawler/cookiejar/all', methods=['GET'])
|
@crawler_splash.route('/crawler/cookiejar/all', methods=['GET'])
|
||||||
#@login_required
|
@login_required
|
||||||
#@login_read_only
|
@login_read_only
|
||||||
def crawler_cookiejar_all():
|
def crawler_cookiejar_all():
|
||||||
user_id = current_user.get_id()
|
user_id = current_user.get_id()
|
||||||
user_cookiejar = crawlers.get_cookiejar_metadata_by_iterator(crawlers.get_user_cookiejar(user_id))
|
user_cookiejar = crawlers.get_cookiejar_metadata_by_iterator(crawlers.get_user_cookiejar(user_id))
|
||||||
|
@ -358,8 +367,8 @@ def crawler_cookiejar_all():
|
||||||
return render_template("all_cookiejar.html", user_cookiejar=user_cookiejar, global_cookiejar=global_cookiejar)
|
return render_template("all_cookiejar.html", user_cookiejar=user_cookiejar, global_cookiejar=global_cookiejar)
|
||||||
|
|
||||||
@crawler_splash.route('/crawler/cookiejar/show', methods=['GET'])
|
@crawler_splash.route('/crawler/cookiejar/show', methods=['GET'])
|
||||||
#@login_required
|
@login_required
|
||||||
#@login_read_only
|
@login_read_only
|
||||||
def crawler_cookiejar_show():
|
def crawler_cookiejar_show():
|
||||||
user_id = current_user.get_id()
|
user_id = current_user.get_id()
|
||||||
cookiejar_uuid = request.args.get('cookiejar_uuid')
|
cookiejar_uuid = request.args.get('cookiejar_uuid')
|
||||||
|
@ -379,8 +388,8 @@ def crawler_cookiejar_show():
|
||||||
l_cookies=l_cookies, l_cookie_uuid=l_cookie_uuid)
|
l_cookies=l_cookies, l_cookie_uuid=l_cookie_uuid)
|
||||||
|
|
||||||
@crawler_splash.route('/crawler/cookiejar/cookie/delete', methods=['GET'])
|
@crawler_splash.route('/crawler/cookiejar/cookie/delete', methods=['GET'])
|
||||||
#@login_required
|
@login_required
|
||||||
#@login_read_only
|
@login_read_only
|
||||||
def crawler_cookiejar_cookie_delete():
|
def crawler_cookiejar_cookie_delete():
|
||||||
user_id = current_user.get_id()
|
user_id = current_user.get_id()
|
||||||
cookiejar_uuid = request.args.get('cookiejar_uuid')
|
cookiejar_uuid = request.args.get('cookiejar_uuid')
|
||||||
|
@ -392,8 +401,8 @@ def crawler_cookiejar_cookie_delete():
|
||||||
return redirect(url_for('crawler_splash.crawler_cookiejar_show', cookiejar_uuid=cookiejar_uuid))
|
return redirect(url_for('crawler_splash.crawler_cookiejar_show', cookiejar_uuid=cookiejar_uuid))
|
||||||
|
|
||||||
@crawler_splash.route('/crawler/cookiejar/delete', methods=['GET'])
|
@crawler_splash.route('/crawler/cookiejar/delete', methods=['GET'])
|
||||||
#@login_required
|
@login_required
|
||||||
#@login_read_only
|
@login_read_only
|
||||||
def crawler_cookiejar_delete():
|
def crawler_cookiejar_delete():
|
||||||
user_id = current_user.get_id()
|
user_id = current_user.get_id()
|
||||||
cookiejar_uuid = request.args.get('cookiejar_uuid')
|
cookiejar_uuid = request.args.get('cookiejar_uuid')
|
||||||
|
|
|
@ -182,6 +182,20 @@ def add_object_id_to_export():
|
||||||
# redirect
|
# redirect
|
||||||
return redirect(url_for('import_export.export_object'))
|
return redirect(url_for('import_export.export_object'))
|
||||||
|
|
||||||
|
@import_export.route("/import_export/investigation", methods=['GET'])
|
||||||
|
@login_required
|
||||||
|
@login_analyst
|
||||||
|
def export_investigation():
|
||||||
|
investigation_uuid = request.args.get("uuid")
|
||||||
|
|
||||||
|
if MispExport.ping_misp():
|
||||||
|
event_metadata = MispExport.create_investigation_event(investigation_uuid)
|
||||||
|
else:
|
||||||
|
return Response(json.dumps({"error": "Can't reach MISP Instance"}, indent=2, sort_keys=True), mimetype='application/json'), 400
|
||||||
|
|
||||||
|
return redirect(url_for('investigations_b.show_investigation', uuid=investigation_uuid))
|
||||||
|
|
||||||
|
|
||||||
# @import_export.route("/import_export/delete_object_id_to_export", methods=['GET'])
|
# @import_export.route("/import_export/delete_object_id_to_export", methods=['GET'])
|
||||||
# @login_required
|
# @login_required
|
||||||
# @login_analyst
|
# @login_analyst
|
||||||
|
|
212
var/www/blueprints/investigations_b.py
Normal file
212
var/www/blueprints/investigations_b.py
Normal file
|
@ -0,0 +1,212 @@
|
||||||
|
#!/usr/bin/env python3
|
||||||
|
# -*-coding:UTF-8 -*
|
||||||
|
|
||||||
|
'''
|
||||||
|
Blueprint Flask: ail_investigations
|
||||||
|
'''
|
||||||
|
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import json
|
||||||
|
|
||||||
|
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file
|
||||||
|
from flask_login import login_required, current_user
|
||||||
|
|
||||||
|
# Import Role_Manager
|
||||||
|
from Role_Manager import login_admin, login_analyst, login_read_only
|
||||||
|
|
||||||
|
sys.path.append('modules')
|
||||||
|
import Flask_config
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
|
||||||
|
import Investigations
|
||||||
|
from lib.objects import ail_objects
|
||||||
|
|
||||||
|
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
|
||||||
|
import Tag
|
||||||
|
|
||||||
|
# ============ BLUEPRINT ============
|
||||||
|
investigations_b = Blueprint('investigations_b', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/investigations'))
|
||||||
|
|
||||||
|
# ============ VARIABLES ============
|
||||||
|
bootstrap_label = Flask_config.bootstrap_label
|
||||||
|
|
||||||
|
# ============ FUNCTIONS ============
|
||||||
|
def create_json_response(data, status_code):
|
||||||
|
return Response(json.dumps(data, indent=2, sort_keys=True), mimetype='application/json'), status_code
|
||||||
|
|
||||||
|
# ============= ROUTES ==============
|
||||||
|
|
||||||
|
@investigations_b.route("/investigations", methods=['GET'])
|
||||||
|
@login_required
|
||||||
|
@login_read_only
|
||||||
|
def investigations_dashboard():
|
||||||
|
investigations = Investigations.get_all_investigations_meta(r_str=True)
|
||||||
|
return render_template("investigations.html", bootstrap_label=bootstrap_label,
|
||||||
|
investigations=investigations)
|
||||||
|
|
||||||
|
|
||||||
|
@investigations_b.route("/investigation", methods=['GET']) ## FIXME: add /view ????
|
||||||
|
@login_required
|
||||||
|
@login_read_only
|
||||||
|
def show_investigation():
|
||||||
|
investigation_uuid = request.args.get("uuid")
|
||||||
|
investigation = Investigations.Investigation(investigation_uuid)
|
||||||
|
metadata = investigation.get_metadata(r_str=True)
|
||||||
|
objs = ail_objects.get_objects_meta(investigation.get_objects(), icon=True)
|
||||||
|
return render_template("view_investigation.html", bootstrap_label=bootstrap_label,
|
||||||
|
metadata=metadata, investigation_objs=objs)
|
||||||
|
|
||||||
|
|
||||||
|
@investigations_b.route("/investigation/add", methods=['GET', 'POST'])
|
||||||
|
@login_required
|
||||||
|
@login_read_only
|
||||||
|
def add_investigation():
|
||||||
|
if request.method == 'POST':
|
||||||
|
user_id = current_user.get_id()
|
||||||
|
name = request.form.get("investigation_name")
|
||||||
|
date = request.form.get("investigation_date")
|
||||||
|
threat_level = request.form.get("threat_level")
|
||||||
|
analysis = request.form.get("analysis")
|
||||||
|
info = request.form.get("investigation_info")
|
||||||
|
# tags
|
||||||
|
taxonomies_tags = request.form.get('taxonomies_tags')
|
||||||
|
if taxonomies_tags:
|
||||||
|
try:
|
||||||
|
taxonomies_tags = json.loads(taxonomies_tags)
|
||||||
|
except Exception:
|
||||||
|
taxonomies_tags = []
|
||||||
|
else:
|
||||||
|
taxonomies_tags = []
|
||||||
|
galaxies_tags = request.form.get('galaxies_tags')
|
||||||
|
if galaxies_tags:
|
||||||
|
try:
|
||||||
|
galaxies_tags = json.loads(galaxies_tags)
|
||||||
|
except Exception:
|
||||||
|
galaxies_tags = []
|
||||||
|
tags = taxonomies_tags + galaxies_tags
|
||||||
|
|
||||||
|
input_dict = {"user_id": user_id, "name": name,
|
||||||
|
"threat_level": threat_level, "date": date,
|
||||||
|
"analysis": analysis, "info": info, "tags": tags}
|
||||||
|
res = Investigations.api_add_investigation(input_dict)
|
||||||
|
if res[1] != 200:
|
||||||
|
return create_json_response(res[0], res[1])
|
||||||
|
|
||||||
|
return redirect(url_for('investigations_b.show_investigation', uuid=res[0]))
|
||||||
|
else:
|
||||||
|
return render_template("add_investigation.html", tags_selector_data=Tag.get_tags_selector_data())
|
||||||
|
|
||||||
|
|
||||||
|
@investigations_b.route("/investigation/edit", methods=['GET', 'POST'])
|
||||||
|
@login_required
|
||||||
|
@login_read_only
|
||||||
|
def edit_investigation():
|
||||||
|
if request.method == 'POST':
|
||||||
|
user_id = current_user.get_id()
|
||||||
|
investigation_uuid = request.form.get("investigation_uuid")
|
||||||
|
name = request.form.get("investigation_name")
|
||||||
|
date = request.form.get("investigation_date")
|
||||||
|
threat_level = request.form.get("threat_level")
|
||||||
|
analysis = request.form.get("analysis")
|
||||||
|
info = request.form.get("investigation_info")
|
||||||
|
|
||||||
|
# tags
|
||||||
|
taxonomies_tags = request.form.get('taxonomies_tags')
|
||||||
|
if taxonomies_tags:
|
||||||
|
try:
|
||||||
|
taxonomies_tags = json.loads(taxonomies_tags)
|
||||||
|
except Exception:
|
||||||
|
taxonomies_tags = []
|
||||||
|
else:
|
||||||
|
taxonomies_tags = []
|
||||||
|
galaxies_tags = request.form.get('galaxies_tags')
|
||||||
|
if galaxies_tags:
|
||||||
|
try:
|
||||||
|
galaxies_tags = json.loads(galaxies_tags)
|
||||||
|
except Exception:
|
||||||
|
galaxies_tags = []
|
||||||
|
tags = taxonomies_tags + galaxies_tags
|
||||||
|
|
||||||
|
input_dict = {"user_id": user_id, "uuid": investigation_uuid,
|
||||||
|
"name": name, "threat_level": threat_level,
|
||||||
|
"analysis": analysis, "info": info, "tags": tags}
|
||||||
|
res = Investigations.api_edit_investigation(input_dict)
|
||||||
|
if res[1] != 200:
|
||||||
|
return create_json_response(res[0], res[1])
|
||||||
|
|
||||||
|
return redirect(url_for('investigations_b.show_investigation', uuid=res[0]))
|
||||||
|
else:
|
||||||
|
investigation_uuid = request.args.get('uuid')
|
||||||
|
investigation = Investigations.Investigation(investigation_uuid)
|
||||||
|
metadata = investigation.get_metadata(r_str=False)
|
||||||
|
taxonomies_tags, galaxies_tags = Tag.sort_tags_taxonomies_galaxies(metadata['tags'])
|
||||||
|
tags_selector_data = Tag.get_tags_selector_data()
|
||||||
|
tags_selector_data['taxonomies_tags'] = taxonomies_tags
|
||||||
|
tags_selector_data['galaxies_tags'] = galaxies_tags
|
||||||
|
return render_template("add_investigation.html", edit=True,
|
||||||
|
tags_selector_data=tags_selector_data, metadata=metadata)
|
||||||
|
|
||||||
|
@investigations_b.route("/investigation/delete", methods=['GET'])
|
||||||
|
@login_required
|
||||||
|
@login_read_only
|
||||||
|
def delete_investigation():
|
||||||
|
investigation_uuid = request.args.get('uuid')
|
||||||
|
input_dict = {"uuid": investigation_uuid}
|
||||||
|
res = Investigations.api_delete_investigation(input_dict)
|
||||||
|
if res[1] != 200:
|
||||||
|
return create_json_response(res[0], res[1])
|
||||||
|
return redirect(url_for('investigations_b.investigations_dashboard'))
|
||||||
|
|
||||||
|
@investigations_b.route("/investigation/object/register", methods=['GET'])
|
||||||
|
@login_required
|
||||||
|
@login_read_only
|
||||||
|
def register_investigation():
|
||||||
|
investigations_uuid = request.args.get('uuids')
|
||||||
|
investigations_uuid = investigations_uuid.split(',')
|
||||||
|
|
||||||
|
object_type = request.args.get('type')
|
||||||
|
object_subtype = request.args.get('subtype')
|
||||||
|
object_id = request.args.get('id')
|
||||||
|
|
||||||
|
for investigation_uuid in investigations_uuid:
|
||||||
|
input_dict = {"uuid": investigation_uuid, "id": object_id,
|
||||||
|
"type": object_type, "subtype": object_subtype}
|
||||||
|
res = Investigations.api_register_object(input_dict)
|
||||||
|
if res[1] != 200:
|
||||||
|
return create_json_response(res[0], res[1])
|
||||||
|
return redirect(url_for('investigations_b.investigations_dashboard', uuid=investigation_uuid))
|
||||||
|
|
||||||
|
@investigations_b.route("/investigation/object/unregister", methods=['GET'])
|
||||||
|
@login_required
|
||||||
|
@login_read_only
|
||||||
|
def unregister_investigation():
|
||||||
|
investigation_uuid = request.args.get('uuid')
|
||||||
|
object_type = request.args.get('type')
|
||||||
|
object_subtype = request.args.get('subtype')
|
||||||
|
object_id = request.args.get('id')
|
||||||
|
input_dict = {"uuid": investigation_uuid, "id": object_id,
|
||||||
|
"type": object_type, "subtype": object_subtype}
|
||||||
|
res = Investigations.api_unregister_object(input_dict)
|
||||||
|
if res[1] != 200:
|
||||||
|
return create_json_response(res[0], res[1])
|
||||||
|
return redirect(url_for('investigations_b.show_investigation', uuid=investigation_uuid))
|
||||||
|
|
||||||
|
|
||||||
|
@investigations_b.route("/investigation/all/selector_json")
|
||||||
|
@login_required
|
||||||
|
@login_read_only
|
||||||
|
def get_investigations_selector_json():
|
||||||
|
return jsonify(Investigations.get_investigations_selector())
|
||||||
|
|
||||||
|
|
||||||
|
#
|
||||||
|
# @investigations_b.route("/object/item") #completely shows the paste in a new tab
|
||||||
|
# @login_required
|
||||||
|
# @login_analyst
|
||||||
|
# def showItem(): # # TODO: support post
|
||||||
|
# item_id = request.args.get('id')
|
||||||
|
# if not item_id or not Item.exist_item(item_id):
|
||||||
|
# abort(404)
|
||||||
|
#
|
||||||
|
# return render_template("show_item.html", bootstrap_label=bootstrap_label)
|
|
@ -288,5 +288,11 @@ def passive_dns_change_state():
|
||||||
passivedns_enabled = d4.change_passive_dns_state(new_state)
|
passivedns_enabled = d4.change_passive_dns_state(new_state)
|
||||||
return redirect(url_for('settings.passive_dns'))
|
return redirect(url_for('settings.passive_dns'))
|
||||||
|
|
||||||
|
@settings.route("/settings/ail", methods=['GET'])
|
||||||
|
@login_required
|
||||||
|
@login_admin
|
||||||
|
def ail_configs():
|
||||||
|
return render_template("ail_configs.html", passivedns_enabled=None)
|
||||||
|
|
||||||
# ========= REGISTRATION =========
|
# ========= REGISTRATION =========
|
||||||
app.register_blueprint(settings, url_prefix=baseUrl)
|
app.register_blueprint(settings, url_prefix=baseUrl)
|
||||||
|
|
93
var/www/modules/settings/templates/ail_configs.html
Normal file
93
var/www/modules/settings/templates/ail_configs.html
Normal file
|
@ -0,0 +1,93 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
<html>
|
||||||
|
|
||||||
|
<head>
|
||||||
|
<title>Passive DNS - AIL</title>
|
||||||
|
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||||
|
|
||||||
|
<!-- Core CSS -->
|
||||||
|
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
|
||||||
|
|
||||||
|
<!-- JS -->
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||||
|
|
||||||
|
</head>
|
||||||
|
<body>
|
||||||
|
|
||||||
|
{% include 'nav_bar.html' %}
|
||||||
|
|
||||||
|
<div class="container-fluid">
|
||||||
|
<div class="row">
|
||||||
|
|
||||||
|
{% include 'settings/menu_sidebar.html' %}
|
||||||
|
|
||||||
|
<div class="col-12 col-lg-10" id="core_content">
|
||||||
|
|
||||||
|
<div class="card my-3">
|
||||||
|
<div class="card-body">
|
||||||
|
|
||||||
|
<div class="container-fluid">
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-12 col-lg-4">
|
||||||
|
<a href="https://www.misp-project.org/">
|
||||||
|
<img src="{{ url_for('static', filename='image/misp-logo.png')}}" alt="MISP project">
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
<div class="col-12 col-lg-8">
|
||||||
|
<form>
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="misp_url">Url</label>
|
||||||
|
<input class="form-control" type="text" id="misp_url" name="misp_url" placeholder="URL of the MISP instance">
|
||||||
|
</div>
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="misp_key">Key</label>
|
||||||
|
<input class="form-control" type="text" id="misp_key" name="misp_key" placeholder="API key of the user you want to use">
|
||||||
|
</div>
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="misp_key">Org UUID</label>
|
||||||
|
<input class="form-control" type="text" id="org_uuid" name="org_uuid" placeholder="Org UUID">
|
||||||
|
</div>
|
||||||
|
<button type="submit" class="btn btn-primary">Update</button>
|
||||||
|
</form>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</body>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
$(document).ready(function(){
|
||||||
|
$("#nav_settings").removeClass("text-muted");
|
||||||
|
$("#nav_ail_configs").addClass("active");
|
||||||
|
} );
|
||||||
|
|
||||||
|
function toggle_sidebar(){
|
||||||
|
if($('#nav_menu').is(':visible')){
|
||||||
|
$('#nav_menu').hide();
|
||||||
|
$('#side_menu').removeClass('border-right')
|
||||||
|
$('#side_menu').removeClass('col-lg-2')
|
||||||
|
$('#core_content').removeClass('col-lg-10')
|
||||||
|
}else{
|
||||||
|
$('#nav_menu').show();
|
||||||
|
$('#side_menu').addClass('border-right')
|
||||||
|
$('#side_menu').addClass('col-lg-2')
|
||||||
|
$('#core_content').addClass('col-lg-10')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
</script>
|
||||||
|
|
||||||
|
</html>
|
|
@ -5,3 +5,16 @@
|
||||||
padding-left: 0.15em;
|
padding-left: 0.15em;
|
||||||
background-color: #2e5;
|
background-color: #2e5;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
.blue {
|
||||||
|
color: #0088cc !important;
|
||||||
|
}
|
||||||
|
|
||||||
|
.bold {
|
||||||
|
font-weight: bold;
|
||||||
|
}
|
||||||
|
|
||||||
|
.object_node_icon {
|
||||||
|
font-size: 16px;
|
||||||
|
pointer-events: none;
|
||||||
|
}
|
||||||
|
|
|
@ -78,6 +78,14 @@
|
||||||
<a class="btn btn-secondary" href="{{ url_for('correlation.show_correlation') }}?object_type=cryptocurrency&type_id=bitcoin&correlation_id={{ dict_object['correlation_id'] }}&expand_card=True&correlation_objects=paste">Expand Bitcoin address</a>
|
<a class="btn btn-secondary" href="{{ url_for('correlation.show_correlation') }}?object_type=cryptocurrency&type_id=bitcoin&correlation_id={{ dict_object['correlation_id'] }}&expand_card=True&correlation_objects=paste">Expand Bitcoin address</a>
|
||||||
{% endif %}
|
{% endif %}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
|
{% with obj_type='cryptocurrency', obj_id=dict_object['correlation_id'], obj_subtype=dict_object["metadata"]["type_id"] %}
|
||||||
|
{% include 'modals/investigations_register_obj.html' %}
|
||||||
|
{% endwith %}
|
||||||
|
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
|
||||||
|
<i class="fas fa-microscope"></i> Investigations
|
||||||
|
</button>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|
|
@ -88,6 +88,14 @@
|
||||||
<button class='btn btn-info'><i class="fas fa-download"></i> Download Decoded file
|
<button class='btn btn-info'><i class="fas fa-download"></i> Download Decoded file
|
||||||
</button>
|
</button>
|
||||||
</a>
|
</a>
|
||||||
|
|
||||||
|
{% with obj_type='decoded', obj_id=dict_object['correlation_id'], obj_subtype='' %}
|
||||||
|
{% include 'modals/investigations_register_obj.html' %}
|
||||||
|
{% endwith %}
|
||||||
|
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
|
||||||
|
<i class="fas fa-microscope"></i> Investigations
|
||||||
|
</button>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|
|
@ -64,5 +64,13 @@
|
||||||
|
|
||||||
</li>
|
</li>
|
||||||
</ul>
|
</ul>
|
||||||
|
|
||||||
|
{% with obj_type='domain', obj_id=dict_object['correlation_id'], obj_subtype='' %}
|
||||||
|
{% include 'modals/investigations_register_obj.html' %}
|
||||||
|
{% endwith %}
|
||||||
|
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
|
||||||
|
<i class="fas fa-microscope"></i> Investigations
|
||||||
|
</button>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -46,5 +46,13 @@
|
||||||
|
|
||||||
</li>
|
</li>
|
||||||
</ul>
|
</ul>
|
||||||
|
|
||||||
|
{% with obj_type='item', obj_id=dict_object['correlation_id'], obj_subtype='' %}
|
||||||
|
{% include 'modals/investigations_register_obj.html' %}
|
||||||
|
{% endwith %}
|
||||||
|
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
|
||||||
|
<i class="fas fa-microscope"></i> Investigations
|
||||||
|
</button>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
@ -40,6 +40,14 @@
|
||||||
</div>
|
</div>
|
||||||
</li>
|
</li>
|
||||||
</ul>
|
</ul>
|
||||||
|
|
||||||
|
{% with obj_type='pgp', obj_id=dict_object['correlation_id'], obj_subtype=dict_object["metadata"]["type_id"] %}
|
||||||
|
{% include 'modals/investigations_register_obj.html' %}
|
||||||
|
{% endwith %}
|
||||||
|
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
|
||||||
|
<i class="fas fa-microscope"></i> Investigations
|
||||||
|
</button>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|
|
@ -78,6 +78,14 @@
|
||||||
</div>
|
</div>
|
||||||
</li>
|
</li>
|
||||||
</ul>
|
</ul>
|
||||||
|
|
||||||
|
{% with obj_type='screenshot', obj_id=dict_object['correlation_id'], obj_subtype='' %}
|
||||||
|
{% include 'modals/investigations_register_obj.html' %}
|
||||||
|
{% endwith %}
|
||||||
|
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
|
||||||
|
<i class="fas fa-microscope"></i> Investigations
|
||||||
|
</button>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|
|
@ -40,6 +40,14 @@
|
||||||
</div>
|
</div>
|
||||||
</li>
|
</li>
|
||||||
</ul>
|
</ul>
|
||||||
|
|
||||||
|
{% with obj_type='username', obj_id=dict_object['correlation_id'], obj_subtype=dict_object["metadata"]["type_id"] %}
|
||||||
|
{% include 'modals/investigations_register_obj.html' %}
|
||||||
|
{% endwith %}
|
||||||
|
<button type="button" class="btn btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
|
||||||
|
<i class="fas fa-microscope"></i> Investigations
|
||||||
|
</button>
|
||||||
|
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
|
|
214
var/www/templates/investigations/add_investigation.html
Normal file
214
var/www/templates/investigations/add_investigation.html
Normal file
|
@ -0,0 +1,214 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>AIL-Framework</title>
|
||||||
|
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
|
||||||
|
<!-- Core CSS -->
|
||||||
|
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/ail-project.css') }}" />
|
||||||
|
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
|
||||||
|
|
||||||
|
<!-- JS -->
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
|
||||||
|
|
||||||
|
</head>
|
||||||
|
|
||||||
|
<body>
|
||||||
|
|
||||||
|
{% include 'nav_bar.html' %}
|
||||||
|
|
||||||
|
<div class="container-fluid">
|
||||||
|
<div class="row">
|
||||||
|
|
||||||
|
{% include 'sidebars/sidebar_objects.html' %}
|
||||||
|
|
||||||
|
<div class="col-12 col-lg-10" id="core_content">
|
||||||
|
|
||||||
|
<div class="card my-3">
|
||||||
|
<div class="card-header bg-dark text-white">
|
||||||
|
<h5 class="card-title">
|
||||||
|
{% if edit %}
|
||||||
|
Edit Investigation
|
||||||
|
{% else %}
|
||||||
|
Create Investigation
|
||||||
|
{% endif %}
|
||||||
|
</h5>
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
|
||||||
|
<form action="{% if edit %}{{ url_for('investigations_b.edit_investigation') }}{% else %}{{ url_for('investigations_b.add_investigation') }}{% endif %}" method='post' onsubmit="SubmitCreateInvestigation();">
|
||||||
|
|
||||||
|
{% if edit %}
|
||||||
|
<input id="investigation_uuid" name="investigation_uuid" type="text" value="{{ metadata['uuid'] }}" hidden>
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-12 col-xl-9">
|
||||||
|
<div class="input-group mb-2 mr-sm-2">
|
||||||
|
<div class="input-group-prepend">
|
||||||
|
<div class="input-group-text bg-dark text-white"><i class="fas fa-quote-right"></i></div>
|
||||||
|
</div>
|
||||||
|
<input id="investigation_info" name="investigation_info" class="form-control" placeholder="Quick Investigation Info" type="text" {% if edit %}value="{{metadata['info']}}"{% endif %} required>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-12 col-xl-6">
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="threat_level">Threat Level:
|
||||||
|
<span id="threat_level_idInfoPopover" class="fas fa-info-circle" data-toggle="popover" data-trigger="hover"></span>
|
||||||
|
<script type="text/javascript">
|
||||||
|
$(function() {
|
||||||
|
$('#threat_level_idInfoPopover').popover({
|
||||||
|
html: true,
|
||||||
|
content: function() {
|
||||||
|
var tempSelector = '#threat_level';
|
||||||
|
return $('<div>').append(
|
||||||
|
$('<span>').attr('class', 'blue bold').text($(tempSelector +" option:selected").text())
|
||||||
|
).append(
|
||||||
|
$('<span>').text(': ' + fieldDesc[$(tempSelector).val()])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
var fieldDesc = {"1":"*high* means sophisticated APT malware or 0-day attack","2":"*medium* means APT malware","3":"*low* means mass-malware","4":"*undefined* no risk"};
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
</label>
|
||||||
|
<select class="form-control" id="threat_level" name="threat_level" required>
|
||||||
|
<option value="1">High</option>
|
||||||
|
<option value="2">Medium</option>
|
||||||
|
<option value="3">Low</option>
|
||||||
|
<option value="4">Undefined</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<div class="col-12 col-xl-6">
|
||||||
|
<div class="form-group">
|
||||||
|
<label for="analysis">Analysis:
|
||||||
|
<span id="analysis_idInfoPopover" class="fas fa-info-circle" data-toggle="popover" data-trigger="hover"></span>
|
||||||
|
<script type="text/javascript">
|
||||||
|
$(function() {
|
||||||
|
$('#analysis_idInfoPopover').popover({
|
||||||
|
html: true,
|
||||||
|
content: function() {
|
||||||
|
var tempSelector = '#analysis';
|
||||||
|
return $('<div>').append(
|
||||||
|
$('<span>').attr('class', 'blue bold').text($(tempSelector +" option:selected").text())
|
||||||
|
).append(
|
||||||
|
$('<span>').text(': ' + fieldDesc[$(tempSelector).val()])
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
var fieldDesc = {"0":"Investigation has just been created and is in an initial state","1":"The analysis is still ongoing","2":"The Investigation creator considers the analysis complete"};
|
||||||
|
});
|
||||||
|
</script>
|
||||||
|
</label>
|
||||||
|
<select class="form-control" id="analysis" name="analysis" required>
|
||||||
|
<option value="0">Initial</option>
|
||||||
|
<option value="1">Ongoing</option>
|
||||||
|
<option value="2">Completed</option>
|
||||||
|
</select>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="input-group" id="ivestigation-date">
|
||||||
|
<div class="input-group-prepend"><span class="input-group-text bg-secondary text-white"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
|
||||||
|
<input class="form-control" id="ivestigation-date-input" placeholder="yyyy-mm-dd" name="investigation_date" autocomplete="off" required>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="card my-4">
|
||||||
|
<div class="card-header bg-secondary text-white">
|
||||||
|
<b>Tags</b>
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
{% include 'tags/block_tags_selector.html' %}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="col-12 col-xl-3">
|
||||||
|
{% if edit %}
|
||||||
|
Edit Investigation
|
||||||
|
{% else %}
|
||||||
|
Create a new Investigation
|
||||||
|
{% endif %}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
<br>
|
||||||
|
<button class="btn btn-primary mt-2">
|
||||||
|
{% if edit %}
|
||||||
|
<i class="fas fa-pencil-alt"></i> Edit Investigation
|
||||||
|
{% else %}
|
||||||
|
<i class="fas fa-plus"></i> Create Investigation
|
||||||
|
{% endif %}
|
||||||
|
</button>
|
||||||
|
|
||||||
|
</form>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</body>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
$(document).ready(function(){
|
||||||
|
$('#nav_investigation').removeClass("text-muted");
|
||||||
|
$("#nav_add_investigation").addClass("active");
|
||||||
|
|
||||||
|
{% if edit %}
|
||||||
|
var investigation_date = "{{metadata['date']}}";
|
||||||
|
{% else %}
|
||||||
|
var investigation_date = getTodayDate();
|
||||||
|
{% endif %}
|
||||||
|
$('#ivestigation-date-input').val(investigation_date);
|
||||||
|
$('#ivestigation-date-input').dateRangePicker({
|
||||||
|
autoClose: true,
|
||||||
|
singleDate : true,
|
||||||
|
showShortcuts: false,
|
||||||
|
startDate: false,
|
||||||
|
});
|
||||||
|
|
||||||
|
{% if edit %}
|
||||||
|
$('#threat_level').val({{metadata['threat_level']}});
|
||||||
|
$('#analysis').val({{metadata['analysis']}});
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
function SubmitCreateInvestigation() {
|
||||||
|
var tags = ltags.getValue();
|
||||||
|
var tagsgalaxy = ltagsgalaxies.getValue();
|
||||||
|
$('#ltags').val(tags);
|
||||||
|
$('#ltagsgalaxies').val(tagsgalaxy);
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getTodayDate() {
|
||||||
|
var date = new Date();
|
||||||
|
var month = date.getMonth()+1;
|
||||||
|
var day = date.getDate();
|
||||||
|
return date.getFullYear() +'-'+ (month<10 ? '0' : '') + month +'-'+ (day<10 ? '0' : '') + day;
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
</script>
|
90
var/www/templates/investigations/investigations.html
Normal file
90
var/www/templates/investigations/investigations.html
Normal file
|
@ -0,0 +1,90 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>Investigations</title>
|
||||||
|
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
|
||||||
|
<!-- Core CSS -->
|
||||||
|
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||||
|
|
||||||
|
<!-- JS -->
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||||
|
|
||||||
|
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
|
||||||
|
|
||||||
|
</head>
|
||||||
|
|
||||||
|
<body>
|
||||||
|
|
||||||
|
{% include 'nav_bar.html' %}
|
||||||
|
|
||||||
|
<div class="container-fluid">
|
||||||
|
<div class="row">
|
||||||
|
|
||||||
|
{% include 'sidebars/sidebar_objects.html' %}
|
||||||
|
|
||||||
|
<div class="col-12 col-lg-10" id="core_content">
|
||||||
|
|
||||||
|
<h3 class="mt-2 text-secondary">
|
||||||
|
<i class="fas fa-microscope"></i> Investigations:
|
||||||
|
</h3>
|
||||||
|
|
||||||
|
<table id="table_investigation" class="table table-striped border-primary">
|
||||||
|
<thead class="bg-dark text-white">
|
||||||
|
<tr>
|
||||||
|
<th>Name</th>
|
||||||
|
<th>Date</th>
|
||||||
|
<th>last modified</th>
|
||||||
|
<td>Info</td>
|
||||||
|
<th>Nb Objects</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody style="font-size: 15px;">
|
||||||
|
{% for dict_investigation in investigations %}
|
||||||
|
<tr class="border-color: blue;">
|
||||||
|
<td>
|
||||||
|
<a href="{{ url_for('investigations_b.show_investigation') }}?uuid={{ dict_investigation['uuid'] }}">
|
||||||
|
{{ dict_investigation['info']}}
|
||||||
|
<div>
|
||||||
|
{% for tag in dict_investigation['tags'] %}
|
||||||
|
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
|
||||||
|
{% endfor %}
|
||||||
|
</div>
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
<td>{{ dict_investigation['date']}}</td>
|
||||||
|
<td>{{ dict_investigation['last_change']}}</td>
|
||||||
|
<td>{{ dict_investigation['info']}}</td>
|
||||||
|
<td>{{ dict_investigation['nb_objects']}}</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</body>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
$(document).ready(function(){
|
||||||
|
$('#nav_sync').removeClass("text-muted");
|
||||||
|
$("#navsync_queues").addClass("active");
|
||||||
|
|
||||||
|
$('#table_investigation').DataTable({
|
||||||
|
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||||
|
"iDisplayLength": 10,
|
||||||
|
"order": [[ 2, "desc" ]]
|
||||||
|
});
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
</script>
|
178
var/www/templates/investigations/view_investigation.html
Normal file
178
var/www/templates/investigations/view_investigation.html
Normal file
|
@ -0,0 +1,178 @@
|
||||||
|
<!DOCTYPE html>
|
||||||
|
|
||||||
|
<html>
|
||||||
|
<head>
|
||||||
|
<title>AIL-Framework</title>
|
||||||
|
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png')}}">
|
||||||
|
<!-- Core CSS -->
|
||||||
|
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
|
||||||
|
<link href="{{ url_for('static', filename='css/ail-project.css') }}" rel="stylesheet">
|
||||||
|
|
||||||
|
<!-- JS -->
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
|
||||||
|
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
|
||||||
|
|
||||||
|
</head>
|
||||||
|
|
||||||
|
<body>
|
||||||
|
|
||||||
|
{% include 'nav_bar.html' %}
|
||||||
|
|
||||||
|
<div class="container-fluid">
|
||||||
|
<div class="row">
|
||||||
|
|
||||||
|
{% include 'sidebars/sidebar_objects.html' %}
|
||||||
|
|
||||||
|
<div class="col-12 col-lg-10" id="core_content">
|
||||||
|
|
||||||
|
<div class="card my-1">
|
||||||
|
<div class="card-header bg-dark text-white">
|
||||||
|
<h4 class="card-title">{{metadata['info']}}</h4>
|
||||||
|
</div>
|
||||||
|
<div class="card-body">
|
||||||
|
<div class="container-fluid">
|
||||||
|
<div class="row">
|
||||||
|
<div class="col-12 col-lg-6">
|
||||||
|
|
||||||
|
<table class="table table-hover">
|
||||||
|
<tr>
|
||||||
|
<th style="width:30%">UUID</th>
|
||||||
|
<td>{{metadata['uuid']}}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<th>Creator</th>
|
||||||
|
<td>{{metadata['user_creator']}}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<th>Tags</th>
|
||||||
|
<td>
|
||||||
|
{% for tag in metadata['tags'] %}
|
||||||
|
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
|
||||||
|
{% endfor %}
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<th>Date</th>
|
||||||
|
<td>{{metadata['date']}}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<th>Threat Level</th>
|
||||||
|
<td>{{metadata['threat_level']}}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<th>Analysis</th>
|
||||||
|
<td>{{metadata['analysis']}}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<th>Info</th>
|
||||||
|
<td>{{metadata['info']}}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<th># Objects</th>
|
||||||
|
<td>{{metadata['nb_objects']}}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<th>Timestamp</th>
|
||||||
|
<td>{{metadata['timestamp']}}</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<th>Last change</th>
|
||||||
|
<td>{{metadata['last_change']}}</td>
|
||||||
|
</tr>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
<div class="col-12 col-lg-6">
|
||||||
|
|
||||||
|
<div class="my-4">
|
||||||
|
<a href="{{ url_for('investigations_b.delete_investigation') }}?uuid={{metadata['uuid']}}">
|
||||||
|
<button type="button" class="btn btn-danger">
|
||||||
|
<i class="fas fa-trash-alt"></i> <b>Delete</b>
|
||||||
|
</button>
|
||||||
|
</a>
|
||||||
|
<a href="{{ url_for('investigations_b.edit_investigation') }}?uuid={{metadata['uuid']}}">
|
||||||
|
<button type="button" class="btn btn-info">
|
||||||
|
<i class="fas fa-pencil-alt"></i> <b>Edit</b>
|
||||||
|
</button>
|
||||||
|
</a>
|
||||||
|
<a class="btn btn-outline-dark" target="_blank" href="{{ url_for('import_export.export_investigation')}}?uuid={{metadata['uuid']}}">
|
||||||
|
<img id="misp-logo" src="{{ url_for('static', filename='image/misp-logo.png')}}" height="25">
|
||||||
|
Export as Event
|
||||||
|
</a>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<h3>Objects</h3>
|
||||||
|
|
||||||
|
<table id="table_sync_queues" class="table table-striped border-primary">
|
||||||
|
<thead class="bg-dark text-white">
|
||||||
|
<tr>
|
||||||
|
<th>Type</th>
|
||||||
|
<th></th>
|
||||||
|
<th>Id</th>
|
||||||
|
<th>Tags</th>
|
||||||
|
<th></th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody style="font-size: 15px;">
|
||||||
|
{% for object in investigation_objs %}
|
||||||
|
<tr class="border-color: blue;">
|
||||||
|
<td>
|
||||||
|
{% with style=object['icon']['style'], icon=object['icon']['icon'] , color=object['icon']['color'] %}
|
||||||
|
{% include 'objects/obj_svg_block.html' %}
|
||||||
|
{% endwith %}
|
||||||
|
{{ object['type']}}
|
||||||
|
</td>
|
||||||
|
<td>
|
||||||
|
{% if object['subtype'] %}
|
||||||
|
{{ object['subtype']}}
|
||||||
|
{% endif %}
|
||||||
|
</td>
|
||||||
|
<td>{{ object['id']}}</td>
|
||||||
|
<td>
|
||||||
|
{% for tag in object['tags'] %}
|
||||||
|
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
|
||||||
|
{% endfor %}
|
||||||
|
</td>
|
||||||
|
<td class="text-right">
|
||||||
|
<a href="{{ url_for('investigations_b.unregister_investigation') }}?uuid={{ metadata['uuid']}}&type={{ object['type'] }}&subtype={{ object['subtype']}}&id={{ object['id']}}">
|
||||||
|
<button type="button" class="btn btn-danger"><i class="fas fa-trash-alt"></i></button>
|
||||||
|
</a>
|
||||||
|
</td>
|
||||||
|
</tr>
|
||||||
|
{% endfor %}
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</body>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
$(document).ready(function(){
|
||||||
|
$('#nav_sync').removeClass("text-muted");
|
||||||
|
|
||||||
|
$('#table_sync_queues').DataTable({
|
||||||
|
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
|
||||||
|
"iDisplayLength": 10,
|
||||||
|
"order": [[ 0, "asc" ]]
|
||||||
|
});
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
|
||||||
|
</script>
|
62
var/www/templates/modals/investigations_register_obj.html
Normal file
62
var/www/templates/modals/investigations_register_obj.html
Normal file
|
@ -0,0 +1,62 @@
|
||||||
|
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
|
||||||
|
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
|
||||||
|
|
||||||
|
<div id="investigations_register_obj_modal" class="modal fade" role="dialog">
|
||||||
|
<div class="modal-dialog modal-lg">
|
||||||
|
|
||||||
|
<div id="investigations_register_obj_content" class="modal-content">
|
||||||
|
<div class="modal-header" style="border-bottom: 4px solid #cccccc; background-color: #cccccc; color: #ffffff;">
|
||||||
|
<h4>Add to Investigations</h4>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="modal-body">
|
||||||
|
|
||||||
|
<div class="input-group" >
|
||||||
|
<input id="linvestigations" type="text" class="form-control" autocomplete="off" style="width: 760px">
|
||||||
|
</div>
|
||||||
|
|
||||||
|
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="modal-footer">
|
||||||
|
<a class="btn btn-info" href="{{ url_for('investigations_b.add_investigation') }}" target="_blank">
|
||||||
|
<i class="fas fa-microscope"></i>
|
||||||
|
<span class="label-icon">Create Investigation </span>
|
||||||
|
</a>
|
||||||
|
<button class="btn btn-primary" onclick="Register_Obj()">
|
||||||
|
<i class="fas fa-plus"></i>
|
||||||
|
Add to Investigations
|
||||||
|
</button>
|
||||||
|
<button type="button" class="btn btn-outline-dark" data-dismiss="modal" >Close</button>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
var linvestigations;
|
||||||
|
|
||||||
|
$('#investigations_register_obj_modal').on('shown.bs.modal', function () {
|
||||||
|
|
||||||
|
$.getJSON("{{ url_for('investigations_b.get_investigations_selector_json') }}",
|
||||||
|
function(data) {
|
||||||
|
console.log(data);
|
||||||
|
|
||||||
|
linvestigations = $('#linvestigations').tagSuggest({
|
||||||
|
data: data,
|
||||||
|
maxDropHeight: 200,
|
||||||
|
name: 'linvestigations',
|
||||||
|
emptyText: 'Select Investigations'
|
||||||
|
});
|
||||||
|
}
|
||||||
|
);
|
||||||
|
|
||||||
|
});
|
||||||
|
|
||||||
|
function Register_Obj() {
|
||||||
|
var uuids = linvestigations.getValue();
|
||||||
|
// TODO: REQUEST
|
||||||
|
window.location.replace("{{ url_for('investigations_b.register_investigation') }}?uuids=" + uuids + "&type={{ obj_type }}&subtype={{ obj_subtype }}&id={{ obj_id }}");
|
||||||
|
}
|
||||||
|
</script>
|
|
@ -97,6 +97,16 @@
|
||||||
</button>
|
</button>
|
||||||
</a>
|
</a>
|
||||||
</div>
|
</div>
|
||||||
|
<div>
|
||||||
|
{% with obj_type='item', obj_id=dict_item['id'], obj_subtype=''%}
|
||||||
|
{% include 'modals/investigations_register_obj.html' %}
|
||||||
|
{% endwith %}
|
||||||
|
<div class="mr-2">
|
||||||
|
<button type="button" class="btn btn-lg btn-primary" data-toggle="modal" data-target="#investigations_register_obj_modal">
|
||||||
|
<i class="fas fa-microscope"></i> Investigations
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
<div class="mx-2">
|
<div class="mx-2">
|
||||||
{% with obj_type='item', obj_id=dict_item['id'], obj_lvl=0%}
|
{% with obj_type='item', obj_id=dict_item['id'], obj_lvl=0%}
|
||||||
{% include 'import_export/block_add_user_object_to_export.html' %}
|
{% include 'import_export/block_add_user_object_to_export.html' %}
|
||||||
|
|
6
var/www/templates/objects/obj_svg_block.html
Normal file
6
var/www/templates/objects/obj_svg_block.html
Normal file
|
@ -0,0 +1,6 @@
|
||||||
|
<svg height="26" width="26">
|
||||||
|
<g class="nodes">
|
||||||
|
<circle cx="13" cy="13" r="13" fill="{{ color }}"></circle>
|
||||||
|
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="object_node_icon {{ style }}">{{ icon }}</text>
|
||||||
|
</g>
|
||||||
|
</svg>
|
After Width: | Height: | Size: 249 B |
|
@ -44,7 +44,13 @@
|
||||||
<span>Settings</span>
|
<span>Settings</span>
|
||||||
</h5>
|
</h5>
|
||||||
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
|
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <!--nav-pills-->
|
||||||
<li class="nav-item">
|
<!-- <li class="nav-item">
|
||||||
|
<a class="nav-link" href="{{url_for('settings.ail_configs')}}" id="nav_ail_configs">
|
||||||
|
<img src="{{ url_for('static', filename='image/ail.png')}}" alt="AIL servers" style="width:25px;">
|
||||||
|
<span>AIL Configs</span>
|
||||||
|
</a>
|
||||||
|
</li> -->
|
||||||
|
<li class="nav-item">
|
||||||
<a class="nav-link" href="{{url_for('settings.passive_dns')}}" id="passive_dns">
|
<a class="nav-link" href="{{url_for('settings.passive_dns')}}" id="passive_dns">
|
||||||
<img src="{{ url_for('static', filename='image/d4-logo.png')}}" alt="D4 project" style="width:25px;">
|
<img src="{{ url_for('static', filename='image/d4-logo.png')}}" alt="D4 project" style="width:25px;">
|
||||||
<span>Passive DNS</span>
|
<span>Passive DNS</span>
|
||||||
|
|
94
var/www/templates/sidebars/sidebar_objects.html
Normal file
94
var/www/templates/sidebars/sidebar_objects.html
Normal file
|
@ -0,0 +1,94 @@
|
||||||
|
<div class="col-12 col-lg-2 p-0 bg-light border-right" id="side_menu">
|
||||||
|
|
||||||
|
<button type="button" class="btn btn-outline-secondary mt-1 ml-3" onclick="toggle_sidebar()">
|
||||||
|
<i class="fas fa-align-left"></i>
|
||||||
|
<span>Toggle Sidebar</span>
|
||||||
|
</button>
|
||||||
|
|
||||||
|
<nav class="navbar navbar-expand navbar-light bg-light flex-md-column flex-row align-items-start py-2" id="nav_investigation">
|
||||||
|
<h5 class="d-flex text-muted w-100">
|
||||||
|
<span>Investigations</span>
|
||||||
|
</h5>
|
||||||
|
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100 mb-4">
|
||||||
|
<li class="nav-item">
|
||||||
|
<a class="nav-link" href="{{url_for('investigations_b.investigations_dashboard')}}" id="nav_investigation_dashboard">
|
||||||
|
<i class="fas fa-microscope"></i>
|
||||||
|
<span>Investigations</span>
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
<li class="nav-item">
|
||||||
|
<a class="nav-link" href="{{url_for('investigations_b.add_investigation')}}" id="nav_add_investigation">
|
||||||
|
<i class="fas fa-plus"></i>
|
||||||
|
<span>Add Investigation</span>
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
|
||||||
|
<h5 class="d-flex text-muted w-100">
|
||||||
|
<span>Objects</span>
|
||||||
|
</h5>
|
||||||
|
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100 mb-4">
|
||||||
|
<li class="nav-item">
|
||||||
|
<a class="nav-link" href="{{url_for('hashDecoded.hashDecoded_page')}}" id="nav_dashboard">
|
||||||
|
<i class="fas fa-lock-open"></i>
|
||||||
|
<span>Decoded</span>
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
<li class="nav-item">
|
||||||
|
<a class="nav-link" href="{{url_for('hashDecoded.pgpdump_page')}}" id="nav_dashboard_pgpdump">
|
||||||
|
<i class="fas fa-key"></i>
|
||||||
|
<span>PGP Dumps</span>
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
<li class="nav-item">
|
||||||
|
<a class="nav-link" href="{{url_for('hashDecoded.cryptocurrency_page')}}" id="nav_dashboard_cryptocurrency">
|
||||||
|
<i class="fas fa-coins"></i>
|
||||||
|
<span>Cryptocurrency</span>
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
<li class="nav-item">
|
||||||
|
<a class="nav-link" href="{{url_for('hashDecoded.username_page')}}" id="nav_dashboard_username">
|
||||||
|
<i class="fas fa-user"></i>
|
||||||
|
<span>Username</span>
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
|
||||||
|
|
||||||
|
<h5 class="d-flex text-muted w-100">
|
||||||
|
<span>
|
||||||
|
<img src="{{ url_for('static', filename='image/misp-logo.png')}}" alt="MISP" style="width:80px;">
|
||||||
|
Format
|
||||||
|
</span>
|
||||||
|
</h5>
|
||||||
|
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100">
|
||||||
|
<li class="nav-item">
|
||||||
|
<a class="nav-link" href="{{url_for('import_export.import_object')}}" id="nav_misp_import">
|
||||||
|
<b>Import</b>
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
<li class="nav-item">
|
||||||
|
<a class="nav-link" href="{{url_for('import_export.export_object')}}" id="nav_misp_export">
|
||||||
|
<b>Export</b>
|
||||||
|
</a>
|
||||||
|
</li>
|
||||||
|
</ul>
|
||||||
|
</nav>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<script>
|
||||||
|
function toggle_sidebar(){
|
||||||
|
if($('#nav_menu').is(':visible')){
|
||||||
|
$('#nav_menu').hide();
|
||||||
|
$('#side_menu').removeClass('border-right')
|
||||||
|
$('#side_menu').removeClass('col-lg-2')
|
||||||
|
$('#core_content').removeClass('col-lg-10')
|
||||||
|
}else{
|
||||||
|
$('#nav_menu').show();
|
||||||
|
$('#side_menu').addClass('border-right')
|
||||||
|
$('#side_menu').addClass('col-lg-2')
|
||||||
|
$('#core_content').addClass('col-lg-10')
|
||||||
|
}
|
||||||
|
}
|
||||||
|
</script>
|
Loading…
Reference in a new issue