chg: [tags] refactor tags + cleanup

This commit is contained in:
Terrtia 2022-11-22 10:47:15 +01:00
parent 104eaae793
commit aac024565f
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
69 changed files with 1871 additions and 3011 deletions

1
.gitignore vendored
View file

@ -2,6 +2,7 @@
*.swp *.swp
*.pyc *.pyc
*.swo *.swo
.idea
# Install Dirs # Install Dirs
AILENV AILENV

3
.gitmodules vendored
View file

@ -4,3 +4,6 @@
[submodule "files/misp-taxonomies"] [submodule "files/misp-taxonomies"]
path = files/misp-taxonomies path = files/misp-taxonomies
url = https://github.com/MISP/misp-taxonomies.git url = https://github.com/MISP/misp-taxonomies.git
[submodule "files/misp-galaxy"]
path = files/misp-galaxy
url = https://github.com/MISP/misp-galaxy.git

View file

@ -73,7 +73,7 @@ old_crawlers.r_serv_onion = r_crawler
# CREATE FUNCTION BY DB/FEATURES # CREATE FUNCTION BY DB/FEATURES
# /!\ ISSUE WITH FILE DUPLICATES => NEED TO BE REFACTORED # /!\ TODO MIGRATE DUPLICATES
def get_item_date(item_id): def get_item_date(item_id):
@ -670,28 +670,38 @@ def get_subtype_object(obj_type, subtype, obj_id):
return Username(obj_id, subtype) return Username(obj_id, subtype)
def migrate_subtype_obj(Obj, obj_type, subtype, obj_id): def migrate_subtype_obj(Obj, obj_type, subtype, obj_id):
first_seen = get_obj_subtype_first_seen(obj_type, subtype, obj_id) # first_seen = get_obj_subtype_first_seen(obj_type, subtype, obj_id)
last_seen = get_obj_subtype_last_seen(obj_type, subtype, obj_id) # last_seen = get_obj_subtype_last_seen(obj_type, subtype, obj_id)
# dates # dates
for item_id in get_item_correlation_obj(obj_type, subtype, obj_id): for item_id in get_item_correlation_obj(obj_type, subtype, obj_id):
date = get_item_date(item_id) date = get_item_date(item_id)
Obj.add(date, item_id) Obj.add(date, item_id)
dict_obj_subtypes = {'cryptocurrency': ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash'], dict_obj_subtypes = {'cryptocurrency': ['bitcoin', 'bitcoin-cash', 'dash', 'ethereum', 'litecoin', 'monero', 'zcash'],
'pgpdump': ['key', 'mail', 'name'], 'pgpdump': ['key', 'mail', 'name'],
'username': ['telegram', 'twitter', 'jabber']} 'username': ['telegram', 'twitter', 'jabber']}
def subtypes_obj_migration(): def subtypes_obj_migration():
print('SUBPTYPE MIGRATION...') print('SUBTYPE MIGRATION...')
pgp_symmetrical_key = '0x0000000000000000'
for obj_type in dict_obj_subtypes: for obj_type in dict_obj_subtypes:
print(f'{obj_type} MIGRATION...') print(f'{obj_type} MIGRATION...')
for subtype in dict_obj_subtypes[obj_type]: for subtype in dict_obj_subtypes[obj_type]:
for obj_id in get_all_subtype_id(obj_type, subtype): for obj_id in get_all_subtype_id(obj_type, subtype):
if obj_type == 'pgp' and subtype == 'key' and obj_id == pgp_symmetrical_key:
pass
else:
Obj = get_subtype_object(obj_type, subtype, obj_id) Obj = get_subtype_object(obj_type, subtype, obj_id)
migrate_subtype_obj(Obj, obj_type, subtype, obj_id) migrate_subtype_obj(Obj, obj_type, subtype, obj_id)
# ADD PGP Symmetrical tag to item
for item_id in get_item_correlation_obj('pgpdump', 'key', pgp_symmetrical_key):
item = Items.Item(item_id)
item.add_tag(f'infoleak:automatic-detection="pgp-symmetric";{item_id}') # TODO SELECT TAG
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
# STATISTICS # STATISTICS
# #
@ -702,10 +712,9 @@ def get_all_provider():
return r_serv_trend.smembers('all_provider_set') return r_serv_trend.smembers('all_provider_set')
def get_item_source_stats_by_date(date, source): def get_item_source_stats_by_date(date, source):
stats = {} stats = {'num': r_serv_trend.hget(f'{source}_num', date),
stats['num'] = r_serv_trend.hget(f'{source}_num', date) 'size': r_serv_trend.hget(f'{source}_size', date),
stats['size'] = r_serv_trend.hget(f'{source}_size', date) 'avg': r_serv_trend.hget(f'{source}_avg', date)}
stats['avg'] = r_serv_trend.hget(f'{source}_avg', date)
return stats return stats
def get_item_stats_size_avg_by_date(date): def get_item_stats_size_avg_by_date(date):

View file

@ -74,7 +74,6 @@ if __name__ == '__main__':
print('Please provide a list of valid IP addresses') print('Please provide a list of valid IP addresses')
sys.exit(0) sys.exit(0)
# Sent to the logging a description of the module # Sent to the logging a description of the module
publisher.info("Run IP module") publisher.info("Run IP module")

View file

@ -4,7 +4,6 @@
import os import os
import sys import sys
import uuid import uuid
import redis
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))

View file

@ -42,7 +42,7 @@ def get_global_id_from_id(global_id):
obj_meta['id'] = global_id[2] obj_meta['id'] = global_id[2]
else: else:
obj_meta['type'] = global_id[0] obj_meta['type'] = global_id[0]
obj_meta['subtype'] = None obj_meta['subtype'] = ''
obj_meta['id'] = global_id[1] obj_meta['id'] = global_id[1]
return obj_meta return obj_meta

View file

@ -433,25 +433,7 @@ def get_obj_global_id(obj_type, obj_id, obj_sub_type=None):
return '{}:{}'.format(obj_type, obj_id) return '{}:{}'.format(obj_type, obj_id)
def get_global_id_from_id(global_id):
obj_meta = {}
global_id = global_id.split(':', 3)
if len(global_id) > 2:
obj_meta['type'] = global_id[0]
obj_meta['subtype'] = global_id[1]
obj_meta['id'] = global_id[2]
else:
obj_meta['type'] = global_id[0]
obj_meta['subtype'] = None
obj_meta['id'] = global_id[1]
return obj_meta
# used by UI
def get_obj_str_type_subtype(obj_type, obj_subtype):
if obj_subtype:
return '{};{}'.format(obj_type, obj_subtype)
else:
return obj_type
def sanitise_correlation_names(correlation_names): def sanitise_correlation_names(correlation_names):
''' '''

View file

@ -36,7 +36,6 @@ config_loader = None
# Duplicates domains != Duplicates items # Duplicates domains != Duplicates items
def get_ssdeep_hash(content): def get_ssdeep_hash(content):
return ssdeep.hash(content) return ssdeep.hash(content)
@ -100,8 +99,6 @@ def add_obj_duplicate(algo, hash, similarity, obj_type, subtype, id, date_ymonth
_add_obj_duplicate(algo, similarity, obj_type, subtype, obj2_id, id) _add_obj_duplicate(algo, similarity, obj_type, subtype, obj2_id, id)
def get_last_x_month_dates(nb_months): def get_last_x_month_dates(nb_months):
now = datetime.datetime.now() now = datetime.datetime.now()
result = [now.strftime("%Y%m")] result = [now.strftime("%Y%m")]
@ -111,20 +108,7 @@ def get_last_x_month_dates(nb_months):
return result return result
if __name__ == '__main__': if __name__ == '__main__':
res = get_last_x_month_dates(7) res = get_last_x_month_dates(7)
print(res) print(res)
#################################

View file

@ -1,11 +1,15 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import os
import sys
import redis import redis
import datetime import datetime
import os
import json
import sys
from glob import glob
sys.path.append(os.environ['AIL_BIN']) sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
# Import Project packages # Import Project packages
@ -24,37 +28,41 @@ config_loader = None
#### CORE FUNCTIONS #### #### CORE FUNCTIONS ####
# # # # UNSAFE TAGS # # # #
def build_unsafe_tags(): def build_unsafe_tags():
unsafe_tags = set() tags = set()
## CE content # CE content
unsafe_tags.add('dark-web:topic="pornography-child-exploitation"') tags.add('dark-web:topic="pornography-child-exploitation"')
# add copine-scale tags # add copine-scale tags
taxonomies = Taxonomies() taxonomies = Taxonomies()
copine_scale = taxonomies.get('copine-scale') copine_scale = taxonomies.get('copine-scale')
if copine_scale: if copine_scale:
for tag in copine_scale.machinetags(): for tag in copine_scale.machinetags():
unsafe_tags.add(tag) tags.add(tag)
return unsafe_tags return tags
def is_tags_safe(ltags):
"""
Check if a list of tags contain unsafe tags (CE, ...)
:param ltags: list of tags
:type ltags: list
:return: is a tag in the set unsafe
:rtype: boolean
"""
return unsafe_tags.isdisjoint(ltags)
# set of unsafe tags # set of unsafe tags
unsafe_tags = build_unsafe_tags() unsafe_tags = build_unsafe_tags()
def is_tags_safe(ltags): # - - - UNSAFE TAGS - - - #
'''
Check if a list of tags contain an unsafe tag (CE, ...)
:param ltags: list of tags
:type ltags: list
:return: is a tag in the unsafe set
:rtype: boolean
'''
return unsafe_tags.isdisjoint(ltags)
# # TODO: verify tags + object_type # # TODO: verify tags + object_type
# get set_keys: intersection # get set_keys: intersection
def get_obj_keys_by_tags(tags, obj_type, subtype='', date=None): def get_obj_keys_by_tags(tags, obj_type, subtype='', date=None):
l_set_keys = [] l_set_keys = []
if obj_type=='item': if obj_type == 'item':
for tag in tags: for tag in tags:
l_set_keys.append(f'{obj_type}:{subtype}:{tag}:{date}') l_set_keys.append(f'{obj_type}:{subtype}:{tag}:{date}')
else: else:
@ -65,7 +73,405 @@ def get_obj_keys_by_tags(tags, obj_type, subtype='', date=None):
def get_obj_by_tag(key_tag): def get_obj_by_tag(key_tag):
return r_tags.smembers(key_tag) return r_tags.smembers(key_tag)
##-- CORE FUNCTIONS --## # -- CORE FUNCTIONS -- #
#### Taxonomies ####
TAXONOMIES = {}
def load_taxonomies():
global TAXONOMIES
manifest = os.path.join(os.environ['AIL_HOME'], 'files/misp-taxonomies/MANIFEST.json')
TAXONOMIES = Taxonomies(manifest_path=manifest)
load_taxonomies()
def get_taxonomies():
return TAXONOMIES.keys()
# r_tags.sadd(f'active_taxonomies', taxonomy)
def is_taxonomy_enabled(taxonomy):
# enabled = r_tags.sismember('taxonomies:enabled', taxonomy)
try:
enabled = r_tags.sismember('taxonomies:enabled', taxonomy)
except redis.exceptions.ResponseError:
enabled = False
return enabled
def enable_taxonomy(taxonomy):
r_tags.sadd('taxonomies:enabled', taxonomy)
def disable_taxonomy(taxonomy):
r_tags.srem('taxonomies:enabled', taxonomy)
def exists_taxonomy(taxonomy):
return TAXONOMIES.get(taxonomy) is not None
def get_taxonomy_description(taxonomy):
return TAXONOMIES.get(taxonomy).description
def get_taxonomy_name(taxonomy):
return TAXONOMIES.get(taxonomy).name
def get_taxonomy_predicates(taxonomy):
meta = {}
predicates = taxonomy.predicates
for predicate in predicates:
meta[predicate] = {}
expanded = predicates[predicate].expanded
if expanded:
meta[predicate]['expanded'] = expanded
description = predicates[predicate].description
if description:
meta[predicate]['description'] = description
return meta
def get_taxonomy_refs(taxonomy):
return TAXONOMIES.get(taxonomy).refs
def get_taxonomy_version(taxonomy):
return TAXONOMIES.get(taxonomy).version
def get_taxonomy_tags(taxonomy, enabled=False):
taxonomy_obj = TAXONOMIES.get(taxonomy)
tags = []
for p, content in taxonomy_obj.items():
if content:
for k, entry in content.items():
tag = f'{taxonomy_obj.name}:{p}="{k}"'
expanded = entry.expanded
meta_tag = {'tag': tag, 'expanded': expanded}
if enabled:
meta_tag['enabled'] = is_taxonomy_tag_enabled(taxonomy, tag)
tags.append(meta_tag)
else:
tag = f'{taxonomy_obj.name}:{p}'
expanded = content.expanded
meta_tag = {'tag': tag, 'expanded': expanded}
if enabled:
meta_tag['enabled'] = is_taxonomy_tag_enabled(taxonomy, tag)
tags.append(meta_tag)
return tags
# TODO GET ACTIVE TAGS
# TODO GET IF TAXONOMY IS ACTIVATED
def get_taxonomy_meta(taxonomy_name, enabled=False, enabled_tags=False, nb_active_tags=False, predicates=False, tags=False, expanded=True):
meta = {}
if not exists_taxonomy(taxonomy_name):
return meta
taxonomy = TAXONOMIES.get(taxonomy_name)
meta['description'] = taxonomy.description
meta['name'] = taxonomy.name
meta['version'] = taxonomy.version
if enabled:
meta['enabled'] = is_taxonomy_enabled(taxonomy_name)
if predicates:
meta['predicates'] = get_taxonomy_predicates(taxonomy)
if taxonomy.expanded:
meta['expanded'] = taxonomy.expanded
if taxonomy.refs:
meta['refs'] = taxonomy.refs
# TODO PERF SAVE IN DB ?????
if tags:
if expanded:
meta['tags'] = get_taxonomy_tags(taxonomy_name, enabled=enabled_tags)
else:
meta['tags'] = taxonomy.machinetags()
meta['nb_tags'] = len(meta['tags'])
if nb_active_tags:
meta['nb_active_tags'] = get_taxonomy_nb_tags_enabled(taxonomy_name)
if not tags:
meta['nb_tags'] = len(taxonomy.machinetags())
return meta
def get_taxonomies_meta():
meta = {}
for taxonomy in get_taxonomies():
meta[taxonomy] = get_taxonomy_meta(taxonomy, enabled=True, nb_active_tags=True)
return meta
#### Enabled Tags ####
# r_tags.sadd('active_taxonomies', taxonomy)
def get_taxonomy_tags_enabled(taxonomy):
return r_tags.smembers(f'taxonomy:tags:enabled:{taxonomy}')
def get_taxonomy_nb_tags_enabled(taxonomy):
return r_tags.scard(f'taxonomy:tags:enabled:{taxonomy}')
def is_taxonomy_tag_enabled(taxonomy, tag):
# r_tags.sismember('list_tags', tag)
try:
enabled = r_tags.sismember(f'taxonomy:tags:enabled:{taxonomy}', tag)
except redis.exceptions.ResponseError:
enabled = False
return enabled
def add_taxonomy_tag_enabled(taxonomy, tag):
r_tags.sadd(f'taxonomy:tags:enabled:{taxonomy}', tag)
def remove_taxonomy_tag_enabled(taxonomy, tag):
r_tags.srem(f'taxonomy:tags:enabled:{taxonomy}', tag)
def disable_taxonomy_tags_enabled(taxonomy):
r_tags.delete(f'taxonomy:tags:enabled:{taxonomy}')
def update_taxonomy_tag_enabled(taxonomy, tags):
if not tags:
return None
enable_taxonomy(taxonomy)
tags = set(tags)
enabled_tags = get_taxonomy_tags_enabled(taxonomy)
if tags != enabled_tags:
# disable tags
for tag in enabled_tags.difference(tags):
remove_taxonomy_tag_enabled(taxonomy, tag)
# enable tags
for tag in tags:
add_taxonomy_tag_enabled(taxonomy, tag)
def api_update_taxonomy_tag_enabled(data):
taxonomy = data.get('taxonomy')
if not exists_taxonomy(taxonomy):
return {'error': f'taxonomy {taxonomy} not found'}, 404
tags = data.get('tags', [])
taxonomy_tags = set(TAXONOMIES.get(taxonomy).machinetags())
for tag in tags:
if tag not in taxonomy_tags:
return {'error': f'tag {tag} not found'}, 404
update_taxonomy_tag_enabled(taxonomy, tags)
def enable_taxonomy_tags(taxonomy):
enable_taxonomy(taxonomy)
for tag in TAXONOMIES.get(taxonomy).machinetags():
add_taxonomy_tag_enabled(taxonomy, tag)
def api_enable_taxonomy_tags(data):
taxonomy = data.get('taxonomy')
if not exists_taxonomy(taxonomy):
return {'error': f'taxonomy {taxonomy} not found'}, 404
enable_taxonomy_tags(taxonomy)
def disable_taxonomy_tags(taxonomy):
disable_taxonomy(taxonomy)
disable_taxonomy_tags_enabled(taxonomy)
def api_disable_taxonomy_tags(data):
taxonomy = data.get('taxonomy')
if not exists_taxonomy(taxonomy):
return {'error': f'taxonomy {taxonomy} not found'}, 404
disable_taxonomy_tags(taxonomy)
# -- Enabled Tags -- #
# -- Taxonomies -- #
#### GALAXIES ####
#
# var galaxy = galaxy type
#
GALAXIES = {}
CLUSTERS = {}
def load_galaxies():
global GALAXIES
galaxies = []
root_dir_galaxies = os.path.join(os.environ['AIL_HOME'], 'files/misp-galaxy/galaxies')
for galaxy_file in glob(os.path.join(root_dir_galaxies, '*.json')):
with open(galaxy_file, 'r') as f:
galaxies.append(json.load(f))
GALAXIES = Galaxies(galaxies=galaxies)
global CLUSTERS
clusters = []
root_dir_clusters = os.path.join(os.environ['AIL_HOME'], 'files/misp-galaxy/clusters')
for cluster_file in glob(os.path.join(root_dir_clusters, '*.json')):
with open(cluster_file, 'r') as f:
clusters.append(json.load(f))
CLUSTERS = Clusters(clusters)
# LOAD GALAXY + CLUSTERS
load_galaxies()
def get_galaxies():
return GALAXIES.keys()
def get_galaxy(galaxy_name):
return GALAXIES.get(galaxy_name)
def exists_galaxy(galaxy):
return CLUSTERS.get(galaxy) is not None
# r_tags.sadd('active_galaxies', galaxy)
def is_galaxy_enabled(galaxy):
try:
enabled = r_tags.sismember('galaxies:enabled', galaxy)
except redis.exceptions.ResponseError:
enabled = False
return enabled
def enable_galaxy(galaxy):
r_tags.sadd('galaxies:enabled', galaxy)
def disable_galaxy(galaxy):
r_tags.srem('galaxies:enabled', galaxy)
def get_galaxy_meta(galaxy_name, nb_active_tags=False):
galaxy = get_galaxy(galaxy_name)
meta = {'name': galaxy.name, 'namespace': galaxy.namespace, 'description': galaxy.description,
'type': galaxy.type, 'version': galaxy.version, 'enabled': is_galaxy_enabled(galaxy.type)}
icon = galaxy.icon
if icon == 'android' or icon == 'optin-monster' or icon == 'internet-explorer' or icon == 'btc':
meta['icon'] = f'fab fa-{icon}'
else:
meta['icon'] = f'fas fa-{icon}'
if nb_active_tags:
meta['nb_active_tags'] = get_galaxy_nb_tags_enabled(galaxy)
meta['nb_tags'] = len(get_galaxy_tags(galaxy.type))
return meta
def get_galaxies_meta():
galaxies = []
for galaxy_name in get_galaxies():
galaxies.append(get_galaxy_meta(galaxy_name, nb_active_tags=True))
return galaxies
def get_galaxy_tag_meta(galaxy_type, tag):
cluster = get_cluster(galaxy_type)
if not cluster:
return {}
try:
tag_val = tag.rsplit('=', 1)[1][1:-1]
except:
return {}
cluster_val = cluster.cluster_values.get(tag_val)
if not cluster_val:
return {}
meta = cluster_val.to_dict()
if 'meta' in meta:
meta['meta'] = json.loads(meta.get('meta').to_json())
meta['meta'] = json.dumps(meta['meta'], ensure_ascii=False, indent=4)
meta['tag'] = f'misp-galaxy:{galaxy_type}="{cluster_val.value}"'
meta['enabled'] = is_galaxy_tag_enabled(galaxy_type, meta['tag'])
return meta
def get_clusters():
return CLUSTERS.keys()
def get_cluster(cluster_type):
return CLUSTERS.get(cluster_type)
def get_galaxy_tags(galaxy_type):
cluster = get_cluster(galaxy_type)
return cluster.machinetags()
# TODO synonym
def get_cluster_tags(cluster_type, enabled=False):
tags = []
cluster = get_cluster(cluster_type)
for cluster_val in cluster.values():
tag = f'misp-galaxy:{cluster_type}="{cluster_val.value}"'
meta_tag = {'tag': tag, 'description': cluster_val.description}
if enabled:
meta_tag['enabled'] = is_galaxy_tag_enabled(cluster_type, tag)
synonyms = cluster_val.meta.synonyms
if not synonyms:
synonyms = []
meta_tag['synonyms'] = synonyms
tags.append(meta_tag)
return tags
def get_cluster_meta(cluster_type, tags=False, enabled=False):
cluster = get_cluster(cluster_type)
if not cluster:
return {}
meta = {'name': cluster.name, 'type': cluster.type, 'source': cluster.source,
'authors': cluster.authors, 'description': cluster.description, 'version': cluster.version,
'category': cluster.category}
if enabled:
meta['enabled'] = is_galaxy_enabled(cluster_type)
if tags:
meta['tags'] = get_cluster_tags(cluster_type, enabled=enabled)
return meta
#### Enabled Tags ####
def get_galaxy_tags_enabled(galaxy):
return r_tags.smembers(f'galaxy:tags:enabled:{galaxy}')
def get_galaxy_nb_tags_enabled(galaxy):
return r_tags.scard(f'galaxy:tags:enabled:{galaxy}')
def is_galaxy_tag_enabled(galaxy, tag):
try:
enabled = r_tags.sismember(f'galaxy:tags:enabled:{galaxy}', tag)
except redis.exceptions.ResponseError:
enabled = False
return enabled
def add_galaxy_tag_enabled(galaxy, tag):
r_tags.sadd(f'galaxy:tags:enabled:{galaxy}', tag)
def remove_galaxy_tag_enabled(galaxy, tag):
r_tags.srem(f'galaxy:tags:enabled:{galaxy}', tag)
def disable_galaxy_tags_enabled(galaxy):
r_tags.delete(f'galaxy:tags:enabled:{galaxy}')
def update_galaxy_tag_enabled(galaxy, tags):
if not tags:
return None
enable_galaxy(galaxy)
tags = set(tags)
enabled_tags = get_galaxy_tags_enabled(galaxy)
if tags != enabled_tags:
# disable tags
for tag in enabled_tags.difference(tags):
remove_galaxy_tag_enabled(galaxy, tag)
# enable tags
for tag in tags:
add_galaxy_tag_enabled(galaxy, tag)
def api_update_galaxy_tag_enabled(data):
galaxy = data.get('galaxy')
if not exists_galaxy(galaxy):
return {'error': f'galaxy {galaxy} not found'}, 404
tags = data.get('tags', [])
galaxy_tags = set(get_galaxy_tags(galaxy))
for tag in tags:
if tag not in galaxy_tags:
return {'error': f'tag {tag} not found'}, 404
update_galaxy_tag_enabled(galaxy, tags)
def enable_galaxy_tags(galaxy):
enable_galaxy(galaxy)
for tag in get_galaxy_tags(galaxy):
add_galaxy_tag_enabled(galaxy, tag)
def api_enable_galaxy_tags(data):
galaxy = data.get('galaxy')
if not exists_galaxy(galaxy):
return {'error': f'galaxy {galaxy} not found'}, 404
enable_galaxy_tags(galaxy)
def disable_galaxy_tags(galaxy):
disable_galaxy(galaxy)
disable_galaxy_tags_enabled(galaxy)
def api_disable_galaxy_tags(data):
galaxy = data.get('galaxy')
if not exists_galaxy(galaxy):
return {'error': f'galaxy {galaxy} not found'}, 404
disable_galaxy_tags(galaxy)
# -- Enabled Tags -- #
# -- GALAXIES -- #
################################################################################ ################################################################################
################################################################################ ################################################################################
@ -74,38 +480,44 @@ def get_obj_by_tag(key_tag):
################################################################################ ################################################################################
def is_obj_tagged(obj_type, obj_id, subtype=''): def is_obj_tagged(obj_type, obj_id, subtype=''):
''' """
Check if a object is tagged Check if a object is tagged
:param object_id: object id :param obj_type: object type
:type domain: str :type obj_type: str
:param subtype: object subtype
:type subtype: str
:param obj_id: object ID
:type obj_id: str
:return: is object tagged :return: is object tagged
:rtype: boolean :rtype: boolean
''' """
return r_tags.exists(f'tag:{obj_type}:{subtype}:{obj_id}') return r_tags.exists(f'tag:{obj_type}:{subtype}:{obj_id}')
def is_obj_tagged_by_tag(obj_type, obj_id, tag, subtype=''): def is_obj_tagged_by_tag(obj_type, obj_id, tag, subtype=''):
''' """
Check if a object is tagged Check if a object is tagged by a specified tag
:param object_id: object id :param obj_type: object type
:type domain: str :type obj_type: str
:param tag: object type :param subtype: object subtype
:type domain: str :type subtype: str
:param obj_id: object ID
:type obj_id: str
:param tag: tag
:type tag: str
:return: is object tagged :return: is object tagged by a specified tag
:rtype: boolean :rtype: boolean
''' """
return r_tags.sismember(f'tag:{obj_type}:{subtype}:{obj_id}', tag) return r_tags.sismember(f'tag:{obj_type}:{subtype}:{obj_id}', tag)
# #
# f'tag:{obj_type}:{subtype}:{id}' f'tag:{id}' # 'tag:{obj_type}:{subtype}:{id}' 'tag:{id}'
# #
# f'list_tags:{obj_type}:{subtype}' f'list_tags:{obj_type}' # 'list_tags:{obj_type}:{subtype}' 'list_tags:{obj_type}'
# #
# graph tags by days ??????????????????????????????? # graph tags by days ???????????????????????????????
# #
@ -217,7 +629,7 @@ def get_tag_objects(obj_type, subtype='', date=''):
def get_object_tags(obj_type, obj_id, subtype=''): def get_object_tags(obj_type, obj_id, subtype=''):
return r_tags.smembers(f'tag:{obj_type}:{subtype}:{obj_id}') return r_tags.smembers(f'tag:{obj_type}:{subtype}:{obj_id}')
def add_object_tag(tag, obj_type, id, subtype=''): ############################# def add_object_tag(tag, obj_type, id, subtype=''):
if r_tags.sadd(f'tag:{obj_type}:{subtype}:{id}', tag) == 1: if r_tags.sadd(f'tag:{obj_type}:{subtype}:{id}', tag) == 1:
r_tags.sadd('list_tags', tag) r_tags.sadd('list_tags', tag)
r_tags.sadd(f'list_tags:{obj_type}', tag) r_tags.sadd(f'list_tags:{obj_type}', tag)
@ -227,7 +639,7 @@ def add_object_tag(tag, obj_type, id, subtype=''): #############################
r_tags.sadd(f'{obj_type}:{subtype}:{tag}:{date}', id) r_tags.sadd(f'{obj_type}:{subtype}:{tag}:{date}', id)
# add domain tag # add domain tag
if item_basic.is_crawled(id) and tag!='infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"': if item_basic.is_crawled(id) and tag != 'infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
domain = item_basic.get_item_domain(id) domain = item_basic.get_item_domain(id)
add_object_tag(tag, "domain", domain) add_object_tag(tag, "domain", domain)
@ -237,18 +649,28 @@ def add_object_tag(tag, obj_type, id, subtype=''): #############################
r_tags.hincrby(f'daily_tags:{datetime.date.today().strftime("%Y%m%d")}', tag, 1) r_tags.hincrby(f'daily_tags:{datetime.date.today().strftime("%Y%m%d")}', tag, 1)
def update_tag_global_by_obj_type(tag, object_type, subtype=''): # obj -> Object()
def confirm_tag(tag, obj):
if tag.startswith('infoleak:automatic-detection'):
tag = tag.replace('automatic-detection', 'analyst-detection', 1)
obj.add_tag(tag)
return True
return False
# FIXME
# TODO REVIEW ME
def update_tag_global_by_obj_type(tag, obj_type, subtype=''):
tag_deleted = False tag_deleted = False
if object_type=='item': if obj_type == 'item':
if not r_tags.exists(f'tag_metadata:{tag}'): if not r_tags.exists(f'tag_metadata:{tag}'):
tag_deleted = True tag_deleted = True
else: else:
if not r_tags.exists(f'{object_type}:{subtype}:{tag}'): if not r_tags.exists(f'{obj_type}:{subtype}:{tag}'):
r_tags.srem(f'list_tags:{obj_type}:{subtype}', tag) r_tags.srem(f'list_tags:{obj_type}:{subtype}', tag)
# Iterate on all subtypes # Iterate on all subtypes
delete_global_obj_tag = True delete_global_obj_tag = True
for obj_subtype in ail_core.get_object_all_subtypes(): for obj_subtype in ail_core.get_object_all_subtypes():
if r_tags.exists(f'list_tags:{obj_type}:{subtype}'): if r_tags.exists(f'list_tags:{obj_type}:{obj_subtype}'):
delete_global_obj_tag = False delete_global_obj_tag = False
break break
if delete_global_obj_tag: if delete_global_obj_tag:
@ -280,6 +702,7 @@ def delete_object_tag(tag, obj_type, id, subtype=''):
################################################################################################################ ################################################################################################################
# TODO: REWRITE OLD
# TODO: rewrite me # TODO: rewrite me
# TODO: other objects # TODO: other objects
def get_obj_by_tags(obj_type, l_tags, date_from=None, date_to=None, nb_obj=50, page=1): def get_obj_by_tags(obj_type, l_tags, date_from=None, date_to=None, nb_obj=50, page=1):
@ -338,7 +761,7 @@ def get_obj_by_tags(obj_type, l_tags, date_from=None, date_to=None, nb_obj=50, p
l_tagged_obj = r_tags.sinter(l_set_keys[0], *l_set_keys[1:]) l_tagged_obj = r_tags.sinter(l_set_keys[0], *l_set_keys[1:])
if not l_tagged_obj: if not l_tagged_obj:
return {"tagged_obj":l_tagged_obj, "page":0, "nb_pages":0} return {"tagged_obj": l_tagged_obj, "page": 0, "nb_pages": 0}
# handle pagination # handle pagination
nb_all_elem = len(l_tagged_obj) nb_all_elem = len(l_tagged_obj)
@ -359,7 +782,7 @@ def get_obj_by_tags(obj_type, l_tags, date_from=None, date_to=None, nb_obj=50, p
for elem in l_tagged_obj: for elem in l_tagged_obj:
if current_index > stop: if current_index > stop:
break break
if start <= current_index and stop >= current_index: if start <= current_index <= stop:
l_obj.append(elem) l_obj.append(elem)
current_index += 1 current_index += 1
l_tagged_obj = l_obj l_tagged_obj = l_obj
@ -375,102 +798,6 @@ def get_obj_by_tags(obj_type, l_tags, date_from=None, date_to=None, nb_obj=50, p
return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop, "nb_all_elem":nb_all_elem} return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop, "nb_all_elem":nb_all_elem}
################################################################################
################################################################################
################################################################################
################################################################################
#### Taxonomies - Galaxies ####
################################################################################
# galaxies = Galaxies()
# clusters = Clusters(skip_duplicates=True)
#
# list_all_tags = {}
# for name, c in clusters.items(): #galaxy name + tags
# list_all_tags[name] = c
#
# list_galaxies = []
# for g in galaxies.values():
# list_galaxies.append(g.to_json())
#
# list_clusters = []
# for c in clusters.values():
# list_clusters.append(c.to_json())
#
# # tags numbers in galaxies
# total_tags = {}
# for name, tags in clusters.items(): #galaxie name + tags
# total_tags[name] = len(tags)
################################################################################
#### Taxonomies ####
def get_taxonomy_tags_from_cluster(taxonomy_name):
taxonomies = Taxonomies()
taxonomy = taxonomies[taxonomy_name]
return taxonomy.machinetags()
# TODO: ADD api handler
def enable_taxonomy(taxonomy):
tags = get_taxonomy_tags_from_cluster(taxonomy)
r_tags.sadd('active_taxonomies', taxonomy)
for tag in tags:
r_tags.sadd(f'active_tag_{taxonomy}', tag)
# def enable_taxonomy(taxonomie, enable_tags=True):
# '''
# Enable a taxonomy. (UI)
#
# :param taxonomie: MISP taxonomy
# :type taxonomie: str
# :param enable_tags: crawled domain
# :type enable_tags: boolean
# '''
# taxonomies = Taxonomies()
# if enable_tags:
# taxonomie_info = taxonomies.get(taxonomie)
# if taxonomie_info:
# # activate taxonomie
# r_tags.sadd('active_taxonomies', taxonomie)
# # activate taxonomie tags
# for tag in taxonomie_info.machinetags():
# r_tags.sadd('active_tag_{}'.format(taxonomie), tag)
# #r_tags.sadd('active_taxonomies_tags', tag)
# else:
# print('Error: {}, please update pytaxonomies'.format(taxonomie))
#### Galaxies ####
def get_galaxy_tags_from_cluster(galaxy_name):
clusters = Clusters(skip_duplicates=True)
cluster = clusters[galaxy_name]
return cluster.machinetags()
def get_galaxy_tags_with_sysnonym_from_cluster(galaxy_name):
tags = {}
clusters = Clusters(skip_duplicates=True)
cluster = clusters[galaxy_name]
for data in cluster.to_dict()['values']:
tag = f'misp-galaxy:{cluster.type}="{data.value}"'
synonyms = data.meta.synonyms
if not synonyms:
synonyms = []
tags[tag] = synonyms
return tags
def enable_galaxy(galaxy):
tags = get_galaxy_tags_with_sysnonym_from_cluster(galaxy)
r_tags.sadd('active_galaxies', galaxy)
for tag in tags:
r_tags.sadd(f'active_tag_galaxies_{galaxy}', tag)
# synonyms
for synonym in tags[tag]:
r_tags.sadd(f'synonym_tag_{tag}', synonym)
################################################################################ ################################################################################
################################################################################ ################################################################################
################################################################################ ################################################################################
@ -490,9 +817,6 @@ def get_galaxy_from_tag(tag):
except IndexError: except IndexError:
return None return None
def get_taxonomies():
return Taxonomies().keys()
def is_taxonomie(taxonomie, taxonomies=[]): def is_taxonomie(taxonomie, taxonomies=[]):
if not taxonomies: if not taxonomies:
taxonomies = get_taxonomies() taxonomies = get_taxonomies()
@ -579,11 +903,11 @@ def is_taxonomie_tag_enabled(taxonomie, tag):
else: else:
return False return False
def is_galaxy_tag_enabled(galaxy, tag): # def is_galaxy_tag_enabled(galaxy, tag):
if tag in r_tags.smembers('active_tag_galaxies_' + galaxy): # if tag in r_tags.smembers('active_tag_galaxies_' + galaxy):
return True # return True
else: # else:
return False # return False
def is_custom_tag_enabled(tag): def is_custom_tag_enabled(tag):
return r_tags.sismember('tags:custom:enabled_tags', tag) return r_tags.sismember('tags:custom:enabled_tags', tag)
@ -748,12 +1072,12 @@ def unpack_str_tags_list(str_tags_list):
return [] return []
# used by modal # used by modal
def get_modal_add_tags(item_id, object_type='item'): def get_modal_add_tags(object_id, object_type='item', object_subtype=''):
''' '''
Modal: add tags to domain or Paste Modal: add tags to domain or Paste
''' '''
return {"active_taxonomies": get_active_taxonomies(), "active_galaxies": get_active_galaxies(), return {"active_taxonomies": get_active_taxonomies(), "active_galaxies": get_active_galaxies(),
"object_id": item_id, "object_type": object_type} "object_id": object_id, "object_type": object_type, "object_subtype": object_subtype}
######## NEW VERSION ######## ######## NEW VERSION ########
def create_custom_tag(tag): def create_custom_tag(tag):
@ -790,6 +1114,27 @@ def get_all_tags():
def get_all_obj_tags(obj_type): def get_all_obj_tags(obj_type):
return list(r_tags.smembers(f'list_tags:{obj_type}')) return list(r_tags.smembers(f'list_tags:{obj_type}'))
# # # UI # # #
def get_enabled_tags_with_synonyms_ui():
list_tags = []
for tag in get_all_tags():
t = tag.split(':')[0]
# add synonym
str_synonyms = ' - synonyms: '
if t == 'misp-galaxy':
synonyms = get_tag_synonyms(tag)
for synonym in synonyms:
str_synonyms = str_synonyms + synonym + ', '
# add real tag
if str_synonyms != ' - synonyms: ':
list_tags.append({'name': tag + str_synonyms, 'id': tag})
else:
list_tags.append({'name': tag, 'id': tag})
return list_tags
# - - UI - - #
## Objects tags ## ## Objects tags ##
################################################################################### ###################################################################################
@ -807,59 +1152,59 @@ def add_global_tag(tag, object_type=None):
Create a set of all tags used in AIL (all + by object) Create a set of all tags used in AIL (all + by object)
:param tag: tag :param tag: tag
:type domain: str :type tag: str
:param object_type: object type :param object_type: object type
:type domain: str :type object_type: str
''' '''
r_tags.sadd('list_tags', tag) r_tags.sadd('list_tags', tag)
if object_type: if object_type:
r_tags.sadd('list_tags:{}'.format(object_type), tag) r_tags.sadd('list_tags:{}'.format(object_type), tag)
def add_obj_tags(object_id, object_type, tags=[], galaxy_tags=[]): def add_obj_tags(object_id, object_subtype, object_type, tags=[], galaxy_tags=[]):
obj_date = get_obj_date(object_type, object_id)
for tag in tags: for tag in tags:
if tag: if tag:
taxonomie = get_taxonomie_from_tag(tag) taxonomy = get_taxonomie_from_tag(tag)
if is_taxonomie_tag_enabled(taxonomie, tag): if is_taxonomie_tag_enabled(taxonomy, tag):
add_object_tag(tag, object_type, object_id) add_object_tag(tag, object_type, object_id, object_subtype)
else: else:
return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled', 'value': tag}, 400) return {'status': 'error', 'reason': 'Tags or Galaxy not enabled', 'value': tag}, 400
for tag in galaxy_tags: for tag in galaxy_tags:
if tag: if tag:
galaxy = get_galaxy_from_tag(tag) galaxy = get_galaxy_from_tag(tag)
if is_galaxy_tag_enabled(galaxy, tag): if is_galaxy_tag_enabled(galaxy, tag):
add_object_tag(tag, object_type, object_id) add_object_tag(tag, object_type, object_id, object_subtype)
else: else:
return ({'status': 'error', 'reason': 'Tags or Galaxy not enabled', 'value': tag}, 400) return {'status': 'error', 'reason': 'Tags or Galaxy not enabled', 'value': tag}, 400
# TEMPLATE + API QUERY # TEMPLATE + API QUERY
def api_add_obj_tags(tags=[], galaxy_tags=[], object_id=None, object_type="item"): # WARNING CHECK IF OBJECT EXISTS
def api_add_obj_tags(tags=[], galaxy_tags=[], object_id=None, object_type="item", object_subtype=''):
res_dict = {} res_dict = {}
if object_id == None: if not object_id:
return ({'status': 'error', 'reason': 'object_id id not found'}, 404) return {'status': 'error', 'reason': 'object_id id not found'}, 404
if not tags and not galaxy_tags: if not tags and not galaxy_tags:
return ({'status': 'error', 'reason': 'Tags or Galaxy not specified'}, 400) return {'status': 'error', 'reason': 'Tags or Galaxy not specified'}, 400
if object_type not in ('item', 'domain', 'image', 'decoded'): # # TODO: put me in another file if object_type not in ail_core.get_all_objects():
return ({'status': 'error', 'reason': 'Incorrect object_type'}, 400) return {'status': 'error', 'reason': 'Incorrect object_type'}, 400
# remove empty tags # remove empty tags
tags = list(filter(bool, tags)) tags = list(filter(bool, tags))
galaxy_tags = list(filter(bool, galaxy_tags)) galaxy_tags = list(filter(bool, galaxy_tags))
res = add_obj_tags(object_id, object_type, tags=tags, galaxy_tags=galaxy_tags) res = add_obj_tags(object_id, object_subtype, object_type, tags=tags, galaxy_tags=galaxy_tags)
if res: if res:
return res return res
res_dict['tags'] = tags + galaxy_tags res_dict['tags'] = tags + galaxy_tags
res_dict['id'] = object_id res_dict['id'] = object_id
res_dict['type'] = object_type res_dict['type'] = object_type
return (res_dict, 200) return res_dict, 200
# def add_tag(object_type, tag, object_id, obj_date=None): # def add_tag(object_type, tag, object_id, obj_date=None):
# # new tag # # new tag
# if not is_obj_tagged(object_id, tag): # if not is_obj_tagged(object_id, tag):
# # # TODO: # FIXME: sanityze object_type # # # TODO: # FIXME: sanitize object_type
# if obj_date: # if obj_date:
# try: # try:
# obj_date = int(obj_date) # obj_date = int(obj_date)
@ -950,5 +1295,7 @@ def get_list_of_solo_tags_to_export_by_type(export_type): # by type
#r_serv_db.smembers('whitelist_hive') #r_serv_db.smembers('whitelist_hive')
# if __name__ == '__main__': # if __name__ == '__main__':
# galaxy = 'infoleak' # taxo = 'accessnow'
# get_taxonomy_tags_from_cluster(galaxy) # # taxo = TAXONOMIES.get(taxo)
# res = is_taxonomy_tag_enabled(taxo, 'test')
# print(res)

View file

@ -15,14 +15,15 @@ config_loader = ConfigLoader()
config_loader = None config_loader = None
AIL_OBJECTS = {'cve', 'cryptocurrency', 'decoded', 'domain', 'item', 'pgp', 'screenshot', 'username'}
def get_ail_uuid(): def get_ail_uuid():
pass pass
#### AIL OBJECTS #### #### AIL OBJECTS ####
# # TODO: check change paste => item
def get_all_objects(): def get_all_objects():
return ['cve', 'domain', 'item', 'pgp', 'cryptocurrency', 'decoded', 'screenshot', 'username'] return AIL_OBJECTS
def get_object_all_subtypes(obj_type): def get_object_all_subtypes(obj_type):
if obj_type == 'cryptocurrency': if obj_type == 'cryptocurrency':

View file

@ -69,10 +69,10 @@ def get_nb_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type):
def get_nb_correlations(obj_type, subtype, obj_id, filter_types=[]): def get_nb_correlations(obj_type, subtype, obj_id, filter_types=[]):
if subtype is None: if subtype is None:
subtype = '' subtype = ''
nb_correlations = 0 obj_correlations = {}
filter_types = sanityze_obj_correl_types(obj_type, filter_types) filter_types = sanityze_obj_correl_types(obj_type, filter_types)
for correl_type in filter_types: for correl_type in filter_types:
obj_correlations += get_nb_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type) obj_correlations[correl_type] = get_nb_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type)
return obj_correlations return obj_correlations
def get_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type): def get_correlation_by_correl_type(obj_type, subtype, obj_id, correl_type):

View file

@ -546,7 +546,7 @@ def blacklist_domain(domain):
def load_blacklist(): def load_blacklist():
try: try:
with open(os.path.join(os.environ['AIL_BIN'], 'torcrawler/blacklist.txt'), 'r') as f: with open(os.path.join(os.environ['AIL_BIN'], 'crawlers/blacklist.txt'), 'r') as f:
r_crawler.delete('blacklist:domain') r_crawler.delete('blacklist:domain')
lines = f.read().splitlines() lines = f.read().splitlines()
for line in lines: for line in lines:

View file

@ -92,7 +92,7 @@ class CryptoCurrency(AbstractSubtypeObject):
meta = self._get_meta() meta = self._get_meta()
meta['id'] = self.id meta['id'] = self.id
meta['subtype'] = self.subtype meta['subtype'] = self.subtype
meta['tags'] = self.get_tags() meta['tags'] = self.get_tags(r_list=True)
return meta return meta

View file

@ -63,7 +63,7 @@ class Cve(AbstractDaterangeObject):
meta = self._get_meta(options=options) meta = self._get_meta(options=options)
meta['id'] = self.id meta['id'] = self.id
meta['subtype'] = self.subtype meta['subtype'] = self.subtype
meta['tags'] = self.get_tags() meta['tags'] = self.get_tags(r_list=True)
return meta return meta
def add(self, date, item_id): def add(self, date, item_id):

View file

@ -53,7 +53,7 @@ class Item(AbstractObject):
super(Item, self).__init__('item', id) super(Item, self).__init__('item', id)
def exists(self): def exists(self):
return os.path.isfile(self.get_filename()) return item_basic.exist_item(self.id)
def get_date(self, separator=False): def get_date(self, separator=False):
""" """
@ -123,6 +123,9 @@ class Item(AbstractObject):
payload = {'raw': self.get_gzip_content(b64=True)} payload = {'raw': self.get_gzip_content(b64=True)}
return payload return payload
def get_parent(self):
return item_basic.get_item_parent(self.id)
def set_father(self, father_id): # UPDATE KEYS ????????????????????????????? def set_father(self, father_id): # UPDATE KEYS ?????????????????????????????
r_serv_metadata.sadd(f'paste_children:{father_id}', self.id) r_serv_metadata.sadd(f'paste_children:{father_id}', self.id)
r_serv_metadata.hset(f'paste_metadata:{self.id}', 'father', father_id) r_serv_metadata.hset(f'paste_metadata:{self.id}', 'father', father_id)
@ -234,7 +237,7 @@ class Item(AbstractObject):
def get_screenshot(self): def get_screenshot(self):
s = self.get_correlation('screenshot') s = self.get_correlation('screenshot')
if s: if s.get('screenshot'):
s = s['screenshot'].pop()[1:] s = s['screenshot'].pop()[1:]
return os.path.join(s[0:2], s[2:4], s[4:6], s[6:8], s[8:10], s[10:12], s[12:]) return os.path.join(s[0:2], s[2:4], s[4:6], s[6:8], s[8:10], s[10:12], s[12:])
@ -267,10 +270,13 @@ class Item(AbstractObject):
if 'lines' in options: if 'lines' in options:
content = meta.get('content') content = meta.get('content')
meta['lines'] = self.get_meta_lines(content=content) meta['lines'] = self.get_meta_lines(content=content)
if 'parent' in options:
meta['parent'] = self.get_parent()
if 'size' in options: if 'size' in options:
meta['size'] = self.get_size(str=True) meta['size'] = self.get_size(str=True)
if 'mimetype' in options:
# # TODO: ADD GET FATHER content = meta.get('content')
meta['mimetype'] = self.get_mimetype(content=content)
# meta['encoding'] = None # meta['encoding'] = None
return meta return meta

View file

@ -45,7 +45,7 @@ class Pgp(AbstractSubtypeObject):
meta = self._get_meta() meta = self._get_meta()
meta['id'] = self.id meta['id'] = self.id
meta['subtype'] = self.subtype meta['subtype'] = self.subtype
meta['tags'] = self.get_tags() meta['tags'] = self.get_tags(r_list=True)
return meta return meta
def get_link(self, flask_context=False): def get_link(self, flask_context=False):

View file

@ -67,7 +67,7 @@ class Username(AbstractSubtypeObject):
meta = self._get_meta() meta = self._get_meta()
meta['id'] = self.id meta['id'] = self.id
meta['subtype'] = self.subtype meta['subtype'] = self.subtype
meta['tags'] = self.get_tags() meta['tags'] = self.get_tags(r_list=True)
return meta return meta
def get_misp_object(self): def get_misp_object(self):

View file

@ -18,7 +18,7 @@ sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
from lib import Tag from lib import Tag
from lib import Duplicate from lib import Duplicate
from lib.correlations_engine import get_correlations, add_obj_correlation, delete_obj_correlation, exists_obj_correlation, is_obj_correlated from lib.correlations_engine import get_nb_correlations, get_correlations, add_obj_correlation, delete_obj_correlation, exists_obj_correlation, is_obj_correlated
from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations from lib.Investigations import is_object_investigated, get_obj_investigations, delete_obj_investigations
from lib.Tracker import is_obj_tracked, get_obj_all_trackers, delete_obj_trackers from lib.Tracker import is_obj_tracked, get_obj_all_trackers, delete_obj_trackers
@ -183,6 +183,9 @@ class AbstractObject(ABC):
""" """
return get_correlations(self.type, self.subtype, self.id) return get_correlations(self.type, self.subtype, self.id)
def get_nb_correlations(self, filter_types=[]):
return get_nb_correlations(self.type, self.subtype, self.id, filter_types=filter_types)
def add_correlation(self, type2, subtype2, id2): def add_correlation(self, type2, subtype2, id2):
""" """
Add object correlation Add object correlation

View file

@ -10,7 +10,7 @@ import os
import sys import sys
from abc import abstractmethod from abc import abstractmethod
#from flask import url_for # from flask import url_for
sys.path.append(os.environ['AIL_BIN']) sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
@ -49,6 +49,13 @@ class AbstractSubtypeObject(AbstractObject):
def exists(self): def exists(self):
return r_metadata.exists(f'{self.type}_metadata_{self.subtype}:{self.id}') return r_metadata.exists(f'{self.type}_metadata_{self.subtype}:{self.id}')
# def exists(self):
# res = r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id)
# if res is not None:
# return True
# else:
# return False
def get_first_seen(self, r_int=False): def get_first_seen(self, r_int=False):
first_seen = r_metadata.hget(f'{self.type}_metadata_{self.subtype}:{self.id}', 'first_seen') first_seen = r_metadata.hget(f'{self.type}_metadata_{self.subtype}:{self.id}', 'first_seen')
if r_int: if r_int:
@ -70,7 +77,7 @@ class AbstractSubtypeObject(AbstractObject):
return last_seen return last_seen
def get_nb_seen(self): def get_nb_seen(self):
return r_metadata.scard(f'set_{self.type}_{self.subtype}:{self.id}') return int(r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id))
# # TODO: CHECK RESULT # # TODO: CHECK RESULT
def get_nb_seen_by_date(self, date_day): def get_nb_seen_by_date(self, date_day):
@ -81,22 +88,11 @@ class AbstractSubtypeObject(AbstractObject):
return int(nb) return int(nb)
def _get_meta(self): def _get_meta(self):
meta_dict = {} meta_dict = {'first_seen': self.get_first_seen(),
meta_dict['first_seen'] = self.get_first_seen() 'last_seen': self.get_last_seen(),
meta_dict['last_seen'] = self.get_last_seen() 'nb_seen': self.get_nb_seen()}
meta_dict['nb_seen'] = self.get_nb_seen()
return meta_dict return meta_dict
# def exists(self):
# res = r_metadata.zscore(f'{self.type}_all:{self.subtype}', self.id)
# if res is not None:
# return True
# else:
# return False
def exists(self):
return r_metadata.exists(f'{self.type}_metadata_{self.subtype}:{self.id}')
def set_first_seen(self, first_seen): def set_first_seen(self, first_seen):
r_metadata.hset(f'{self.type}_metadata_{self.subtype}:{self.id}', 'first_seen', first_seen) r_metadata.hset(f'{self.type}_metadata_{self.subtype}:{self.id}', 'first_seen', first_seen)
@ -137,7 +133,7 @@ class AbstractSubtypeObject(AbstractObject):
# daily # daily
r_metadata.hincrby(f'{self.type}:{self.subtype}:{date}', self.id, 1) r_metadata.hincrby(f'{self.type}:{self.subtype}:{date}', self.id, 1)
# all subtypes # all subtypes
r_metadata.zincrby(f'{self.type}_all:{self.subtype}', self.id, 1) r_metadata.zincrby(f'{self.type}_all:{self.subtype}', 1, self.id)
####################################################################### #######################################################################
####################################################################### #######################################################################

View file

@ -109,7 +109,7 @@ def get_object_card_meta(obj_type, subtype, id, related_btc=False):
meta["vt"]["status"] = obj.is_vt_enabled() meta["vt"]["status"] = obj.is_vt_enabled()
# TAGS MODAL # TAGS MODAL
if obj.get_type() == 'screenshot' or obj.get_type() == 'decoded': if obj.get_type() == 'screenshot' or obj.get_type() == 'decoded':
meta["add_tags_modal"] = Tag.get_modal_add_tags(obj.id, object_type=obj.get_type()) meta["add_tags_modal"] = Tag.get_modal_add_tags(obj.id, obj.get_type(), obj.get_subtype(r_str=True))
return meta return meta
def get_ui_obj_tag_table_keys(obj_type): def get_ui_obj_tag_table_keys(obj_type):

@ -1 +1 @@
Subproject commit edc390c4a8d93a028e29938e92aacb399e270cc4 Subproject commit 3ccbec3145d3dec0ebf46fb996eb79d0932388d9

View file

@ -1,55 +0,0 @@
#!/bin/bash
install_docker() {
# install docker
sudo apt install docker.io;
# pull splah docker
sudo docker pull scrapinghub/splash;
}
# install_python_requirement() {
# . ./AILENV/bin/activate;
# pip3 install -U -r crawler_requirements.txt;
# }
install_all() {
read -p "Do you want to install docker? (use local splash server) [y/n] " -n 1 -r
echo # (optional) move to a new line
if [[ $REPLY =~ ^[Yy]$ ]]
then
install_docker;
fi
}
usage() {
echo "Usage: crawler_hidden_services_install.sh [-y | -n]" 1>&2;
echo " -y: install docker"
echo " -n: don't install docker"
echo ""
echo "example:"
echo "crawler_hidden_services_install.sh -y"
exit 1;
}
if [[ $1 == "" ]]; then
install_all;
exit;
else
key="$1"
case $key in
"")
install_all;
;;
-y|--yes)
install_docker;
install_python_requirement;
;;
-n|--no)
install_python_requirement;
;;
*) # unknown option
usage;
;;
esac
fi

1
files/misp-galaxy Submodule

@ -0,0 +1 @@
Subproject commit aba1321b34e18122ec1825b54e2fc8176a4bd25c

@ -1 +1 @@
Subproject commit f2fbd0e5fb02decb350232de16c21251db0e7517 Subproject commit 7aeaa0b890e16f6d9b349f6db3577bf25a9a40ad

View file

@ -8,7 +8,6 @@ import json
import time import time
import uuid import uuid
import redis
import random import random
import logging import logging
import logging.handlers import logging.handlers
@ -16,7 +15,6 @@ import logging.handlers
from flask import Flask, render_template, jsonify, request, Request, Response, session, redirect, url_for from flask import Flask, render_template, jsonify, request, Request, Response, session, redirect, url_for
from flask_login import LoginManager, current_user, login_user, logout_user, login_required from flask_login import LoginManager, current_user, login_user, logout_user, login_required
import flask
import importlib import importlib
from os.path import join from os.path import join
@ -90,7 +88,7 @@ if not os.path.isdir(log_dir):
# ========= TLS =========# # ========= TLS =========#
ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2) ssl_context = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
ssl_context.load_cert_chain(certfile=os.path.join(Flask_dir, 'server.crt'), keyfile=os.path.join(Flask_dir, 'server.key')) ssl_context.load_cert_chain(certfile=os.path.join(Flask_dir, 'server.crt'), keyfile=os.path.join(Flask_dir, 'server.key'))
#print(ssl_context.get_ciphers()) # print(ssl_context.get_ciphers())
# ========= =========# # ========= =========#
Flask_config.app = Flask(__name__, static_url_path=baseUrl+'/static/') Flask_config.app = Flask(__name__, static_url_path=baseUrl+'/static/')
@ -132,6 +130,7 @@ def load_user(user_id):
# ========= HEADER GENERATION ======== DEPRECATED # ========= HEADER GENERATION ======== DEPRECATED
# Get headers items that should be ignored (not displayed) # Get headers items that should be ignored (not displayed)
toIgnoreModule = set() toIgnoreModule = set()
try: try:
@ -166,7 +165,6 @@ for root, dirs, files in os.walk(os.path.join(Flask_dir, 'modules')):
to_add_to_header_dico[module_name] = f.read() to_add_to_header_dico[module_name] = f.read()
# create header.html # create header.html
complete_header = ""
with open(os.path.join(Flask_dir, 'templates', 'header_base.html'), 'r') as f: with open(os.path.join(Flask_dir, 'templates', 'header_base.html'), 'r') as f:
complete_header = f.read() complete_header = f.read()
modified_header = complete_header modified_header = complete_header
@ -250,21 +248,20 @@ def page_not_found(e):
# ========== INITIAL taxonomies ============ # ========== INITIAL taxonomies ============
default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"] default_taxonomies = ["infoleak", "gdpr", "fpf", "dark-web"]
# enable default taxonomies # enable default taxonomies
for taxo in default_taxonomies: for taxonomy in default_taxonomies:
Tag.enable_taxonomy(taxo) Tag.enable_taxonomy_tags(taxonomy)
# ========== INITIAL tags auto export ============ # ========== INITIAL tags auto export ============
taxonomies = Taxonomies() # taxonomies = Taxonomies()
#
infoleak_tags = taxonomies.get('infoleak').machinetags() # infoleak_tags = taxonomies.get('infoleak').machinetags()
infoleak_automatic_tags = [] # infoleak_automatic_tags = []
for tag in taxonomies.get('infoleak').machinetags(): # for tag in taxonomies.get('infoleak').machinetags():
if tag.split('=')[0][:] == 'infoleak:automatic-detection': # if tag.split('=')[0][:] == 'infoleak:automatic-detection':
r_serv_db.sadd('list_export_tags', tag) # r_serv_db.sadd('list_export_tags', tag)
#
r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"') # r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"')
# ============ MAIN ============ # ============ MAIN ============
if __name__ == "__main__": if __name__ == "__main__":

View file

@ -8,7 +8,6 @@
import os import os
import sys import sys
import json import json
import random
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, make_response from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, make_response
from flask_login import login_required, current_user, login_user, logout_user from flask_login import login_required, current_user, login_user, logout_user
@ -19,15 +18,15 @@ import Flask_config
# Import Role_Manager # Import Role_Manager
from Role_Manager import login_admin, login_analyst, login_read_only from Role_Manager import login_admin, login_analyst, login_read_only
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) sys.path.append(os.environ['AIL_BIN'])
import Tag ##################################
# Import Project packages
##################################
from core import ail_2_ail
from lib import item_basic
from lib import Tag
from lib import Tracker
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import item_basic
import Tracker
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'core'))
import ail_2_ail
bootstrap_label = Flask_config.bootstrap_label bootstrap_label = Flask_config.bootstrap_label
@ -37,7 +36,6 @@ ail_2_ail_sync = Blueprint('ail_2_ail_sync', __name__, template_folder=os.path.j
# ============ VARIABLES ============ # ============ VARIABLES ============
# ============ FUNCTIONS ============ # ============ FUNCTIONS ============
def api_validator(api_response): def api_validator(api_response):
if api_response: if api_response:

View file

@ -136,7 +136,6 @@ def show_correlation():
"filter": filter_types, "filter_str": ",".join(filter_types), "filter": filter_types, "filter_str": ",".join(filter_types),
"metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, flask_context=True) "metadata": ail_objects.get_object_meta(obj_type, subtype, obj_id, flask_context=True)
} }
print(dict_object)
if subtype: if subtype:
dict_object["metadata"]['type_id'] = subtype dict_object["metadata"]['type_id'] = subtype
dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc) dict_object["metadata_card"] = ail_objects.get_object_card_meta(obj_type, subtype, obj_id, related_btc=related_btc)
@ -196,4 +195,4 @@ def subtype_search():
obj_type = request.form.get('object_type') obj_type = request.form.get('object_type')
obj_subtype = request.form.get('object_subtype') obj_subtype = request.form.get('object_subtype')
obj_id = request.form.get('object_id') obj_id = request.form.get('object_id')
return redirect(url_for('correlation.show_correlation', object_type=obj_type, type_id=obj_subtype, correlation_id=obj_id)) return redirect(url_for('correlation.show_correlation', type=obj_type, subtype=obj_subtype, id=obj_id))

View file

@ -19,9 +19,13 @@ import Flask_config
# Import Role_Manager # Import Role_Manager
from Role_Manager import login_admin, login_analyst, login_read_only from Role_Manager import login_admin, login_analyst, login_read_only
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) sys.path.append(os.environ['AIL_BIN'])
import item_basic ##################################
import Tracker # Import Project packages
##################################
from lib import item_basic
from lib import Tracker
bootstrap_label = Flask_config.bootstrap_label bootstrap_label = Flask_config.bootstrap_label

View file

@ -20,14 +20,16 @@ import Flask_config
# Import Role_Manager # Import Role_Manager
from Role_Manager import login_admin, login_analyst, login_read_only from Role_Manager import login_admin, login_analyst, login_read_only
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'export')) sys.path.append(os.environ['AIL_BIN'])
import MispImport ##################################
import MispExport # Import Project packages
##################################
from export import MispExport
from export import MispImport
from export import AILObjects ####################### # # # # ########################## ## # ####################################
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) from lib.objects import ail_objects
import Correlate_object
import AILObjects
# ============ BLUEPRINT ============ # ============ BLUEPRINT ============
import_export = Blueprint('import_export', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export')) import_export = Blueprint('import_export', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/import_export'))
@ -66,10 +68,11 @@ def import_object_file():
map_uuid_global_id = MispImport.import_objs_from_file(filename) map_uuid_global_id = MispImport.import_objs_from_file(filename)
os.remove(filename) os.remove(filename)
for obj_uuid in map_uuid_global_id: for obj_uuid in map_uuid_global_id:
dict_obj = Correlate_object.get_global_id_from_id(map_uuid_global_id[obj_uuid]) dict_obj = MispImport.get_global_id_from_id(map_uuid_global_id[obj_uuid])
obj = ail_objects.get_object(dict_obj['type'], dict_obj['subtype'], dict_obj['id'])
dict_obj['uuid'] = obj_uuid dict_obj['uuid'] = obj_uuid
dict_obj['url'] = Correlate_object.get_item_url(dict_obj['type'], dict_obj['id'], correlation_type=dict_obj['subtype']) dict_obj['url'] = obj.get_link(flask_context=True)
dict_obj['node'] = Correlate_object.get_correlation_node_icon(dict_obj['type'], correlation_type=dict_obj['subtype'], value=dict_obj['id']) dict_obj['node'] = obj.get_svg_icon()
all_imported_obj.append(dict_obj) all_imported_obj.append(dict_obj)
if not all_imported_obj: if not all_imported_obj:
@ -129,7 +132,7 @@ def export_object_file():
else: else:
dict_misp_event_export[str(obj_tuple)] = request.form.get(obj_tuple) dict_misp_event_export[str(obj_tuple)] = request.form.get(obj_tuple)
#print(dict_misp_event_export) # print(dict_misp_event_export)
if dict_misp_event_export.get('export_to_misp', None): if dict_misp_event_export.get('export_to_misp', None):
export_to_misp = True export_to_misp = True
@ -142,7 +145,9 @@ def export_object_file():
for obj_dict in l_obj_invalid: # set uuid input for obj_dict in l_obj_invalid: # set uuid input
obj_dict['uuid'] = str(uuid.uuid4()) obj_dict['uuid'] = str(uuid.uuid4())
obj_dict['type'] = Correlate_object.get_obj_str_type_subtype(obj_dict['type'], obj_dict.get('subtype', None)) subtype = obj_dict.get('subtype', None)
if subtype:
obj_dict['type'] = f'{obj_dict["type"]};{subtype}'
return render_template("export_object.html", l_obj_to_export=l_obj_to_export, return render_template("export_object.html", l_obj_to_export=l_obj_to_export,
l_obj_invalid=l_obj_invalid, dict_misp_event_export=dict_misp_event_export) l_obj_invalid=l_obj_invalid, dict_misp_event_export=dict_misp_event_export)

View file

@ -18,12 +18,13 @@ from Role_Manager import login_admin, login_analyst, login_read_only
sys.path.append('modules') sys.path.append('modules')
import Flask_config import Flask_config
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) sys.path.append(os.environ['AIL_BIN'])
import Investigations ##################################
# Import Project packages
##################################
from lib import Investigations
from lib.objects import ail_objects from lib.objects import ail_objects
from lib import Tag
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import Tag
# ============ BLUEPRINT ============ # ============ BLUEPRINT ============
investigations_b = Blueprint('investigations_b', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/investigations')) investigations_b = Blueprint('investigations_b', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/investigations'))

View file

@ -5,20 +5,21 @@
Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ... Blueprint Flask: crawler splash endpoints: dashboard, onion crawler ...
''' '''
import difflib
import os import os
import sys import sys
import json
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response, abort, send_file, send_from_directory
from flask_login import login_required, current_user from flask_login import login_required, current_user
# Import Role_Manager # Import Role_Manager
from Role_Manager import login_admin, login_analyst, login_read_only from Role_Manager import login_admin, login_analyst, login_read_only, no_cache
sys.path.append(os.environ['AIL_BIN']) sys.path.append(os.environ['AIL_BIN'])
################################## ##################################
# Import Project packages # Import Project packages
################################## ##################################
from lib import ConfigLoader
from lib import item_basic from lib import item_basic
from lib.objects.Items import Item from lib.objects.Items import Item
from lib import Tag from lib import Tag
@ -29,13 +30,25 @@ from export import Export
objects_item = Blueprint('objects_item', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/item')) objects_item = Blueprint('objects_item', __name__, template_folder=os.path.join(os.environ['AIL_FLASK'], 'templates/objects/item'))
# ============ VARIABLES ============ # ============ VARIABLES ============
config_loader = ConfigLoader.ConfigLoader()
bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info'] bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
DiffMaxLineLength = config_loader.get_config_int('Flask', 'DiffMaxLineLength')
max_preview_modal = config_loader.get_config_int('Flask', 'max_preview_modal')
SCREENSHOT_FOLDER = ConfigLoader.get_screenshots_dir()
config_loader = None
# ============ FUNCTIONS ============ # ============ FUNCTIONS ============
# ============= ROUTES ============== # ============= ROUTES ==============
@objects_item.route('/screenshot/<path:filename>')
@login_required
@login_read_only
@no_cache
def screenshot(filename):
return send_from_directory(SCREENSHOT_FOLDER, f'{filename}.png', as_attachment=True)
@objects_item.route("/object/item") @objects_item.route("/object/item")
@login_required @login_required
@login_read_only @login_read_only
@ -95,3 +108,60 @@ def item_download(): # # TODO: support post
abort(404) abort(404)
item = Item(item_id) item = Item(item_id)
return send_file(item.get_raw_content(), download_name=item_id, as_attachment=True) return send_file(item.get_raw_content(), download_name=item_id, as_attachment=True)
@objects_item.route("/object/item/content/more")
@login_required
@login_read_only
def item_content_more():
item_id = request.args.get('id', '')
item = Item(item_id)
item_content = item.get_content()
to_return = item_content[max_preview_modal-1:]
return to_return
@objects_item.route("/object/item/diff")
@login_required
@login_analyst
def object_item_diff():
id1 = request.args.get('s1', '')
id2 = request.args.get('s2', '')
item1 = Item(id1)
item2 = Item(id2)
item1_content = item1.get_content()
item2_content = item2.get_content()
i1_max_len = item1.get_meta_lines(content=item1_content)['max_length']
i2_max_len = item2.get_meta_lines(content=item2_content)['max_length']
if i1_max_len > DiffMaxLineLength or i2_max_len > DiffMaxLineLength:
return jsonify({'error': "Can't make the difference as the lines are too long."}), 400
lines1 = item1_content.splitlines()
lines2 = item2_content.splitlines()
htmldiff = difflib.HtmlDiff()
diff = htmldiff.make_file(lines1, lines2)
return diff
@objects_item.route("/object/item/preview")
@login_required
@login_read_only
def item_preview():
item_id = request.args.get('id')
if not item_id or not item_basic.exist_item(item_id):
abort(404)
item = Item(item_id)
meta = item.get_meta(options={'content', 'crawler', 'lines', 'mimetype', 'parent', 'size'})
initsize = len(meta['content'])
if len(meta['content']) > max_preview_modal:
meta['content'] = meta['content'][0:max_preview_modal]
meta['nb_correlations'] = item.get_nb_correlations()
misp_eventid = None # TODO SHOW MISP EVENT
misp_url = None # TODO SHOW MISP EVENT
hive_caseid = None # TODO SHOW HIVE CASE
hive_url = None # TODO SHOW HIVE CASE
return render_template("show_item_min.html", bootstrap_label=bootstrap_label,
meta=meta, initsize=initsize,
misp_eventid=misp_eventid, misp_url=misp_url,
hive_caseid=hive_caseid, hive_url=hive_url)

View file

@ -8,9 +8,8 @@
import os import os
import sys import sys
import json import json
import random
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, Response from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for, abort
from flask_login import login_required, current_user, login_user, logout_user from flask_login import login_required, current_user, login_user, logout_user
sys.path.append('modules') sys.path.append('modules')
@ -39,6 +38,140 @@ tags_ui = Blueprint('tags_ui', __name__, template_folder=os.path.join(os.environ
# ============= ROUTES ============== # ============= ROUTES ==============
@tags_ui.route('/tag/taxonomies')
@login_required
@login_read_only
def tags_taxonomies():
taxonomies = Tag.get_taxonomies_meta()
return render_template("tags/taxonomies.html", taxonomies=taxonomies)
@tags_ui.route('/tag/taxonomy')
@login_required
@login_read_only
def tags_taxonomy():
taxonomy_name = request.args.get('taxonomy')
taxonomy = Tag.get_taxonomy_meta(taxonomy_name, enabled=True, predicates=True, tags=True, enabled_tags=True)
if not taxonomy:
abort(404)
return render_template("tags/taxonomy.html", taxonomy=taxonomy)
@tags_ui.route('/tag/taxonomy/enable')
@login_required
@login_read_only
def taxonomy_enable():
taxonomy = request.args.get('taxonomy')
res = Tag.api_enable_taxonomy_tags({'taxonomy': taxonomy})
if res:
return jsonify(res[0]), res[1]
else:
return redirect(url_for('tags_ui.tags_taxonomy', taxonomy=taxonomy))
@tags_ui.route('/tag/taxonomy/disable')
@login_required
@login_read_only
def taxonomy_disable():
taxonomy = request.args.get('taxonomy')
res = Tag.api_disable_taxonomy_tags({'taxonomy': taxonomy})
if res:
return jsonify(res[0]), res[1]
else:
return redirect(url_for('tags_ui.tags_taxonomy', taxonomy=taxonomy))
@tags_ui.route('/tag/taxonomy/enable_tags')
@login_required
@login_read_only
def taxonomy_enable_tags():
taxonomy = request.args.get('taxonomy')
tags = request.args.getlist('tags')
res = Tag.api_update_taxonomy_tag_enabled({'taxonomy': taxonomy, 'tags': tags})
if res:
return jsonify(res[0]), 1
else:
return redirect(url_for('tags_ui.tags_taxonomy', taxonomy=taxonomy))
@tags_ui.route('/tag/galaxies')
@login_required
@login_read_only
def tags_galaxies():
galaxies = Tag.get_galaxies_meta()
return render_template("tags/galaxies.html", galaxies=galaxies)
@tags_ui.route('/tag/galaxy')
@login_required
@login_read_only
def tags_galaxy():
galaxy_name = request.args.get('galaxy')
galaxy = Tag.get_cluster_meta(galaxy_name, enabled=True, tags=True)
if not galaxy:
abort(404)
return render_template("tags/galaxy.html", galaxy=galaxy)
@tags_ui.route('/tag/galaxy/tag')
@login_required
@login_read_only
def tags_galaxy_tag():
galaxy_type = request.args.get('galaxy')
tag = request.args.get('tag')
tag_meta = Tag.get_galaxy_tag_meta(galaxy_type, tag)
if not tag_meta:
abort(404)
return render_template("tags/galaxy_tag.html", galaxy=galaxy_type, tag=tag_meta)
@tags_ui.route('/tag/galaxy/enable')
@login_required
@login_read_only
def galaxy_enable():
galaxy = request.args.get('galaxy')
res = Tag.api_enable_galaxy_tags({'galaxy': galaxy})
if res:
return jsonify(res[0]), res[1]
else:
return redirect(url_for('tags_ui.tags_galaxy', galaxy=galaxy))
@tags_ui.route('/tag/galaxy/disable')
@login_required
@login_read_only
def galaxy_disable():
galaxy = request.args.get('galaxy')
res = Tag.api_disable_galaxy_tags({'galaxy': galaxy})
if res:
return jsonify(res[0]), res[1]
else:
return redirect(url_for('tags_ui.tags_galaxy', galaxy=galaxy))
@tags_ui.route('/tag/galaxy/enable_tags')
@login_required
@login_read_only
def galaxy_enable_tags():
galaxy = request.args.get('galaxy')
tags = request.args.getlist('tags')
res = Tag.api_update_galaxy_tag_enabled({'galaxy': galaxy, 'tags': tags})
if res:
return jsonify(res[0]), 1
else:
return redirect(url_for('tags_ui.tags_galaxy', galaxy=galaxy))
@tags_ui.route('/tag/enabled')
@login_required
@login_read_only
def get_all_tags_enabled():
return jsonify(Tags.get_enabled_tags_with_synonyms_ui())
@tags_ui.route('/tag/confirm')
@login_required
@login_read_only
def tag_confirm():
tag = request.args.get('tag')
obj_type = request.args.get('type')
subtype = request.args.get('subtype', '')
obj_id = request.args.get('id', '')
obj = ail_objects.get_object(obj_type, subtype, obj_id)
if not obj.exists():
abort(404)
Tag.confirm_tag(tag, obj)
return redirect(obj.get_link(flask_context=True))
@tags_ui.route('/tag/add_tags') @tags_ui.route('/tag/add_tags')
@login_required @login_required
@login_analyst @login_analyst
@ -46,19 +179,20 @@ def add_tags():
tags = request.args.get('tags') tags = request.args.get('tags')
tagsgalaxies = request.args.get('tagsgalaxies') tagsgalaxies = request.args.get('tagsgalaxies')
object_id = request.args.get('object_id') object_type = request.args.get('type')
object_type = request.args.get('object_type') object_subtype = request.args.get('subtype')
subtype = '' # TODO: handle subtype object object_id = request.args.get('id')
list_tag = tags.split(',') list_tag = tags.split(',')
list_tag_galaxies = tagsgalaxies.split(',') list_tag_galaxies = tagsgalaxies.split(',')
res = Tag.api_add_obj_tags(tags=list_tag, galaxy_tags=list_tag_galaxies, object_id=object_id, object_type=object_type) res = Tag.api_add_obj_tags(tags=list_tag, galaxy_tags=list_tag_galaxies,
object_id=object_id, object_type=object_type, object_subtype=object_subtype)
# error # error
if res[1] != 200: if res[1] != 200:
return str(res[0]) return str(res[0])
return redirect(ail_objects.get_object_link(object_type, subtype, object_id, flask_context=True)) return redirect(ail_objects.get_object_link(object_type, object_subtype, object_id, flask_context=True))
@tags_ui.route('/tag/delete_tag') @tags_ui.route('/tag/delete_tag')
@login_required @login_required
@ -95,7 +229,7 @@ def get_all_obj_tags():
object_type = request.args.get('object_type') object_type = request.args.get('object_type')
res = ail_objects.api_sanitize_object_type(object_type) res = ail_objects.api_sanitize_object_type(object_type)
if res: if res:
return jsonify(res) return jsonify(res[0]), res[1]
return jsonify(Tag.get_all_obj_tags(object_type)) return jsonify(Tag.get_all_obj_tags(object_type))
@tags_ui.route('/tag/taxonomies/tags/enabled/json') @tags_ui.route('/tag/taxonomies/tags/enabled/json')
@ -137,7 +271,7 @@ def tag_galaxy_tags_enabled_json():
@login_read_only @login_read_only
def tags_search_items(): def tags_search_items():
object_type = 'item' object_type = 'item'
dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"} dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s"}
dict_tagged['date'] = Date.sanitise_date_range('', '', separator='-') dict_tagged['date'] = Date.sanitise_date_range('', '', separator='-')
return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
@ -146,7 +280,7 @@ def tags_search_items():
@login_read_only @login_read_only
def tags_search_domains(): def tags_search_domains():
object_type = 'domain' object_type = 'domain'
dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"} dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s"}
return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
@tags_ui.route('/tag/search/decoded') @tags_ui.route('/tag/search/decoded')
@ -154,15 +288,15 @@ def tags_search_domains():
@login_read_only @login_read_only
def tags_search_decoded(): def tags_search_decoded():
object_type = 'decoded' object_type = 'decoded'
dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"} dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s"}
return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
@tags_ui.route('/tag/search/image') @tags_ui.route('/tag/search/screenshot')
@login_required @login_required
@login_read_only @login_read_only
def tags_search_images(): def tags_search_screenshot():
object_type = 'image' object_type = 'screenshot'
dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"} dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s"}
return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged) return render_template("tags/search_obj_by_tags.html", bootstrap_label=bootstrap_label, dict_tagged=dict_tagged)
@tags_ui.route('/tag/search/get_obj_by_tags') @tags_ui.route('/tag/search/get_obj_by_tags')
@ -170,7 +304,7 @@ def tags_search_images():
@login_read_only @login_read_only
def get_obj_by_tags(): def get_obj_by_tags():
# # TODO: sanityze all # # TODO: sanitize all
object_type = request.args.get('object_type') object_type = request.args.get('object_type')
subtype = '' # TODO: handle subtype subtype = '' # TODO: handle subtype
ltags = request.args.get('ltags') ltags = request.args.get('ltags')
@ -188,7 +322,7 @@ def get_obj_by_tags():
list_tags = ltags.split(',') list_tags = ltags.split(',')
list_tag = [] list_tag = []
for tag in list_tags: for tag in list_tags:
list_tag.append(tag.replace('"','\"')) list_tag.append(tag.replace('"', '\"'))
# object_type # object_type
res = ail_objects.api_sanitize_object_type(object_type) res = ail_objects.api_sanitize_object_type(object_type)
@ -198,19 +332,21 @@ def get_obj_by_tags():
# page # page
try: try:
page = int(page) page = int(page)
except: except (TypeError, ValueError):
page = 1 page = 1
# TODO REPLACE ME
dict_obj = Tag.get_obj_by_tags(object_type, list_tag, date_from=date_from, date_to=date_to, page=page) dict_obj = Tag.get_obj_by_tags(object_type, list_tag, date_from=date_from, date_to=date_to, page=page)
print(dict_obj)
if dict_obj['tagged_obj']: if dict_obj['tagged_obj']:
dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s", dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s",
"tagged_obj":[], "page":dict_obj['page'] ,"nb_pages":dict_obj['nb_pages'], "tagged_obj": [], "page": dict_obj['page'], "nb_pages": dict_obj['nb_pages'],
"nb_first_elem":dict_obj['nb_first_elem'], "nb_last_elem":dict_obj['nb_last_elem'], "nb_all_elem":dict_obj['nb_all_elem']} "nb_first_elem": dict_obj['nb_first_elem'], "nb_last_elem": dict_obj['nb_last_elem'],
"nb_all_elem": dict_obj['nb_all_elem']}
for obj_id in dict_obj['tagged_obj']: for obj_id in dict_obj['tagged_obj']:
obj_metadata = ail_objects.get_object_meta(object_type, subtype, obj_id, flask_context=True) obj_metadata = ail_objects.get_object_meta(object_type, subtype, obj_id, flask_context=True)
#ail_objects.
obj_metadata['id'] = obj_id obj_metadata['id'] = obj_id
dict_tagged["tagged_obj"].append(obj_metadata) dict_tagged["tagged_obj"].append(obj_metadata)
@ -222,9 +358,9 @@ def get_obj_by_tags():
dict_tagged['current_tags'] = list_tag dict_tagged['current_tags'] = list_tag
dict_tagged['current_tags_str'] = ltags dict_tagged['current_tags_str'] = ltags
#return jsonify(dict_tagged) # return jsonify(dict_tagged)
else: else:
dict_tagged = {"object_type":object_type, "object_name":object_type.title() + "s"} dict_tagged = {"object_type": object_type, "object_name": object_type.title() + "s"}
if 'date' in dict_obj: if 'date' in dict_obj:
dict_tagged['date'] = dict_obj['date'] dict_tagged['date'] = dict_obj['date']

View file

@ -29,6 +29,9 @@ r_serv = config_loader.get_redis_conn("Redis_Queues")
r_cache = config_loader.get_redis_conn("Redis_Cache") r_cache = config_loader.get_redis_conn("Redis_Cache")
r_serv_log = config_loader.get_redis_conn("Redis_Log") r_serv_log = config_loader.get_redis_conn("Redis_Log")
r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit") r_serv_log_submit = config_loader.get_redis_conn("Redis_Log_submit")
r_serv_charts = config_loader.get_redis_conn("ARDB_Trending") r_serv_charts = config_loader.get_redis_conn("ARDB_Trending")
r_serv_sentiment = config_loader.get_redis_conn("ARDB_Sentiment") r_serv_sentiment = config_loader.get_redis_conn("ARDB_Sentiment")
r_serv_term = config_loader.get_redis_conn("ARDB_Tracker") r_serv_term = config_loader.get_redis_conn("ARDB_Tracker")

View file

@ -13,8 +13,6 @@ from flask_login import login_required
import json import json
import datetime import datetime
import Paste
from pytaxonomies import Taxonomies from pytaxonomies import Taxonomies
from pymispgalaxies import Galaxies, Clusters from pymispgalaxies import Galaxies, Clusters
@ -25,38 +23,10 @@ from lib import Tag
app = Flask_config.app app = Flask_config.app
baseUrl = Flask_config.baseUrl baseUrl = Flask_config.baseUrl
r_serv_tags = Flask_config.r_serv_tags r_serv_tags = Flask_config.r_serv_tags
r_serv_metadata = Flask_config.r_serv_metadata
r_serv_statistics = Flask_config.r_serv_statistics
max_preview_char = Flask_config.max_preview_char
max_preview_modal = Flask_config.max_preview_modal
bootstrap_label = Flask_config.bootstrap_label
max_tags_result = Flask_config.max_tags_result
Tags = Blueprint('Tags', __name__, template_folder='templates') Tags = Blueprint('Tags', __name__, template_folder='templates')
galaxies = Galaxies()
clusters = Clusters(skip_duplicates=True)
list_all_tags = {}
for name, c in clusters.items(): #galaxy name + tags
list_all_tags[name] = c
list_galaxies = []
for g in galaxies.values():
list_galaxies.append(g.to_json())
list_clusters = []
for c in clusters.values():
list_clusters.append(c.to_json())
# tags numbers in galaxies
total_tags = {}
for name, tags in clusters.items(): #galaxie name + tags
total_tags[name] = len(tags)
# ============ FUNCTIONS ============ # ============ FUNCTIONS ============
def one():
return 1
# TODO: # TODO:
# TODO: # TODO:
@ -77,230 +47,22 @@ def get_tags_with_synonyms(tag):
else: else:
return {'name':tag,'id':tag} return {'name':tag,'id':tag}
def get_item_date(item_filename):
l_directory = item_filename.split('/')
return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
def substract_date(date_from, date_to):
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
delta = date_to - date_from # timedelta
l_date = []
for i in range(delta.days + 1):
date = date_from + datetime.timedelta(i)
l_date.append( date.strftime('%Y%m%d') )
return l_date
def get_all_dates_range(date_from, date_to):
all_dates = {}
date_range = []
if date_from is not None and date_to is not None:
#change format
try:
if len(date_from) != 8:
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
date_range = substract_date(date_from, date_to)
except:
pass
if not date_range:
date_range.append(datetime.date.today().strftime("%Y%m%d"))
date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8]
date_to = date_from
else:
date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
all_dates['date_from'] = date_from
all_dates['date_to'] = date_to
all_dates['date_range'] = date_range
return all_dates
def get_last_seen_from_tags_list(list_tags):
min_last_seen = 99999999
for tag in list_tags:
tag_last_seen = r_serv_tags.hget('tag_metadata:{}'.format(tag), 'last_seen')
if tag_last_seen:
tag_last_seen = int(tag_last_seen)
if tag_last_seen < min_last_seen:
min_last_seen = tag_last_seen
return str(min_last_seen)
# ============= ROUTES ============== # ============= ROUTES ==============
# @Tags.route("/tags/", methods=['GET'])
# @login_required
# @login_read_only
# def Tags_page():
# date_from = request.args.get('date_from')
# date_to = request.args.get('date_to')
# tags = request.args.get('ltags')
#
# if tags is None:
# dates = get_all_dates_range(date_from, date_to)
# return render_template("Tags.html", date_from=dates['date_from'], date_to=dates['date_to'])
#
# # unpack tags
# list_tags = tags.split(',')
# list_tag = []
# for tag in list_tags:
# list_tag.append(tag.replace('"','\"'))
#
# #no search by date, use last_seen for date_from/date_to
# if date_from is None and date_to is None and tags is not None:
# date_from = get_last_seen_from_tags_list(list_tags)
# date_to = date_from
#
# # TODO verify input
#
# dates = get_all_dates_range(date_from, date_to)
#
# if(type(list_tags) is list):
# # no tag
# if list_tags is False:
# print('empty')
# # 1 tag
# elif len(list_tags) < 2:
# tagged_pastes = []
# for date in dates['date_range']:
# tagged_pastes.extend(r_serv_tags.smembers('{}:{}'.format(list_tags[0], date)))
#
# # 2 tags or more
# else:
# tagged_pastes = []
# for date in dates['date_range']:
# tag_keys = []
# for tag in list_tags:
# tag_keys.append('{}:{}'.format(tag, date))
#
# if len(tag_keys) > 1:
# daily_items = r_serv_tags.sinter(tag_keys[0], *tag_keys[1:])
# else:
# daily_items = r_serv_tags.sinter(tag_keys[0])
# tagged_pastes.extend(daily_items)
#
# else :
# return 'INCORRECT INPUT'
#
# all_content = []
# paste_date = []
# paste_linenum = []
# all_path = []
# allPastes = list(tagged_pastes)
# paste_tags = []
#
# try:
# page = int(request.args.get('page'))
# except:
# page = 1
# if page <= 0:
# page = 1
# nb_page_max = len(tagged_pastes)/(max_tags_result)
# if not nb_page_max.is_integer():
# nb_page_max = int(nb_page_max)+1
# else:
# nb_page_max = int(nb_page_max)
# if page > nb_page_max:
# page = nb_page_max
# start = max_tags_result*(page -1)
# stop = max_tags_result*page
#
# for path in allPastes[start:stop]:
# all_path.append(path)
# paste = Paste.Paste(path)
# content = paste.get_p_content()
# content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
# all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " "))
# curr_date = str(paste._get_p_date())
# curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
# paste_date.append(curr_date)
# paste_linenum.append(paste.get_lines_info()[0])
# p_tags = r_serv_metadata.smembers('tag:'+path)
# complete_tags = []
# l_tags = []
# for tag in p_tags:
# complete_tag = tag
#
# tag = tag.split('=')
# if len(tag) > 1:
# if tag[1] != '':
# tag = tag[1][1:-1]
# # no value
# else:
# tag = tag[0][1:-1]
# # use for custom tags
# else:
# tag = tag[0]
#
# l_tags.append( (tag,complete_tag) )
#
# paste_tags.append(l_tags)
#
# if len(allPastes) > 10:
# finished = False
# else:
# finished = True
#
# if len(list_tag) == 1:
# tag_nav=tags.replace('"', '').replace('=', '').replace(':', '')
# else:
# tag_nav='empty'
#
# return render_template("Tags.html",
# all_path=all_path,
# tags=tags,
# tag_nav=tag_nav,
# list_tag = list_tag,
# date_from=dates['date_from'],
# date_to=dates['date_to'],
# page=page, nb_page_max=nb_page_max,
# paste_tags=paste_tags,
# bootstrap_label=bootstrap_label,
# content=all_content,
# paste_date=paste_date,
# paste_linenum=paste_linenum,
# char_to_display=max_preview_modal,
# finished=finished)
@Tags.route("/Tags/get_all_tags")
@login_required
@login_read_only
def get_all_tags():
all_tags = r_serv_tags.smembers('list_tags')
list_tags = []
for tag in all_tags:
t = tag.split(':')[0]
# add synonym
str_synonyms = ' - synonyms: '
if t == 'misp-galaxy':
synonyms = r_serv_tags.smembers('synonym_tag_' + tag)
for synonym in synonyms:
str_synonyms = str_synonyms + synonym + ', '
# add real tag
if str_synonyms != ' - synonyms: ':
list_tags.append({'name':tag + str_synonyms,'id':tag})
else:
list_tags.append({'name':tag,'id':tag})
return jsonify(list_tags)
@Tags.route("/Tags/get_all_tags_taxonomies") @Tags.route("/Tags/get_all_tags_taxonomies")
@login_required @login_required
@login_read_only @login_read_only
def get_all_tags_taxonomies(): def get_all_tags_taxonomies():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
active_taxonomie = r_serv_tags.smembers('active_taxonomies') active_taxonomie = r_serv_tags.smembers('active_taxonomies')
list_tags = [] list_tags = []
for taxonomie in active_taxonomie: for taxonomie in active_taxonomie:
#l_tags = taxonomies.get(taxonomie).machinetags() # l_tags = taxonomies.get(taxonomie).machinetags()
l_tags = r_serv_tags.smembers('active_tag_' + taxonomie) l_tags = r_serv_tags.smembers('active_tag_' + taxonomie)
for tag in l_tags: for tag in l_tags:
list_tags.append( tag ) list_tags.append( tag )
@ -372,568 +134,36 @@ def get_tags_galaxy():
else: else:
return 'this galaxy is disable' return 'this galaxy is disable'
@Tags.route("/Tags/remove_tag") # @Tags.route("/Tags/tag_validation")
@login_required # @login_required
@login_analyst # @login_analyst
def remove_tag(): #TODO remove me , used by showpaste # def tag_validation():
#
# path = request.args.get('paste')
# tag = request.args.get('tag')
# status = request.args.get('status')
#
# if (status == 'fp' or status == 'tp') and r_serv_tags.sismember('list_tags', tag):
#
# if status == 'tp':
# r_serv_statistics.sadd('tp:'+tag, path)
# r_serv_statistics.srem('fp:'+tag, path)
# else:
# r_serv_statistics.sadd('fp:'+tag, path)
# r_serv_statistics.srem('tp:'+tag, path)
#
# return redirect(url_for('objects_item.showItem', id=path))
# else:
# return 'input error'
path = request.args.get('paste')
tag = request.args.get('tag')
res = Tag.api_delete_obj_tags(tags=[tag], object_id=path, object_type="item")
if res[1] != 200:
return str(res[0])
return redirect(url_for('objects_item.showItem', id=path))
@Tags.route("/Tags/confirm_tag")
@login_required
@login_analyst
def confirm_tag():
#TODO verify input
path = request.args.get('paste')
tag = request.args.get('tag')
if(tag[9:28] == 'automatic-detection'):
Tag.api_delete_obj_tags(tags=[tag], object_id=path, object_type="item")
tag = tag.replace('automatic-detection','analyst-detection', 1)
#add analyst tag
Tag.add_tag('item', tag, path)
return redirect(url_for('objects_item.showItem', id=path))
return 'incompatible tag'
@Tags.route("/Tags/tag_validation")
@login_required
@login_analyst
def tag_validation():
path = request.args.get('paste')
tag = request.args.get('tag')
status = request.args.get('status')
if (status == 'fp' or status == 'tp') and r_serv_tags.sismember('list_tags', tag):
if status == 'tp':
r_serv_statistics.sadd('tp:'+tag, path)
r_serv_statistics.srem('fp:'+tag, path)
else:
r_serv_statistics.sadd('fp:'+tag, path)
r_serv_statistics.srem('tp:'+tag, path)
return redirect(url_for('objects_item.showItem', id=path))
else:
return 'input error'
@Tags.route("/Tags/taxonomies")
@login_required
@login_read_only
def taxonomies():
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
id = []
name = []
description = []
version = []
enabled = []
n_tags = []
for taxonomie in list_taxonomies:
id.append(taxonomie)
name.append(taxonomies.get(taxonomie).name)
description.append(taxonomies.get(taxonomie).description)
version.append(taxonomies.get(taxonomie).version)
if taxonomie in active_taxonomies:
enabled.append(True)
else:
enabled.append(False)
n = str(r_serv_tags.scard('active_tag_' + taxonomie))
n_tags.append(n + '/' + str(len(taxonomies.get(taxonomie).machinetags())) )
return render_template("taxonomies.html",
id=id,
all_name = name,
description = description,
version = version,
enabled = enabled,
n_tags=n_tags)
@Tags.route("/Tags/edit_taxonomie")
@login_required
@login_analyst
def edit_taxonomie():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
id = request.args.get('taxonomie')
#verify input
if id in list(taxonomies.keys()):
active_tag = r_serv_tags.smembers('active_tag_' + id)
list_tag = taxonomies.get(id).machinetags()
list_tag_desc = taxonomies.get(id).machinetags_expanded()
active_taxonomies = r_serv_tags.smembers('active_taxonomies')
if id in active_taxonomies:
active = True
else:
active = False
n = str(r_serv_tags.scard('active_tag_' + id))
badge = n + '/' + str(len(taxonomies.get(id).machinetags()))
name = taxonomies.get(id).name
description = taxonomies.get(id).description
version = taxonomies.get(id).version
status = []
for tag in list_tag:
if tag in active_tag:
status.append(True)
else:
status.append(False)
return render_template("edit_taxonomie.html",
id=id,
name=name,
badge = badge,
description = description,
version = version,
active=active,
all_tags = list_tag,
list_tag_desc=list_tag_desc,
status = status)
else:
return 'INVALID TAXONOMIE'
@Tags.route("/Tags/disable_taxonomie")
@login_required
@login_analyst
def disable_taxonomie():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
id = request.args.get('taxonomie')
if id in list_taxonomies:
r_serv_tags.srem('active_taxonomies', id)
for tag in taxonomies.get(id).machinetags():
r_serv_tags.srem('active_tag_' + id, tag)
return redirect(url_for('Tags.taxonomies'))
else:
return "INCORRECT INPUT"
@Tags.route("/Tags/active_taxonomie")
@login_required
@login_analyst
def active_taxonomie():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
id = request.args.get('taxonomie')
# verify input
if id in list_taxonomies:
r_serv_tags.sadd('active_taxonomies', id)
for tag in taxonomies.get(id).machinetags():
r_serv_tags.sadd('active_tag_' + id, tag)
return redirect(url_for('Tags.taxonomies'))
else:
return "INCORRECT INPUT"
@Tags.route("/Tags/edit_taxonomie_tag")
@login_required
@login_analyst
def edit_taxonomie_tag():
taxonomies = Taxonomies()
list_taxonomies = list(taxonomies.keys())
arg1 = request.args.getlist('tag_enabled')
arg2 = request.args.getlist('tag_disabled')
id = request.args.get('taxonomie')
#verify input
if id in list_taxonomies:
list_tag = taxonomies.get(id).machinetags()
#check tags validity
if ( all(elem in list_tag for elem in arg1) or (len(arg1) == 0) ) and ( all(elem in list_tag for elem in arg2) or (len(arg2) == 0) ):
active_tag = r_serv_tags.smembers('active_tag_' + id)
diff = list(set(arg1) ^ set(list_tag))
#remove tags
for tag in diff:
r_serv_tags.srem('active_tag_' + id, tag)
#all tags unchecked
if len(arg1) == 0 and len(arg2) == 0:
r_serv_tags.srem('active_taxonomies', id)
#add new tags
for tag in arg2:
r_serv_tags.sadd('active_taxonomies', id)
r_serv_tags.sadd('active_tag_' + id, tag)
return redirect(url_for('Tags.taxonomies'))
else:
return "INCORRECT INPUT"
else:
return "INCORRECT INPUT"
@Tags.route("/Tags/galaxies")
@login_required
@login_read_only
def galaxies():
active_galaxies = r_serv_tags.smembers('active_galaxies')
name = []
icon = []
version = []
all_type = []
namespace = []
description = []
enabled = []
n_tags = []
for galaxie_json in list_galaxies:
galaxie = json.loads(galaxie_json)
name.append(galaxie['name'])
icon.append(galaxie['icon'])
version.append(galaxie['version'])
type = galaxie['type']
all_type.append(type)
namespace.append(galaxie['namespace'])
description.append(galaxie['description'])
if type in active_galaxies:
enabled.append(True)
else:
enabled.append(False)
n = str(r_serv_tags.scard('active_tag_galaxies_' + type))
n_tags.append(n + '/' + str(total_tags[type]) )
return render_template("galaxies.html",
name=name,
icon = icon,
version = version,
description = description,
namespace = namespace,
all_type = all_type,
enabled = enabled,
n_tags=n_tags)
@Tags.route("/Tags/edit_galaxy")
@login_required
@login_analyst
def edit_galaxy():
id = request.args.get('galaxy')
for clusters_json in list_clusters:
#get clusters
cluster = json.loads(clusters_json)
if cluster['type'] == id:
type = id
active_tag = r_serv_tags.smembers('active_tag_galaxies_' + type)
n = str(r_serv_tags.scard('active_tag_galaxies_' + type))
badge = n + '/' + str(total_tags[type])
name = cluster['name']
description = cluster['description']
version = cluster['version']
source = cluster['source']
val = cluster['values']
tags = []
for data in val:
try:
meta = data['meta']
except KeyError:
meta = []
tag_name = data['value']
tag_name = 'misp-galaxy:{}="{}"'.format(type, tag_name)
try:
tag_description = data['description']
except KeyError:
tag_description = ''
tags.append( (tag_name, tag_description, meta) )
status = []
for tag in tags:
if tag[0] in active_tag:
status.append(True)
else:
status.append(False)
active_galaxies = r_serv_tags.smembers('active_galaxies')
if id in active_galaxies:
active = True
else:
active = False
return render_template("edit_galaxy.html",
id = type,
name = name,
badge = badge,
description = description,
version = version,
active = active,
tags = tags,
status = status)
return 'INVALID GALAXY'
@Tags.route("/Tags/active_galaxy")
@login_required
@login_analyst
def active_galaxy():
id = request.args.get('galaxy')
# verify input
try:
l_tags = list_all_tags[id]
except KeyError:
return "INCORRECT INPUT"
r_serv_tags.sadd('active_galaxies', id)
for tag in l_tags:
r_serv_tags.sadd('active_tag_galaxies_' + id, 'misp-galaxy:{}="{}"'.format(id, tag))
#save synonyms
for clusters_json in list_clusters:
#get clusters
cluster = json.loads(clusters_json)
if cluster['type'] == id:
val = cluster['values']
tags = []
for data in val:
try:
meta = data['meta']
synonyms = meta['synonyms']
tag_name = data['value']
tag_name = 'misp-galaxy:{}="{}"'.format(id, tag_name)
#save synonyms
for synonym in synonyms:
r_serv_tags.sadd('synonym_tag_' + tag_name, synonym)
except KeyError:
pass
break
return redirect(url_for('Tags.galaxies'))
@Tags.route("/Tags/disable_galaxy")
@login_required
@login_analyst
def disable_galaxy():
id = request.args.get('galaxy')
# verify input
try:
l_tags = list_all_tags[id]
except KeyError:
return "INCORRECT INPUT"
r_serv_tags.srem('active_galaxies', id)
for tag in l_tags:
tag_name = 'misp-galaxy:{}="{}"'.format(id, tag)
r_serv_tags.srem('active_tag_galaxies_' + id, tag_name)
r_serv_tags.delete('synonym_tag_' + tag_name)
return redirect(url_for('Tags.galaxies'))
@Tags.route("/Tags/edit_galaxy_tag")
@login_required
@login_analyst
def edit_galaxy_tag():
arg1 = request.args.getlist('tag_enabled')
arg2 = request.args.getlist('tag_disabled')
id = request.args.get('galaxy')
#verify input
try:
l_tags = list_all_tags[id]
except KeyError:
return "INCORRECT INPUT"
#get full tags
list_tag = []
for tag in l_tags:
list_tag.append('misp-galaxy:{}="{}"'.format(id, tag))
#check tags validity
if ( all(elem in list_tag for elem in arg1) or (len(arg1) == 0) ) and ( all(elem in list_tag for elem in arg2) or (len(arg2) == 0) ):
active_tag = r_serv_tags.smembers('active_tag_galaxies_' + id)
diff = list(set(arg1) ^ set(list_tag))
#remove tags
for tag in diff:
r_serv_tags.srem('active_tag_galaxies_' + id, tag)
r_serv_tags.delete('synonym_tag_' + tag)
#all tags unchecked
if len(arg1) == 0 and len(arg2) == 0:
r_serv_tags.srem('active_galaxies', id)
#add new tags
for tag in arg2:
r_serv_tags.sadd('active_galaxies', id)
r_serv_tags.sadd('active_tag_galaxies_' + id, tag)
#get tags synonyms
for clusters_json in list_clusters:
#get clusters
cluster = json.loads(clusters_json)
if cluster['type'] == id:
val = cluster['values']
tags = []
for data in val:
try:
meta = data['meta']
synonyms = meta['synonyms']
tag_name = data['value']
tag_name = 'misp-galaxy:{}="{}"'.format(id, tag_name)
if tag_name in arg2:
#save synonyms
for synonym in synonyms:
r_serv_tags.sadd('synonym_tag_' + tag_name, synonym)
except KeyError:
pass
break
return redirect(url_for('Tags.galaxies'))
else:
return "INCORRECT INPUT"
@Tags.route("/Tags/tag_galaxy_info")
@login_required
@login_read_only
def tag_galaxy_info():
galaxy = request.args.get('galaxy')
tag = request.args.get('tag')
full_tag = tag
title = tag.split(':')[1]
tag = tag.split('=')[1]
tag = tag[1:-1]
#get clusters
for clusters_json in list_clusters:
cluster = json.loads(clusters_json)
if cluster['type'] == galaxy:
val = cluster['values']
source = cluster['source']
for data in val:
if tag == data['value']:
try:
description = data['description']
except KeyError:
description = ''
if r_serv_tags.sismember('active_tag_galaxies_' + galaxy, full_tag):
active = True
else:
active = False
synonyms = []
metadata = []
list_metadata = []
try:
meta = data['meta']
for key in meta:
if key != 'synonyms':
if type(meta[key]) is list:
for item in meta[key]:
list_metadata.append(key + ' : ' + item)
else:
list_metadata.append(key + ' : ' + meta[key])
try:
synonyms = meta['synonyms']
bool_synonyms = True
except KeyError:
synonyms = []
bool_synonyms = False
except KeyError:
pass
if synonyms:
bool_synonyms = True
else:
bool_synonyms = False
if list_metadata:
metadata = True
else:
metadata = False
return render_template("tag_galaxy_info.html",
title = title,
description = description,
source = source,
active = active,
synonyms = synonyms,
bool_synonyms = bool_synonyms,
metadata = metadata,
list_metadata = list_metadata)
return 'INVALID INPUT'
# ========= REGISTRATION ========= # ========= REGISTRATION =========
app.register_blueprint(Tags, url_prefix=baseUrl) app.register_blueprint(Tags, url_prefix=baseUrl)

View file

@ -1,407 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<title>Tags - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
<style>
.rotate{
-moz-transition: all 0.1s linear;
-webkit-transition: all 0.1s linear;
transition: all 0.1s linear;
}
.rotate.down{
-moz-transform:rotate(180deg);
-webkit-transform:rotate(180deg);
transform:rotate(180deg);
}
</style>
</head>
<body>
{% include 'nav_bar.html' %}
<!-- Modal -->
<div id="mymodal" class="modal fade" role="dialog">
<div class="modal-dialog modal-lg">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div id="mymodalbody" class="modal-body" max-width="850px">
<p>Loading paste information...</p>
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
</div>
<div class="modal-footer">
<a id="button_show_path" target="_blank" href=""><button type="button" class="btn btn-info">Show saved paste</button></a>
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
<div class="container-fluid">
<div class="row">
{% include 'tags/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="card mb-3 mt-1">
<div class="card-header text-white bg-dark">
<h5 class="card-title">Search Tags by date range :</h5>
</div>
<div class="card-body">
<div class="row mb-3">
<div class="col-md-6">
<div class="input-group" id="date-range-from">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-from-input" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from" autocomplete="off">
</div>
</div>
<div class="col-md-6">
<div class="input-group" id="date-range-to">
<div class="input-group-prepend"><span class="input-group-text"><i class="far fa-calendar-alt" aria-hidden="true"></i></span></div>
<input class="form-control" id="date-range-to-input" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to" autocomplete="off">
</div>
</div>
</div>
<div class="input-group mb-3">
<div class="input-group-prepend">
<button class="btn btn-outline-danger" type="button" id="button-clear-tags" style="z-index: 1;" onclick="emptyTags()">
<i class="fas fa-eraser"></i>
</button>
</div>
<input id="ltags" name="ltags" type="text" class="form-control" aria-describedby="button-clear-tags" autocomplete="off">
</div>
<button class="btn btn-primary" type="button" id="button-search-tags" onclick="searchTags()">
<i class="fas fa-search"></i> Search Tags
</button>
</div>
</div>
<div>
{%if all_path%}
<table class="table table-bordered table-hover" id="myTable_">
<thead class="thead-dark">
<tr>
<th>Date</th>
<th style="max-width: 800px;">Path</th>
<th># of lines</th>
<th>Action</th>
</tr>
</thead>
<tbody>
{% for path in all_path %}
<tr>
<td class="pb-0">{{ paste_date[loop.index0] }}</td>
<td class="pb-0"><a target="_blank" href="{{ url_for('objects_item.showItem') }}?id={{path}}" class="text-secondary">
<div style="line-height:0.9;">{{ path }}</div>
</a>
<div class="mb-2">
{% for tag in paste_tags[loop.index0] %}
<a href="{{ url_for('Tags.Tags_page') }}?ltags={{ tag[1] }}">
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
</a>
{% endfor %}
</div>
</td class="pb-0">
<td class="pb-0">{{ paste_linenum[loop.index0] }}</td>
<td class="pb-0"><p>
<span class="fas fa-info-circle" data-toggle="tooltip" data-placement="left" title="{{ content[loop.index0] }} "></span>
<button type="button" class="btn btn-light" data-num="{{ loop.index0 + 1 }}" data-toggle="modal" data-target="#mymodal" data-url="{{ url_for('showsavedpastes.showsaveditem_min') }}?paste={{ path }}&num={{ loop.index0+1 }}" data-path="{{ path }}">
<span class="fas fa-search-plus"></span>
</button></p>
</td>
</tr>
{% endfor %}
</tbody>
</table>
<div class="d-flex justify-content-center border-top mt-2">
<nav class="mt-4" aria-label="...">
<ul class="pagination">
<li class="page-item {%if page==1%}disabled{%endif%}">
<a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">Previous</a>
</li>
{%if page>3%}
<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page=1&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">1</a></li>
<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page-1}}</a></li>
<li class="page-item active"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page}}</a></li>
{%else%}
{%if page>2%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-2}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page-2}}</a></li>{%endif%}
{%if page>1%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page-1}}</a></li>{%endif%}
<li class="page-item active"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page}}</a></li>
{%endif%}
{%if nb_page_max-page>3%}
<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page+1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page+1}}</a></li>
<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{nb_page_max}}</a></li>
{%else%}
{%if nb_page_max-page>2%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max-2}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{nb_page_max-2}}</a></li>{%endif%}
{%if nb_page_max-page>1%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max-1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{nb_page_max-1}}</a></li>{%endif%}
{%if nb_page_max-page>0%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{nb_page_max}}</a></li>{%endif%}
{%endif%}
<li class="page-item {%if page==nb_page_max%}disabled{%endif%}">
<a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page+1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}" aria-disabled="true">Next</a>
</li>
</ul>
</nav>
</div>
{%endif%}
</div>
<div>
<br></br>
<a class="btn btn-light text-secondary" href="{{ url_for('Tags.taxonomies') }}" target="_blank">
<i class="fas fa-wrench fa-2x"></i>
<br></br>
<span class="label-icon">Edit Taxonomies List </span>
</a>
<a class="btn btn-light text-secondary" href="{{ url_for('Tags.galaxies') }}" target="_blank">
<i class="fas fa-rocket fa-2x"></i>
<br></br>
<span class="label-icon">Edit Galaxies List</span>
</a>
</div>
<div>
<br></br>
<a class="btn btn-light text-secondary" href="{{ url_for('PasteSubmit.edit_tag_export') }}" target="_blank">
<i class="fas fa-cogs fa-2x"></i>
<br></br>
<span class="label-icon">MISP and Hive, auto push</span>
</a>
</div>
</div>
</div>
</div>
</body>
<script>
var ltags;
var search_table;
var last_clicked_paste;
var can_change_modal_content = true;
$(document).ready(function(){
$("#nav_tags_search").removeClass("text-muted");
$("#nav_tags_search_item").addClass("active");
search_table = $('#myTable_').DataTable({ "order": [[ 0, "asc" ]] });
// Use to bind the button with the new displayed data
// (The bind do not happens if the dataTable is in tabs and the clicked data is in another page)
search_table.on( 'draw.dt', function () {
// Bind tooltip each time we draw a new page
$('[data-toggle="tooltip"]').tooltip();
// On click, get html content from url and update the corresponding modal
$("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
get_html_and_update_modal(event, $(this));
});
} );
search_table.draw()
var valueData = [
{% for tag in list_tag %}
'{{tag|safe}}',
{% endfor %}
];
$.getJSON("{{ url_for('Tags.get_all_tags') }}",
function(data) {
ltags = $('#ltags').tagSuggest({
data: data,
value: valueData,
sortOrder: 'name',
maxDropHeight: 200,
name: 'ltags'
});
});
$('#date-range-from').dateRangePicker({
separator : ' to ',
getValue: function(){
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
else
return '';
},
setValue: function(s,s1,s2){
$('#date-range-from-input').val(s1);
$('#date-range-to-input').val(s2);
}
});
$('#date-range-to').dateRangePicker({
separator : ' to ',
getValue: function(){
if ($('#date-range-from-input').val() && $('#date-range-to-input').val() )
return $('#date-range-from-input').val() + ' to ' + $('#date-range-to-input').val();
else
return '';
},
setValue: function(s,s1,s2){
$('#date-range-from-input').val(s1);
$('#date-range-to-input').val(s2);
}
});
});
</script>
<script>
function searchTags() {
var data = ltags.getValue();
var date_from = $('#date-range-from-input').val();
var date_to =$('#date-range-to-input').val();
window.location.replace("{{ url_for('Tags.Tags_page') }}?date_from="+date_from+"&date_to="+date_to+"&ltags=" + data);
}
function emptyTags() {
ltags.clear();
}
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
<!-- Dynamically update the modal -->
<script>
// static data
var alert_message = '<div class="alert alert-info alert-dismissable"><button type="button" class="close" data-dismiss="alert" aria-hidden="true">×</button><strong>No more data.</strong> Full paste displayed.</div>';
var complete_paste = null;
var char_to_display = {{ char_to_display }};
var start_index = 0;
// When the modal goes out, refresh it to normal content
$("#mymodal").on('hidden.bs.modal', function () {
can_change_modal_content = true;
$("#mymodalbody").html("<p>Loading paste information...</p>");
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
$("#mymodalbody").append(loading_gif); // Show the loading GIF
$("#button_show_path").attr('href', '');
$("#button_show_path").hide();
complete_paste = null;
start_index = 0;
});
// Update the paste preview in the modal
function update_preview() {
if (start_index + char_to_display > complete_paste.length-1){ // end of paste reached
var final_index = complete_paste.length-1;
var flag_stop = true;
} else {
var final_index = start_index + char_to_display;
}
if (final_index != start_index){ // still have data to display
// Append the new content using text() and not append (XSS)
$("#mymodalbody").find("#paste-holder").text($("#mymodalbody").find("#paste-holder").text()+complete_paste.substring(start_index+1, final_index+1));
start_index = final_index;
if (flag_stop)
nothing_to_display();
} else {
nothing_to_display();
}
}
// Update the modal when there is no more data
function nothing_to_display() {
var new_content = $(alert_message).hide();
$("#mymodalbody").find("#panel-body").append(new_content);
new_content.show('fast');
$("#load-more-button").hide();
}
function get_html_and_update_modal(event, truemodal) {
event.preventDefault();
var modal=truemodal;
var url = " {{ url_for('showsavedpastes.showpreviewpaste') }}?paste=" + modal.attr('data-path') + "&num=" + modal.attr('data-num');
last_clicked_paste = modal.attr('data-num');
$.get(url, function (data) {
// verify that the reveived data is really the current clicked paste. Otherwise, ignore it.
var received_num = parseInt(data.split("|num|")[1]);
if (received_num == last_clicked_paste && can_change_modal_content) {
can_change_modal_content = false;
// clear data by removing html, body, head tags. prevent dark modal background stack bug.
var cleared_data = data.split("<body>")[1].split("</body>")[0];
$("#mymodalbody").html(cleared_data);
var button = $('<button type="button" id="load-more-button" class="btn btn-outline-primary rounded-circle px-1 py-0" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><i class="fas fa-arrow-circle-down mt-1"></i></button>');
button.tooltip(button);
$("#container-show-more").append(button);
$("#button_show_path").attr('href', '{{ url_for('objects_item.showItem') }}?id=' + $(modal).attr('data-path'));
$("#button_show_path").show('fast');
$("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
nothing_to_display();
}
// collapse decoded
$('#collapseDecoded').collapse('hide');
// On click, donwload all paste's content
$("#load-more-button").on("click", function (event) {
if (complete_paste == null) { //Donwload only once
$.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
complete_paste = data;
update_preview();
});
} else {
update_preview();
}
});
} else if (can_change_modal_content) {
$("#mymodalbody").html("Ignoring previous not finished query of paste #" + received_num);
}
});
}
</script>
</html>

View file

@ -1,317 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Tags - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<script src="{{ url_for('static', filename='js/jquery.js') }}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
<style>
.tooltip-inner {
text-align: left;
height: 200%;
width: 200%;
max-width: 500px;
max-height: 500px;
font-size: 13px;
}
xmp {
white-space:pre-wrap;
word-wrap:break-word;
}
.test thead{
background: #d91f2d;
color: #fff;
}
</style>
</head>
<body>
{% include 'navbar.html' %}
<!-- Modal -->
<div id="mymodal" class="modal fade" role="dialog">
<div class="modal-dialog modal-lg">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div id="mymodalbody" class="modal-body" max-width="850px">
<p>Loading paste information...</p>
<img id="loading-gif-modal" src="{{url_for('static', filename='image/loading.gif') }}" height="26" width="26" style="margin: 4px;">
</div>
<div class="modal-footer">
<a id="button_show_path" target="_blank" href=""><button type="button" class="btn btn-info">Show saved paste</button></a>
<button type="button" class="btn btn-default" data-dismiss="modal">Close</button>
</div>
</div>
</div>
</div>
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-tags" >Tags</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="form-group input-group" >
<input id="ltags" style="width:100%;" type="text" name="ltags" autocomplete="off">
<div class="input-group-btn">
<button type="button" class="btn btn-search btn-primary btn-tags" onclick="searchTags()">
<span class="glyphicon glyphicon-search "></span>
<span class="label-icon">Search Tags</span>
</button>
</div>
</div>
<button class="btn btn-primary" onclick="emptyTags()" style="margin-bottom: 30px;">
<span class="glyphicon glyphicon-remove"></span>
<span class="label-icon">Clear Tags</span>
</button>
<table class="test table table-striped table-bordered table-hover table-responsive " id="myTable_">
<thead>
<tr>
<th>#</th>
<th style="max-width: 800px;">Path</th>
<th>Date</th>
<th># of lines</th>
<th>Action</th>
</tr>
</thead>
<tbody>
{% for path in all_path %}
<tr>
<td> {{ loop.index0 }}</td>
<td><a target="_blank" href="{{ url_for('objects_item.showItem') }}?id={{path}}">{{ path }}</a>
<div>
{% for tag in paste_tags[loop.index0] %}
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag[1] }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag[0] }}</span>
</a>
{% endfor %}
</div>
</td>
<td>{{ paste_date[loop.index0] }}</td>
<td>{{ paste_linenum[loop.index0] }}</td>
<td><p><span class="glyphicon glyphicon-info-sign" data-toggle="tooltip" data-placement="left" title="{{ content[loop.index0] }} "></span> <button type="button" class="btn-link" data-num="{{ loop.index0 + 1 }}" data-toggle="modal" data-target="#mymodal" data-url="{{ url_for('objects_item.showItem') }}?id={{ path }}&num={{ loop.index0+1 }}" data-path="{{ path }}"><span class="fa fa-search-plus"></span></button></p></td>
</tr>
{% endfor %}
</tbody>
</table>
</br>
<div id="nbr_entry" class="alert alert-info">
</div>
<div id="div_stil_data">
<button id="load_more_json_button1" type="button" class="btn btn-default" onclick="add_entries(100)" style="display: True">Load 100 entries</button>
<button id="load_more_json_button2" type="button" class="btn btn-warning" onclick="add_entries(300)" style="display: True">Load 300 entries</button>
<img id="loading_gif_browse" src="{{url_for('static', filename='image/loading.gif') }}" heigt="20" width="20" style="margin: 2px;"></div>
</br>
<div>
<br></br>
<a class="btn btn-tags" href="{{ url_for('Tags.taxonomies') }}" target="_blank">
<i class="fa fa-wrench fa-2x"></i>
<br></br>
<span class="label-icon">Edit Taxonomies List </span>
</a>
<a class="btn btn-tags" href="{{ url_for('Tags.galaxies') }}" target="_blank">
<i class="fa fa-rocket fa-2x"></i>
<br></br>
<span class="label-icon">Edit Galaxies List</span>
</a>
</div>
<div>
<br></br>
<a class="btn btn-tags" href="{{ url_for('PasteSubmit.edit_tag_export') }}" target="_blank">
<i class="fa fa-cogs fa-2x"></i>
<br></br>
<span class="label-icon">MISP and Hive, auto push</span>
</a>
</div>
</div>
</body>
<script>
var search_table;
var ltags;
var last_clicked_paste;
var can_change_modal_content = true;
$("#myTable_").attr('data-numElem', "{{ all_path|length }}");
$(document).ready(function(){
search_table = $('#myTable_').DataTable({ "order": [[ 2, "desc" ]] });
$('[data-toggle="tooltip"]').tooltip();
$("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
get_html_and_update_modal(event, $(this));
});
$("#load_more_json_button1").hide();
$("#load_more_json_button2").hide();
$("#nbr_entry").hide();
$("#loading_gif_browse").hide();
$.getJSON("{{ url_for('Tags.get_all_tags') }}",
function(data) {
activePage = "page-Tags"
$("#"+activePage).addClass("active");
var valueData = [
{% for tag in list_tag %}
'{{tag|safe}}',
{% endfor %}
];
ltags = $('#ltags').tagSuggest({
data: data,
value: valueData,
sortOrder: 'name',
maxDropHeight: 200,
name: 'ltags',
});
});
});
</script>
<script>
function searchTags() {
var data = ltags.getValue();
window.location.replace("{{ url_for('Tags.get_tagged_paste') }}?ltags=" + data);
}
function emptyTags() {
ltags.clear();
}
</script>
<!-- Dynamically update the modal -->
<script>
// static data
var alert_message = '<div class="alert alert-info alert-dismissable"><button type="button" class="close" data-dismiss="alert" aria-hidden="true">×</button><strong>No more data.</strong> Full paste displayed.</div>';
var complete_paste = null;
var char_to_display = {{ char_to_display }};
var start_index = 0;
// When the modal goes out, refresh it to normal content
$("#mymodal").on('hidden.bs.modal', function () {
can_change_modal_content = true;
$("#mymodalbody").html("<p>Loading paste information...</p>");
var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
$("#mymodalbody").append(loading_gif); // Show the loading GIF
$("#button_show_path").attr('href', '');
$("#button_show_path").hide();
complete_paste = null;
start_index = 0;
});
// Update the paste preview in the modal
function update_preview() {
if (start_index + char_to_display > complete_paste.length-1){ // end of paste reached
var final_index = complete_paste.length-1;
var flag_stop = true;
} else {
var final_index = start_index + char_to_display;
}
if (final_index != start_index){ // still have data to display
// Append the new content using text() and not append (XSS)
$("#mymodalbody").find("#paste-holder").text($("#mymodalbody").find("#paste-holder").text()+complete_paste.substring(start_index+1, final_index+1));
start_index = final_index;
if (flag_stop)
nothing_to_display();
} else {
nothing_to_display();
}
}
// Update the modal when there is no more data
function nothing_to_display() {
var new_content = $(alert_message).hide();
$("#mymodalbody").find("#panel-body").append(new_content);
new_content.show('fast');
$("#load-more-button").hide();
}
function get_html_and_update_modal(event, truemodal) {
event.preventDefault();
var modal=truemodal;
var url = " {{ url_for('showsavedpastes.showpreviewpaste') }}?paste=" + modal.attr('data-path') + "&num=" + modal.attr('data-num');
last_clicked_paste = modal.attr('data-num');
$.get(url, function (data) {
// verify that the reveived data is really the current clicked paste. Otherwise, ignore it.
var received_num = parseInt(data.split("|num|")[1]);
if (received_num == last_clicked_paste && can_change_modal_content) {
can_change_modal_content = false;
// clear data by removing html, body, head tags. prevent dark modal background stack bug.
var cleared_data = data.split("<body>")[1].split("</body>")[0];
$("#mymodalbody").html(cleared_data);
var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
button.tooltip();
$("#mymodalbody").children(".panel-default").append(button);
$("#button_show_path").attr('href', $(modal).attr('data-url'));
$("#button_show_path").show('fast');
$("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
nothing_to_display();
}
// On click, donwload all paste's content
$("#load-more-button").on("click", function (event) {
if (complete_paste == null) { //Donwload only once
$.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
complete_paste = data;
update_preview();
});
} else {
update_preview();
}
});
} else if (can_change_modal_content) {
$("#mymodalbody").html("Ignoring previous not finished query of paste #" + received_num);
}
});
}
// Use to bind the button with the new displayed data
// (The bind do not happens if the dataTable is in tabs and the clicked data is in another page)
search_table.on( 'draw.dt', function () {
// Bind tooltip each time we draw a new page
$('[data-toggle="tooltip"]').tooltip();
// On click, get html content from url and update the corresponding modal
$("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
get_html_and_update_modal(event, $(this));
});
} );
</script>

View file

@ -28,7 +28,6 @@ import Flask_config
app = Flask_config.app app = Flask_config.app
config_loader = Flask_config.config_loader config_loader = Flask_config.config_loader
baseUrl = Flask_config.baseUrl baseUrl = Flask_config.baseUrl
r_serv = Flask_config.r_serv
r_serv_log = Flask_config.r_serv_log r_serv_log = Flask_config.r_serv_log
max_dashboard_logs = Flask_config.max_dashboard_logs max_dashboard_logs = Flask_config.max_dashboard_logs
@ -53,7 +52,7 @@ def event_stream():
if msg['type'] == 'pmessage' and level != "DEBUG": if msg['type'] == 'pmessage' and level != "DEBUG":
yield 'data: %s\n\n' % json.dumps(msg) yield 'data: %s\n\n' % json.dumps(msg)
def get_queues(r): def get_queues():
# We may want to put the llen in a pipeline to do only one query. # We may want to put the llen in a pipeline to do only one query.
return queues_modules.get_all_modules_queues_stats() return queues_modules.get_all_modules_queues_stats()
@ -63,21 +62,21 @@ def get_date_range(date_from, num_day):
for i in range(0, num_day+1): for i in range(0, num_day+1):
new_date = date.substract_day(i) new_date = date.substract_day(i)
date_list.append(new_date[0:4] +'-'+ new_date[4:6] +'-'+ new_date[6:8]) date_list.append(f'{new_date[0:4]}-{new_date[4:6]}-{new_date[6:8]}')
return date_list return date_list
def dashboard_alert(log): def dashboard_alert(log):
# check if we need to display this log # check if we need to display this log
if len(log)>50: if len(log) > 50:
date = log[1:5]+log[6:8]+log[9:11] date = log[1:5]+log[6:8]+log[9:11]
utc_str = log[1:20] utc_str = log[1:20]
log = log[46:].split(';') log = log[46:].split(';')
if len(log) == 6: if len(log) == 6:
time = datetime_from_utc_to_local(utc_str) date_time = datetime_from_utc_to_local(utc_str)
path = url_for('objects_item.showItem',id=log[5]) path = url_for('objects_item.showItem', id=log[5])
res = {'date': date, 'time': time, 'script': log[0], 'domain': log[1], 'date_paste': log[2], res = {'date': date, 'time': date_time, 'script': log[0], 'domain': log[1], 'date_paste': log[2],
'paste': log[3], 'message': log[4], 'path': path} 'paste': log[3], 'message': log[4], 'path': path}
return res return res
else: else:
@ -116,7 +115,7 @@ def get_last_logs_json():
date_range = get_date_range(date, max_day_search) date_range = get_date_range(date, max_day_search)
while max_day_search != day_search and warning_found != warning_to_found: while max_day_search != day_search and warning_found != warning_to_found:
filename_warning_log = 'logs/Script_warn-'+ date_range[day_search] +'.log' filename_warning_log = f'logs/Script_warn-{date_range[day_search]}.log'
filename_log = os.path.join(os.environ['AIL_HOME'], filename_warning_log) filename_log = os.path.join(os.environ['AIL_HOME'], filename_warning_log)
try: try:
@ -147,7 +146,7 @@ def get_last_logs_json():
@login_required @login_required
@login_read_only @login_read_only
def stuff(): def stuff():
return jsonify(row1=get_queues(r_serv)) return jsonify(row1=get_queues())
# TODO: ADD UPDATE NOTE BY USER # TODO: ADD UPDATE NOTE BY USER
@ -169,9 +168,11 @@ def index():
background_update = True background_update = True
update_message = ail_updates.get_update_background_message() update_message = ail_updates.get_update_background_message()
return render_template("index.html", default_minute = default_minute, threshold_stucked_module=threshold_stucked_module, return render_template("index.html", default_minute = default_minute,
threshold_stucked_module=threshold_stucked_module,
log_select=log_select, selected=max_dashboard_logs, log_select=log_select, selected=max_dashboard_logs,
background_update=background_update, update_message=update_message) background_update=background_update, update_message=update_message)
# ========= REGISTRATION ========= # ========= REGISTRATION =========
app.register_blueprint(dashboard, url_prefix=baseUrl) app.register_blueprint(dashboard, url_prefix=baseUrl)

View file

@ -17,7 +17,7 @@ from Role_Manager import login_admin, login_analyst, login_read_only, no_cache
from flask_login import login_required from flask_login import login_required
from Date import Date from Date import Date
from HiddenServices import HiddenServices sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import crawlers import crawlers
# ============ VARIABLES ============ # ============ VARIABLES ============
@ -30,9 +30,6 @@ r_serv_onion = Flask_config.r_serv_onion
r_serv_metadata = Flask_config.r_serv_metadata r_serv_metadata = Flask_config.r_serv_metadata
bootstrap_label = Flask_config.bootstrap_label bootstrap_label = Flask_config.bootstrap_label
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import crawlers
hiddenServices = Blueprint('hiddenServices', __name__, template_folder='templates') hiddenServices = Blueprint('hiddenServices', __name__, template_folder='templates')
faup = Faup() faup = Faup()

View file

@ -1,34 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Flask functions and routes for the trending modules page
'''
import redis
from flask import Flask, render_template, jsonify, request, Blueprint
from Role_Manager import login_admin, login_analyst, login_read_only
from flask_login import login_required
# ============ VARIABLES ============
import Flask_config
app = Flask_config.app
rawSkeleton = Blueprint('rawSkeleton', __name__, template_folder='templates')
# ============ FUNCTIONS ============
def one():
return 1
# ============= ROUTES ==============
@rawSkeleton.route("/rawSkeleton/", methods=['GET'])
@login_required
@login_read_only
def skeleton_page():
return render_template("rawSkeleton.html")
# ========= REGISTRATION =========
app.register_blueprint(rawSkeleton)

View file

@ -1 +0,0 @@
<li id='page-rawSkeleton'><a href="{{ url_for('rawSkeleton.skeleton_page') }}"><i class="glyphicon glyphicon-new-window"></i> rawSkeleton page</a></li>

View file

@ -1,47 +0,0 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.pie.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
</head>
<body>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-rawSkeleton" >rawSkeleton</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
<script>
$(document).ready(function(){
activePage = "page-rawSkeleton"
$("#"+activePage).addClass("active");
});
</script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
</body>
</html>

View file

@ -311,7 +311,7 @@
// On click, donwload all paste's content // On click, donwload all paste's content
$("#load-more-button").off('click.download').on("click.download", function (event) { $("#load-more-button").off('click.download').on("click.download", function (event) {
if (complete_paste == null) { //Donwload only once if (complete_paste == null) { //Donwload only once
$.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){ $.get("{{ url_for('objects_item.item_content_more') }}"+"?id="+$(modal).attr('data-path'), function(data, status){
complete_paste = data; complete_paste = data;
update_preview(); update_preview();
}); });

View file

@ -1,283 +0,0 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Flask functions and routes for the trending modules page
'''
import json
import os
import sys
import flask
from flask import Flask, render_template, jsonify, request, Blueprint, make_response, Response, send_from_directory, redirect, url_for, abort
from Role_Manager import login_admin, login_analyst, login_read_only, no_cache
from flask_login import login_required
import difflib
import Paste
import requests
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Item
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
# ============ VARIABLES ============
import Flask_config
app = Flask_config.app
baseUrl = Flask_config.baseUrl
r_serv_metadata = Flask_config.r_serv_metadata
r_serv_tags = Flask_config.r_serv_tags
r_serv_statistics = Flask_config.r_serv_statistics
max_preview_char = Flask_config.max_preview_char
max_preview_modal = Flask_config.max_preview_modal
DiffMaxLineLength = Flask_config.DiffMaxLineLength
bootstrap_label = Flask_config.bootstrap_label
misp_event_url = Flask_config.misp_event_url
hive_case_url = Flask_config.hive_case_url
vt_enabled = Flask_config.vt_enabled
PASTES_FOLDER = Flask_config.PASTES_FOLDER
SCREENSHOT_FOLDER = Flask_config.SCREENSHOT_FOLDER
showsavedpastes = Blueprint('showsavedpastes', __name__, template_folder='templates')
# ============ FUNCTIONS ============
def get_item_screenshot_path(item):
screenshot = r_serv_metadata.hget('paste_metadata:{}'.format(item), 'screenshot')
if screenshot:
screenshot = os.path.join(screenshot[0:2], screenshot[2:4], screenshot[4:6], screenshot[6:8], screenshot[8:10], screenshot[10:12], screenshot[12:])
else:
screenshot = ''
return screenshot
def get_item_basic_info(item):
item_basic_info = {}
item_basic_info['date'] = str(item.get_p_date())
item_basic_info['date'] = '{}/{}/{}'.format(item_basic_info['date'][0:4], item_basic_info['date'][4:6], item_basic_info['date'][6:8])
item_basic_info['source'] = item.get_item_source()
item_basic_info['size'] = item.get_item_size()
## TODO: FIXME ##performance
item_basic_info['encoding'] = item._get_p_encoding()
## TODO: FIXME ##performance
#item_basic_info['language'] = item._get_p_language()
## TODO: FIXME ##performance
info_line = item.get_lines_info()
item_basic_info['nb_lines'] = info_line[0]
item_basic_info['max_length_line'] = info_line[1]
return item_basic_info
def show_item_min(requested_path , content_range=0):
relative_path = None
if PASTES_FOLDER not in requested_path:
relative_path = requested_path
requested_path = os.path.join(PASTES_FOLDER, requested_path)
else:
relative_path = requested_path.replace(PASTES_FOLDER, '', 1)
# remove old full path
# requested_path = requested_path.replace(PASTES_FOLDER, '')
# escape directory transversal
if os.path.commonprefix((os.path.realpath(requested_path),PASTES_FOLDER)) != PASTES_FOLDER:
return 'path transversal detected'
item_info ={}
paste = Paste.Paste(requested_path)
item_basic_info = get_item_basic_info(paste)
item_info['nb_duplictates'] = paste.get_nb_duplicate()
## TODO: use this for fix ?
item_content = paste.get_p_content()
char_to_display = len(item_content)
if content_range != 0:
item_content = item_content[0:content_range]
vt_enabled = Flask_config.vt_enabled
p_hashtype_list = []
print(requested_path)
l_tags = r_serv_metadata.smembers('tag:'+relative_path)
if relative_path is not None:
l_tags.union( r_serv_metadata.smembers('tag:'+relative_path) )
item_info['tags'] = l_tags
item_info['name'] = relative_path.replace('/', ' / ')
l_64 = []
# load hash files
if r_serv_metadata.scard('hash_paste:'+relative_path) > 0:
set_b64 = r_serv_metadata.smembers('hash_paste:'+relative_path)
for hash in set_b64:
nb_in_file = r_serv_metadata.zscore('nb_seen_hash:'+hash, relative_path)
# item list not updated
if nb_in_file is None:
l_pastes = r_serv_metadata.zrange('nb_seen_hash:'+hash, 0, -1)
for paste_name in l_pastes:
# dynamic update
if PASTES_FOLDER in paste_name:
score = r_serv_metadata.zscore('nb_seen_hash:{}'.format(hash), paste_name)
r_serv_metadata.zrem('nb_seen_hash:{}'.format(hash), paste_name)
paste_name = paste_name.replace(PASTES_FOLDER, '', 1)
r_serv_metadata.zadd('nb_seen_hash:{}'.format(hash), score, paste_name)
nb_in_file = r_serv_metadata.zscore('nb_seen_hash:{}'.format(hash), relative_path)
nb_in_file = int(nb_in_file)
estimated_type = r_serv_metadata.hget('metadata_hash:'+hash, 'estimated_type')
file_type = estimated_type.split('/')[0]
# set file icon
if file_type == 'application':
file_icon = 'fa-file '
elif file_type == 'audio':
file_icon = 'fa-file-video '
elif file_type == 'image':
file_icon = 'fa-file-image'
elif file_type == 'text':
file_icon = 'fa-file-alt'
else:
file_icon = 'fa-file'
saved_path = r_serv_metadata.hget('metadata_hash:'+hash, 'saved_path')
if r_serv_metadata.hexists('metadata_hash:'+hash, 'vt_link'):
b64_vt = True
b64_vt_link = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_link')
b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report')
else:
b64_vt = False
b64_vt_link = ''
b64_vt_report = r_serv_metadata.hget('metadata_hash:'+hash, 'vt_report')
# hash never refreshed
if b64_vt_report is None:
b64_vt_report = ''
l_64.append( (file_icon, estimated_type, hash, saved_path, nb_in_file, b64_vt, b64_vt_link, b64_vt_report) )
crawler_metadata = {}
if 'infoleak:submission="crawler"' in l_tags:
crawler_metadata['get_metadata'] = True
crawler_metadata['domain'] = r_serv_metadata.hget('paste_metadata:'+relative_path, 'domain')
crawler_metadata['domain'] = crawler_metadata['domain'].rsplit(':', 1)[0]
crawler_metadata['paste_father'] = r_serv_metadata.hget('paste_metadata:'+relative_path, 'father')
crawler_metadata['real_link'] = r_serv_metadata.hget('paste_metadata:'+relative_path,'real_link')
crawler_metadata['screenshot'] = get_item_screenshot_path(relative_path)
# crawler_metadata['har_file'] = Item.get_item_har(relative_path)
else:
crawler_metadata['get_metadata'] = False
item_parent = Item.get_item_parent(requested_path)
misp_event = r_serv_metadata.get('misp_events:' + requested_path)
if misp_event is None:
misp_eventid = False
misp_url = ''
else:
misp_eventid = True
misp_url = misp_event_url + misp_event
hive_case = r_serv_metadata.get('hive_cases:' + requested_path)
if hive_case is None:
hive_caseid = False
hive_url = ''
else:
hive_caseid = True
hive_url = hive_case_url.replace('id_here', hive_case)
return render_template("show_saved_item_min.html", bootstrap_label=bootstrap_label, content=item_content,
item_basic_info=item_basic_info, item_info=item_info,
item_parent=item_parent,
initsize=len(item_content),
hashtype_list = p_hashtype_list,
crawler_metadata=crawler_metadata,
l_64=l_64, vt_enabled=vt_enabled, misp_eventid=misp_eventid, misp_url=misp_url, hive_caseid=hive_caseid, hive_url=hive_url)
# ============ ROUTES ============
@showsavedpastes.route("/showsaveditem_min/") #completely shows the paste in a new tab
@login_required
@login_read_only
def showsaveditem_min():
requested_path = request.args.get('paste', '')
return show_item_min(requested_path)
@showsavedpastes.route("/showsavedrawpaste/") #shows raw
@login_required
@login_read_only
def showsavedrawpaste():
requested_path = request.args.get('paste', '')
paste = Paste.Paste(requested_path)
content = paste.get_p_content()
return Response(content, mimetype='text/plain')
@showsavedpastes.route("/showpreviewpaste/")
@login_required
@login_read_only
def showpreviewpaste():
num = request.args.get('num', '')
requested_path = request.args.get('paste', '')
return "|num|"+num+"|num|"+show_item_min(requested_path, content_range=max_preview_modal)
@showsavedpastes.route("/getmoredata/")
@login_required
@login_read_only
def getmoredata():
requested_path = request.args.get('paste', '')
paste = Paste.Paste(requested_path)
p_content = paste.get_p_content()
to_return = p_content[max_preview_modal-1:]
return to_return
@showsavedpastes.route("/showDiff/")
@login_required
@login_analyst
def showDiff():
s1 = request.args.get('s1', '')
s2 = request.args.get('s2', '')
p1 = Paste.Paste(s1)
p2 = Paste.Paste(s2)
maxLengthLine1 = p1.get_lines_info()[1]
maxLengthLine2 = p2.get_lines_info()[1]
if maxLengthLine1 > DiffMaxLineLength or maxLengthLine2 > DiffMaxLineLength:
return "Can't make the difference as the lines are too long."
htmlD = difflib.HtmlDiff()
lines1 = p1.get_p_content().splitlines()
lines2 = p2.get_p_content().splitlines()
the_html = htmlD.make_file(lines1, lines2)
return the_html
@showsavedpastes.route('/screenshot/<path:filename>')
@login_required
@login_read_only
@no_cache
def screenshot(filename):
return send_from_directory(SCREENSHOT_FOLDER, filename+'.png', as_attachment=True)
@showsavedpastes.route('/send_file_to_vt/', methods=['POST'])
@login_required
@login_analyst
def send_file_to_vt():
b64_path = request.form['b64_path']
paste = request.form['paste']
hash = request.form['hash']
## TODO: # FIXME: path transversal
b64_full_path = os.path.join(os.environ['AIL_HOME'], b64_path)
b64_content = ''
with open(b64_full_path, 'rb') as f:
b64_content = f.read()
files = {'file': (hash, b64_content)}
response = requests.post('https://www.virustotal.com/vtapi/v2/file/scan', files=files, params=vt_auth)
json_response = response.json()
print(json_response)
vt_b64_link = json_response['permalink'].split('analysis')[0] + 'analysis/'
r_serv_metadata.hset('metadata_hash:'+hash, 'vt_link', vt_b64_link)
return redirect(url_for('showsavedpastes.showsavedpaste', paste=paste))
# ========= REGISTRATION =========
app.register_blueprint(showsavedpastes, url_prefix=baseUrl)

View file

@ -1,581 +0,0 @@
<!DOCTYPE html>
<html lang="en">
<head>
<title>Paste information - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/tags.css') }}" rel="stylesheet" type="text/css" />
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>
<script src="{{ url_for('static', filename='js/tags.js') }}"></script>
<style>
.scrollable-menu {
height: auto;
max-height: 200px;
overflow-x: hidden;
width:100%;
}
.red_table thead{
background: #d91f2d;
color: #fff;
}
</style>
</head>
<body>
<div class="panel panel-default">
<div class="panel-heading">
<h1 class="page-header" >Paste: {{ request.args.get('paste') }}</h1>
<h2 class="page-header" >
<div>
<div id="mymodal" class="modal fade" role="dialog">
<div class="modal-dialog modal-lg">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #cccccc; background-color: #cccccc; color: #ffffff;">
<p class="heading">Edit this tag</p>
</div>
<div class="modal-body">
<div class="input-group" >
<input id="ltags" type="text" class="form-control" autocomplete="off" style="width: 760px" name="ltags">
</div>
<div class="btn-group btn-block">
<button type="button" class="btn btn-primary dropdown-toggle btn-block" data-toggle="dropdown">Taxonomie Selection
<i class="fa fa-chevron-down"></i>
</button>
<ul class="dropdown-menu scrollable-menu" role="menu">
<li><a href="#" id="all-tags-taxonomies">All Tags <i class="fa fa-tags"></i></a></li>
<li role="separator" class="divider"></li>
{% for taxo in active_taxonomies %}
<li><a href="#" id="{{ taxo }}-id{{ loop.index0 }}">{{ taxo }}</a></li>
{% endfor %}
</ul>
</div>
<br/><br/>
<div class="input-group">
<input id="ltagsgalaxies" type="text" class="form-control" autocomplete="off" style="width: 760px" name="ltagsgalaxies">
</div>
<div class="btn-group btn-block">
<button type="button" class="btn btn-primary dropdown-toggle btn-block" data-toggle="dropdown">Galaxy Selection
<i class="fa fa-chevron-down"></i>
</button>
<ul class="dropdown-menu scrollable-menu" role="menu">
<li><a href="#" id="all-tags-galaxies">All Tags <i class="fa fa-tags"></i></a></li>
<li role="separator" class="divider"></li>
{% for galaxy in active_galaxies %}
<li><a href="#" id="{{ galaxy }}-idgalax{{ loop.index0 }}">{{ galaxy }}</a></li>
{% endfor %}
</ul>
</div>
</div>
<div class="modal-footer">
<a class="btn btn-tags pull-left" href="{{ url_for('Tags.taxonomies') }}" target="_blank">
<span class="label-icon">Edit Taxonomies List </span>
<i class="fa fa-wrench fa-2x"></i>
</a>
<a class="btn btn-tags pull-left" href="{{ url_for('Tags.galaxies') }}" target="_blank">
<span class="label-icon">Edit Galaxies List</span>
<i class="fa fa-rocket fa-2x"></i>
</a>
<button class="btn btn-primary btn-tags" onclick="addTags()">
<span class="glyphicon glyphicon-plus"></span>
<span class="label-icon">Add Tags</span>
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
{% for tag in list_tags %}
<span class="btn btn-{{ bootstrap_label[loop.index0 % 5] }} btn-lg pull-left" data-toggle="modal" data-target="#myModal_{{ loop.index0 }}">{{ tag[0] }}</span>
<!-- Modal edit this tag -->
<div class="modal fade" id="myModal_{{ loop.index0 }}" role="dialog">
<div class="modal-dialog">
<div class="modal-content text-center">
<div class="modal-header" style="border-bottom: 4px solid #cccccc; background-color: #cccccc; color: #ffffff;">
<p class="heading">Edit this tag</p>
</div>
<div class="modal-body">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }}" >{{ tag[0] }}</span>
</div>
<div class="modal-footer center">
{% if not tag[2] %}
<a href="{{ url_for('Tags.tag_validation') }}?paste={{ request.args.get('paste') }}&tag={{ tag[0] }}&status=tp" class="btn btn-success pull-left" data-toggle="tooltip" title="Good Detection">
<span class="glyphicon glyphicon-thumbs-up "></span>
</a>
{% endif %}
{% if tag[2] %}
<button class="btn btn-success pull-left disabled" data-toggle="tooltip" title="Good Detection">
<span class="glyphicon glyphicon-thumbs-up "></span>
</button>
{% endif %}
{% if not tag[3] %}
<a href="{{ url_for('Tags.tag_validation') }}?paste={{ request.args.get('paste') }}&tag={{ tag[0] }}&status=fp" class="btn btn-danger pull-left" data-toggle="tooltip" title="Bad Detection">
<span class="glyphicon glyphicon-thumbs-down "></span>
</a>
{% endif %}
{% if tag[3] %}
<button class="btn btn-danger pull-left disabled" data-toggle="tooltip" title="Bad Detection">
<span class="glyphicon glyphicon-thumbs-down "></span>
</button>
{% endif %}
{% if tag[1] %}
<a href="{{ url_for('Tags.confirm_tag') }}?paste={{ request.args.get('paste') }}&tag={{ tag[0] }}" class="btn btn-primary">
<span class="glyphicon glyphicon-ok "></span> Confirm this Tag
</a>
{% endif %}
<a href="{{ url_for('Tags.remove_tag') }}?paste={{ request.args.get('paste') }}&tag={{ tag[0] }}" class="btn btn-danger">
<span class="glyphicon glyphicon-trash "></span> Delete this Tag
</a>
</div>
</div>
</div>
</div>
{% endfor %}
<button type="button" class="btn btn-light btn-lg" data-toggle="modal" data-target="#mymodal" data-url="{{ url_for('Tags.taxonomies') }}">
<span class="glyphicon glyphicon-plus "></span>
</button>
</div>
</h2>
<table class="table table-condensed">
<thead>
<tr>
<th>Date</th>
<th>Source</th>
<th>Encoding</th>
<th>Language</th>
<th>Size (Kb)</th>
<th>Mime</th>
<th>Number of lines</th>
<th>Max line length</th>
</tr>
</thead>
<tbody>
<tr>
<td>{{ date }}</td>
<td>{{ source }}</td>
<td>{{ encoding }}</td>
<td>{{ language }}</td>
<td>{{ size }}</td>
<td>{{ mime }}</td>
<td>{{ lineinfo.0 }}</td>
<td>{{ lineinfo.1 }}</td>
</tr>
</tbody>
</table>
{% if item_parent %}
<div>
<a href="{{ url_for('showsavedpastes.showsavedpaste')}}?paste={{item_parent}}" target="_blank">{{item_parent}}</a>
</div>
{% endif %}
<div>
{% with obj_type='item', obj_id=request.args.get('paste'), obj_lvl=0%}
{% include 'import_export/block_add_user_object_to_export.html' %}
{% endwith %}
{% if hive %}
<button class="btn btn-primary btn-lg" data-toggle="modal" data-target="#myModal_hive">
Create
<img id="thehive-icon" src="{{url_for('static', filename='image/thehive_icon.png') }}">
Case
</button>
<!-- Modal HIVE-->
<div class="modal fade" id="myModal_hive" role="dialog">
<div class="modal-dialog">
<div class="modal-content">
<div class="modal-header text-center">
<img id="misp-logo" src="{{url_for('static', filename='image/thehive-logo.png') }}" width="500" >
</div>
<div class="modal-body">
<form method="post" action="{{ url_for('PasteSubmit.create_hive_case') }}" target="_blank">
<div class="input clear required">
<label for="EventThreatLevelId">Threat Level</label>
<select name="threat_level_hive" id="EventThreatLevelId" required="required">
<option value="1">High</option>
<option value="2" selected="selected">Medium</option>
<option value="3">Low</option>
</select>
</div>
<div class="input select required">
<label for="TLP">TLP</label>
<select name="hive_tlp" id="hive_tlp" required="required" class="selectpicker">
<option value="0">White</option>
<option value="1">Green</option>
<option value="2" selected="selected">Amber</option>
<option value="3">Red</option>
</select>
</div>
<div class="clear required">
<label for="hive_case_title">Title</label>
<input name="hive_case_title" class="form-control span6" placeholder="Title" type="text" id="hive_case_title" required="required"/>
</div>
<div class="clear required">
<label for="hive_description">Description</label>
<input name="hive_description" class="form-control span6" placeholder="Quick Case Description" type="text" id="hive_description" required="required"/>
</div>
<input type="hidden" id="paste" name="paste" value="{{ request.args.get('paste') }}">
{% if hive_caseid %}
<br>
<div class="list-group" id="misp_event">
<li class="list-group-item active">The Hive Case already Created</li>
<a target="_blank" href="{{ hive_url }}" class="list-group-item">{{ hive_url }}</a></div>
{% endif %}
</div>
<div class="modal-footer center">
<button href="" class="btn btn-primary">
<span class="glyphicon glyphicon-ok "></span> Create Case
</button>
</form>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
{% endif %}
</div>
<div>
<a href="{{ url_for('correlation.show_correlation')}}?object_type=paste&correlation_id={{ request.args.get('paste') }}&correlation_objects=paste" target="_blank" style="font-size: 15px">
<button class="btn btn-info"><i class="fa fa-search"></i> Show Paste Correlation
</button>
</a>
</div>
</div>
<div class="panel-body" id="panel-body">
{% if duplicate_list|length == 0 %}
{% else %}
<h3> Duplicate list: </h3>
<table id="tableDup" class="table table-striped table-bordered">
{% set i = 0 %}
<thead>
<tr>
<th>Hash type</th>
<th>Paste info</th>
<th>Date</th>
<th>Path</th>
<th>Action</th>
</tr>
</thead>
<tbody>
{% for dup_path in duplicate_list %}
<tr>
<td>{{ hashtype_list[loop.index - 1] }}</td>
<td>Similarity: {{ simil_list[loop.index - 1] }}%</td>
<td>{{ date_list[loop.index - 1] }}</td>
<td><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{ dup_path }}" id='dup_path'>{{ dup_path }}</a></td>
<td><a target="_blank" href="{{ url_for('showsavedpastes.showDiff') }}?s1={{ request.args.get('paste') }}&s2={{ dup_path }}" class="fa fa-columns" title="Show differences"></a></td>
</tr>
{% endfor %}
</tbody>
</table>
{% endif %}
{% if l_64|length != 0 %}
<h3> Hash files: </h3>
<table id="tableb64" class="red_table table table-striped table-bordered">
<thead>
<tr>
<th>estimated type</th>
<th>hash</th>
<th>saved_path</th>
<th>Virus Total</th>
</tr>
</thead>
<tbody>
{% for b64 in l_64 %}
<tr>
<td><i class="fa {{ b64[0] }}"></i>&nbsp;&nbsp;{{ b64[1] }}</td>
<td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?object_type=decoded&correlation_id={{ b64[2] }}&correlation_objects=paste">{{ b64[2] }}</a> ({{ b64[4] }})</td>
<td>{{ b64[3] }}</td>
<td style="text-align:center;">
{% if vt_enabled %}
{% if not b64[5] %}
<darkbutton_{{ b64[2] }}>
<button id="submit_vt_{{ b64[2] }}" class="btn btn-primary" onclick="sendFileToVT('{{ b64[2] }}')">
<i class="fa fa-paper-plane"></i>&nbsp;Send this file to VT
</button>
</darkbutton_{{ b64[2] }}>
{% else %}
<a class="btn btn-primary" target="_blank" href="{{ b64[6] }}"><i class="fa fa-link"> VT Report</i></a>
{% endif %}
<button class="btn btn-default" onclick="updateVTReport('{{ b64[2] }}')">
<div id="report_vt_{{ b64[2] }}"><span class="glyphicon glyphicon-refresh"></span>&nbsp;{{ b64[7] }}</div>
</button>
{% else %}
Virus Total submission is disabled
{% endif %}
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endif %}
{% if crawler_metadata['get_metadata'] %}
<div class="row">
<div class="col-md-5">
<div class="row">
<div class="panel panel-info">
<div class="panel-heading">
<i class="fa fa-eye-slash"></i> Crawled Paste
</div>
<table class="table table-hover table-striped">
<tbody>
<tr>
<td>Domain</td>
<td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td>
</tr>
<tr>
<td>Father</td>
<td><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{ crawler_metadata['paste_father'] }}" id='paste_father'>{{ crawler_metadata['paste_father'] }}</a></td>
</tr>
<tr>
<td>Source link</td>
<td>{{ crawler_metadata['real_link'] }}</td>
</tr>
</tbody>
</table>
{% if crawler_metadata['har_file'] %}
button
{% endif %}
</div>
</div>
</div>
<div class="col-md-7">
<div class="panel panel-info" style="text-align:center;">
<div class="panel-heading">
<div class="row">
<div class="col-md-8">
<input class="center" id="blocks" type="range" min="1" max="50" value="{%if tags_safe%}13{%else%}0{%endif%}">
</div>
<div class="col-md-4">
<button class="btn {%if tags_safe%}btn-primary{%else%}btn-danger{%endif%} btn-tags" onclick="blocks.value=50;pixelate();">
<span class="glyphicon glyphicon-zoom-in"></span>
<span class="label-icon">Full resolution</span>
</button>
</div>
</div>
</div>
</div>
<canvas id="canvas" style="width:100%;"></canvas>
</div>
</div>
{% endif %}
<h3> Content: </h3>
<a href="{{ url_for('showsavedpastes.showsavedrawpaste') }}?paste={{ request.args.get('paste') }}" id='raw_paste' > [Raw content] </a>
<p data-initsize="{{ initsize }}"> <pre id="paste-holder">{{ content }}</pre></p>
</div>
</div>
</body>
<script>
var ltags
var ltagsgalaxies
$(document).ready(function(){
$.getJSON("{{ url_for('Tags.get_all_tags_taxonomies') }}",
function(data) {
ltags = $('#ltags').tagSuggest({
data: data,
maxDropHeight: 200,
name: 'ltags'
});
});
$.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}",
function(data) {
ltagsgalaxies = $('#ltagsgalaxies').tagSuggest({
data: data,
maxDropHeight: 200,
name: 'ltagsgalaxies'
});
});
$('#tableDup').DataTable();
$('#tableb64').DataTable({
"aLengthMenu": [[5, 10, 15, -1], [5, 10, 15, "All"]],
"iDisplayLength": 5,
"order": [[ 1, "asc" ]]
});
});
</script>
<script>
function updateVTReport(hash) {
//updateReport
$.getJSON("{{ url_for('hashDecoded.update_vt_result') }}?hash="+hash,
function(data) {
content = '<span class="glyphicon glyphicon-refresh"></span> ' +data['report_vt']
$( "#report_vt_"+hash ).html(content);
});
}
function sendFileToVT(hash) {
//send file to vt
$.getJSON("{{ url_for('hashDecoded.send_file_to_vt_js') }}?hash="+hash,
function(data) {
var content = '<a id="submit_vt_'+hash+'" class="btn btn-primary" target="_blank" href="'+ data['vt_link'] +'"><i class="fa fa-link"> '+ ' VT Report' +'</i></a>';
$('#submit_vt_'+hash).remove();
$('darkbutton_'+hash).append(content);
});
}
</script>
<script>
jQuery("#all-tags-taxonomies").click(function(e){
//change input tags list
$.getJSON("{{ url_for('Tags.get_all_tags_taxonomies') }}",
function(data) {
ltags.setData(data)
});
});
</script>
<script>
jQuery("#all-tags-galaxies").click(function(e){
$.getJSON("{{ url_for('Tags.get_all_tags_galaxy') }}",
function(data) {
ltagsgalaxies.setData(data)
});
});
</script>
<script>
function addTags() {
var tags = ltags.getValue()
var tagsgalaxy = ltagsgalaxies.getValue()
var path = '{{ request.args.get('paste') }}'
window.location.replace("{{ url_for('tags_ui.add_tags') }}?tags=" + tags + "&tagsgalaxies=" + tagsgalaxy + "&object_id=" + path + "&object_type=item");
}
</script>
<script>
{% for taxo in active_taxonomies %}
jQuery("#{{ taxo }}-id{{ loop.index0 }}").click(function(e){
$.getJSON("{{ url_for('Tags.get_tags_taxonomie') }}?taxonomie={{ taxo }}",
function(data) {
ltags.setData(data)
});
});
{% endfor %}
</script>
<script>
{% for galaxy in active_galaxies %}
jQuery("#{{ galaxy }}-idgalax{{ loop.index0 }}").click(function(e){
$.getJSON("{{ url_for('Tags.get_tags_galaxy') }}?galaxy={{ galaxy }}",
function(data) {
ltagsgalaxies.setData(data)
});
});
{% endfor %}
</script>
{% if crawler_metadata['get_metadata'] %}
<script>
var ctx = canvas.getContext('2d'), img = new Image();
/// turn off image smoothing
ctx.webkitImageSmoothingEnabled = false;
ctx.imageSmoothingEnabled = false;
img.onload = pixelate;
img.addEventListener("error", img_error);
var draw_img = false;
img.src = "{{ url_for('showsavedpastes.screenshot', filename=crawler_metadata['screenshot']) }}";
function pixelate() {
/// use slider value
if( blocks.value == 50 ){
size = 1;
} else {
var size = (blocks.value) * 0.01;
}
canvas.width = img.width;
canvas.height = img.height;
/// cache scaled width and height
w = canvas.width * size;
h = canvas.height * size;
/// draw original image to the scaled size
ctx.drawImage(img, 0, 0, w, h);
/// pixelated
ctx.drawImage(canvas, 0, 0, w, h, 0, 0, canvas.width, canvas.height);
}
function img_error() {
img.onerror=null;
img.src="{{ url_for('static', filename='image/AIL.png') }}";
blocks.value = 50;
pixelate;
}
blocks.addEventListener('change', pixelate, false);
</script>
{% endif %}
</html>

View file

@ -22,7 +22,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
{{ dict_object["metadata"]["type_id"] }} {{ dict_object["metadata"]["type_id"] }}

View file

@ -30,7 +30,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
</td> </td>

View file

@ -32,7 +32,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
{{ dict_object["metadata"]["file_type"] }} {{ dict_object["metadata"]["file_type"] }}
@ -56,7 +56,7 @@
Tags: Tags:
{% for tag in dict_object["metadata"]['tags'] %} {% for tag in dict_object["metadata"]['tags'] %}
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal" <button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
data-tagid="{{ tag }}" data-objtype="decoded" data-objid="{{ dict_object["correlation_id"] }}"> data-tagid="{{ tag }}" data-objtype="decoded" data-objsubtype="" data-objid="{{ dict_object["correlation_id"] }}">
{{ tag }} {{ tag }}
</button> </button>
{% endfor %} {% endfor %}

View file

@ -21,7 +21,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
{{ dict_object["metadata"]["type_id"] }} {{ dict_object["metadata"]["type_id"] }}

View file

@ -22,7 +22,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
{{ dict_object["metadata"]["type_id"] }} {{ dict_object["metadata"]["type_id"] }}
@ -39,6 +39,16 @@
</div> </div>
</div> </div>
</li> </li>
{% if dict_object["metadata"]['tags'] %}
<li class="list-group-item py-0">
<div class="my-3">
<b>Tags:</b>
{% for tag in dict_object["metadata"]['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{% endfor %}
</div>
</li>
{% endif %}
</ul> </ul>
{% with obj_type='pgp', obj_id=dict_object['correlation_id'], obj_subtype=dict_object["metadata"]["type_id"] %} {% with obj_type='pgp', obj_id=dict_object['correlation_id'], obj_subtype=dict_object["metadata"]["type_id"] %}

View file

@ -68,7 +68,7 @@
Tags: Tags:
{% for tag in dict_object["metadata"]['tags'] %} {% for tag in dict_object["metadata"]['tags'] %}
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal" <button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
data-tagid="{{ tag }}" data-objtype="image" data-objid="{{ dict_object["correlation_id"] }}"> data-tagid="{{ tag }}" data-objtype="screenshot" data-objsubtype="" data-objid="{{ dict_object["correlation_id"] }}">
{{ tag }} {{ tag }}
</button> </button>
{% endfor %} {% endfor %}
@ -99,7 +99,7 @@ img.onload = pixelate;
img.addEventListener("error", img_error); img.addEventListener("error", img_error);
var draw_img = false; var draw_img = false;
img.src = "{{ url_for('showsavedpastes.screenshot', filename=dict_object['metadata']['img']) }}"; img.src = "{{ url_for('objects_item.screenshot', filename=dict_object['metadata']['img']) }}";
function pixelate() { function pixelate() {
/// use slider value /// use slider value

View file

@ -22,7 +22,7 @@
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="orange"></circle> <circle cx="13" cy="13" r="13" fill="orange"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["icon_class"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon_text"] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_object["metadata_card"]["icon"]["style"] }}" font-size="16px">{{ dict_object["metadata_card"]["icon"]["icon"] }}</text>
</g> </g>
</svg> </svg>
{{ dict_object["metadata"]["type_id"] }} {{ dict_object["metadata"]["type_id"] }}

View file

@ -511,7 +511,7 @@ if (d.popover) {
desc = desc + "</dl>" desc = desc + "</dl>"
if (data["img"]) { if (data["img"]) {
desc = desc + "<img src={{ url_for('showsavedpastes.screenshot', filename="") }}" + data["img"] +" class=\"img-thumbnail\" />"; desc = desc + "<img src={{ url_for('objects_item.screenshot', filename="") }}" + data["img"] +" class=\"img-thumbnail\" />";
} }
if (data["tags"]) { if (data["tags"]) {

View file

@ -11,8 +11,8 @@
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script> <script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script> <script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script> <script src="{{ url_for('static', filename='js/bootstrap4.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script> <script src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script> <script src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<style> <style>
.card-columns { .card-columns {
@ -217,7 +217,7 @@ blocks.addEventListener('change', pixelate_all, false);
{% for dict_domain in dict_data['list_elem'] %} {% for dict_domain in dict_data['list_elem'] %}
{% if 'screenshot' in dict_domain %} {% if 'screenshot' in dict_domain %}
{% if dict_domain['is_tags_safe'] %} {% if dict_domain['is_tags_safe'] %}
var screenshot_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}{{dict_domain['screenshot']}}"; var screenshot_url = "{{ url_for('objects_item.screenshot', filename="") }}{{dict_domain['screenshot']}}";
{% else %} {% else %}
var screenshot_url = "{{ url_for('static', filename='image/AIL.png') }}"; var screenshot_url = "{{ url_for('static', filename='image/AIL.png') }}";
{% endif %} {% endif %}

View file

@ -85,7 +85,7 @@
<div> <div>
{% include 'modals/edit_tag.html' %} {% include 'modals/edit_tag.html' %}
{% for tag in dict_domain['tags'] %} {% for tag in dict_domain['tags'] %}
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal" data-tagid="{{ tag }}" data-objtype="domain" data-objid="{{ dict_domain['domain'] }}"> <button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal" data-tagid="{{ tag }}" data-objtype="domain" data-objsubtype="" data-objid="{{ dict_domain['domain'] }}">
{{ tag }} {{ tag }}
</button> </button>
{% endfor %} {% endfor %}
@ -526,7 +526,7 @@ function toggle_sidebar(){
<script> <script>
var ctx = canvas.getContext('2d'), img = new Image(); var ctx = canvas.getContext('2d'), img = new Image();
var base_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}"; var base_url = "{{ url_for('objects_item.screenshot', filename="") }}";
var screenshot_href = "{{ url_for('objects_item.showItem') }}?id="; var screenshot_href = "{{ url_for('objects_item.showItem') }}?id=";
/// turn off image smoothing /// turn off image smoothing

View file

@ -174,7 +174,7 @@ blocks.addEventListener('change', pixelate_all, false);
{% for dict_domain in l_dict_domains['list_elem'] %} {% for dict_domain in l_dict_domains['list_elem'] %}
{% if 'screenshot' in dict_domain %} {% if 'screenshot' in dict_domain %}
{% if dict_domain['is_tags_safe'] %} {% if dict_domain['is_tags_safe'] %}
var screenshot_url = "{{ url_for('showsavedpastes.screenshot', filename="") }}{{dict_domain['screenshot']}}"; var screenshot_url = "{{ url_for('objects_item.screenshot', filename="") }}{{dict_domain['screenshot']}}";
{% else %} {% else %}
var screenshot_url = "{{ url_for('static', filename='image/AIL.png') }}"; var screenshot_url = "{{ url_for('static', filename='image/AIL.png') }}";
{% endif %} {% endif %}

View file

@ -84,8 +84,8 @@
<div class="my-1"> <div class="my-1">
<svg height="26" width="26"> <svg height="26" width="26">
<g class="nodes"> <g class="nodes">
<circle cx="13" cy="13" r="13" fill="{{ dict_obj['node']['node_color'] }}"></circle> <circle cx="13" cy="13" r="13" fill="{{ dict_obj['node']['color'] }}"></circle>
<text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_obj['node']['icon_class'] }}" font-size="16px">{{ dict_obj['node']['icon_text'] }}</text> <text x="13" y="13" text-anchor="middle" dominant-baseline="central" class="graph_node_icon {{ dict_obj['node']['style'] }}" font-size="16px">{{ dict_obj['node']['icon'] }}</text>
</g> </g>
</svg> </svg>
</div> </div>

View file

@ -126,6 +126,6 @@ jQuery("#all-tags-galaxies").click(function(e){
function addTags() { function addTags() {
var tags = ltags.getValue() var tags = ltags.getValue()
var tagsgalaxy = ltagsgalaxies.getValue() var tagsgalaxy = ltagsgalaxies.getValue()
window.location.replace("{{ url_for('tags_ui.add_tags') }}?tags=" + tags + "&tagsgalaxies=" + tagsgalaxy + "&object_id={{ modal_add_tags['object_id'] }}&object_type={{ modal_add_tags['object_type'] }}"); window.location.replace("{{ url_for('tags_ui.add_tags') }}?tags=" + tags + "&tagsgalaxies=" + tagsgalaxy + "&type={{ modal_add_tags['object_type'] }}&subtype={{ modal_add_tags['object_subtype'] }}&id={{ modal_add_tags['object_id'] }}");
} }
</script> </script>

View file

@ -15,17 +15,17 @@
<div class="modal-footer"> <div class="modal-footer">
<!-- <a href="{{ url_for('Tags.tag_validation') }}?object_id=rrrrr&tag=eeeeee&status=tp" class="btn btn-outline-success mr-0" data-toggle="tooltip" title="Good Detection"> <!-- <a href="?object_id=rrrrr&tag=eeeeee&status=tp" class="btn btn-outline-success mr-0" data-toggle="tooltip" title="Good Detection">
<i class="fas fa-thumbs-up"></i> <i class="fas fa-thumbs-up"></i>
</a> </a>
<a href="{{ url_for('Tags.tag_validation') }}?object_id=rrrrr&tag=eeeeee&status=fp" class="btn btn-outline-danger mr-auto" data-toggle="tooltip" title="Bad Detection"> <a href="?object_id=rrrrr&tag=eeeeee&status=fp" class="btn btn-outline-danger mr-auto" data-toggle="tooltip" title="Bad Detection">
<i class="fas fa-thumbs-down"></i> <i class="fas fa-thumbs-down"></i>
</a>
<a href="{{ url_for('Tags.confirm_tag') }}?object_id=rrrrr&tag=eeeeee" class="btn btn-primary">
<i class="fas fa-check"></i> Confirm this Tag
</a> --> </a> -->
<a href="#" class="btn btn-primary" id="modal_tag_confirm">
<i class="fas fa-check"></i> Confirm this Tag
</a>
<a href="#" class="btn btn-danger" id="modal_tag_edit_delete_tag"> <a href="#" class="btn btn-danger" id="modal_tag_edit_delete_tag">
<i class="fas fa-trash-alt"></i> Delete this Tag <i class="fas fa-trash-alt"></i> Delete this Tag
</a> </a>
@ -38,12 +38,19 @@
<script> <script>
// tagid + objtype + objid // tagid + objtype + objid
$('#edit_tags_modal').on('show.bs.modal', function (event) { $('#edit_tags_modal').on('show.bs.modal', function (event) {
var tag_confirm = $('#modal_tag_confirm');
tag_confirm.hide();
var button = $(event.relatedTarget); var button = $(event.relatedTarget);
var tagid = button.data('tagid') var tagid = button.data('tagid')
var objtype = button.data('objtype') var objtype = button.data('objtype')
var objsubtype = button.data('objsubtype')
var objid = button.data('objid') var objid = button.data('objid')
var modal = $(this) var modal = $(this)
modal.find('#modal_tag_edit_tag_id').text(tagid) modal.find('#modal_tag_edit_tag_id').text(tagid)
modal.find('#modal_tag_edit_delete_tag').prop("href", "{{ url_for('tags_ui.delete_tag') }}?object_type="+ objtype +"&object_id="+ objid +"&tag="+ tagid) if (tagid.startsWith("infoleak:automatic-detection")) {
tag_confirm.show();
modal.find('#modal_tag_confirm').prop("href", "{{ url_for('tags_ui.tag_confirm') }}?type="+ objtype +"&subtype="+ objsubtype +"&id="+ objid +"&tag="+ tagid);
}
modal.find('#modal_tag_edit_delete_tag').prop("href", "{{ url_for('tags_ui.delete_tag') }}?object_type="+ objtype +"&object_id="+ objid +"&tag="+ tagid);
}) })
</script> </script>

View file

@ -69,13 +69,10 @@ function get_html_and_update_modal(event, truemodal) {
event.preventDefault(); event.preventDefault();
var modal=truemodal; var modal=truemodal;
var url = " {{ url_for('showsavedpastes.showpreviewpaste') }}?paste=" + modal.attr('data-path') + "&num=" + modal.attr('data-num'); var url = " {{ url_for('objects_item.item_preview') }}?id=" + modal.attr('data-path');
last_clicked_paste = modal.attr('data-num');
$.get(url, function (data) { $.get(url, function (data) {
// verify that the reveived data is really the current clicked item. Otherwise, ignore it. // verify that the reveived data is really the current clicked item. Otherwise, ignore it.
var received_num = parseInt(data.split("|num|")[1]);
if (received_num == last_clicked_paste && can_change_modal_content) {
can_change_modal_content = false; can_change_modal_content = false;
// clear data by removing html, body, head tags. prevent dark modal background stack bug. // clear data by removing html, body, head tags. prevent dark modal background stack bug.
@ -97,7 +94,7 @@ function get_html_and_update_modal(event, truemodal) {
// On click, donwload all item's content // On click, donwload all item's content
$("#load-more-button").on("click", function (event) { $("#load-more-button").on("click", function (event) {
if (complete_item == null) { //Donwload only once if (complete_item == null) { //Donwload only once
$.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){ $.get("{{ url_for('objects_item.item_content_more') }}"+"?id="+$(modal).attr('data-path'), function(data, status){
complete_item = data; complete_item = data;
update_preview(); update_preview();
}); });
@ -105,9 +102,6 @@ function get_html_and_update_modal(event, truemodal) {
update_preview(); update_preview();
} }
}); });
} else if (can_change_modal_content) {
$("#modal_show_min_item_body").html("Ignoring previous not finished query of item #" + received_num);
}
}); });
} }

View file

@ -66,11 +66,11 @@
<div> <div>
<h5> <h5>
<div> <span>
{% include 'modals/edit_tag.html' %} {% include 'modals/edit_tag.html' %}
{% for tag in meta['tags'] %} {% for tag in meta['tags'] %}
<button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal" <button class="btn btn-{{ bootstrap_label[loop.index0 % 5] }}" data-toggle="modal" data-target="#edit_tags_modal"
data-tagid="{{ tag }}" data-objtype="item" data-objid="{{ meta['id'] }}"> data-tagid="{{ tag }}" data-objtype="item" data-objsubtype="" data-objid="{{ meta['id'] }}">
{{ tag }} {{ tag }}
</button> </button>
@ -80,7 +80,7 @@
<button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal"> <button type="button" class="btn btn-light" data-toggle="modal" data-target="#add_tags_modal">
<i class="far fa-plus-square"></i> <i class="far fa-plus-square"></i>
</button> </button>
</div> </span>
</h5> </h5>
</div> </div>
@ -200,7 +200,7 @@
</a> </a>
</td> </td>
<td> <td>
<a target="_blank" href="{{ url_for('showsavedpastes.showDiff') }}?s1={{meta['id']}}&s2={{duplicate_id}}" class="fa fa-columns" title="Show diff"></a> <a target="_blank" href="{{ url_for('objects_item.object_item_diff') }}?id1={{meta['id']}}&id2={{duplicate_id}}" class="fa fa-columns" title="Show diff"></a>
</td> </td>
</tr> </tr>
{% endfor %} {% endfor %}
@ -362,7 +362,7 @@
<a class="nav-link dropdown-toggle" data-toggle="dropdown" href="#">Others</a> <a class="nav-link dropdown-toggle" data-toggle="dropdown" href="#">Others</a>
<div class="dropdown-menu"> <div class="dropdown-menu">
<a class="dropdown-item" href="{{ url_for('objects_item.item_raw_content', id=meta['id']) }}"><i class="far fa-file"></i> &nbsp;Raw Content</a> <a class="dropdown-item" href="{{ url_for('objects_item.item_raw_content', id=meta['id']) }}"><i class="far fa-file"></i> &nbsp;Raw Content</a>
<a class="dropdown-item" href="{{ url_for('objects_item.item_download', id=meta['id']) }}"><i class="fas fa-download"></i> &nbsp;Download</i></a> <a class="dropdown-item" href="{{ url_for('objects_item.item_download', id=meta['id']) }}"><i class="fas fa-download"></i> &nbsp;Download</a>
</div> </div>
</li> </li>
</ul> </ul>
@ -416,7 +416,11 @@
img.addEventListener("error", img_error); img.addEventListener("error", img_error);
var draw_img = false; var draw_img = false;
img.src = "{{ url_for('showsavedpastes.screenshot', filename=meta['crawler']['screenshot']) }}"; {% if meta['crawler']['screenshot'] %}
img.src = "{{ url_for('objects_item.screenshot', filename=meta['crawler']['screenshot']) }}";
{% else %}
img.src = "";
{% endif %}
function pixelate() { function pixelate() {

View file

@ -10,7 +10,7 @@
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet"> <link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet"> <link href="{{ url_for('static', filename='css/dataTables.bootstrap4.min.css') }}" rel="stylesheet">
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script> <script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script> <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script> <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js') }}"></script> <script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js') }}"></script>
@ -34,7 +34,7 @@
<div class="card mb-2"> <div class="card mb-2">
<div class="card-header bg-dark"> <div class="card-header bg-dark">
<h3 class="text-white text-center" >{{ item_info['name'] }}</h3> <h3 class="text-white text-center" >{{ meta['id'] }}</h3>
</div> </div>
<div class="card-body pb-1"> <div class="card-body pb-1">
<table class="table table-condensed table-responsive"> <table class="table table-condensed table-responsive">
@ -50,19 +50,19 @@
</thead> </thead>
<tbody> <tbody>
<tr> <tr>
<td>{{ item_basic_info['date'] }}</td> <td>{{ meta['date'] }}</td>
<td>{{ item_basic_info['source'] }}</td> <td>{{ meta['source'] }}</td>
<td>{{ item_basic_info['encoding'] }}</td> <td>{{ meta['mimetype'] }}</td>
<td>{{ item_basic_info['size'] }}</td> <td>{{ meta['size'] }}</td>
<td>{{ item_basic_info['nb_lines'] }}</td> <td>{{ meta['lines']['nb'] }}</td>
<td>{{ item_basic_info['max_length_line'] }}</td> <td>{{ meta['lines']['max_length'] }}</td>
</tr> </tr>
</tbody> </tbody>
</table> </table>
<div> <div>
<h5> <h5>
{% for tag in item_info['tags'] %} {% for tag in meta['tags'] %}
<span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span> <span class="badge badge-{{ bootstrap_label[loop.index0 % 5] }}">{{ tag }}</span>
{% endfor %} {% endfor %}
</h5> </h5>
@ -71,9 +71,9 @@
</div> </div>
</div> </div>
{% if item_parent %} {% if meta['parent'] %}
<div class="list-group" id="item_parent"> <div class="list-group" id="item_parent">
<a href="{{ url_for('objects_item.showItem')}}?id={{item_parent}}" target="_blank">{{item_parent}}</a> parent: <a href="{{ url_for('objects_item.showItem')}}?id={{meta['parent']}}" target="_blank">{{meta['parent']}}</a>
</div> </div>
{% endif %} {% endif %}
@ -91,13 +91,13 @@
</div> </div>
{% endif %} {% endif %}
{% if item_info['nb_duplictates'] != 0 %} {% if meta['duplicates'] != 0 %}
<div id="accordionDuplicate" class="mb-2"> <div id="accordionDuplicate" class="mb-2">
<div class="card"> <div class="card">
<div class="card-header py-1" id="headingDuplicate"> <div class="card-header py-1" id="headingDuplicate">
<div class="my-1"> <div class="my-1">
<i class="far fa-clone"></i> duplicates&nbsp;&nbsp; <i class="far fa-clone"></i> duplicates&nbsp;&nbsp;
<div class="badge badge-warning">{{item_info['nb_duplictates']}}</div> <div class="badge badge-warning">{{meta['duplicates']}}</div>
</div> </div>
</div> </div>
</div> </div>
@ -105,52 +105,43 @@
{% endif %} {% endif %}
{% if l_64|length != 0 %} {% if meta['nb_correlations'] %}
<div id="accordionDecoded" class="mb-3"> <div id="accordionCorrelation" class="mb-2">
<div class="card"> <div class="card">
<div class="card-header py-1" id="headingDecoded"> <div class="card-header py-1" id="headingCorrelation">
<div class="row"> <div class="my-1">
<div class="col-11"> <h5>
<div class="mt-2"> {% for obj_type in meta['nb_correlations'] %}
<i class="fas fa-lock-open"></i> Decoded Files&nbsp;&nbsp; {% if meta['nb_correlations'][obj_type] %}
<div class="badge badge-warning">{{l_64|length}}</div> <span class="badge badge-dark">
</div> {% if obj_type=='cve' %}
</div> <i class="fas fa-bug"></i>
<div class="col-1"> {% elif obj_type=='cryptocurrency' %}
<button class="btn btn-link py-2 rotate" data-toggle="collapse" data-target="#collapseDecoded" aria-expanded="true" aria-controls="collapseDecoded"> <i class="fas fa-coins"></i>
<i class="fas fa-chevron-circle-down"></i> {% elif obj_type=='decoded' %}
</button> <i class="fas fa-lock-open"></i>
</div> {% elif obj_type=='pgp' %}
</div> <i class="fas fa-key"></i>
</div> {% elif obj_type=='username' %}
<i class="fas fa-user"></i>
<div id="collapseDecoded" class="collapse show" aria-labelledby="headingDecoded" data-parent="#accordionDecoded"> {% elif obj_type=='screenshot' %}
<div class="card-body"> <i class="fas fa-image"></i>
{% else %}
<table id="tableb64" class="red_table table table-striped"> <i class="fas fa-cube"></i>
<thead> {% endif %}
<tr> {{ obj_type }}&nbsp;
<th>estimated type</th> <div class="badge badge-warning">{{meta['nb_correlations'][obj_type]}}</div>
<th>hash</th> </span>
</tr> {% endif %}
</thead>
<tbody>
{% for b64 in l_64 %}
<tr>
<td><i class="fas {{ b64[0] }}"></i>&nbsp;&nbsp;{{ b64[1] }}</td>
<td><a target="_blank" href="{{ url_for('correlation.show_correlation') }}?object_type=decoded&correlation_id={{ b64[2] }}&correlation_objects=paste"</a> {{b64[2]}} ({{ b64[4] }})</td>
</tr>
{% endfor %} {% endfor %}
</tbody> </h5>
</table>
</div> </div>
</div> </div>
</div> </div>
</div> </div>
{% endif %} {% endif %}
{% if crawler_metadata['get_metadata'] %} {% if 'crawler' in meta %}
<div id="accordionCrawler" class="mb-3"> <div id="accordionCrawler" class="mb-3">
<div class="card"> <div class="card">
<div class="card-header py-1" id="headingCrawled"> <div class="card-header py-1" id="headingCrawled">
@ -175,15 +166,15 @@
<tbody> <tbody>
<tr> <tr>
<td>Domain</td> <td>Domain</td>
<td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ crawler_metadata['domain'] }}" id='domain'>{{ crawler_metadata['domain'] }}</a></td> <td><a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{ meta['crawler']['domain'] }}" id='domain'>{{ meta['crawler']['domain'] }}</a></td>
</tr> </tr>
<tr> <tr>
<td>Father</td> <td>Father</td>
<td><a target="_blank" href="{{ url_for('objects_item.showItem') }}?id={{ crawler_metadata['paste_father'] }}" id='paste_father'>{{ crawler_metadata['paste_father'] }}</a></td> <td><a target="_blank" href="{{ url_for('objects_item.showItem') }}?id={{ meta['parent'] }}" id='paste_father'>{{ meta['parent'] }}</a></td>
</tr> </tr>
<tr> <tr>
<td>Url</td> <td>Url</td>
<td>{{ crawler_metadata['real_link'] }}</td> <td>{{ meta['crawler']['url'] }}</td>
</tr> </tr>
</tbody> </tbody>
</table> </table>
@ -229,7 +220,7 @@
</div> </div>
<p class="my-0" data-initsize="{{ initsize }}"> <pre id="paste-holder" class="border">{{ content }}</pre></p> <p class="my-0" data-initsize="{{ initsize }}"> <pre id="paste-holder" class="border">{{ meta['content'] }}</pre></p>
<script> <script>
var ltags var ltags
@ -247,7 +238,7 @@
}); });
</script> </script>
{% if crawler_metadata['get_metadata'] %} {% if 'crawler' in meta %}
<script> <script>
var ctx = canvas.getContext('2d'), img = new Image(); var ctx = canvas.getContext('2d'), img = new Image();
@ -259,7 +250,7 @@
img.addEventListener("error", img_error); img.addEventListener("error", img_error);
var draw_img = false; var draw_img = false;
img.src = "{{ url_for('showsavedpastes.screenshot', filename=crawler_metadata['screenshot']) }}"; img.src = "{{ url_for('objects_item.screenshot', filename=meta['crawler']['screenshot']) }}";
function pixelate() { function pixelate() {

View file

@ -0,0 +1,123 @@
<!DOCTYPE html>
<html>
<head>
<title>Galaxies - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'tags/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<h3 class="mt-2 text-secondary">
<i class="fas fa-rocket"></i> Galaxies:
</h3>
<table class="table table-bordered table-striped table-hover" id="myTable_">
<thead class="thead-dark">
<tr>
<th>Name</th>
<th>Description</th>
<th>Namespace</th>
<th>Enabled</th>
<th>Active Tags</th>
<th></th>
</tr>
</thead>
<tbody>
{% for galaxy in galaxies %}
<tr>
<td><i class="{{ galaxy['icon'] }}"></i> {{ galaxy['name'] }}</td>
<td>{{ galaxy['description'] }}</td>
<td>{{ galaxy['namespace'] }}</td>
<td class="text-center">
{% if galaxy['enebled'] %}
<div style="color:Green;">
<i class="fas fa-check-circle fa-2x"></i>
</div>
{% else %}
<div style="color:Red;">
<i class="fas fa-times-circle fa-2x"></i>
</div>
{% endif %}
</td>
<td>
<h5>
<span class="badge badge-dark">
<b>{{ galaxy['nb_active_tags'] }}</b>/{{ galaxy['nb_tags'] }}
</span>
</h5>
</td>
<td>
<a class="btn btn-outline-secondary" href="{{ url_for('tags_ui.tags_galaxy') }}?galaxy={{ galaxy['type'] }}">
<i class="fas fa-tag"></i>
</a>
</td>
</tr>
{% endfor %}
</tbody>
</table>
<br>
<br>
<br>
<br>
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$("#navbar-tags").addClass("active");
$("#nav_tags_config").removeClass("text-muted");
$("#nav_galaxies").addClass("active");
$('#myTable_').DataTable({ "lengthMenu": [ 5, 10, 25, 50, 100 ], "pageLength": 15, "order": [[ 0, "asc" ]] });
});
</script>
<script>
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</html>

View file

@ -0,0 +1,219 @@
<!DOCTYPE html>
<html>
<head>
<title>Galaxy - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'tags/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="my-3">
<a href="{{ url_for('tags_ui.tags_galaxies') }}" class="pull-left">
<i class="fa fa-arrow-left fa"></i> List Galaxies
</a>
</div>
<div class="card">
<div class="card-header bg-secondary text-white">
<div class="d-flex">
<span class="h3">{{ galaxy['name'] }}</span>
<span class="h5 ml-auto">
{% if galaxy['enabled'] %}
<span class="badge badge-success">Enabled</span>
{% else %}
<span class="badge badge-danger">Disabled</span>
{% endif %}
</span>
</div>
</div>
<div class="card-body">
<table class="table table-sm table-hover">
<tbody>
<tr>
<td><b>Description</b></td>
<td>{{ galaxy['description'] }}</td>
</tr>
{% if galaxy['namespace'] %}
<tr>
<td><b>Namespace</b></td>
<td>{{ galaxy['namespace'] }}</td>
</tr>
{% endif %}
{% if galaxy['category'] %}
<tr>
<td><b>Category</b></td>
<td>{{ galaxy['category'] }}</td>
</tr>
{% endif %}
{% if galaxy['type'] %}
<tr>
<td><b>Type</b></td>
<td>{{ galaxy['type'] }}</td>
</tr>
{% endif %}
{% if galaxy['source'] %}
<tr>
<td><b>Source</b></td>
<td>{{ galaxy['source'] }}</td>
</tr>
{% endif %}
<tr>
<td><b>Version</b></td>
<td>{{ galaxy['version'] }}</td>
</tr>
</tbody>
</table>
<div class="mt-4">
{% if galaxy['enabled'] %}
<a href="{{ url_for('tags_ui.galaxy_disable') }}?galaxy={{ galaxy['type'] }}" class="btn btn-danger">
<i class="fas fa-times"></i> Disable Galaxy
</a>
{% else %}
<a href="{{ url_for('tags_ui.galaxy_enable') }}?galaxy={{ galaxy['type'] }}" class="btn btn-success">
<i class="fas fa-check"></i> Enable Galaxy
</a>
{% endif %}
</div>
</div>
</div>
<form action="{{ url_for('tags_ui.galaxy_enable_tags') }}" id="checkboxForm">
<input type="hidden" value="{{ galaxy['type'] }}" name="galaxy">
<table class="table table-bordered table-hover my-4">
<thead class="bg-dark text-white">
<tr>
<th></th>
<th>Tag</th>
<th>Description</th>
<th class="text-center">Enabled</th>
</tr>
</thead>
<tbody>
{% for tag in galaxy['tags'] %}
<tr>
<td class="text-center">
<div class="custom-control custom-switch">
<input class="custom-control-input" type="checkbox" name="tags" id="{{ tag['tag'] }}" value="{{ tag['tag'] }}" {% if tag['enabled'] %}checked{% endif %}>
<label class="custom-control-label" for="{{ tag['tag'] }}"></label>
</div>
</td>
<td>
<a href="{{ url_for('tags_ui.tags_galaxy_tag') }}?galaxy={{ galaxy['type'] }}&tag={{ tag['tag'] }}" class="badge badge-info">{{ tag['tag'] }}</a>
{% if tag['synonyms'] %}
<div>
Synonym: <i>
{% for synonym in tag['synonyms'] %}
{{ synonym }},
{% endfor %}
</i>
</div>
{% endif %}
</td>
<td>
{% if 'description' in tag %}
{{ tag['description'] }}
{% endif %}
</td>
<td>
<div class="text-center">
{% if tag['enabled'] %}
<div style="color:Green;">
<i class="fas fa-check-circle"></i>
</div>
{% else %}
<div style="color:Red;">
<i class="fas fa-times-circle"></i>
</div>
{% endif %}
</div>
</td>
</tr>
{% endfor %}
</tbody>
</table>
<div class="input-group-btn">
<button class="btn btn-primary btn-lg" onclick="submitEnabledTags()">
<i class="fas fa-check-square"></i> Update Tags
</button>
</div>
</form>
<br><br><br><br>
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$("#navbar-tags").addClass("active");
$("#nav_tags_config").removeClass("text-muted");
table = $('#myTable_').DataTable(
{
"aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
"iDisplayLength": 10,
"order": [[ 3, "desc" ]]
}
);
});
</script>
<script>
function submitEnabledTags() {
table.destroy();
table = $('#myTable_').DataTable(
{
"iDisplayLength": -1,
}
);
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</html>

View file

@ -0,0 +1,146 @@
<!DOCTYPE html>
<html>
<head>
<title>Galaxy - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'tags/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="my-3">
<a href="{{ url_for('tags_ui.tags_galaxy') }}?galaxy={{ galaxy }}" class="pull-left">
<i class="fa fa-arrow-left fa"></i> {{ galaxy }} Galaxy
</a>
</div>
<div class="card">
<div class="card-header bg-secondary text-white">
<div class="d-flex">
<span class="h3">
{{ tag['value'] }}
</span>
<span class="h5 ml-auto">
{% if tag['enabled'] %}
<span class="badge badge-success">Enabled</span>
{% else %}
<span class="badge badge-danger">Disabled</span>
{% endif %}
</span>
</div>
</div>
<div class="card-body">
<h3><div class="mb-4">
<span class="badge badge-primary">{{ tag['tag'] }}</span>
</div></h3>
<table class="table table-sm table-hover">
<tbody>
{% if 'description' in tag %}
<tr>
<td><b>Description</b></td>
<td>{{ tag['description'] }}</td>
</tr>
{% endif %}
{% if 'refs' in tag %}
<tr>
<td><b>Refs</b></td>
<td>{{ tag['refs'] }}</td>
</tr>
{% endif %}
</tbody>
</table>
{% if 'meta' in tag %}
<hr>
<b>Meta</b>
<pre class="border">{{ tag['meta'] }}</pre>
{% endif %}
{# <div class="mt-4">#}
{# {% if galaxy['enabled'] %}#}
{# <a href="{{ url_for('tags_ui.galaxy_disable') }}?galaxy={{ galaxy['type'] }}" class="btn btn-danger">#}
{# <i class="fas fa-times"></i> Disable Galaxy#}
{# </a>#}
{# {% else %}#}
{# <a href="{{ url_for('tags_ui.galaxy_enable') }}?galaxy={{ galaxy['type'] }}" class="btn btn-success">#}
{# <i class="fas fa-check"></i> Enable Galaxy#}
{# </a>#}
{# {% endif %}#}
{# </div>#}
</div>
</div>
<br><br><br><br>
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$("#navbar-tags").addClass("active");
$("#nav_tags_config").removeClass("text-muted");
table = $('#myTable_').DataTable(
{
"aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
"iDisplayLength": 10,
"order": [[ 3, "desc" ]]
}
);
});
</script>
<script>
function submitEnabledTags() {
table.destroy();
table = $('#myTable_').DataTable(
{
"iDisplayLength": -1,
}
);
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</html>

View file

@ -29,26 +29,26 @@
</a> </a>
</li> </li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="{{ url_for('tags_ui.tags_search_images') }}" id="nav_tags_search_image"> <a class="nav-link" href="{{ url_for('tags_ui.tags_search_screenshot') }}" id="nav_tags_search_screenshot">
<i class="fas fa-image"></i> <i class="fas fa-image"></i>
Search Images by Tags Search Screenshots by Tags
</a> </a>
</li> </li>
</ul> </ul>
<h5 class="d-flex text-muted w-100"> <h5 class="d-flex text-muted w-100" id="nav_tags_config">
<span>Tags Management </span> <span>Tags Management </span>
</h5> </h5>
<ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100"> <ul class="nav flex-md-column flex-row navbar-nav justify-content-between w-100">
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="{{ url_for('Tags.taxonomies') }}" id="nav_taxonomies"> <a class="nav-link" href="{{ url_for('tags_ui.tags_taxonomies') }}" id="nav_taxonomies">
<i class="fas fa-wrench"></i> <i class="fas fa-tags"></i>
Edit Taxonomies List Taxonomies
</a> </a>
</li> </li>
<li class="nav-item"> <li class="nav-item">
<a class="nav-link" href="{{ url_for('Tags.galaxies') }}" id="nav_onion_galaxies"> <a class="nav-link" href="{{ url_for('tags_ui.tags_galaxies') }}" id="nav_galaxies">
<i class="fas fa-rocket"></i> <i class="fas fa-rocket"></i>
Edit Galaxies List Galaxies
</a> </a>
</li> </li>
</ul> </ul>

View file

@ -61,7 +61,7 @@
<th>Last seen</th> <th>Last seen</th>
<th>Nb seen</th> <th>Nb seen</th>
<th>Size</th> <th>Size</th>
{%elif dict_tagged["object_type"]=="image"%} {%elif dict_tagged["object_type"]=="screenshot"%}
<th style="max-width: 800px;">Image</th> <th style="max-width: 800px;">Image</th>
{%elif dict_tagged["object_type"]=="item"%} {%elif dict_tagged["object_type"]=="item"%}
<th>Date</th> <th>Date</th>
@ -106,7 +106,7 @@
</tr> </tr>
{% endfor %} {% endfor %}
{%elif dict_tagged["object_type"]=="image"%} {%elif dict_tagged["object_type"]=="screenshot"%}
{% for dict_obj in dict_tagged["tagged_obj"] %} {% for dict_obj in dict_tagged["tagged_obj"] %}
<tr> <tr>
<td class="pb-0"> <td class="pb-0">
@ -141,7 +141,7 @@
</div> </div>
</td> </td>
<td class="pb-0"> <td class="pb-0">
<button type="button" class="btn btn-light" data-num="{{ loop.index0 + 1 }}" data-toggle="modal" data-target="#modal_show_min_item" data-url="{{ url_for('showsavedpastes.showsaveditem_min') }}?paste={{ dict_obj['id'] }}" data-path="{{ dict_obj['id'] }}"> <button type="button" class="btn btn-light" data-num="{{ loop.index0 + 1 }}" data-toggle="modal" data-target="#modal_show_min_item" data-url="{{ url_for('objects_item.item_preview') }}?id={{ dict_obj['id'] }}" data-path="{{ dict_obj['id'] }}">
<span class="fas fa-search-plus"></span> <span class="fas fa-search-plus"></span>
</button> </button>

View file

@ -0,0 +1,123 @@
<!DOCTYPE html>
<html>
<head>
<title>Taxonomies - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'tags/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<h3 class="mt-2 text-secondary">
<i class="fas fa-tags"></i> Taxonomies:
</h3>
<table class="table table-bordered table-striped table-hover" id="myTable_">
<thead class="thead-dark">
<tr>
<th>Namespace</th>
<th>Description</th>
<th>Version</th>
<th>Enabled</th>
<th>Active Tags</th>
<th></th>
</tr>
</thead>
<tbody>
{% for taxonomy in taxonomies %}
<tr>
<td>{{ taxonomies[taxonomy]['name'] }}</td>
<td>{{ taxonomies[taxonomy]['description'] }}</td>
<td>{{ taxonomies[taxonomy]['version'] }}</td>
<td>
{% if taxonomies[taxonomy]['enabled'] %}
<div style="color:Green;">
<i class="fas fa-check-circle fa-2x"></i>
</div>
{% else %}
<div style="color:Red;">
<i class="fas fa-times-circle fa-2x"></i>
</div>
{% endif %}
</td>
<td>
<h5>
<span class="badge badge-dark">
<b>{{ taxonomies[taxonomy]['nb_active_tags'] }}</b>/{{ taxonomies[taxonomy]['nb_tags'] }}
</span>
</h5>
</td>
<td>
<a class="btn btn-outline-secondary" href="{{ url_for('tags_ui.tags_taxonomy') }}?taxonomy={{ taxonomy }}">
<i class="fas fa-tag"></i>
</a>
</td>
</tr>
{% endfor %}
</tbody>
</table>
<br>
<br>
<br>
<br>
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$("#navbar-tags").addClass("active");
$("#nav_tags_config").removeClass("text-muted");
$("#nav_taxonomies").addClass("active");
$('#myTable_').DataTable({ "lengthMenu": [ 5, 10, 25, 50, 100 ], "pageLength": 15, "order": [[ 0, "asc" ]] });
});
</script>
<script>
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</html>

View file

@ -0,0 +1,196 @@
<!DOCTYPE html>
<html>
<head>
<title>Taxonomy - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap4.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/font-awesome.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.min.css') }}" rel="stylesheet">
<!-- JS -->
<script src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/popper.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/bootstrap4.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js')}}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.min.js')}}"></script>
</head>
<body>
{% include 'nav_bar.html' %}
<div class="container-fluid">
<div class="row">
{% include 'tags/menu_sidebar.html' %}
<div class="col-12 col-lg-10" id="core_content">
<div class="my-3">
<a href="{{ url_for('tags_ui.tags_taxonomies') }}" class="pull-left">
<i class="fa fa-arrow-left fa"></i> List Taxonomies
</a>
</div>
<div class="card">
<div class="card-header bg-secondary text-white">
<div class="d-flex">
<span class="h3">{{ taxonomy['name'] }}</span>
<span class="h5 ml-auto">
{% if taxonomy['enabled'] %}
<span class="badge badge-success">Enabled</span>
{% else %}
<span class="badge badge-danger">Disabled</span>
{% endif %}
</span>
</div>
</div>
<div class="card-body">
{% if taxonomy['expended'] %}
<div>{{ taxonomy['expended'] }}</div>
{% endif %}
{{ taxonomy['description'] }}
<div>Version: {{ taxonomy['version'] }}</div>
{% if taxonomy['predicates'] %}
<br>
Predicates:
<table class="table table-sm table-hover">
<tbody>
{% for predicate in taxonomy['predicates'] %}
<tr>
<td><b>{{ predicate }}</b></td>
<td>{{ taxonomy['predicates'][predicate]['expanded'] }}</td>
<td>{{ taxonomy['predicates'][predicate]['description'] }}</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endif %}
{% if taxonomy['refs'] %}
<div>Refs: {{ taxonomy['refs'] }}</div>
<br><br>
{% endif %}
<div class="mt-4">
{% if taxonomy['enabled'] %}
<a href="{{ url_for('tags_ui.taxonomy_disable') }}?taxonomy={{ taxonomy['name'] }}" class="btn btn-danger">
<i class="fas fa-times"></i> Disable Taxonomy
</a>
{% else %}
<a href="{{ url_for('tags_ui.taxonomy_enable') }}?taxonomy={{ taxonomy['name'] }}" class="btn btn-success">
<i class="fas fa-check"></i> Enable Taxonomy
</a>
{% endif %}
</div>
</div>
</div>
<form action="{{ url_for('tags_ui.taxonomy_enable_tags') }}" id="checkboxForm">
<input type="hidden" value="{{ taxonomy['name'] }}" name="taxonomy">
<table class="table table-bordered table-hover my-4">
<thead class="bg-dark text-white">
<tr>
<th></th>
<th>Tag</th>
<th>Description</th>
<th class="text-center">Enabled</th>
</tr>
</thead>
<tbody>
{% for tag in taxonomy['tags'] %}
<tr>
<td class="text-center">
<div class="custom-control custom-switch">
<input class="custom-control-input" type="checkbox" name="tags" id="{{ tag['tag'] }}" value="{{ tag['tag'] }}" {% if tag['enabled'] %}checked{% endif %}>
<label class="custom-control-label" for="{{ tag['tag'] }}"></label>
</div>
</td>
<td><span class="badge badge-info">{{ tag['tag'] }}</span></td>
<td>
{% if 'expanded' in tag %}
{{ tag['expanded'] }}
{% endif %}
</td>
<td>
<div class="text-center">
{% if tag['enabled'] %}
<div style="color:Green;">
<i class="fas fa-check-circle"></i>
</div>
{% else %}
<div style="color:Red;">
<i class="fas fa-times-circle"></i>
</div>
{% endif %}
</div>
</td>
</tr>
{% endfor %}
</tbody>
</table>
<div class="input-group-btn">
<button class="btn btn-primary btn-lg" onclick="submitEnabledTags()">
<i class="fas fa-check-square"></i> Update Tags
</button>
</div>
</form>
<br><br><br><br>
</div>
</div>
</div>
</body>
<script>
$(document).ready(function(){
$("#navbar-tags").addClass("active");
$("#nav_tags_config").removeClass("text-muted");
table = $('#myTable_').DataTable(
{
"aLengthMenu": [[5, 10, 15, 20, -1], [5, 10, 15, 20, "All"]],
"iDisplayLength": 10,
"order": [[ 3, "desc" ]]
}
);
});
</script>
<script>
function submitEnabledTags() {
table.destroy();
table = $('#myTable_').DataTable(
{
"iDisplayLength": -1,
}
);
function toggle_sidebar(){
if($('#nav_menu').is(':visible')){
$('#nav_menu').hide();
$('#side_menu').removeClass('border-right')
$('#side_menu').removeClass('col-lg-2')
$('#core_content').removeClass('col-lg-10')
}else{
$('#nav_menu').show();
$('#side_menu').addClass('border-right')
$('#side_menu').addClass('col-lg-2')
$('#core_content').addClass('col-lg-10')
}
}
</script>
</html>