AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-12 17:48:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into i2p

Browse source
This commit is contained in:
David Cruciani 2022-06-28 14:31:42 +02:00
commit 51a48a4c1a
38 changed files with 746 additions and 137 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
bin/LAUNCH.sh
View file

@ -19,15 +19,6 @@ if [ -e "${DIR}/AILENV/bin/python" ]; then
ENV_PY="${DIR}/AILENV/bin/python"
export AIL_VENV=${AIL_HOME}/AILENV/
. ./AILENV/bin/activate
elif [ ! -z "$TRAVIS" ]; then
echo "Travis detected"
ENV_PY="~/virtualenv/python3.6/bin/python"
export AIL_VENV="~/virtualenv/python3.6/"
export AIL_BIN=${AIL_HOME}/bin/
export AIL_FLASK=${AIL_HOME}/var/www/
export AIL_REDIS=${AIL_HOME}/redis/src/
export AIL_ARDB=${AIL_HOME}/ardb/src/
else
echo "Please make sure you have a AIL-framework environment, au revoir"
exit 1
@ -209,8 +200,6 @@ function launching_scripts {
sleep 0.1
screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Decoder.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./DomClassifier.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Keys.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Onion.py; read x"
@ -220,17 +209,25 @@ function launching_scripts {
screen -S "Script_AIL" -X screen -t "Telegram" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Telegram.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Hosts" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Hosts.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./DomClassifier.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Urls" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Urls.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./SQLInjectionDetection.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./LibInjection.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Zerobins" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Zerobins.py; read x"
sleep 0.1
##################################
# TRACKERS MODULES #
##################################
screen -S "Script_AIL" -X screen -t "Tracker_Typo_Squatting" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Typo_Squatting.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Tracker_Term" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Term.py; read x"
sleep 0.1
screen -S "Script_AIL" -X screen -t "Tracker_Regex" bash -c "cd ${AIL_BIN}/trackers; ${ENV_PY} ./Tracker_Regex.py; read x"
@ -439,12 +436,12 @@ function launch_feeder {
}
function killscript {
if [[ $islogged || $isqueued || $is_ail_core || $isscripted || $isflasked || $isfeeded || $iscrawler ]]; then
if [[ $islogged || $isqueued || $is_ail_core || $isscripted || $isflasked || $isfeeded || $iscrawler || $is_ail_2_ail ]]; then
echo -e $GREEN"Killing Script"$DEFAULT
kill $islogged $isqueued $is_ail_core $isscripted $isflasked $isfeeded $iscrawler
kill $islogged $isqueued $is_ail_core $isscripted $isflasked $isfeeded $iscrawler $is_ail_2_ail
sleep 0.2
echo -e $ROSE`screen -ls`$DEFAULT
echo -e $GREEN"\t* $islogged $isqueued $is_ail_core $isscripted $isflasked $isfeeded $iscrawler killed."$DEFAULT
echo -e $GREEN"\t* $islogged $isqueued $is_ail_core $isscripted $isflasked $isfeeded $iscrawler $is_ail_2_ail killed."$DEFAULT
else
echo -e $RED"\t* No script to kill"$DEFAULT
fi

1
bin/Languages.py
View file

@ -3,7 +3,6 @@
import os
import sys
import cld3
import time
from packages import Item

2
bin/core/Sync_importer.py
View file

@ -79,7 +79,7 @@ class Sync_importer(AbstractModule):
b64_gzip_content = ail_stream['payload']['raw']
# # TODO: create default id
item_id = ail_stream['meta']['ail:id'] + 'test'
item_id = ail_stream['meta']['ail:id']
message = f'{item_id} {b64_gzip_content}'
print(item_id)

11
bin/core/Sync_module.py
View file

@ -48,8 +48,6 @@ class Sync_module(AbstractModule):
def compute(self, message):
print(message)
### REFRESH DICT
if self.last_refresh < ail_2_ail.get_last_updated_sync_config():
self.last_refresh = time.time()
@ -71,17 +69,16 @@ class Sync_module(AbstractModule):
tags = obj.get_tags(r_set=True)
# check filter + tags
#print(message)
for queue_uuid in self.dict_sync_queues:
filter_tags = self.dict_sync_queues[queue_uuid]['filter']
print(tags)
print(filter_tags)
print(tags.issubset(filter_tags))
if filter_tags and tags:
if tags.issubset(filter_tags):
#print(f'tags: {tags} filter: {filter_tags}')
if filter_tags.issubset(tags):
obj_dict = obj.get_default_meta()
# send to queue push and/or pull
for dict_ail in self.dict_sync_queues[queue_uuid]['ail_instances']:
print(f'ail_uuid: {dict_ail["ail_uuid"]} obj: {message}')
ail_2_ail.add_object_to_sync_queue(queue_uuid, dict_ail['ail_uuid'], obj_dict,
push=dict_ail['push'], pull=dict_ail['pull'])

20
bin/core/ail_2_ail.py
View file

@ -1028,8 +1028,9 @@ def api_create_sync_queue(json_dict):
tags = json_dict.get('tags')
if not tags:
return {"status": "error", "reason": "no tags provided"}, 400
if not Tag.are_enabled_tags(tags):
return {"status": "error", "reason": "Invalid/Disabled tags"}, 400
# FIXME: add custom tags
# if not Tag.are_enabled_tags(tags):
# return {"status": "error", "reason": "Invalid/Disabled tags"}, 400
max_size = json_dict.get('max_size')
if not max_size:
@ -1064,8 +1065,9 @@ def api_edit_sync_queue(json_dict):
tags = json_dict.get('tags')
if tags:
if not Tag.are_enabled_tags(tags):
return {"status": "error", "reason": "Invalid/Disabled tags"}, 400
# FIXME: add custom tags
# if not Tag.are_enabled_tags(tags):
# return {"status": "error", "reason": "Invalid/Disabled tags"}, 400
edit_sync_queue_filter_tags(queue_uuid, tags)
max_size = json_dict.get('max_size')
@ -1203,11 +1205,13 @@ def create_ail_stream(Object):
'type': Object.get_type()}
# OBJECT META
ail_stream['meta'] = {'ail_mime-type': 'text/plain'}
ail_stream['meta'] = {'ail:mime-type': 'text/plain'}
ail_stream['meta']['compress'] = 'gzip'
ail_stream['meta']['encoding'] = 'base64'
ail_stream['meta']['ail:id'] = Object.get_id()
ail_stream['meta']['ail:tags'] = Object.get_tags()
# GLOBAL PAYLOAD
ail_stream['meta']['ail:uuid'] = get_ail_uuid()
ail_stream['meta']['tags'] = Object.get_tags()
# GLOBAL META
ail_stream['meta']['uuid_org'] = get_ail_uuid()
# OBJECT PAYLOAD
ail_stream['payload'] = Object.get_ail_2_ail_payload()

14
bin/core/ail_2_ail_client.py
View file

@ -20,6 +20,16 @@ sys.path.append(os.environ['AIL_BIN'])
# Import Project packages
##################################
from core import ail_2_ail
from lib.ConfigLoader import ConfigLoader
config_loader = ConfigLoader()
local_addr = config_loader.get_config_str('AIL_2_AIL', 'local_addr')
if not local_addr or local_addr == None:
local_addr = None
else:
local_addr = (local_addr, 0)
config_loader = None
#### LOGS ####
redis_logger = publisher
@ -68,10 +78,9 @@ async def push(websocket, ail_uuid):
Obj, queue_uuid = ail_2_ail.get_sync_queue_object_and_queue_uuid(ail_uuid)
if Obj:
obj_ail_stream = ail_2_ail.create_ail_stream(Obj)
print(obj_ail_stream['meta'])
obj_ail_stream = json.dumps(obj_ail_stream)
sys.stdout.write(obj_ail_stream)
# send objects
await websocket.send(obj_ail_stream)
await asyncio.sleep(0.1)
@ -112,6 +121,7 @@ async def ail_to_ail_client(ail_uuid, sync_mode, api, ail_key=None, client_id=No
async with websockets.connect(
uri,
ssl=ssl_context,
local_addr=local_addr,
#open_timeout=10, websockers 10.0 /!\ python>=3.7
extra_headers={"Authorization": f"{ail_key}"}
) as websocket:

11
bin/core/ail_2_ail_server.py
View file

@ -17,6 +17,12 @@ sys.path.append(os.environ['AIL_BIN'])
##################################
from pubsublogger import publisher
from core import ail_2_ail
from lib.ConfigLoader import ConfigLoader
config_loader = ConfigLoader()
host = config_loader.get_config_str('AIL_2_AIL', 'server_host')
port = config_loader.get_config_int('AIL_2_AIL', 'server_port')
config_loader = None
# # TODO: refactor logging
#### LOGS ####
@ -303,9 +309,6 @@ class AIL_2_AIL_Protocol(websockets.WebSocketServerProtocol):
if __name__ == '__main__':
host = '0.0.0.0'
port = 4443
print('Launching Server...')
redis_logger.info('Launching Server...')
@ -315,7 +318,7 @@ if __name__ == '__main__':
cert_dir = os.environ['AIL_FLASK']
ssl_context.load_cert_chain(certfile=os.path.join(cert_dir, 'server.crt'), keyfile=os.path.join(cert_dir, 'server.key'))
start_server = websockets.serve(ail_to_ail_serv, host, port, ssl=ssl_context, create_protocol=AIL_2_AIL_Protocol)
start_server = websockets.serve(ail_to_ail_serv, host, port, ssl=ssl_context, create_protocol=AIL_2_AIL_Protocol, max_size=None)
print(f'Server Launched: wss://{host}:{port}')
redis_logger.info(f'Server Launched: wss://{host}:{port}')

59
bin/import/ail_json_importer/Ail_feeder_telegram.py Executable file
View file

@ -0,0 +1,59 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The JSON Receiver Module
================
Recieve Json Items (example: Twitter feeder)
"""
import os
import json
import sys
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import item_basic
import Username
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer'))
from Default_json import Default_json
class Ail_feeder_telegram(Default_json):
"""Twitter Feeder functions"""
def __init__(self, name, json_item):
super().__init__(name, json_item)
def get_feeder_name(self):
return 'telegram'
# define item id
def get_item_id(self):
# use twitter timestamp ?
item_date = datetime.date.today().strftime("%Y/%m/%d")
channel_id = str(self.json_item['meta']['channel_id'])
message_id = str(self.json_item['meta']['message_id'])
item_id = f'{channel_id}_{message_id}'
return os.path.join('telegram', item_date, item_id) + '.gz'
def process_json_meta(self, process, item_id):
'''
Process JSON meta filed.
'''
channel_id = str(self.json_item['meta']['channel_id'])
message_id = str(self.json_item['meta']['message_id'])
telegram_id = f'{channel_id}_{message_id}'
item_basic.add_map_obj_id_item_id(telegram_id, item_id, 'telegram_id')
#print(self.json_item['meta'])
username = None
if self.json_item['meta'].get('user'):
username = str(self.json_item['meta']['user'])
else:
if self.json_item['meta'].get('channel'):
username = str(self.json_item['meta']['channel']['username'])
if username:
#print(username)
item_date = item_basic.get_item_date(item_id)
Username.save_item_correlation('telegram', username, item_id, item_date)
return None

13
bin/lib/Investigations.py
View file

@ -270,7 +270,7 @@ def exists_investigation(investigation_uuid):
# created by user
def get_user_all_investigations(user_id):
return r_tracking.smembers('investigations:user:{user_id}')
return r_tracking.smembers(f'investigations:user:{user_id}')
def is_object_investigated(obj_id, obj_type, subtype=''):
return r_tracking.exists(f'obj:investigations:{obj_type}:{subtype}:{obj_id}')
@ -278,6 +278,15 @@ def is_object_investigated(obj_id, obj_type, subtype=''):
def get_obj_investigations(obj_id, obj_type, subtype=''):
return r_tracking.smembers(f'obj:investigations:{obj_type}:{subtype}:{obj_id}')
def delete_obj_investigations(obj_id, obj_type, subtype=''):
unregistred = False
for investigation_uuid in get_obj_investigations(obj_id, obj_type, subtype=subtype):
investigation = Investigation(investigation_uuid)
investigation.unregister_object(obj_id, obj_type, subtype)
unregistred = True
return unregistred
# # TODO: fix default threat_level analysis
# # TODO: limit description + name
# # TODO: sanityze tags
@ -286,7 +295,7 @@ def create_investigation(user_id, date, name, threat_level, analysis, info, tags
investigation_uuid = generate_uuid()
r_tracking.sadd('investigations:all', investigation_uuid)
# user map
r_tracking.sadd('investigations:user:{user_id}', investigation_uuid)
r_tracking.sadd(f'investigations:user:{user_id}', investigation_uuid)
# metadata
r_tracking.hset(f'investigations:data:{investigation_uuid}', 'creator_user', user_id)

68
bin/lib/Tracker.py
View file

@ -11,11 +11,15 @@ import yara
import datetime
import base64
from ail_typo_squatting import runAll
import math
from flask import escape
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Date
import Tag
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
@ -26,6 +30,10 @@ r_cache = config_loader.get_redis_conn("Redis_Cache")
r_serv_db = config_loader.get_redis_conn("ARDB_DB")
r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker")
items_dir = config_loader.get_config_str("Directories", "pastes")
if items_dir[-1] == '/':
items_dir = items_dir[:-1]
config_loader = None
email_regex = r'[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}'
@ -75,8 +83,8 @@ def get_all_tracker_type():
def get_all_tracker_uuid():
return r_serv_tracker.smembers(f'trackers:all')
def get_all_tracker_by_type(tracker_type):
r_serv_tracker.smembers(f'trackers:all:{tracker_type}')
def get_all_tracker_uuid_by_type(tracker_type):
return r_serv_tracker.smembers(f'trackers:all:{tracker_type}')
# def get_all_tracker():
# l_keys_name = []
@ -211,6 +219,20 @@ def get_tracker_items_by_daterange(tracker_uuid, date_from, date_to):
all_item_id |= r_serv_tracker.smembers(f'tracker:item:{tracker_uuid}:{date_day}')
return all_item_id
def get_tracker_typosquatting_domains(tracker_uuid):
return r_serv_tracker.smembers(f'tracker:typosquatting:{tracker_uuid}')
def get_typosquatting_tracked_words_list():
all_typo = dict()
typos_uuid = get_all_tracker_uuid_by_type("typosquatting")
for typo_uuid in typos_uuid:
tracker = get_tracker_by_uuid(typo_uuid)
all_typo[tracker] = get_tracker_typosquatting_domains(typo_uuid)
return all_typo
def add_tracked_item(tracker_uuid, item_id):
item_date = item_basic.get_item_date(item_id)
# track item
@ -248,7 +270,6 @@ def update_tracker_daterange(tracker_uuid, date, op='add'):
if op == 'del':
pass
def remove_tracked_item(item_id):
item_date = item_basic.get_item_date(item_id)
for tracker_uuid in get_item_all_trackers_uuid(item_id):
@ -267,6 +288,11 @@ def is_obj_tracked(obj_type, subtype, id):
def get_obj_all_trackers(obj_type, subtype, id):
return r_serv_tracker.smembers(f'obj:trackers:{obj_type}:{obj_id}')
# # TODO: ADD all Objects + Subtypes
def delete_obj_trackers(obj_type, subtype, id):
if obj_type == 'item':
remove_tracked_item(id)
def get_email_subject(tracker_uuid):
tracker_description = get_tracker_description(tracker_uuid)
if not tracker_description:
@ -400,6 +426,15 @@ def api_validate_tracker_to_add(tracker , tracker_type, nb_words=1):
tracker = ",".join(words_set)
tracker = "{};{}".format(tracker, nb_words)
elif tracker_type == 'typosquatting':
tracker = tracker.lower()
# Take only the first term
domain = tracker.split(" ")
if len(domain) > 1:
return {"status": "error", "reason": "Only one domain is accepted at a time"}, 400
if not "." in tracker:
return {"status": "error", "reason": "Invalid domain name"}, 400
elif tracker_type=='yara_custom':
if not is_valid_yara_rule(tracker):
@ -440,6 +475,12 @@ def create_tracker(tracker, tracker_type, user_id, level, tags, mails, descripti
tracker = save_yara_rule(tracker_type, tracker, tracker_uuid=tracker_uuid)
tracker_type = 'yara'
elif tracker_type == 'typosquatting':
domain = tracker.split(" ")[0]
typo_generation = runAll(domain=domain, limit=math.inf, formatoutput="text", pathOutput="-", verbose=False)
for typo in typo_generation:
r_serv_tracker.sadd(f'tracker:typosquatting:{tracker_uuid}', typo)
# create metadata
r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'tracked', tracker)
r_serv_tracker.hset('tracker:{}'.format(tracker_uuid), 'type', tracker_type)
@ -1039,7 +1080,10 @@ def get_retro_hunt_dir_day_to_analyze(task_uuid, date, filter_last=False, source
# # TODO: move me
def get_items_to_analyze(dir, last=None):
full_dir = os.path.join(os.environ['AIL_HOME'], 'PASTES', dir) # # TODO: # FIXME: use item config dir
if items_dir == 'PASTES':
full_dir = os.path.join(os.environ['AIL_HOME'], 'PASTES', dir)
else:
full_dir = os.path.join(items_dir, dir)
if os.path.isdir(full_dir):
all_items = sorted([os.path.join(dir, f) for f in os.listdir(full_dir) if os.path.isfile(os.path.join(full_dir, f))])
# remove processed items
@ -1265,7 +1309,21 @@ def api_delete_retro_hunt_task(task_uuid):
else:
return (delete_retro_hunt_task(task_uuid), 200)
# if __name__ == '__main__':
#### DB FIX ####
def get_trackers_custom_tags():
tags = set()
for tracker_uuid in get_all_tracker_uuid():
for tag in get_tracker_tags(tracker_uuid):
tags.add(tag)
for task_uuid in get_all_retro_hunt_tasks():
for tag in get_retro_hunt_task_tags(task_uuid):
tags.add(tag)
return tags
#### -- ####
if __name__ == '__main__':
print(get_trackers_custom_tags())
# fix_all_tracker_uuid_list()
# res = get_all_tracker_uuid()
# print(len(res))

7
bin/lib/item_basic.py
View file

@ -116,6 +116,9 @@ def is_domain_root(item_id):
else:
return True
def get_item_url(item_id):
return r_serv_metadata.hget(f'paste_metadata:{item_id}', 'real_link')
def get_nb_children(item_id):
return r_serv_metadata.scard('paste_children:{}'.format(item_id))
@ -166,7 +169,7 @@ def add_item_parent(parent_item_id, item_id):
#### UNKNOW SECTION ####
def get_obj_id_item_id(parent_type, parent_id):
all_parents_type = ['twitter_id', 'jabber_id']
all_parents_type = ['twitter_id', 'jabber_id', 'telegram_id']
if parent_type in all_parents_type:
return r_serv_metadata.hget('map:{}:item_id'.format(parent_type), parent_id)
else:
@ -177,6 +180,8 @@ def add_map_obj_id_item_id(obj_id, item_id, obj_type):
r_serv_metadata.hset('map:twitter_id:item_id', obj_id, item_id)
if obj_type == 'jabber_id':
r_serv_metadata.hset('map:jabber_id:item_id', obj_id, item_id)
if obj_type == 'telegram_id':
r_serv_metadata.hset('map:telegram_id:item_id', obj_id, item_id)
# delete twitter id

53
bin/lib/objects/Items.py
View file

@ -11,25 +11,33 @@ import html2text
from io import BytesIO
from pymisp import MISPObject
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from export.Export import get_ail_uuid # # TODO: REPLACE
from lib.objects.abstract_object import AbstractObject
from lib.ConfigLoader import ConfigLoader
from lib import item_basic
from lib import domain_basic
from packages import Tag
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Tag
import Cryptocurrency
import Pgp
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import item_basic
import domain_basic
import ConfigLoader
import Correlate_object
import Decoded
import Screenshot
import Username
from abstract_object import AbstractObject
from item_basic import *
from flask import url_for
config_loader = ConfigLoader.ConfigLoader()
config_loader = ConfigLoader()
# get and sanityze PASTE DIRECTORY
# # TODO: rename PASTES_FOLDER
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/'
@ -89,6 +97,12 @@ class Item(AbstractObject):
"""
return item_basic.get_item_content(self.id)
def get_raw_content(self):
filepath = self.get_filename()
with open(filepath, 'rb') as f:
raw_content = BytesIO(f.read())
return raw_content
def get_gzip_content(self, b64=False):
with open(self.get_filename(), 'rb') as f:
content = f.read()
@ -97,8 +111,7 @@ class Item(AbstractObject):
return content.decode()
def get_ail_2_ail_payload(self):
payload = {'raw': self.get_gzip_content(b64=True),
'compress': 'gzip'}
payload = {'raw': self.get_gzip_content(b64=True)}
return payload
# # TODO:
@ -108,6 +121,7 @@ class Item(AbstractObject):
# # WARNING: UNCLEAN DELETE /!\ TEST ONLY /!\
# TODO: DELETE ITEM CORRELATION + TAGS + METADATA + ...
def delete(self):
self._delete()
try:
os.remove(self.get_filename())
return True
@ -128,9 +142,19 @@ class Item(AbstractObject):
color = '#332288'
return {'style': '', 'icon': '', 'color': color, 'radius':5}
############################################################################
############################################################################
############################################################################
def get_misp_object(self):
obj_date = self.get_date()
obj = MISPObject('ail-leak', standalone=True)
obj.first_seen = obj_date
obj_attrs = []
obj_attrs.append( obj.add_attribute('first-seen', value=obj_date) )
obj_attrs.append( obj.add_attribute('raw-data', value=self.id, data=self.get_raw_content()) )
obj_attrs.append( obj.add_attribute('sensor', value=get_ail_uuid()) )
for obj_attr in obj_attrs:
for tag in self.get_tags():
obj_attr.add_tag(tag)
return obj
def exist_correlation(self):
pass
@ -688,4 +712,7 @@ def delete_domain_node(item_id):
delete_item(child_id)
#if __name__ == '__main__':
# if __name__ == '__main__':
#
# item = Item('')
# print(item.get_misp_object().to_json())

43
bin/modules/DomClassifier.py
View file

@ -53,37 +53,38 @@ class DomClassifier(AbstractModule):
def compute(self, message, r_result=False):
item = Item(message)
host, id = message.split()
item_content = item.get_content()
item = Item(id)
item_basename = item.get_basename()
item_date = item.get_date()
item_source = item.get_source()
try:
mimetype = item_basic.get_item_mimetype(item.get_id())
if mimetype.split('/')[0] == "text":
self.c.text(rawtext=item_content)
self.c.potentialdomain()
self.c.validdomain(passive_dns=True, extended=False)
#self.redis_logger.debug(self.c.vdomain)
self.c.text(rawtext=host)
print(self.c.domain)
self.c.validdomain(passive_dns=True, extended=False)
#self.redis_logger.debug(self.c.vdomain)
if self.c.vdomain and d4.is_passive_dns_enabled():
for dns_record in self.c.vdomain:
self.send_message_to_queue(dns_record)
print(self.c.vdomain)
print()
localizeddomains = self.c.include(expression=self.cc_tld)
if localizeddomains:
print(localizeddomains)
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{item.get_id()}")
if self.c.vdomain and d4.is_passive_dns_enabled():
for dns_record in self.c.vdomain:
self.send_message_to_queue(dns_record)
localizeddomains = self.c.localizedomain(cc=self.cc)
if localizeddomains:
print(localizeddomains)
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{item.get_id()}")
localizeddomains = self.c.include(expression=self.cc_tld)
if localizeddomains:
print(localizeddomains)
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc_tld};{item.get_id()}")
if r_result:
return self.c.vdomain
localizeddomains = self.c.localizedomain(cc=self.cc)
if localizeddomains:
print(localizeddomains)
self.redis_logger.warning(f"DomainC;{item_source};{item_date};{item_basename};Checked {localizeddomains} located in {self.cc};{item.get_id()}")
if r_result:
return self.c.vdomain
except IOError as err:
self.redis_logger.error(f"Duplicate;{item_source};{item_date};{item_basename};CRC Checksum Failed")

77
bin/modules/Hosts.py Executable file
View file

@ -0,0 +1,77 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The Hosts Module
======================
This module is consuming the Redis-list created by the Global module.
It is looking for Hosts
"""
##################################
# Import External packages
##################################
import os
import re
import sys
import time
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.ConfigLoader import ConfigLoader
from lib import regex_helper
#from lib.objects.Items import Item
from packages.Item import Item
class Hosts(AbstractModule):
"""
Hosts module for AIL framework
"""
def __init__(self):
super(Hosts, self).__init__()
config_loader = ConfigLoader()
self.r_cache = config_loader.get_redis_conn("Redis_Cache")
self.redis_cache_key = regex_helper.generate_redis_cache_key(self.module_name)
# regex timeout
self.regex_timeout = 30
# Waiting time in secondes between to message proccessed
self.pending_seconds = 1
self.host_regex = r'\b([a-zA-Z\d-]{,63}(?:\.[a-zA-Z\d-]{,63})+)\b'
re.compile(self.host_regex)
self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, message):
item = Item(message)
# mimetype = item_basic.get_item_mimetype(item.get_id())
# if mimetype.split('/')[0] == "text":
content = item.get_content()
hosts = regex_helper.regex_findall(self.module_name, self.redis_cache_key, self.host_regex, item.get_id(), content)
for host in hosts:
#print(host)
msg = f'{host} {item.get_id()}'
self.send_message_to_queue(msg, 'Host')
if __name__ == '__main__':
module = Hosts()
module.run()

2
bin/modules/Tags.py
View file

@ -54,7 +54,7 @@ class Tags(AbstractModule):
self.send_message_to_queue(message, 'MISP_The_Hive_feeder')
message = f'{item.get_type()};{item.get_subtype(r_str=True)};{item.get_id()}'
self.send_message_to_queue(message, 'Sync_module')
self.send_message_to_queue(message, 'SyncModule')
else:
# Malformed message

74
bin/modules/Zerobins.py Executable file
View file

@ -0,0 +1,74 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The Zerobins Module
======================
This module spots zerobins-like services for further processing
"""
##################################
# Import External packages
##################################
import os
import sys
import time
import pdb
import re
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib import crawlers
class Zerobins(AbstractModule):
"""
Zerobins module for AIL framework
"""
def __init__(self):
super(Zerobins, self).__init__()
binz = [
r'^https:\/\/(zerobin||privatebin)\..*$', # historical ones
]
self.regex = re.compile('|'.join(binz))
# Pending time between two computation (computeNone) in seconds
self.pending_seconds = 10
# Send module state to logs
self.redis_logger.info(f'Module {self.module_name} initialized')
def computeNone(self):
"""
Compute when no message in queue
"""
self.redis_logger.debug("No message in queue")
def compute(self, message):
"""regex_helper.regex_findall(self.module_name, self.redis_cache_key
Compute a message in queue
"""
print(message)
url, id = message.split()
# Extract zerobins addresses
matching_binz = self.regex_findall(self.regex, id, url)
if len(matching_binz) > 0:
for bin in matching_binz:
print("send {} to crawler".format(bin))
crawlers.create_crawler_task(bin, screenshot=False, har=False, depth_limit=1, max_pages=1, auto_crawler=False, crawler_delta=3600, crawler_type=None, cookiejar_uuid=None, user_agent=None)
self.redis_logger.debug("Compute message in queue")
if __name__ == '__main__':
module = Zerobins()
module.run()

7
bin/modules/abstract_module.py
View file

@ -46,6 +46,9 @@ class AbstractModule(ABC):
# If provided could be a namespaced channel like script:<ModuleName>
self.redis_logger.channel = logger_channel
#Cache key
self.redis_cache_key = regex_helper.generate_redis_cache_key(self.module_name)
self.max_execution_time = 30
# Run module endlessly
self.proceed = True
@ -102,6 +105,7 @@ class AbstractModule(ABC):
self.compute(message)
except Exception as err:
trace = traceback.format_tb(err.__traceback__)
trace = ''.join(trace)
self.redis_logger.critical(f"Error in module {self.module_name}: {err}")
self.redis_logger.critical(f"Module {self.module_name} input message: {message}")
self.redis_logger.critical(trace)
@ -109,8 +113,7 @@ class AbstractModule(ABC):
print(f"ERROR: {err}")
print(f'MESSAGE: {message}')
print('TRACEBACK:')
for line in trace:
print(line)
print(trace)
# remove from set_module
## check if item process == completed

6
bin/packages/Item.py
View file

@ -637,8 +637,7 @@ class Item(AbstractObject):
return content.decode()
def get_ail_2_ail_payload(self):
payload = {'raw': self.get_gzip_content(b64=True),
'compress': 'gzip'}
payload = {'raw': self.get_gzip_content(b64=True)}
return payload
# # TODO:
@ -667,6 +666,9 @@ class Item(AbstractObject):
def get_svg_icon(self):
pass
def get_misp_object(self):
pass
############################################################################
############################################################################
############################################################################

2
bin/packages/Term.py
View file

@ -103,7 +103,7 @@ def get_text_word_frequency(item_content, filtering=True):
# # TODO: create all tracked words
def get_tracked_words_list():
return list(r_serv_term.smembers('all:tracker:word'))
def get_set_tracked_words_list():
set_list = r_serv_term.smembers('all:tracker:set')
all_set_list = []

13
bin/packages/modules.cfg
View file

@ -22,8 +22,12 @@ subscribe = Redis_Duplicate
[Indexer]
subscribe = Redis_Global
[DomClassifier]
[Hosts]
subscribe = Redis_Global
publish = Redis_Host
[DomClassifier]
subscribe = Redis_Host
publish = Redis_D4_client
[D4_client]
@ -33,6 +37,10 @@ subscribe = Redis_D4_client
subscribe = Redis
publish = Redis_Tags
[Tracker_Typo_Squatting]
subscribe = Redis_Host
publish = Redis_Tags
[Tracker_Term]
subscribe = Redis_Global
publish = Redis_Tags
@ -161,3 +169,6 @@ publish = Redis_Mixer,Redis_Tags
[IP]
subscribe = Redis_Global
publish = Redis_Duplicate,Redis_Tags
[Zerobins]
subscribe = Redis_Url

18
bin/trackers/Retro_Hunt.py
View file

@ -58,13 +58,16 @@ class Retro_Hunt(AbstractModule):
# end_time
def compute(self, task_uuid):
print(task_uuid)
self.redis_logger.warning(f'{self.module_name}, starting Retro hunt task {task_uuid}')
print(f'starting Retro hunt task {task_uuid}')
self.task_uuid = task_uuid
self.progress = 0
# First launch
# restart
rule = Tracker.get_retro_hunt_task_rule(task_uuid, r_compile=True)
timeout = Tracker.get_retro_hunt_task_timeout(task_uuid)
self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid} timeout {timeout}')
sources = Tracker.get_retro_hunt_task_sources(task_uuid, r_sort=True)
self.date_from = Tracker.get_retro_hunt_task_date_from(task_uuid)
@ -85,6 +88,7 @@ class Retro_Hunt(AbstractModule):
# # TODO: Filter previous item
for dir in dirs_date:
print(dir)
self.redis_logger.debug(f'{self.module_name}, Retro Hunt searching in directory {dir}')
l_obj = Tracker.get_items_to_analyze(dir)
for id in l_obj:
#print(f'{dir} / {id}')
@ -92,6 +96,8 @@ class Retro_Hunt(AbstractModule):
# save current item in cache
Tracker.set_cache_retro_hunt_task_id(task_uuid, id)
self.redis_logger.debug(f'{self.module_name}, Retro Hunt rule {task_uuid}, searching item {id}')
yara_match = rule.match(data=self.item.get_content(), callback=self.yara_rules_match, which_callbacks=yara.CALLBACK_MATCHES, timeout=timeout)
# save last item
@ -120,6 +126,8 @@ class Retro_Hunt(AbstractModule):
Tracker.set_retro_hunt_nb_match(task_uuid)
Tracker.clear_retro_hunt_task_cache(task_uuid)
print(f'Retro Hunt {task_uuid} completed')
self.redis_logger.warning(f'{self.module_name}, Retro Hunt {task_uuid} completed')
# # TODO: stop
@ -133,10 +141,12 @@ class Retro_Hunt(AbstractModule):
# Tracker.set_retro_hunt_task_progress(task_uuid, progress)
def yara_rules_match(self, data):
#print(data)
task_uuid = data['namespace']
id = self.item.get_id()
#print(data)
task_uuid = data['namespace']
self.redis_logger.info(f'{self.module_name}, Retro hunt {task_uuid} match found: {id}')
print(f'Retro hunt {task_uuid} match found: {id}')
Tracker.save_retro_hunt_match(task_uuid, id)

2
bin/trackers/Tracker_Term.py
View file

@ -113,7 +113,7 @@ class Tracker_Term(AbstractModule):
nb_uniq_word += 1
if nb_uniq_word >= nb_words_threshold:
self.new_term_found(word_set, 'set', item)
def new_term_found(self, term, term_type, item):
uuid_list = Term.get_term_uuid_list(term, term_type)

116
bin/trackers/Tracker_Typo_Squatting.py Normal file
View file

@ -0,0 +1,116 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
The Tracker_Typo_Squatting Module
===================
"""
##################################
# Import External packages
##################################
import os
import sys
import time
import requests
sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
import NotificationHelper
from packages.Item import Item
from packages import Term
from lib import Tracker
class Tracker_Typo_Squatting(AbstractModule):
mail_body_template = "AIL Framework,\nNew occurrence for tracked Typo: {}\nitem id: {}\nurl: {}{}"
"""
Tracker_Typo_Squatting module for AIL framework
"""
def __init__(self):
super(Tracker_Typo_Squatting, self).__init__()
self.pending_seconds = 5
self.full_item_url = self.process.config.get("Notifications", "ail_domain") + "/object/item?id="
# loads typosquatting
self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
self.last_refresh_typosquat = time.time()
self.redis_logger.info(f"Module: {self.module_name} Launched")
def compute(self, message):
# refresh Tracked typo
if self.last_refresh_typosquat < Term.get_tracked_term_last_updated_by_type('typosquatting'):
self.typosquat_tracked_words_list = Tracker.get_typosquatting_tracked_words_list()
self.last_refresh_typosquat = time.time()
self.redis_logger.debug('Tracked typosquatting refreshed')
print('Tracked typosquatting refreshed')
host, id = message.split()
item = Item(id)
# Cast message as Item
for key in self.typosquat_tracked_words_list.keys():
#print(key)
if host in self.typosquat_tracked_words_list[key]:
self.new_term_found(key, 'typosquatting', item)
def new_term_found(self, term, term_type, item):
uuid_list = Term.get_term_uuid_list(term, term_type)
item_id = item.get_id()
item_date = item.get_date()
item_source = item.get_source()
self.redis_logger.info(f'new tracked typo found: {term} in {item_id}')
print(f'new tracked typo found: {term} in {item_id}')
for term_uuid in uuid_list:
tracker_sources = Tracker.get_tracker_uuid_sources(term_uuid)
if not tracker_sources or item_source in tracker_sources:
Tracker.add_tracked_item(term_uuid, item_id)
tags_to_add = Term.get_term_tags(term_uuid)
for tag in tags_to_add:
msg = '{};{}'.format(tag, item_id)
self.send_message_to_queue(msg, 'Tags')
mail_to_notify = Term.get_term_mails(term_uuid)
if mail_to_notify:
mail_subject = Tracker.get_email_subject(term_uuid)
mail_body = Tracker_Typo_Squatting.mail_body_template.format(term, item_id, self.full_item_url, item_id)
for mail in mail_to_notify:
self.redis_logger.debug(f'Send Mail {mail_subject}')
print(f'S print(item_content)end Mail {mail_subject}')
NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body)
# Webhook
webhook_to_post = Term.get_term_webhook(term_uuid)
if webhook_to_post:
json_request = {"trackerId": term_uuid,
"itemId": item_id,
"itemURL": self.full_item_url + item_id,
"term": term,
"itemSource": item_source,
"itemDate": item_date,
"tags": tags_to_add,
"emailNotification": f'{mail_to_notify}',
"trackerType": term_type
}
try:
response = requests.post(webhook_to_post, json=json_request)
if response.status_code >= 400:
self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: {response.reason}")
except:
self.redis_logger.error(f"Webhook request failed for {webhook_to_post}\nReason: Something went wrong")
if __name__ == '__main__':
module = Tracker_Typo_Squatting()
module.run()

5
configs/core.cfg.sample
View file

@ -57,6 +57,11 @@ minute_processed_paste = 10
#Maximum line length authorized to make a diff between duplicates
DiffMaxLineLength = 10000
[AIL_2_AIL]
server_host = 0.0.0.0
server_port = 4443
local_addr =
#### Modules ####
[BankAccount]
max_execution_time = 60

16
install_virtualenv.sh
View file

@ -18,22 +18,10 @@ if [ -z "$VIRTUAL_ENV" ]; then
fi
if [ ! -z "$TRAVIS" ]; then
echo "Travis detected"
ENV_PY="~/virtualenv/python3.6/bin/python"
export AIL_VENV="~/virtualenv/python3.6/"
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd |sed 's/bin//' )"
export AIL_HOME="${DIR}"
# activate virtual environment
. ./AILENV/bin/activate
export AIL_BIN=${AIL_HOME}/bin/
export AIL_FLASK=${AIL_HOME}/var/www/
export AIL_REDIS=${AIL_HOME}/redis/src/
export AIL_ARDB=${AIL_HOME}/ardb/src/
else
# activate virtual environment
. ./AILENV/bin/activate
fi
pip3 install -U pip
pip3 install 'git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client'

3
installing_deps.sh
View file

@ -14,9 +14,6 @@ sudo apt-get install python3-pip virtualenv python3-dev python3-tk libfreetype6-
#Needed for downloading jemalloc
sudo apt-get install wget -qq
#optional tor install
sudo apt-get install tor -qq
#Needed for bloom filters
sudo apt-get install libssl-dev libfreetype6-dev python3-numpy -qq

2
requirements.txt
View file

@ -70,6 +70,8 @@ flask>=1.1.4
flask-login
bcrypt>3.1.6
# Ail typo squatting
ail_typo_squatting
# Tests
nose>=1.3.7

7
tests/test_modules.py
View file

@ -59,9 +59,10 @@ class Test_Module_Categ(unittest.TestCase):
def test_module(self):
item_id = 'tests/2021/01/01/categ.gz'
test_categ = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve']
test_categ = ['CreditCards', 'Mail', 'Onion', 'Urls', 'Credential', 'Cve']
result = self.module_obj.compute(item_id, r_result=True)
print(result)
self.assertCountEqual(result, test_categ)
class Test_Module_CreditCards(unittest.TestCase):
@ -87,8 +88,10 @@ class Test_Module_DomClassifier(unittest.TestCase):
self.module_obj = DomClassifier()
def test_module(self):
test_host = 'foo.be'
item_id = 'tests/2021/01/01/domain_classifier.gz'
result = self.module_obj.compute(item_id, r_result=True)
msg = f'{test_host} {item_id}'
result = self.module_obj.compute(msg, r_result=True)
self.assertTrue(len(result))
class Test_Module_Global(unittest.TestCase):

26
update/v4.2/Update.py Executable file
View file

@ -0,0 +1,26 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
import os
import re
import sys
import time
import redis
import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
sys.path.append(os.path.join(os.environ['AIL_HOME'], 'update', 'bin'))
from ail_updater import AIL_Updater
class Updater(AIL_Updater):
"""default Updater."""
def __init__(self, version):
super(Updater, self).__init__(version)
if __name__ == '__main__':
updater = Updater('v4.2')
updater.run_update()

33
update/v4.2/Update.sh Executable file
View file

@ -0,0 +1,33 @@
#!/bin/bash
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
export PATH=$AIL_HOME:$PATH
export PATH=$AIL_REDIS:$PATH
export PATH=$AIL_ARDB:$PATH
export PATH=$AIL_BIN:$PATH
export PATH=$AIL_FLASK:$PATH
GREEN="\\033[1;32m"
DEFAULT="\\033[0;39m"
echo -e $GREEN"Shutting down AIL ..."$DEFAULT
bash ${AIL_BIN}/LAUNCH.sh -ks
wait
# SUBMODULES #
git submodule update
echo ""
echo -e $GREEN"Installing typo-squatting ..."$DEFAULT
pip3 install -U ail_typo_squatting
echo ""
echo -e $GREEN"Updating d4-client ..."$DEFAULT
pip3 install -U d4-pyclient
exit 0

9
var/www/blueprints/correlation.py
View file

@ -261,3 +261,12 @@ def graph_node_json():
res = Correlate_object.get_graph_node_object_correlation(object_type, correlation_id, mode, correlation_names, correlation_objects, requested_correl_type=type_id, max_nodes=max_nodes)
return jsonify(res)
@correlation.route('/correlation/subtype_search', methods=['POST'])
@login_required
@login_read_only
def subtype_search():
obj_type = request.form.get('object_type')
obj_subtype = request.form.get('object_subtype')
obj_id = request.form.get('object_id')
return redirect(url_for('correlation.show_correlation', object_type=obj_type, type_id=obj_subtype, correlation_id=obj_id))

26
var/www/modules/hashDecoded/templates/DaysCorrelation.html
View file

@ -75,8 +75,34 @@
<div class="col-xl-10">
<div class="mt-1" id="barchart_type">
</div>
<div class="card border-secondary my-2">
<div class="card-body text-dark">
<h5 class="card-title">Search {{correlation_type}} by name:</h5>
<form action="{{ url_for('correlation.subtype_search') }}" id="search_subtype_onj" method='post'>
<div class="input-group mb-1">
<input type="text" class="form-control" name="object_type" value="{{correlation_type}}" hidden>
<select class="custom-select col-2" name="object_subtype" value="{{obj_type}}" required>
<option value="">{{correlation_type}} Type...</option>
{% for typ in l_type %}
<option value="{{typ}}">{{typ}}</option>
{% endfor %}
</select>
<input type="text" class="form-control col-8" name="object_id" value="" placeholder="{{correlation_type}} ID" required>
<button class="btn btn-primary input-group-addon search-obj col-2"><i class="fas fa-search"></i></button>
</div>
</form>
</div>
</div>
</div>
<div class="col-xl-2">
<div class="card mb-3 mt-2" style="background-color:#d9edf7;">

18
var/www/modules/hunter/Flask_hunter.py
View file

@ -85,6 +85,16 @@ def tracked_menu_yara():
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
@hunter.route("/trackers/typosquatting")
@login_required
@login_read_only
def tracked_menu_typosquatting():
filter_type = 'typosquatting'
user_id = current_user.get_id()
user_term = Term.get_all_user_tracked_terms(user_id, filter_type=filter_type)
global_term = Term.get_all_global_tracked_terms(filter_type=filter_type)
return render_template("trackersManagement.html", user_term=user_term, global_term=global_term, bootstrap_label=bootstrap_label, filter_type=filter_type)
@hunter.route("/tracker/add", methods=['GET', 'POST'])
@login_required
@ -207,6 +217,13 @@ def show_tracker():
yara_rule_content = Tracker.get_yara_rule_content(tracker_metadata['tracker'])
else:
yara_rule_content = None
if tracker_metadata['type'] == 'typosquatting':
typo_squatting = list(Tracker.get_tracker_typosquatting_domains(tracker_uuid))
typo_squatting.sort()
else:
typo_squatting = None
if date_from:
res = Term.parse_get_tracker_term_item({'uuid': tracker_uuid, 'date_from': date_from, 'date_to': date_to}, user_id)
@ -224,6 +241,7 @@ def show_tracker():
return render_template("showTracker.html", tracker_metadata=tracker_metadata,
yara_rule_content=yara_rule_content,
typo_squatting=typo_squatting,
bootstrap_label=bootstrap_label)
@hunter.route("/tracker/update_tracker_description", methods=['POST'])

7
var/www/modules/hunter/templates/edit_tracker.html
View file

@ -94,6 +94,7 @@
<option value="set">Set</option>
<option value="regex">Regex</option>
<option value="yara">YARA rule</option>
<option value="typosquatting">Typo-squatting</option>
</select>
<p id="tracker_desc">Terms to track (space separated)</p>
@ -199,6 +200,12 @@ $(document).ready(function(){
$("#tracker").hide();
$("#nb_word").hide();
$("#yara_rule").show();
} else if (tracker_type=="typosquatting") {
$("#tracker_desc").text("Generation of variation for domain name. Only one domain name at a time.");
$("#tracker_desc").show();
$("#tracker").show();
$("#nb_word").hide();
$("#yara_rule").hide();
}
});

56
var/www/modules/hunter/templates/showTracker.html
View file

@ -69,29 +69,47 @@
<div class="col-md-10">
<table class="table">
<thead>
<tr>
<th>Type</th>
<th>Tracker</th>
<th>Date added</th>
<th>Access Level</th>
<th>Created by</th>
<th>First seen</th>
<th>Last seen</th>
{% if tracker_metadata['webhook'] %}
<th>Webhook</th>
{% endif %}
<th>Tags <span class="btn-link btn-interaction mouse_pointer"
title="Edit Tags List" onclick="edit_tags();"><i
class="fas fa-pencil-alt" style="color:Red;"></i></span></th>
<th>Email <span class="btn-link btn-interaction mouse_pointer"
title="Edit Email List" onclick="edit_mails();"><i
class="fas fa-pencil-alt" style="color:Red;"></i></span></th>
</tr>
<tr>
<th>Type</th>
<th>Tracker</th>
<th>Date added</th>
<th>Access Level</th>
<th>Created by</th>
<th>First seen</th>
<th>Last seen</th>
{% if tracker_metadata['webhook'] %}
<th>Webhook</th>
{% endif %}
<th>Tags <span class="btn-link btn-interaction mouse_pointer"
title="Edit Tags List" onclick="edit_tags();"><i
class="fas fa-pencil-alt" style="color:Red;"></i></span></th>
<th>Email <span class="btn-link btn-interaction mouse_pointer"
title="Edit Email List" onclick="edit_mails();"><i
class="fas fa-pencil-alt" style="color:Red;"></i></span></th>
</tr>
</thead>
<tbody>
<tr>
<td>{{ tracker_metadata['type'] }}</td>
<td>{{ tracker_metadata['tracker'] }}</td>
{% if tracker_metadata['type'] == 'typosquatting' %}
<td>
<a class="btn btn-primary" data-toggle="collapse" href="#collapseTypo" role="button" aria-expanded="false" aria-controls="collapseTypo">
{{ tracker_metadata['tracker'].split(",")[0] }}
</a>
<div class="collapse" id="collapseTypo">
<div class="card card-body">
{% if typo_squatting %}
{% for typo in typo_squatting %}
{{typo}}
<br/>
{% endfor %}
{%endif%}
</div>
</div>
</td>
{% else %}
<td>{{ tracker_metadata['tracker'] }}</td>
{% endif %}
<td>{{ tracker_metadata['date'][0:4] }}/{{ tracker_metadata['date'][4:6] }}/{{ tracker_metadata['date'][6:8] }}</td>
<td>
{% if tracker_metadata['level'] == 0 %}

20
var/www/modules/hunter/templates/trackersManagement.html
View file

@ -70,10 +70,14 @@
<span>
<a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}">
{% if dict_uuid['term'] %}
{% if dict_uuid['term']|length > 256 %}
{{ dict_uuid['term'][0:256] }}...
{% if dict_uuid['term']|length > 2000 %}
{{ dict_uuid['term'][0:50] }}...
{% else %}
{{ dict_uuid['term'] }}
{% if dict_uuid['term']|length > 100 %}
{{ dict_uuid['term'][0:100] }}...
{% else %}
{{ dict_uuid['term'] }}
{% endif %}
{% endif %}
{% endif %}
</a>
@ -135,10 +139,14 @@
<span>
<a target="_blank" href="{{ url_for('hunter.show_tracker') }}?uuid={{ dict_uuid['uuid'] }}">
{% if dict_uuid['term'] %}
{% if dict_uuid['term']|length > 256 %}
{{ dict_uuid['term'][0:256] }}...
{% if dict_uuid['term']|length > 2000 %}
{{ dict_uuid['term'][0:50] }}...
{% else %}
{{ dict_uuid['term'] }}
{% if dict_uuid['term']|length > 100 %}
{{ dict_uuid['term'][0:100] }}...
{% else %}
{{ dict_uuid['term'] }}
{% endif %}
{% endif %}
{% endif %}
</a>

6
var/www/templates/hunter/menu_sidebar.html
View file

@ -42,6 +42,12 @@
<span class="bg-danger text-white font-weight-bold" style="font-size: 120%">&nbsp;{ </span>
<span>&nbsp;YARA</span>
</a>
</li>
<li class="nav-item">
<a class="nav-link" href="{{url_for('hunter.tracked_menu_typosquatting')}}" id="nav_tracker_typosquatting">
<i class="fa fa-clone"></i>
<span>Typo-squatting</span>
</a>
</li>
</ul>
<h5 class="d-flex text-muted w-100" id="nav_title_retro_hunt">

3
var/www/templates/tags/block_tags_selector.html
View file

@ -35,6 +35,7 @@
</div>
<!--
<button class="btn btn-primary" onclick="tagsSelector()">
<i class="fas fa-plus"></i>
@ -47,7 +48,7 @@
var ltags;
var ltagsgalaxies;
$.getJSON("{{ url_for('tags_ui.tag_taxonomies_tags_enabled_json') }}",
$.getJSON("{{ url_for('Tags.get_all_tags') }}",
function(data) {
{% if 'taxonomies_tags' in tags_selector_data %}
var valueData = [