fix: [MISP auto export] fix MISP_TheHive_feeder

This commit is contained in:
Terrtia 2020-06-19 13:36:03 +02:00
parent b50cc92573
commit 762b517150
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
7 changed files with 118 additions and 114 deletions

View file

@ -20,8 +20,12 @@ from Helper import Process
from packages import Paste
import ailleakObject
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages'))
import Tag
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib'))
import ConfigLoader
import item_basic
from pymisp import PyMISP
@ -94,26 +98,22 @@ def create_the_hive_alert(source, path, tag):
def feeder(message, count=0):
if flag_the_hive or flag_misp:
tag, path = message.split(';')
tag, item_id = message.split(';')
## FIXME: remove it
if PASTES_FOLDER not in path:
path = os.path.join(PASTES_FOLDER, path)
try:
paste = Paste.Paste(path)
except FileNotFoundError:
if not item_basic.exist_item(item_id):
if count < 10:
r_serv_db.zincrby('mess_not_saved_export', message, 1)
return 0
else:
r_serv_db.zrem('mess_not_saved_export', message)
print('Error: {} do not exist, tag= {}'.format(path, tag))
print('Error: {} do not exist, tag= {}'.format(item_id, tag))
return 0
source = '/'.join(paste.p_path.split('/')[-6:])
source = item_basic.get_source(item_id)
if HiveApi != False:
if int(r_serv_db.get('hive:auto-alerts')) == 1:
whitelist_hive = r_serv_db.scard('whitelist_hive')
if r_serv_db.sismember('whitelist_hive', tag):
create_the_hive_alert(source, path, tag)
else:
@ -121,7 +121,7 @@ def feeder(message, count=0):
if flag_misp:
if int(r_serv_db.get('misp:auto-events')) == 1:
if r_serv_db.sismember('whitelist_misp', tag):
misp_wrapper.pushToMISP(uuid_ail, path, tag)
misp_wrapper.pushToMISP(uuid_ail, item_id, tag)
else:
print('misp, auto events creation disable')
@ -161,15 +161,15 @@ if __name__ == "__main__":
print('Not connected to MISP')
if flag_misp:
try:
misp_wrapper = ailleakObject.ObjectWrapper(pymisp)
r_serv_db.set('ail:misp', True)
print('Connected to MISP:', misp_url)
except Exception as e:
flag_misp = False
r_serv_db.set('ail:misp', False)
print(e)
print('Not connected to MISP')
#try:
misp_wrapper = ailleakObject.ObjectWrapper(pymisp)
r_serv_db.set('ail:misp', True)
print('Connected to MISP:', misp_url)
#except Exception as e:
# flag_misp = False
# r_serv_db.set('ail:misp', False)
# print(e)
# print('Not connected to MISP')
# create The HIVE connection
if flag_the_hive:

View file

@ -45,4 +45,5 @@ if __name__ == '__main__':
tag, item_id = message.split(';')
Tag.add_tag("item", tag, item_id)
p.populate_set_out(message, 'MISP_The_Hive_feeder')

View file

@ -4,7 +4,10 @@
import os
import sys
from pymisp import MISPEvent, MISPObject
from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator
MISPEvent
from packages import Paste
import datetime
import json
@ -12,28 +15,10 @@ from io import BytesIO
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
import item_basic
class AilLeakObject(AbstractMISPObjectGenerator):
def __init__(self, uuid_ail, p_source, p_date, p_content, p_duplicate, p_duplicate_number):
super(AbstractMISPObjectGenerator, self).__init__('ail-leak')
self._uuid = uuid_ail
self._p_source = p_source
self._p_date = p_date
self._p_content = p_content
self._p_duplicate = p_duplicate
self._p_duplicate_number = p_duplicate_number
self.generate_attributes()
def generate_attributes(self):
self.add_attribute('origin', value=self._p_source, type='text')
self.add_attribute('last-seen', value=self._p_date, type='datetime')
if self._p_duplicate_number > 0:
self.add_attribute('duplicate', value=self._p_duplicate, type='text')
self.add_attribute('duplicate_number', value=self._p_duplicate_number, type='counter')
self._pseudofile = BytesIO(self._p_content.encode())
res = self.add_attribute('raw-data', value=self._p_source, data=self._pseudofile, type="attachment")# , ShadowAttribute=self.p_tag)
#res.add_shadow_attributes(tag)
self.add_attribute('sensor', value=self._uuid, type="text")
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'export'))
import MispExport
class ObjectWrapper:
def __init__(self, pymisp):
@ -45,53 +30,48 @@ class ObjectWrapper:
config_loader = None
self.attribute_to_tag = None
def add_new_object(self, uuid_ail, path, p_source, tag):
def add_new_object(self, uuid_ail, item_id, tag):
self.uuid_ail = uuid_ail
self.path = path
self.p_source = p_source
self.paste = Paste.Paste(path)
self.p_date = self.date_to_str(self.paste.p_date)
self.p_content = self.paste.get_p_content()
self.p_tag = tag
temp = self.paste._get_p_duplicate()
# self.paste = Paste.Paste(path)
# temp = self.paste._get_p_duplicate()
#
# #beautifier
# if not temp:
# temp = ''
#
# p_duplicate_number = len(temp) if len(temp) >= 0 else 0
#
# to_ret = ""
# for dup in temp[:10]:
# dup = dup.replace('\'','\"').replace('(','[').replace(')',']')
# dup = json.loads(dup)
# algo = dup[0]
# path = dup[1].split('/')[-6:]
# path = '/'.join(path)[:-3] # -3 removes .gz
# if algo == 'tlsh':
# perc = 100 - int(dup[2])
# else:
# perc = dup[2]
# to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
# p_duplicate = to_ret
#beautifier
if not temp:
temp = ''
p_duplicate_number = len(temp) if len(temp) >= 0 else 0
to_ret = ""
for dup in temp[:10]:
dup = dup.replace('\'','\"').replace('(','[').replace(')',']')
dup = json.loads(dup)
algo = dup[0]
path = dup[1].split('/')[-6:]
path = '/'.join(path)[:-3] # -3 removes .gz
if algo == 'tlsh':
perc = 100 - int(dup[2])
else:
perc = dup[2]
to_ret += "{}: {} [{}%]\n".format(path, algo, perc)
p_duplicate = to_ret
self.mispObject = AilLeakObject(self.uuid_ail, self.p_source, self.p_date, self.p_content, p_duplicate, p_duplicate_number)
return MispExport.export_ail_item(item_id, [tag])
def date_to_str(self, date):
return "{0}-{1}-{2}".format(date.year, date.month, date.day)
def get_all_related_events(self):
to_search = "Daily AIL-leaks"
result = self.pymisp.search_all(to_search)
def get_all_related_events(self, to_search):
result = self.pymisp.search(controller='events', eventinfo=to_search, metadata=False)
events = []
for e in result['response']:
events.append({'id': e['Event']['id'], 'org_id': e['Event']['org_id'], 'info': e['Event']['info']})
if result:
for e in result:
events.append({'id': e['Event']['id'], 'org_id': e['Event']['org_id'], 'info': e['Event']['info']})
return events
def get_daily_event_id(self):
to_match = "Daily AIL-leaks {}".format(datetime.date.today())
events = self.get_all_related_events()
events = self.get_all_related_events(to_match)
for dic in events:
info = dic['info']
e_id = dic['id']
@ -99,8 +79,8 @@ class ObjectWrapper:
print('Found: ', info, '->', e_id)
self.currentID_date = datetime.date.today()
return e_id
created_event = self.create_daily_event()['Event']
new_id = created_event['id']
created_event = self.create_daily_event()
new_id = created_event['Event']['id']
print('New event created:', new_id)
self.currentID_date = datetime.date.today()
return new_id
@ -120,17 +100,20 @@ class ObjectWrapper:
orgc_id = None
sharing_group_id = None
date = None
event = self.pymisp.new_event(distribution, threat,
analysis, info, date,
published, orgc_id, org_id, sharing_group_id)
eventUuid = event['Event']['uuid']
self.pymisp.tag(eventUuid, 'infoleak:output-format="ail-daily"')
return event
event = MISPEvent()
event.distribution = distribution
event.info = info
event.analysis = analysis
event.threat = threat
event.published = published
event.add_tag('infoleak:output-format="ail-daily"')
existing_event = self.pymisp.add_event(event)
return existing_event
# Publish object to MISP
def pushToMISP(self, uuid_ail, path, tag):
self._p_source = path.split('/')[-5:]
self._p_source = '/'.join(self._p_source)[:-3]
def pushToMISP(self, uuid_ail, item_id, tag):
if self.currentID_date != datetime.date.today(): #refresh id
self.eventID_to_push = self.get_daily_event_id()
@ -138,42 +121,37 @@ class ObjectWrapper:
mispTYPE = 'ail-leak'
# paste object already exist
if self.paste_object_exist(self.eventID_to_push, self._p_source):
if self.paste_object_exist(self.eventID_to_push, item_id):
# add new tag
self.tag(self.attribute_to_tag, tag)
print(self._p_source + ' tagged: ' + tag)
print(item_id + ' tagged: ' + tag)
#create object
else:
self.add_new_object(uuid_ail, path, self._p_source, tag)
misp_obj = self.add_new_object(uuid_ail, item_id, tag)
# deprecated
# try:
# templateID = [x['ObjectTemplate']['id'] for x in self.pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0]
# except IndexError:
# valid_types = ", ".join([x['ObjectTemplate']['name'] for x in self.pymisp.get_object_templates_list()])
# print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types))
try:
templateID = [x['ObjectTemplate']['id'] for x in self.pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0]
except IndexError:
valid_types = ", ".join([x['ObjectTemplate']['name'] for x in self.pymisp.get_object_templates_list()])
print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types))
r = self.pymisp.add_object(self.eventID_to_push, templateID, self.mispObject)
r = self.pymisp.add_object(self.eventID_to_push, misp_obj, pythonify=True)
if 'errors' in r:
print(r)
else:
# tag new object
self.set_attribute_to_tag_uuid(self.eventID_to_push, self._p_source)
self.tag(self.attribute_to_tag, tag)
print('Pushed:', tag, '->', self._p_source)
print('Pushed:', tag, '->', item_id)
def paste_object_exist(self, eventId, source):
res = self.pymisp.search(controller='attributes', eventid=eventId, values=source)
def paste_object_exist(self, eventId, item_id):
res = self.pymisp.search(controller='attributes', eventid=eventId, value=item_id)
# object already exist
if res['response']:
self.attribute_to_tag = res['response']['Attribute'][0]['uuid']
if res.get('Attribute', []):
self.attribute_to_tag = res['Attribute'][0]['uuid']
return True
# new object
else:
return False
def set_attribute_to_tag_uuid(self, eventId, source):
res = self.pymisp.search(controller='attributes', eventid=eventId, values=source)
self.attribute_to_tag = res['response']['Attribute'][0]['uuid']
def tag(self, uuid, tag):
self.pymisp.tag(uuid, tag)

View file

@ -19,6 +19,7 @@ import Screenshot
import Correlate_object
import AILObjects
import Export
# # TODO: # FIXME: REFRACTOR ME => use UI/Global config
sys.path.append('../../configs/keys')
@ -59,8 +60,12 @@ def tag_misp_object_attributes(l_ref_obj_attr, tags):
for tag in tags:
obj_attr.add_tag(tag)
def export_ail_item(item_id):
def export_ail_item(item_id, tags=[]):
dict_metadata = Item.get_item({'id': item_id, 'date':True, 'tags':True, 'raw_content':True})[0]
# force tags
for tag in tags:
if tag not in dict_metadata['tags']:
dict_metadata['tags'].append(tag)
#obj = MISPObject('ail-item', standalone=True)
obj = MISPObject('ail-leak', standalone=True)
@ -69,6 +74,7 @@ def export_ail_item(item_id):
l_obj_attr = []
l_obj_attr.append( obj.add_attribute('first-seen', value=dict_metadata['date']) )
l_obj_attr.append( obj.add_attribute('raw-data', value=item_id, data=dict_metadata['raw_content']) )
l_obj_attr.append( obj.add_attribute('sensor', value=Export.get_ail_uuid()) )
# add tags
if dict_metadata['tags']:

View file

@ -33,6 +33,9 @@ def get_item_date(item_id, add_separator=False):
else:
return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
def get_source(item_id):
return item_id.split('/')[-5]
# # TODO: add an option to check the tag
def is_crawled(item_id):
return item_id.startswith('crawled')

View file

@ -48,7 +48,7 @@ def get_item_date(item_id, add_separator=False):
return item_basic.get_item_date(item_id, add_separator=add_separator)
def get_source(item_id):
return item_id.split('/')[-5]
return item_basic.get_source(item_id)
def get_item_basename(item_id):
return os.path.basename(item_id)

View file

@ -8,10 +8,10 @@ import datetime
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
import Date
import Item
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/'))
import ConfigLoader
import item_basic
from pytaxonomies import Taxonomies
from pymispgalaxies import Galaxies, Clusters
@ -383,8 +383,8 @@ def add_obj_tag(object_type, object_id, tag, obj_date=None):
r_serv_tags.sadd('{}:{}'.format(tag, obj_date), object_id)
# add domain tag
if Item.is_crawled(object_id) and tag!='infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
domain = Item.get_item_domain(object_id)
if item_basic.is_crawled(object_id) and tag!='infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"':
domain = item_basic.get_item_domain(object_id)
add_tag("domain", tag, domain)
else:
r_serv_metadata.sadd('tag:{}'.format(object_id), tag)
@ -432,7 +432,7 @@ def delete_tag(object_type, tag, object_id, obj_date=None):
# # TODO: move me
def get_obj_date(object_type, object_id):
if object_type == "item":
return int(Item.get_item_date(object_id))
return int(item_basic.get_item_date(object_id))
else:
return None
@ -573,3 +573,19 @@ def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50
l_tagged_obj = list(l_tagged_obj)
return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop, "nb_all_elem":nb_all_elem}
#### TAGS EXPORT ####
# # TODO:
def is_updated_tags_to_export(): # by type
return False
def get_list_of_solo_tags_to_export_by_type(export_type): # by type
if export_type in ['misp', 'thehive']:
return r_serv_db.smembers('whitelist_{}'.format(export_type))
else:
return None
#r_serv_db.smembers('whitelist_hive')
#### -- ####