From 5c1de2d09f61725e4788c927c24bfc6566af0607 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 17 Jun 2020 16:20:03 +0200 Subject: [PATCH 01/28] fix: [install] unshallow clone --- install_virtualenv.sh | 5 ++--- installing_deps.sh | 14 ++++++++------ 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/install_virtualenv.sh b/install_virtualenv.sh index 9c80b08c..bde03b7b 100755 --- a/install_virtualenv.sh +++ b/install_virtualenv.sh @@ -15,11 +15,10 @@ if [ -z "$VIRTUAL_ENV" ]; then echo export AIL_FLASK=$(pwd)/var/www/ >> ./AILENV/bin/activate echo export AIL_REDIS=$(pwd)/redis/src/ >> ./AILENV/bin/activate echo export AIL_ARDB=$(pwd)/ardb/src/ >> ./AILENV/bin/activate - - . ./AILENV/bin/activate - fi +. ./AILENV/bin/activate + pip3 install -U pip pip3 install 'git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client' pip3 install -U -r requirements.txt diff --git a/installing_deps.sh b/installing_deps.sh index 49e7cb2e..28de889e 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -88,16 +88,18 @@ fi # create AILENV + intall python packages ./install_virtualenv.sh +# force virtualenv activation +. ./AILENV/bin/activate -pushd ${AIL_BIN}helper/gen_cert +pushd ${AIL_BIN}/helper/gen_cert ./gen_root.sh wait ./gen_cert.sh wait popd -cp ${AIL_BIN}helper/gen_cert/server.crt ${AIL_FLASK}server.crt -cp ${AIL_BIN}helper/gen_cert/server.key ${AIL_FLASK}server.key +cp ${AIL_BIN}/helper/gen_cert/server.crt ${AIL_FLASK}/server.crt +cp ${AIL_BIN}/helper/gen_cert/server.key ${AIL_FLASK}/server.key mkdir -p $AIL_HOME/PASTES @@ -109,14 +111,14 @@ $AIL_HOME/doc/generate_modules_data_flow_graph.sh # init update version pushd ${AIL_HOME} # shallow clone -git fetch --tags --prune --unshallow +git fetch --tags --prune --depth=10000 git describe --abbrev=0 --tags | tr -d '\n' > ${AIL_HOME}/update/current_version echo "AIL current version:" git describe --abbrev=0 --tags popd # LAUNCH ARDB -bash ${AIL_BIN}LAUNCH.sh -lav & +bash ${AIL_BIN}/LAUNCH.sh -lav & wait echo "" @@ -125,6 +127,6 @@ pushd ${AIL_FLASK} python3 create_default_user.py popd -bash ${AIL_BIN}LAUNCH.sh -k & +bash ${AIL_BIN}/LAUNCH.sh -k & wait echo "" From 060337ee9d0fb5df6d1f6cd1292f9d0eb2f7437a Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 18 Jun 2020 08:42:22 +0200 Subject: [PATCH 02/28] fix: [travis] virtualenv --- install_virtualenv.sh | 4 ++-- installing_deps.sh | 4 +++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/install_virtualenv.sh b/install_virtualenv.sh index bde03b7b..f20cb93f 100755 --- a/install_virtualenv.sh +++ b/install_virtualenv.sh @@ -15,9 +15,9 @@ if [ -z "$VIRTUAL_ENV" ]; then echo export AIL_FLASK=$(pwd)/var/www/ >> ./AILENV/bin/activate echo export AIL_REDIS=$(pwd)/redis/src/ >> ./AILENV/bin/activate echo export AIL_ARDB=$(pwd)/ardb/src/ >> ./AILENV/bin/activate -fi -. ./AILENV/bin/activate + . ./AILENV/bin/activate +fi pip3 install -U pip pip3 install 'git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client' diff --git a/installing_deps.sh b/installing_deps.sh index 28de889e..dfa8558e 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -89,7 +89,9 @@ fi ./install_virtualenv.sh # force virtualenv activation -. ./AILENV/bin/activate +if [ -z "$VIRTUAL_ENV" ]; then + . ./AILENV/bin/activate +fi pushd ${AIL_BIN}/helper/gen_cert ./gen_root.sh From 52494d5c0117dd5744fcdc8ed89fe646e0f6a83c Mon Sep 17 00:00:00 2001 From: ChilliSec <16847981+ChilliSec@users.noreply.github.com> Date: Fri, 19 Jun 2020 00:17:20 +0100 Subject: [PATCH 03/28] Update HOWTO.md --- HOWTO.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/HOWTO.md b/HOWTO.md index 9e72c77e..3978408f 100644 --- a/HOWTO.md +++ b/HOWTO.md @@ -109,11 +109,11 @@ There are two types of installation. You can install a *local* or a *remote* Spl ### Installation/Configuration -1. *(Splash host)* Launch ``crawler_hidden_services_install.sh`` to install all requirements (type ``y`` if a localhost splah server is used or use the ``-y`` option) +1. *(Splash host)* Launch ``crawler_hidden_services_install.sh`` to install all requirements (type ``y`` if a localhost splash server is used or use the ``-y`` option) 2. *(Splash host)* To install and setup your tor proxy: - Install the tor proxy: ``sudo apt-get install tor -y`` - (Not required if ``Splah host == AIL host`` - The tor proxy is installed by default in AIL) + (Not required if ``Splash host == AIL host`` - The tor proxy is installed by default in AIL) (Warning: Some v3 onion address are not resolved with the tor proxy provided via apt get. Use the tor proxy provided by [The torproject](https://2019.www.torproject.org/docs/debian) to solve this issue) - Allow Tor to bind to any interface or to the docker interface (by default binds to 127.0.0.1 only) in ``/etc/tor/torrc`` From 762b517150ac46898331e52878496752681be167 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 19 Jun 2020 13:36:03 +0200 Subject: [PATCH 04/28] fix: [MISP auto export] fix MISP_TheHive_feeder --- bin/MISP_The_Hive_feeder.py | 40 +++++----- bin/Tags.py | 1 + bin/ailleakObject.py | 154 ++++++++++++++++-------------------- bin/export/MispExport.py | 8 +- bin/lib/item_basic.py | 3 + bin/packages/Item.py | 2 +- bin/packages/Tag.py | 24 +++++- 7 files changed, 118 insertions(+), 114 deletions(-) diff --git a/bin/MISP_The_Hive_feeder.py b/bin/MISP_The_Hive_feeder.py index 0949471c..23043788 100755 --- a/bin/MISP_The_Hive_feeder.py +++ b/bin/MISP_The_Hive_feeder.py @@ -20,8 +20,12 @@ from Helper import Process from packages import Paste import ailleakObject -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) +import Tag + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) import ConfigLoader +import item_basic from pymisp import PyMISP @@ -94,26 +98,22 @@ def create_the_hive_alert(source, path, tag): def feeder(message, count=0): if flag_the_hive or flag_misp: - tag, path = message.split(';') + tag, item_id = message.split(';') + ## FIXME: remove it - if PASTES_FOLDER not in path: - path = os.path.join(PASTES_FOLDER, path) - try: - paste = Paste.Paste(path) - except FileNotFoundError: + if not item_basic.exist_item(item_id): if count < 10: r_serv_db.zincrby('mess_not_saved_export', message, 1) return 0 else: r_serv_db.zrem('mess_not_saved_export', message) - print('Error: {} do not exist, tag= {}'.format(path, tag)) + print('Error: {} do not exist, tag= {}'.format(item_id, tag)) return 0 - source = '/'.join(paste.p_path.split('/')[-6:]) + source = item_basic.get_source(item_id) if HiveApi != False: if int(r_serv_db.get('hive:auto-alerts')) == 1: - whitelist_hive = r_serv_db.scard('whitelist_hive') if r_serv_db.sismember('whitelist_hive', tag): create_the_hive_alert(source, path, tag) else: @@ -121,7 +121,7 @@ def feeder(message, count=0): if flag_misp: if int(r_serv_db.get('misp:auto-events')) == 1: if r_serv_db.sismember('whitelist_misp', tag): - misp_wrapper.pushToMISP(uuid_ail, path, tag) + misp_wrapper.pushToMISP(uuid_ail, item_id, tag) else: print('misp, auto events creation disable') @@ -161,15 +161,15 @@ if __name__ == "__main__": print('Not connected to MISP') if flag_misp: - try: - misp_wrapper = ailleakObject.ObjectWrapper(pymisp) - r_serv_db.set('ail:misp', True) - print('Connected to MISP:', misp_url) - except Exception as e: - flag_misp = False - r_serv_db.set('ail:misp', False) - print(e) - print('Not connected to MISP') + #try: + misp_wrapper = ailleakObject.ObjectWrapper(pymisp) + r_serv_db.set('ail:misp', True) + print('Connected to MISP:', misp_url) + #except Exception as e: + # flag_misp = False + # r_serv_db.set('ail:misp', False) + # print(e) + # print('Not connected to MISP') # create The HIVE connection if flag_the_hive: diff --git a/bin/Tags.py b/bin/Tags.py index 88e0ef0e..b38d6309 100755 --- a/bin/Tags.py +++ b/bin/Tags.py @@ -45,4 +45,5 @@ if __name__ == '__main__': tag, item_id = message.split(';') Tag.add_tag("item", tag, item_id) + p.populate_set_out(message, 'MISP_The_Hive_feeder') diff --git a/bin/ailleakObject.py b/bin/ailleakObject.py index 5fbf9f75..fd07e6cc 100755 --- a/bin/ailleakObject.py +++ b/bin/ailleakObject.py @@ -4,7 +4,10 @@ import os import sys +from pymisp import MISPEvent, MISPObject from pymisp.tools.abstractgenerator import AbstractMISPObjectGenerator +MISPEvent + from packages import Paste import datetime import json @@ -12,28 +15,10 @@ from io import BytesIO sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader +import item_basic -class AilLeakObject(AbstractMISPObjectGenerator): - def __init__(self, uuid_ail, p_source, p_date, p_content, p_duplicate, p_duplicate_number): - super(AbstractMISPObjectGenerator, self).__init__('ail-leak') - self._uuid = uuid_ail - self._p_source = p_source - self._p_date = p_date - self._p_content = p_content - self._p_duplicate = p_duplicate - self._p_duplicate_number = p_duplicate_number - self.generate_attributes() - - def generate_attributes(self): - self.add_attribute('origin', value=self._p_source, type='text') - self.add_attribute('last-seen', value=self._p_date, type='datetime') - if self._p_duplicate_number > 0: - self.add_attribute('duplicate', value=self._p_duplicate, type='text') - self.add_attribute('duplicate_number', value=self._p_duplicate_number, type='counter') - self._pseudofile = BytesIO(self._p_content.encode()) - res = self.add_attribute('raw-data', value=self._p_source, data=self._pseudofile, type="attachment")# , ShadowAttribute=self.p_tag) - #res.add_shadow_attributes(tag) - self.add_attribute('sensor', value=self._uuid, type="text") +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'export')) +import MispExport class ObjectWrapper: def __init__(self, pymisp): @@ -45,53 +30,48 @@ class ObjectWrapper: config_loader = None self.attribute_to_tag = None - def add_new_object(self, uuid_ail, path, p_source, tag): + def add_new_object(self, uuid_ail, item_id, tag): self.uuid_ail = uuid_ail - self.path = path - self.p_source = p_source - self.paste = Paste.Paste(path) - self.p_date = self.date_to_str(self.paste.p_date) - self.p_content = self.paste.get_p_content() - self.p_tag = tag - temp = self.paste._get_p_duplicate() + # self.paste = Paste.Paste(path) + # temp = self.paste._get_p_duplicate() + # + # #beautifier + # if not temp: + # temp = '' + # + # p_duplicate_number = len(temp) if len(temp) >= 0 else 0 + # + # to_ret = "" + # for dup in temp[:10]: + # dup = dup.replace('\'','\"').replace('(','[').replace(')',']') + # dup = json.loads(dup) + # algo = dup[0] + # path = dup[1].split('/')[-6:] + # path = '/'.join(path)[:-3] # -3 removes .gz + # if algo == 'tlsh': + # perc = 100 - int(dup[2]) + # else: + # perc = dup[2] + # to_ret += "{}: {} [{}%]\n".format(path, algo, perc) + # p_duplicate = to_ret - #beautifier - if not temp: - temp = '' - - p_duplicate_number = len(temp) if len(temp) >= 0 else 0 - - to_ret = "" - for dup in temp[:10]: - dup = dup.replace('\'','\"').replace('(','[').replace(')',']') - dup = json.loads(dup) - algo = dup[0] - path = dup[1].split('/')[-6:] - path = '/'.join(path)[:-3] # -3 removes .gz - if algo == 'tlsh': - perc = 100 - int(dup[2]) - else: - perc = dup[2] - to_ret += "{}: {} [{}%]\n".format(path, algo, perc) - p_duplicate = to_ret - - self.mispObject = AilLeakObject(self.uuid_ail, self.p_source, self.p_date, self.p_content, p_duplicate, p_duplicate_number) + return MispExport.export_ail_item(item_id, [tag]) def date_to_str(self, date): return "{0}-{1}-{2}".format(date.year, date.month, date.day) - def get_all_related_events(self): - to_search = "Daily AIL-leaks" - result = self.pymisp.search_all(to_search) + def get_all_related_events(self, to_search): + result = self.pymisp.search(controller='events', eventinfo=to_search, metadata=False) events = [] - for e in result['response']: - events.append({'id': e['Event']['id'], 'org_id': e['Event']['org_id'], 'info': e['Event']['info']}) + if result: + for e in result: + events.append({'id': e['Event']['id'], 'org_id': e['Event']['org_id'], 'info': e['Event']['info']}) return events def get_daily_event_id(self): to_match = "Daily AIL-leaks {}".format(datetime.date.today()) - events = self.get_all_related_events() + events = self.get_all_related_events(to_match) for dic in events: info = dic['info'] e_id = dic['id'] @@ -99,8 +79,8 @@ class ObjectWrapper: print('Found: ', info, '->', e_id) self.currentID_date = datetime.date.today() return e_id - created_event = self.create_daily_event()['Event'] - new_id = created_event['id'] + created_event = self.create_daily_event() + new_id = created_event['Event']['id'] print('New event created:', new_id) self.currentID_date = datetime.date.today() return new_id @@ -120,17 +100,20 @@ class ObjectWrapper: orgc_id = None sharing_group_id = None date = None - event = self.pymisp.new_event(distribution, threat, - analysis, info, date, - published, orgc_id, org_id, sharing_group_id) - eventUuid = event['Event']['uuid'] - self.pymisp.tag(eventUuid, 'infoleak:output-format="ail-daily"') - return event + + event = MISPEvent() + event.distribution = distribution + event.info = info + event.analysis = analysis + event.threat = threat + event.published = published + + event.add_tag('infoleak:output-format="ail-daily"') + existing_event = self.pymisp.add_event(event) + return existing_event # Publish object to MISP - def pushToMISP(self, uuid_ail, path, tag): - self._p_source = path.split('/')[-5:] - self._p_source = '/'.join(self._p_source)[:-3] + def pushToMISP(self, uuid_ail, item_id, tag): if self.currentID_date != datetime.date.today(): #refresh id self.eventID_to_push = self.get_daily_event_id() @@ -138,42 +121,37 @@ class ObjectWrapper: mispTYPE = 'ail-leak' # paste object already exist - if self.paste_object_exist(self.eventID_to_push, self._p_source): + if self.paste_object_exist(self.eventID_to_push, item_id): # add new tag self.tag(self.attribute_to_tag, tag) - print(self._p_source + ' tagged: ' + tag) + print(item_id + ' tagged: ' + tag) #create object else: - self.add_new_object(uuid_ail, path, self._p_source, tag) + misp_obj = self.add_new_object(uuid_ail, item_id, tag) + + # deprecated + # try: + # templateID = [x['ObjectTemplate']['id'] for x in self.pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0] + # except IndexError: + # valid_types = ", ".join([x['ObjectTemplate']['name'] for x in self.pymisp.get_object_templates_list()]) + # print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types)) - try: - templateID = [x['ObjectTemplate']['id'] for x in self.pymisp.get_object_templates_list() if x['ObjectTemplate']['name'] == mispTYPE][0] - except IndexError: - valid_types = ", ".join([x['ObjectTemplate']['name'] for x in self.pymisp.get_object_templates_list()]) - print ("Template for type %s not found! Valid types are: %s" % (mispTYPE, valid_types)) - r = self.pymisp.add_object(self.eventID_to_push, templateID, self.mispObject) + r = self.pymisp.add_object(self.eventID_to_push, misp_obj, pythonify=True) if 'errors' in r: print(r) else: - # tag new object - self.set_attribute_to_tag_uuid(self.eventID_to_push, self._p_source) - self.tag(self.attribute_to_tag, tag) - print('Pushed:', tag, '->', self._p_source) + print('Pushed:', tag, '->', item_id) - def paste_object_exist(self, eventId, source): - res = self.pymisp.search(controller='attributes', eventid=eventId, values=source) + def paste_object_exist(self, eventId, item_id): + res = self.pymisp.search(controller='attributes', eventid=eventId, value=item_id) # object already exist - if res['response']: - self.attribute_to_tag = res['response']['Attribute'][0]['uuid'] + if res.get('Attribute', []): + self.attribute_to_tag = res['Attribute'][0]['uuid'] return True # new object else: return False - def set_attribute_to_tag_uuid(self, eventId, source): - res = self.pymisp.search(controller='attributes', eventid=eventId, values=source) - self.attribute_to_tag = res['response']['Attribute'][0]['uuid'] - def tag(self, uuid, tag): self.pymisp.tag(uuid, tag) diff --git a/bin/export/MispExport.py b/bin/export/MispExport.py index fe1fc304..4ee6bf89 100755 --- a/bin/export/MispExport.py +++ b/bin/export/MispExport.py @@ -19,6 +19,7 @@ import Screenshot import Correlate_object import AILObjects +import Export # # TODO: # FIXME: REFRACTOR ME => use UI/Global config sys.path.append('../../configs/keys') @@ -59,8 +60,12 @@ def tag_misp_object_attributes(l_ref_obj_attr, tags): for tag in tags: obj_attr.add_tag(tag) -def export_ail_item(item_id): +def export_ail_item(item_id, tags=[]): dict_metadata = Item.get_item({'id': item_id, 'date':True, 'tags':True, 'raw_content':True})[0] + # force tags + for tag in tags: + if tag not in dict_metadata['tags']: + dict_metadata['tags'].append(tag) #obj = MISPObject('ail-item', standalone=True) obj = MISPObject('ail-leak', standalone=True) @@ -69,6 +74,7 @@ def export_ail_item(item_id): l_obj_attr = [] l_obj_attr.append( obj.add_attribute('first-seen', value=dict_metadata['date']) ) l_obj_attr.append( obj.add_attribute('raw-data', value=item_id, data=dict_metadata['raw_content']) ) + l_obj_attr.append( obj.add_attribute('sensor', value=Export.get_ail_uuid()) ) # add tags if dict_metadata['tags']: diff --git a/bin/lib/item_basic.py b/bin/lib/item_basic.py index 98a6891c..6b606dda 100755 --- a/bin/lib/item_basic.py +++ b/bin/lib/item_basic.py @@ -33,6 +33,9 @@ def get_item_date(item_id, add_separator=False): else: return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2]) +def get_source(item_id): + return item_id.split('/')[-5] + # # TODO: add an option to check the tag def is_crawled(item_id): return item_id.startswith('crawled') diff --git a/bin/packages/Item.py b/bin/packages/Item.py index dc8fdccd..28a7349b 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -48,7 +48,7 @@ def get_item_date(item_id, add_separator=False): return item_basic.get_item_date(item_id, add_separator=add_separator) def get_source(item_id): - return item_id.split('/')[-5] + return item_basic.get_source(item_id) def get_item_basename(item_id): return os.path.basename(item_id) diff --git a/bin/packages/Tag.py b/bin/packages/Tag.py index 0c6a695b..f609be0b 100755 --- a/bin/packages/Tag.py +++ b/bin/packages/Tag.py @@ -8,10 +8,10 @@ import datetime sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) import Date -import Item sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader +import item_basic from pytaxonomies import Taxonomies from pymispgalaxies import Galaxies, Clusters @@ -383,8 +383,8 @@ def add_obj_tag(object_type, object_id, tag, obj_date=None): r_serv_tags.sadd('{}:{}'.format(tag, obj_date), object_id) # add domain tag - if Item.is_crawled(object_id) and tag!='infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"': - domain = Item.get_item_domain(object_id) + if item_basic.is_crawled(object_id) and tag!='infoleak:submission="crawler"' and tag != 'infoleak:submission="manual"': + domain = item_basic.get_item_domain(object_id) add_tag("domain", tag, domain) else: r_serv_metadata.sadd('tag:{}'.format(object_id), tag) @@ -432,7 +432,7 @@ def delete_tag(object_type, tag, object_id, obj_date=None): # # TODO: move me def get_obj_date(object_type, object_id): if object_type == "item": - return int(Item.get_item_date(object_id)) + return int(item_basic.get_item_date(object_id)) else: return None @@ -573,3 +573,19 @@ def get_obj_by_tags(object_type, l_tags, date_from=None, date_to=None, nb_obj=50 l_tagged_obj = list(l_tagged_obj) return {"tagged_obj":l_tagged_obj, "page":page, "nb_pages":nb_pages, "nb_first_elem":start+1, "nb_last_elem":stop, "nb_all_elem":nb_all_elem} + + +#### TAGS EXPORT #### +# # TODO: +def is_updated_tags_to_export(): # by type + return False + +def get_list_of_solo_tags_to_export_by_type(export_type): # by type + if export_type in ['misp', 'thehive']: + return r_serv_db.smembers('whitelist_{}'.format(export_type)) + else: + return None + #r_serv_db.smembers('whitelist_hive') + + +#### -- #### From 270a3d1d2a7bbe1b7d269b96f83ff1e9cdff3d85 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 19 Jun 2020 14:15:59 +0200 Subject: [PATCH 05/28] fix: [MISP auto Export] fix import path --- bin/export/MispExport.py | 1 + 1 file changed, 1 insertion(+) diff --git a/bin/export/MispExport.py b/bin/export/MispExport.py index 4ee6bf89..f1f1cf60 100755 --- a/bin/export/MispExport.py +++ b/bin/export/MispExport.py @@ -7,6 +7,7 @@ import sys import uuid import redis +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'export')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) import Item From 1d80ca2087bba8165f9d70a5dca12ddef44b5663 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 19 Jun 2020 14:41:27 +0200 Subject: [PATCH 06/28] fix: [UI popper version] use popper v1 (https://github.com/twbs/bootstrap/issues/29842) --- var/www/update_thirdparty.sh | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh index 53b4e6bf..6fcabc3a 100755 --- a/var/www/update_thirdparty.sh +++ b/var/www/update_thirdparty.sh @@ -76,9 +76,11 @@ wget -q https://cdn.datatables.net/1.10.20/css/dataTables.bootstrap4.min.css -O wget -q https://cdn.datatables.net/1.10.20/js/dataTables.bootstrap4.min.js -O ./static/js/dataTables.bootstrap.min.js #Ressources for bootstrap popover -POPPER_VERSION="2.0.6" -wget -q https://unpkg.com/@popperjs/core@${POPPER_VERSION}/dist/umd/popper.min.js -O ./static/js/popper.min.js -wget -q https://unpkg.com/@popperjs/core@${POPPER_VERSION}/dist/umd/popper.min.js.map -O ./static/js/popper.min.js.map +POPPER_VERSION="1.16.1" +wget https://github.com/FezVrasta/popper.js/archive/v${POPPER_VERSION}.zip -O temp/popper.zip +unzip temp/popper.zip -d temp/ +mv temp/popper-core-${POPPER_VERSION}/dist/umd/popper.min.js ./static/js/ +mv temp/popper-core-${POPPER_VERSION}/dist/umd/popper.min.js.map ./static/js/ #Ressource for graph wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb6109df5c1e8003/jquery.flot.js -O ./static/js/jquery.flot.js @@ -112,11 +114,11 @@ then fi #Update MISP Taxonomies and Galaxies -python3 -m pip install git+https://github.com/MISP/PyTaxonomies --upgrade -python3 -m pip install git+https://github.com/MISP/PyMISPGalaxies --upgrade +pip3 install git+https://github.com/MISP/PyTaxonomies --upgrade +pip3 install git+https://github.com/MISP/PyMISPGalaxies --upgrade #Update PyMISP -python3 -m pip install git+https://github.com/MISP/PyMISP --upgrade +pip3 install git+https://github.com/MISP/PyMISP --upgrade #Update the Hive -python3 -m pip install thehive4py --upgrade +pip3 install thehive4py --upgrade From 5b49e06060da2a0c71052e00011cb7a1c87eaba3 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 19 Jun 2020 15:03:33 +0200 Subject: [PATCH 07/28] fix: [update_thirparty] fix popper install --- var/www/update_thirdparty.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/var/www/update_thirdparty.sh b/var/www/update_thirdparty.sh index 6fcabc3a..c2ed08e5 100755 --- a/var/www/update_thirdparty.sh +++ b/var/www/update_thirdparty.sh @@ -62,8 +62,6 @@ mv temp/d3.min.js ./static/js/ mv temp/moment-2.24.0/min/moment.min.js ./static/js/ mv temp/jquery-date-range-picker-0.20.0/dist/jquery.daterangepicker.min.js ./static/js/ -rm -rf temp - JQVERSION="3.4.1" wget -q http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js @@ -77,8 +75,8 @@ wget -q https://cdn.datatables.net/1.10.20/js/dataTables.bootstrap4.min.js -O ./ #Ressources for bootstrap popover POPPER_VERSION="1.16.1" -wget https://github.com/FezVrasta/popper.js/archive/v${POPPER_VERSION}.zip -O temp/popper.zip -unzip temp/popper.zip -d temp/ +wget -q https://github.com/FezVrasta/popper.js/archive/v${POPPER_VERSION}.zip -O temp/popper.zip +unzip -qq temp/popper.zip -d temp/ mv temp/popper-core-${POPPER_VERSION}/dist/umd/popper.min.js ./static/js/ mv temp/popper-core-${POPPER_VERSION}/dist/umd/popper.min.js.map ./static/js/ @@ -90,7 +88,6 @@ wget -q https://raw.githubusercontent.com/flot/flot/958e5fd43c6dff4bab3e1fd5cb61 #Ressources for sparkline and canvasJS and slider wget -q http://omnipotent.net/jquery.sparkline/2.1.2/jquery.sparkline.min.js -O ./static/js/jquery.sparkline.min.js -mkdir temp wget -q http://canvasjs.com/fdm/chart/ -O temp/canvasjs.zip unzip -qq temp/canvasjs.zip -d temp/ mv temp/jquery.canvasjs.min.js ./static/js/jquery.canvasjs.min.js @@ -99,6 +96,8 @@ wget -q https://jqueryui.com/resources/download/jquery-ui-1.12.1.zip -O temp/jqu unzip -qq temp/jquery-ui.zip -d temp/ mv temp/jquery-ui-1.12.1/jquery-ui.min.js ./static/js/jquery-ui.min.js mv temp/jquery-ui-1.12.1/jquery-ui.min.css ./static/css/jquery-ui.min.css + + rm -rf temp mkdir -p ./static/image From 9bee423eaf0f66d1a7cfbf365e0119656213b5f8 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 19 Jun 2020 15:07:47 +0200 Subject: [PATCH 08/28] fix: [Exporter] ad missing file --- bin/export/Export.py | 46 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 46 insertions(+) create mode 100755 bin/export/Export.py diff --git a/bin/export/Export.py b/bin/export/Export.py new file mode 100755 index 00000000..67d631f2 --- /dev/null +++ b/bin/export/Export.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import redis +from uuid import uuid4 + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) +import ConfigLoader + +## LOAD CONFIG ## +config_loader = ConfigLoader.ConfigLoader() +r_serv_cache = config_loader.get_redis_conn("Redis_Cache") +r_serv_db = config_loader.get_redis_conn("ARDB_DB") +r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") +config_loader = None +## -- ## + +def get_ail_uuid(): + uuid_ail = r_serv_db.get('ail:uuid') + if uuid_ail is None: + uuid_ail = str(uuid4()) + r_serv_db.set('ail:uuid', uuid_ail) + return uuid_ail + +def load_tags_to_export_in_cache(): + all_exports = ['misp', 'thehive'] + for export_target in all_exports: + # save solo tags in cache + all_tags_to_export = Tag.get_list_of_solo_tags_to_export_by_type() + if len(all_tags_to_export) > 1: + r_serv_cache.sadd('to_export:solo_tags:{}'.format(export_target), *all_tags_to_export) + elif all_tags_to_export: + r_serv_cache.sadd('to_export:solo_tags:{}'.format(export_target), all_tags_to_export[0]) + + # save combinaison of tags in cache + pass + +########################################################### +# # set default +# if r_serv_db.get('hive:auto-alerts') is None: +# r_serv_db.set('hive:auto-alerts', 0) +# +# if r_serv_db.get('misp:auto-events') is None: +# r_serv_db.set('misp:auto-events', 0) From 2ad7d912b3d0b782862622600c1fb82374a5892b Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 24 Jun 2020 14:27:10 +0200 Subject: [PATCH 09/28] fix: [reset_AIL] add helper + fix soft reset --- reset_AIL.sh | 71 ++++++++++++++++++++++++++++++++++------------------ 1 file changed, 47 insertions(+), 24 deletions(-) diff --git a/reset_AIL.sh b/reset_AIL.sh index aac94352..50832ac5 100755 --- a/reset_AIL.sh +++ b/reset_AIL.sh @@ -6,6 +6,29 @@ GREEN="\\033[1;32m" [ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; +function helptext { + echo -e $GREEN" + + .o. ooooo ooooo + .888. \`888' \`888' + .8\"888. 888 888 + .8' \`888. 888 888 + .88ooo8888. 888 888 + .8' \`888. 888 888 o + o88o o8888o o o888o o o888ooooood8 + + Analysis Information Leak framework + "$DEFAULT" + Use this script to reset AIL (DB + stored items): + + Usage: + ----- + reset_AIL.sh + [--softReset] Keep All users accounts + [-h | --help] Help + " +} + function reset_dir { # Access dirs and delete cd $AIL_HOME @@ -23,16 +46,6 @@ function reset_dir { popd fi - if [ $userInput -eq $num ] - then - if [ -d DATA_ARDB/ ]; then - pushd DATA_ARDB/ - rm -r * - echo 'cleaned DATA_ARDB' - popd - fi - fi - if [ -d logs/ ]; then pushd logs/ rm * @@ -97,29 +110,36 @@ function flush_DB_keep_user { bash ${AIL_BIN}LAUNCH.sh -k } +function validate_reset { + echo -e $RED"WARNING: DELETE AIL DATA"$DEFAULT + + # Make sure the reseting is intentional + num=$(( ( RANDOM % 100 ) + 1 )) + + echo -e $RED"To reset the platform, enter the following number: "$DEFAULT $num + read userInput + + if [ $userInput -eq $num ] + then + echo "Reseting AIL..." + else + echo "Wrong number" + exit 1; + fi +} + function soft_reset { + validate_reset; reset_dir; flush_DB_keep_user; } #If no params, [[ $@ ]] || { - # Make sure the reseting is intentional - num=$(( ( RANDOM % 100 ) + 1 )) - - echo -e $RED"To reset the platform, enter the following number: "$DEFAULT $num - read userInput - - if [ $userInput -eq $num ] - then - echo "Reseting AIL..." - else - echo "Wrong number" - exit 1; - fi + validate_reset; num=$(( ( RANDOM % 100 ) + 1 )) - echo -e $RED"If yes you want to delete the DB , enter the following number: "$DEFAULT $num + echo -e $RED"If you want to delete the DB , enter the following number: "$DEFAULT $num read userInput reset_dir; @@ -143,6 +163,9 @@ while [ "$1" != "" ]; do case $1 in --softReset ) soft_reset; ;; + -h | --help ) helptext; + exit + ;; * ) exit 1 esac shift From b1d0d067f9a701155cfcc5998ba71668d84f9d41 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 24 Jun 2020 15:07:45 +0200 Subject: [PATCH 10/28] fix: [regex tracker] fix timeout --- bin/RegexTracker.py | 25 +++++++------------------ bin/lib/regex_helper.py | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 18 deletions(-) diff --git a/bin/RegexTracker.py b/bin/RegexTracker.py index 2f7e5b9f..d1bdc26e 100755 --- a/bin/RegexTracker.py +++ b/bin/RegexTracker.py @@ -10,7 +10,6 @@ import os import re import sys import time -import signal from Helper import Process from pubsublogger import publisher @@ -20,18 +19,15 @@ import NotificationHelper from packages import Item from packages import Term +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) +import regex_helper + full_item_url = "/showsavedpaste/?paste=" mail_body_template = "AIL Framework,\nNew occurrence for term tracked regex: {}\nitem id: {}\nurl: {}{}" dict_regex_tracked = Term.get_regex_tracked_words_dict() last_refresh = time.time() -class TimeoutException(Exception): - pass -def timeout_handler(signum, frame): - raise TimeoutException -signal.signal(signal.SIGALRM, timeout_handler) - def new_term_found(term, term_type, item_id, item_date): uuid_list = Term.get_term_uuid_list(term, 'regex') print('new tracked term found: {} in {}'.format(term, item_id)) @@ -56,11 +52,14 @@ if __name__ == "__main__": publisher.info("Script RegexTracker started") config_section = 'RegexTracker' + module_name = "RegexTracker" p = Process(config_section) max_execution_time = p.config.getint(config_section, "max_execution_time") ull_item_url = p.config.get("Notifications", "ail_domain") + full_item_url + redis_cache_key = regex_helper.generate_redis_cache_key(module_name) + # Regex Frequency while True: @@ -72,20 +71,10 @@ if __name__ == "__main__": item_content = Item.get_item_content(item_id) for regex in dict_regex_tracked: - - signal.alarm(max_execution_time) - try: - matched = dict_regex_tracked[regex].search(item_content) - except TimeoutException: - print ("{0} processing timeout".format(item_id)) - continue - else: - signal.alarm(0) - + matched = regex_helper.regex_search(module_name, redis_cache_key, dict_regex_tracked[regex], item_id, item_content, max_time=max_execution_time) if matched: new_term_found(regex, 'regex', item_id, item_date) - else: time.sleep(5) diff --git a/bin/lib/regex_helper.py b/bin/lib/regex_helper.py index 4f38c39c..67256376 100755 --- a/bin/lib/regex_helper.py +++ b/bin/lib/regex_helper.py @@ -73,3 +73,30 @@ def regex_findall(module_name, redis_key, regex, item_id, item_content, max_time print("Caught KeyboardInterrupt, terminating workers") proc.terminate() sys.exit(0) + +def _regex_search(redis_key, regex, item_content): + first_occ = regex.search(item_content) + if r_set: + r_serv_cache.set(redis_key, first_occ) + +def regex_search(module_name, redis_key, regex, item_id, item_content, max_time=30): + proc = Proc(target=_regex_search, args=(redis_key, regex, item_content, )) + try: + proc.start() + proc.join(max_time) + if proc.is_alive(): + proc.terminate() + Statistics.incr_module_timeout_statistic(module_name) + err_mess = "{}: processing timeout: {}".format(module_name, item_id) + print(err_mess) + publisher.info(err_mess) + return None + else: + first_occ = r_serv_cache.get(redis_key) + r_serv_cache.delete(redis_key) + proc.terminate() + return first_occ + except KeyboardInterrupt: + print("Caught KeyboardInterrupt, terminating workers") + proc.terminate() + sys.exit(0) From ccb957e6d957ed3f3fff3bdcb871a1b4966d0007 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 24 Jun 2020 15:13:56 +0200 Subject: [PATCH 11/28] chg: [core] disable Release module --- bin/LAUNCH.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 92da0bdb..9c855668 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -200,8 +200,8 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Phone.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x" - sleep 0.1 + #screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x" + #sleep 0.1 screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cve.py; read x" sleep 0.1 screen -S "Script_AIL" -X screen -t "WebStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./WebStats.py; read x" From 9ce2eb1e79376aaf7c8589087c9824bf98d6e117 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 26 Jun 2020 15:44:07 +0200 Subject: [PATCH 12/28] fix: [regex_helper] --- bin/lib/regex_helper.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/lib/regex_helper.py b/bin/lib/regex_helper.py index 67256376..b8cd5417 100755 --- a/bin/lib/regex_helper.py +++ b/bin/lib/regex_helper.py @@ -76,8 +76,7 @@ def regex_findall(module_name, redis_key, regex, item_id, item_content, max_time def _regex_search(redis_key, regex, item_content): first_occ = regex.search(item_content) - if r_set: - r_serv_cache.set(redis_key, first_occ) + r_serv_cache.set(redis_key, first_occ) def regex_search(module_name, redis_key, regex, item_id, item_content, max_time=30): proc = Proc(target=_regex_search, args=(redis_key, regex, item_content, )) From f20df8944616f4af94ac7e9d7e0a47739b912db3 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 29 Jun 2020 11:01:30 +0200 Subject: [PATCH 13/28] fix:[RegexTracker] fix search regex helper --- bin/lib/regex_helper.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/lib/regex_helper.py b/bin/lib/regex_helper.py index b8cd5417..f3c4379d 100755 --- a/bin/lib/regex_helper.py +++ b/bin/lib/regex_helper.py @@ -76,7 +76,8 @@ def regex_findall(module_name, redis_key, regex, item_id, item_content, max_time def _regex_search(redis_key, regex, item_content): first_occ = regex.search(item_content) - r_serv_cache.set(redis_key, first_occ) + if first_occ: + r_serv_cache.set(redis_key, first_occ) def regex_search(module_name, redis_key, regex, item_id, item_content, max_time=30): proc = Proc(target=_regex_search, args=(redis_key, regex, item_content, )) From 25420005e7b92fdbf217046e78ff482be9544681 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 30 Jun 2020 10:42:10 +0200 Subject: [PATCH 14/28] chg: [whosh index] add data retention fct --- bin/lib/index_whoosh.py | 103 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 103 insertions(+) create mode 100755 bin/lib/index_whoosh.py diff --git a/bin/lib/index_whoosh.py b/bin/lib/index_whoosh.py new file mode 100755 index 00000000..b5657697 --- /dev/null +++ b/bin/lib/index_whoosh.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import redis + +from shutil import rmtree + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + +config_loader = ConfigLoader.ConfigLoader() +INDEX_PATH = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Indexer", "path")) +all_index_file = os.path.join(INDEX_PATH, 'all_index.txt') +config_loader = None + +def get_first_index_name(): + with open(all_index_file) as f: + first_index = f.readline().replace('\n', '') + return first_index + +def get_last_index_name(): + with open(all_index_file) as f: + for line in f: # # FIXME: replace by tail ? + pass + last_index = line.replace('\n', '') + return last_index + +def get_all_index(): + all_index = [] + with open(all_index_file) as f: + for line in f: + line = line.replace('\n', '') + if line: + all_index.append(line) + return all_index + +def get_index_full_path(index_name): + return os.path.join(INDEX_PATH, index_name) + +# remove empty line +def check_index_list_integrity(): + with open(all_index_file, 'r') as f: + lines = f.readlines() + with open(all_index_file, 'w') as f: + for line in lines: + if line != '\n': + f.write(line) + +def _remove_index_name_from_all_index(index_name): + with open(all_index_file, 'r') as f: + lines = f.readlines() + with open(all_index_file, 'w') as f: + for line in lines: + if line.replace('\n', '') != index_name: + f.write(line) + +def delete_index_by_name(index_name): + index_path = get_index_full_path(index_name) + index_path = os.path.realpath(index_path) + # incorrect filename + if not os.path.commonprefix([index_path, INDEX_PATH]) == INDEX_PATH: + raise Exception('Path traversal detected {}'.format(index_path)) + if not os.path.isdir(index_path): + print('Error: The index directory {} doesn\'t exist'.format(index_path)) + return None + res = rmtree(index_path) + _remove_index_name_from_all_index(index_name) + +def delete_first_index(): + index_name = get_first_index_name() + delete_index_by_name(index_name) + +def delete_last_index(): + index_name = get_last_index_name() + delete_index_by_name(index_name) + +#### DATA RETENTION #### + +#keep time most recent index +def delete_older_index_by_time(int_time): + all_index = get_all_index() + if all_index: + if int(all_index[-1]) > int_time: # make sure to keep one files + for index_name in all_index: + if int(index_name) < int_time: + print('deleting index {} ...'.format(index_name)) + delete_index_by_name(index_name) + +# keep x most recent index +def delete_older_index(number_of_index_to_keep): + if number_of_index_to_keep > 1: + all_index = get_all_index() + if len(get_all_index()) > number_of_index_to_keep: + for index_name in all_index[0:-number_of_index_to_keep]: + print('deleting index {} ...'.format(index_name)) + delete_index_by_name(index_name) + +##-- DATA RETENTION --## + +# if __name__ == '__main__': +# delete_older_index(3) From be3446ff977a5dd24ad6364531543164f7b4b72c Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 2 Jul 2020 08:31:47 +0200 Subject: [PATCH 15/28] chg: [feeder pystemon] add debug --- bin/feeder/pystemon-feeder.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/bin/feeder/pystemon-feeder.py b/bin/feeder/pystemon-feeder.py index 2b06e494..3a8e5fb9 100755 --- a/bin/feeder/pystemon-feeder.py +++ b/bin/feeder/pystemon-feeder.py @@ -57,14 +57,20 @@ topic = '102' while True: time.sleep(base_sleeptime + sleep_inc) - paste = r.lpop("pastes") - if paste is None: + item_id = r.lpop("pastes") + if item_id is None: continue try: - print(paste) - with open(pystemonpath+paste, 'rb') as f: #.read() + print(item_id) + full_item_path = os.path.join(pystemonpath, item_id) + if not os.path.isfile(full_item_path): + print('Error: {}, file not found'.format(full_item_path)) + sleep_inc = 1 + continue + + with open(full_item_path, 'rb') as f: #.read() messagedata = f.read() - path_to_send = os.path.join(pastes_directory,paste) + path_to_send = os.path.join(pastes_directory, item_id) s = b' '.join( [ topic.encode(), path_to_send.encode(), base64.b64encode(messagedata) ] ) socket.send(s) From 1518d091069a1d9f52e2228d014d6cb781581ffb Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 6 Jul 2020 14:43:04 +0200 Subject: [PATCH 16/28] Typo fixed as ail repo is now lower-case Typo fixed as ail repo is now lower-case --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index ec1a015c..7ac8d3ff 100644 --- a/README.md +++ b/README.md @@ -82,10 +82,10 @@ Installation Type these command lines for a fully automated installation and start AIL framework: ```bash git clone https://github.com/ail-project/ail-framework.git -cd AIL-framework +cd ail-framework ./installing_deps.sh -cd ~/AIL-framework/ +cd ~/ail-framework/ cd bin/ ./LAUNCH.sh -l ``` From 8abfe2a6193df6fce2724382886323dd34dde7f5 Mon Sep 17 00:00:00 2001 From: C00kie- Date: Mon, 6 Jul 2020 17:09:42 +0200 Subject: [PATCH 17/28] Update core.cfg.sample --- configs/core.cfg.sample | 2 ++ 1 file changed, 2 insertions(+) diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index 876bd5ff..e4c509ff 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -39,6 +39,8 @@ sender_user = [Flask] #Proxying requests to the app baseUrl = / +#Host to bind to +host = 0.0.0.0 #Flask server port port = 7000 #Number of logs to display in the dashboard From b9b50d68a43264db4d33bbbcb34bb6a242a37ea7 Mon Sep 17 00:00:00 2001 From: C00kie- Date: Mon, 6 Jul 2020 17:13:36 +0200 Subject: [PATCH 18/28] Update Flask_server.py --- var/www/Flask_server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 9894ba76..5f1f22d4 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -49,6 +49,7 @@ Flask_dir = os.environ['AIL_FLASK'] # CONFIG # config_loader = ConfigLoader.ConfigLoader() baseUrl = config_loader.get_config_str("Flask", "baseurl") +host = config_loader.get_config_str("Flask", "host") baseUrl = baseUrl.replace('/', '') if baseUrl != '': baseUrl = '/'+baseUrl @@ -253,4 +254,4 @@ r_serv_db.sadd('list_export_tags', 'infoleak:submission="manual"') # ============ MAIN ============ if __name__ == "__main__": - app.run(host='0.0.0.0', port=FLASK_PORT, threaded=True, ssl_context=ssl_context) + app.run(host=host, port=FLASK_PORT, threaded=True, ssl_context=ssl_context) From 8a6e72f48717ccc9b2006867c99f11eceba1bd11 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 7 Jul 2020 11:23:23 +0200 Subject: [PATCH 19/28] chg: [Item delete] delete father/child link + remove from domain tree + delete all child from the same domain --- bin/lib/domain_basic.py | 29 +++++++ bin/lib/item_basic.py | 112 ++++++++++++++++++++++++++- bin/packages/Item.py | 56 +++++++------- var/www/blueprints/crawler_splash.py | 2 +- 4 files changed, 168 insertions(+), 31 deletions(-) create mode 100755 bin/lib/domain_basic.py diff --git a/bin/lib/domain_basic.py b/bin/lib/domain_basic.py new file mode 100755 index 00000000..5bf24a72 --- /dev/null +++ b/bin/lib/domain_basic.py @@ -0,0 +1,29 @@ +#!/usr/bin/python3 + +""" +``basic domain lib`` +=================== + + +""" + +import os +import sys +import redis + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + +config_loader = ConfigLoader.ConfigLoader() +r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") +config_loader = None + +def get_domain_type(domain): + if str(domain).endswith('.onion'): + return 'onion' + else: + return 'regular' + +def delete_domain_item_core(item_id, domain, port): + domain_type = get_domain_type(domain) + r_serv_onion.zrem('crawler_history_{}:{}:{}'.format(domain_type, domain, port), item_id) diff --git a/bin/lib/item_basic.py b/bin/lib/item_basic.py index 6b606dda..c1005b49 100755 --- a/bin/lib/item_basic.py +++ b/bin/lib/item_basic.py @@ -3,6 +3,7 @@ import os import sys +import gzip sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader @@ -12,6 +13,7 @@ config_loader = ConfigLoader.ConfigLoader() PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' PASTES_FOLDER = os.path.join(os.path.realpath(PASTES_FOLDER), '') +r_cache = config_loader.get_redis_conn("Redis_Cache") r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") config_loader = None @@ -43,6 +45,102 @@ def is_crawled(item_id): def get_item_domain(item_id): return item_id[19:-36] +def get_item_content(item_id): + item_full_path = os.path.join(PASTES_FOLDER, item_id) + try: + item_content = r_cache.get(item_full_path) + except UnicodeDecodeError: + item_content = None + except Exception as e: + item_content = None + if item_content is None: + try: + with gzip.open(item_full_path, 'r') as f: + item_content = f.read().decode() + r_cache.set(item_full_path, item_content) + r_cache.expire(item_full_path, 300) + except: + item_content = '' + return str(item_content) + +#### TREE CHILD/FATHER #### +def is_father(item_id): + return r_serv_metadata.exists('paste_children:{}'.format(item_id)) + +def is_children(item_id): + return r_serv_metadata.hexists('paste_metadata:{}'.format(item_id), 'father') + +def is_root_node(): + if is_father(item_id) and not is_children(item_id): + return True + else: + return False + +def is_node(item_id): + if is_father(item_id) or is_children(item_id): + return True + else: + return False + +def is_leaf(item_id): + if not is_father(item_id) and is_children(item_id): + return True + else: + return False + +def is_domain_root(item_id): + if not is_crawled(item_id): + return False + else: + domain = get_item_domain(item_id) + item_father = get_item_parent(item_id) + if not is_crawled(item_father): + return True + else: + # same domain + if get_item_domain(item_father) == domain: + return False + else: + return True + +def get_nb_children(item_id): + return r_serv_metadata.scard('paste_children:{}'.format(item_id)) + + +def get_item_parent(item_id): + return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'father') + +def get_item_children(item_id): + return list(r_serv_metadata.smembers('paste_children:{}'.format(item_id))) + +def add_item_parent(item_parent, item_id): + return item_basic.add_item_parent(item_parent, item_id) + +# # TODO: handle domain last origin in domain lib +def _delete_node(item_id): + # only if item isn't deleted + #if is_crawled(item_id): + # r_serv_metadata.hrem('paste_metadata:{}'.format(item_id), 'real_link') + for chidren_id in get_item_children(item_id): + r_serv_metadata.hdel('paste_metadata:{}'.format(chidren_id), 'father') + r_serv_metadata.delete('paste_children:{}'.format(item_id)) + + # delete regular + # simple if leaf + + # delete item node + +def get_all_domain_node_by_item_id(item_id, l_nodes=[]): + domain = get_item_domain(item_id) + for child_id in get_item_children(item_id): + if get_item_domain(child_id) == domain: + l_nodes.append(child_id) + l_nodes = get_all_domain_node_by_item_id(child_id, l_nodes) + return l_nodes + +##-- --## + + def add_item_parent_by_parent_id(parent_type, parent_id, item_id): parent_item_id = get_obj_id_item_id(parent_type, parent_id) if parent_item_id: @@ -53,9 +151,9 @@ def add_item_parent(parent_item_id, item_id): r_serv_metadata.sadd('paste_children:{}'.format(parent_item_id), item_id) return True -def add_map_obj_id_item_id(obj_id, item_id, obj_type): - if obj_type == 'twitter_id': - r_serv_metadata.hset('map:twitter_id:item_id', obj_id, item_id) +# TODO: +# FIXME: +#### UNKNOW SECTION #### def get_obj_id_item_id(parent_type, parent_id): all_parents_type = ['twitter_id'] @@ -63,3 +161,11 @@ def get_obj_id_item_id(parent_type, parent_id): return r_serv_metadata.hget('map:twitter_id:item_id', parent_id) else: return None + +def add_map_obj_id_item_id(obj_id, item_id, obj_type): + if obj_type == 'twitter_id': + r_serv_metadata.hset('map:twitter_id:item_id', obj_id, item_id) + +# delete twitter id + +##-- --## diff --git a/bin/packages/Item.py b/bin/packages/Item.py index 28a7349b..a551157f 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -3,7 +3,6 @@ import os import sys -import gzip import redis from io import BytesIO @@ -16,12 +15,15 @@ import Pgp sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import item_basic +import domain_basic import ConfigLoader import Correlate_object import Decoded import Screenshot import telegram +from item_basic import * + config_loader = ConfigLoader.ConfigLoader() # get and sanityze PASTE DIRECTORY PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "pastes")) + '/' @@ -30,6 +32,7 @@ PASTES_FOLDER = os.path.join(os.path.realpath(PASTES_FOLDER), '') r_cache = config_loader.get_redis_conn("Redis_Cache") r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata") screenshot_directory = os.path.join(os.environ['AIL_HOME'], config_loader.get_config_str("Directories", "crawled_screenshot")) + config_loader = None def exist_item(item_id): @@ -71,22 +74,7 @@ def get_lines_info(item_id, item_content=None): def get_item_content(item_id): - item_full_path = os.path.join(PASTES_FOLDER, item_id) - try: - item_content = r_cache.get(item_full_path) - except UnicodeDecodeError: - item_content = None - except Exception as e: - item_content = None - if item_content is None: - try: - with gzip.open(item_full_path, 'r') as f: - item_content = f.read().decode() - r_cache.set(item_full_path, item_content) - r_cache.expire(item_full_path, 300) - except: - item_content = '' - return str(item_content) + return item_basic.get_item_content(item_id) # API def get_item(request_dict): @@ -292,14 +280,8 @@ def get_domain(item_id): item_id = item_id[-1] return item_id[:-36] -def get_item_parent(item_id): - return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'father') - -def get_item_children(item_id): - return list(r_serv_metadata.smembers('paste_children:{}'.format(item_id))) - -def add_item_parent(item_parent, item_id): - return item_basic.add_item_parent(item_parent, item_id) +def get_item_domain_with_port(item_id): + return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'domain') def get_item_link(item_id): return r_serv_metadata.hget('paste_metadata:{}'.format(item_id), 'real_link') @@ -423,12 +405,32 @@ def delete_item(obj_id): else: for obj2_id in obj_correlations[correlation]: Correlate_object.delete_obj_relationship(correlation, obj2_id, 'item', obj_id) + + # delete father/child + delete_node(obj_id) + + # delete item metadata + r_serv_metadata.delete('paste_metadata:{}'.format(obj_id)) + return True - ### REQUIRE MORE WORK - # delete child/son !!! ### TODO in inport V2 # delete from tracked items # delete from queue ### return False + +#### #### +def delete_node(item_id): + if is_node(item_id): + if is_crawled(item_id): + delete_domain_node(item_id) + item_basic._delete_node(item_id) + +def delete_domain_node(item_id): + if is_domain_root(item_id): + # remove from domain history + domain, port = get_item_domain_with_port(item_id).split(':') + domain_basic.delete_domain_item_core(item_id, domain, port) + for child_id in get_all_domain_node_by_item_id(item_id): + delete_item(child_id) diff --git a/var/www/blueprints/crawler_splash.py b/var/www/blueprints/crawler_splash.py index d2d3c65a..7d006c3d 100644 --- a/var/www/blueprints/crawler_splash.py +++ b/var/www/blueprints/crawler_splash.py @@ -115,7 +115,7 @@ def showDomain(): dict_domain['tags_safe'] = Tag.is_tags_safe(dict_domain['tags']) dict_domain['history'] = domain.get_domain_history_with_status() dict_domain['crawler_history'] = domain.get_domain_items_crawled(items_link=True, epoch=epoch, item_screenshot=True, item_tag=True) # # TODO: handle multiple port - if dict_domain['crawler_history']['items']: + if dict_domain['crawler_history'].get('items', []): dict_domain['crawler_history']['random_item'] = random.choice(dict_domain['crawler_history']['items']) return render_template("showDomain.html", dict_domain=dict_domain, bootstrap_label=bootstrap_label, From 11c4ba69913b317f4b540a471c73346c5ae868dc Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 9 Jul 2020 17:50:43 +0200 Subject: [PATCH 20/28] chg: [username correlation + login redirection] add twitter username correlation + redirect to the requested page on login --- .../ail_json_importer/Ail_feeder_twitter.py | 5 ++++- bin/lib/Correlate_object.py | 13 +++++++----- bin/lib/Username.py | 21 +++++++++++++++++++ bin/lib/telegram.py | 8 ++----- bin/packages/Item.py | 10 ++++----- var/www/blueprints/correlation.py | 4 ++-- var/www/blueprints/root.py | 11 ++++++---- .../modules/hashDecoded/Flask_hashDecoded.py | 6 +++++- .../correlation/legend_graph_correlation.html | 11 +++++++++- var/www/templates/login.html | 1 + 10 files changed, 65 insertions(+), 25 deletions(-) create mode 100755 bin/lib/Username.py diff --git a/bin/import/ail_json_importer/Ail_feeder_twitter.py b/bin/import/ail_json_importer/Ail_feeder_twitter.py index ec9fb0a2..a622a800 100755 --- a/bin/import/ail_json_importer/Ail_feeder_twitter.py +++ b/bin/import/ail_json_importer/Ail_feeder_twitter.py @@ -14,6 +14,7 @@ import datetime sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) import item_basic +import Username sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer')) from Default_json import Default_json @@ -34,11 +35,13 @@ class Ail_feeder_twitter(Default_json): item_id = str(self.json_item['meta']['twitter:tweet_id']) return os.path.join('twitter', item_date, item_id) + '.gz' - # # TODO: def process_json_meta(self, process, item_id): ''' Process JSON meta filed. ''' twitter_id = str(self.json_item['meta']['twitter:tweet_id']) item_basic.add_map_obj_id_item_id(twitter_id, item_id, 'twitter_id') + username = str(self.json_item['meta']['twitter:id']) + item_date = item_basic.get_item_date(item_id) + Username.save_item_correlation('twitter', username, item_id, item_date) return None diff --git a/bin/lib/Correlate_object.py b/bin/lib/Correlate_object.py index 90fcea3d..c0d959f5 100755 --- a/bin/lib/Correlate_object.py +++ b/bin/lib/Correlate_object.py @@ -13,7 +13,7 @@ import ConfigLoader import Decoded import Domain import Screenshot -import telegram +import Username sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) import Pgp @@ -36,7 +36,7 @@ def is_valid_object_subtype(object_type, object_subtype): elif object_type == 'cryptocurrency': return Cryptocurrency.cryptocurrency.is_valid_obj_subtype(object_subtype) elif object_type == 'username': - return telegram.correlation.is_valid_obj_subtype(object_subtype) + return Username.correlation.is_valid_obj_subtype(object_subtype) elif object_subtype == None: return True else: @@ -69,7 +69,7 @@ def exist_object(object_type, correlation_id, type_id=None): # => work on object elif object_type == 'cryptocurrency': return Cryptocurrency.cryptocurrency.exist_correlation(type_id, correlation_id) elif object_type == 'username': - return telegram.correlation.exist_correlation(type_id, correlation_id) + return Username.correlation.exist_correlation(type_id, correlation_id) elif object_type == 'screenshot' or object_type == 'image': return Screenshot.exist_screenshot(correlation_id) else: @@ -88,7 +88,7 @@ def get_object_metadata(object_type, correlation_id, type_id=None): elif object_type == 'cryptocurrency': return Cryptocurrency.cryptocurrency.get_metadata(type_id, correlation_id) elif object_type == 'username': - return telegram.correlation.get_metadata(type_id, correlation_id) + return Username.correlation.get_metadata(type_id, correlation_id) elif object_type == 'screenshot' or object_type == 'image': return Screenshot.get_metadata(correlation_id) @@ -104,7 +104,7 @@ def get_object_correlation(object_type, value, correlation_names=None, correlati elif object_type == 'cryptocurrency': return Cryptocurrency.cryptocurrency.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects) elif object_type == 'username': - return telegram.correlation.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects) + return Username.correlation.get_correlation_all_object(requested_correl_type, value, correlation_objects=correlation_objects) elif object_type == 'screenshot' or object_type == 'image': return Screenshot.get_screenshot_correlated_object(value, correlation_objects=correlation_objects) return {} @@ -157,6 +157,9 @@ def get_correlation_node_icon(correlation_name, correlation_type=None, value=Non if correlation_type == 'telegram': icon_class = 'fab' icon_text = '\uf2c6' + elif correlation_type == 'twitter': + icon_class = 'fab' + icon_text = '\uf099' else: icon_text = '\uf007' diff --git a/bin/lib/Username.py b/bin/lib/Username.py new file mode 100755 index 00000000..0448a167 --- /dev/null +++ b/bin/lib/Username.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import redis + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) +import Correlation + +config_loader = ConfigLoader.ConfigLoader() +r_serv_crawler = config_loader.get_redis_conn("ARDB_Onion") +config_loader = None + +correlation = Correlation.Correlation('username', ['telegram', 'twitter']) + +def save_item_correlation(subtype, username, item_id, item_date): + correlation.save_item_correlation(subtype, username, item_id, item_date) diff --git a/bin/lib/telegram.py b/bin/lib/telegram.py index 08eafeff..cfad88ed 100755 --- a/bin/lib/telegram.py +++ b/bin/lib/telegram.py @@ -5,20 +5,16 @@ import os import sys import redis -sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) -import Correlation - sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader +import Username config_loader = ConfigLoader.ConfigLoader() r_serv_crawler = config_loader.get_redis_conn("ARDB_Onion") config_loader = None -correlation = Correlation.Correlation('username', ['telegram']) - def save_item_correlation(username, item_id, item_date): - correlation.save_item_correlation('telegram', username, item_id, item_date) + Username.save_item_correlation('telegram', username, item_id, item_date) def save_telegram_invite_hash(invite_hash, item_id): r_serv_crawler.sadd('telegram:invite_code', '{};{}'.format(invite_hash, item_id)) diff --git a/bin/packages/Item.py b/bin/packages/Item.py index a551157f..d898d1e0 100755 --- a/bin/packages/Item.py +++ b/bin/packages/Item.py @@ -20,7 +20,7 @@ import ConfigLoader import Correlate_object import Decoded import Screenshot -import telegram +import Username from item_basic import * @@ -156,15 +156,15 @@ def get_item_pgp(item_id, currencies_type=None, get_nb=False): ''' return Pgp.pgp.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb) -def get_item_username(item_id, currencies_type=None, get_nb=False): +def get_item_username(item_id, sub_type=None, get_nb=False): ''' Return all pgp of a given item. :param item_id: item id - :param currencies_type: list of cryptocurrencies type - :type currencies_type: list, optional + :param sub_type: list of username type + :type sub_type: list, optional ''' - return telegram.correlation.get_item_correlation_dict(item_id, correlation_type=currencies_type, get_nb=get_nb) + return Username.correlation.get_item_correlation_dict(item_id, correlation_type=sub_type, get_nb=get_nb) def get_item_decoded(item_id): ''' diff --git a/var/www/blueprints/correlation.py b/var/www/blueprints/correlation.py index 359ac612..03c7ea5e 100644 --- a/var/www/blueprints/correlation.py +++ b/var/www/blueprints/correlation.py @@ -25,7 +25,7 @@ import Correlate_object import Domain import Screenshot import btc_ail -import telegram +import Username sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages')) import Cryptocurrency @@ -110,7 +110,7 @@ def get_card_metadata(object_type, correlation_id, type_id=None, expand_card=Fal card_dict["sparkline"] = Pgp.pgp.get_list_nb_previous_correlation_object(type_id, correlation_id, 6) card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, type_id) elif object_type == 'username': - card_dict["sparkline"] = telegram.correlation.get_list_nb_previous_correlation_object(type_id, correlation_id, 6) + card_dict["sparkline"] = Username.correlation.get_list_nb_previous_correlation_object(type_id, correlation_id, 6) card_dict["icon"] = Correlate_object.get_correlation_node_icon(object_type, type_id) elif object_type == 'decoded': card_dict["sparkline"] = Decoded.get_list_nb_previous_hash(correlation_id, 6) diff --git a/var/www/blueprints/root.py b/var/www/blueprints/root.py index 83446dc6..2c69be46 100644 --- a/var/www/blueprints/root.py +++ b/var/www/blueprints/root.py @@ -54,7 +54,7 @@ def login(): if request.method == 'POST': username = request.form.get('username') password = request.form.get('password') - #next_page = request.form.get('next_page') + next_page = request.form.get('next_page') if username is not None: user = User.get(username) @@ -74,7 +74,10 @@ def login(): if user.request_password_change(): return redirect(url_for('root.change_password')) else: - return redirect(url_for('dashboard.index')) + if next_page and next_page!='None': + return redirect(next_page) + else: + return redirect(url_for('dashboard.index')) # login failed else: # set brute force protection @@ -91,9 +94,9 @@ def login(): return 'please provide a valid username' else: - #next_page = request.args.get('next') + next_page = request.args.get('next') error = request.args.get('error') - return render_template("login.html" , error=error) + return render_template("login.html" , next_page=next_page, error=error) @root.route('/change_password', methods=['POST', 'GET']) @login_required diff --git a/var/www/modules/hashDecoded/Flask_hashDecoded.py b/var/www/modules/hashDecoded/Flask_hashDecoded.py index 0ad24fdf..4186e431 100644 --- a/var/www/modules/hashDecoded/Flask_hashDecoded.py +++ b/var/www/modules/hashDecoded/Flask_hashDecoded.py @@ -39,7 +39,7 @@ hashDecoded = Blueprint('hashDecoded', __name__, template_folder='templates') ## TODO: put me in option all_cryptocurrency = ['bitcoin', 'ethereum', 'bitcoin-cash', 'litecoin', 'monero', 'zcash', 'dash'] all_pgpdump = ['key', 'name', 'mail'] -all_username = ['telegram'] +all_username = ['telegram', 'twitter'] # ============ FUNCTIONS ============ @@ -128,6 +128,8 @@ def get_icon(correlation_type, type_id): elif correlation_type == 'username': if type_id == 'telegram': icon_text = 'fab fa-telegram-plane' + elif type_id == 'twitter': + icon_text = 'fab fa-twitter' return icon_text def get_icon_text(correlation_type, type_id): @@ -153,6 +155,8 @@ def get_icon_text(correlation_type, type_id): elif correlation_type == 'username': if type_id == 'telegram': icon_text = '\uf2c6' + elif type_id == 'twitter': + icon_text = '\uf099' return icon_text def get_all_types_id(correlation_type): diff --git a/var/www/templates/correlation/legend_graph_correlation.html b/var/www/templates/correlation/legend_graph_correlation.html index 15fee4d9..b8d095fa 100644 --- a/var/www/templates/correlation/legend_graph_correlation.html +++ b/var/www/templates/correlation/legend_graph_correlation.html @@ -152,7 +152,16 @@ telegram - +
+ + + + + + + twitter +
+ f099
diff --git a/var/www/templates/login.html b/var/www/templates/login.html index a0c01462..4b941520 100644 --- a/var/www/templates/login.html +++ b/var/www/templates/login.html @@ -73,6 +73,7 @@
{{error}} From 0c23c24a060599b8c8e0a99318b841fefebc98c7 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 9 Jul 2020 17:57:36 +0200 Subject: [PATCH 21/28] fix: [username correlation] fix domain correlation --- bin/lib/Domain.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/lib/Domain.py b/bin/lib/Domain.py index d6c7247f..bfc0b4cb 100755 --- a/bin/lib/Domain.py +++ b/bin/lib/Domain.py @@ -25,7 +25,7 @@ sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader import Correlate_object import Screenshot -import telegram +import Username config_loader = ConfigLoader.ConfigLoader() r_serv_onion = config_loader.get_redis_conn("ARDB_Onion") @@ -564,7 +564,7 @@ def get_domain_username(domain, currencies_type=None, get_nb=False): :param currencies_type: list of pgp type :type currencies_type: list, optional ''' - return telegram.correlation.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb) + return Username.correlation.get_domain_correlation_dict(domain, correlation_type=currencies_type, get_nb=get_nb) def get_domain_decoded(domain): ''' From 4ca02a7243ae38fb7494c5ad9e0e1fdcbf60b784 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 10 Jul 2020 15:54:14 +0200 Subject: [PATCH 22/28] chg: [Trackers] email notifications: add tracker description in email subject --- bin/NotificationHelper.py | 9 ++++----- bin/RegexTracker.py | 4 +++- bin/TermTrackerMod.py | 5 ++++- bin/lib/Tracker.py | 26 ++++++++++++++++++++++++++ 4 files changed, 37 insertions(+), 7 deletions(-) create mode 100755 bin/lib/Tracker.py diff --git a/bin/NotificationHelper.py b/bin/NotificationHelper.py index b9c90104..5e08f7e4 100755 --- a/bin/NotificationHelper.py +++ b/bin/NotificationHelper.py @@ -23,7 +23,7 @@ config_loader = ConfigLoader.ConfigLoader() publisher.port = 6380 publisher.channel = "Script" -def sendEmailNotification(recipient, alert_name, content): +def sendEmailNotification(recipient, mail_subject, mail_body): sender = config_loader.get_config_str("Notifications", "sender") sender_user = config_loader.get_config_str("Notifications", "sender_user") @@ -60,14 +60,13 @@ def sendEmailNotification(recipient, alert_name, content): mime_msg = MIMEMultipart() mime_msg['From'] = sender mime_msg['To'] = recipient - mime_msg['Subject'] = "AIL Framework " + alert_name + " Alert" + mime_msg['Subject'] = mail_subject - body = content - mime_msg.attach(MIMEText(body, 'plain')) + mime_msg.attach(MIMEText(mail_body, 'plain')) smtp_server.sendmail(sender, recipient, mime_msg.as_string()) smtp_server.quit() - print('Send notification ' + alert_name + ' to '+recipient) + print('Send notification: ' + mail_subject + ' to '+recipient) except Exception as err: traceback.print_tb(err.__traceback__) diff --git a/bin/RegexTracker.py b/bin/RegexTracker.py index d1bdc26e..270df81f 100755 --- a/bin/RegexTracker.py +++ b/bin/RegexTracker.py @@ -20,6 +20,7 @@ from packages import Item from packages import Term sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) +import Tracker import regex_helper full_item_url = "/showsavedpaste/?paste=" @@ -42,9 +43,10 @@ def new_term_found(term, term_type, item_id, item_date): mail_to_notify = Term.get_term_mails(term_uuid) if mail_to_notify: + mail_subject = Tracker.get_email_subject(tracker_uuid) mail_body = mail_body_template.format(term, item_id, full_item_url, item_id) for mail in mail_to_notify: - NotificationHelper.sendEmailNotification(mail, 'Term Tracker', mail_body) + NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) if __name__ == "__main__": publisher.port = 6380 diff --git a/bin/TermTrackerMod.py b/bin/TermTrackerMod.py index ce4dcf20..931b5979 100755 --- a/bin/TermTrackerMod.py +++ b/bin/TermTrackerMod.py @@ -18,6 +18,8 @@ import NotificationHelper from packages import Item from packages import Term +from lib import Tracker + full_item_url = "/showsavedpaste/?paste=" mail_body_template = "AIL Framework,\nNew occurrence for term tracked term: {}\nitem id: {}\nurl: {}{}" @@ -48,9 +50,10 @@ def new_term_found(term, term_type, item_id, item_date): mail_to_notify = Term.get_term_mails(term_uuid) if mail_to_notify: + mail_subject = Tracker.get_email_subject(tracker_uuid) mail_body = mail_body_template.format(term, item_id, full_item_url, item_id) for mail in mail_to_notify: - NotificationHelper.sendEmailNotification(mail, 'Term Tracker', mail_body) + NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) if __name__ == "__main__": diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py new file mode 100755 index 00000000..edf59198 --- /dev/null +++ b/bin/lib/Tracker.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* + +import os +import sys +import time +import redis + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) +import ConfigLoader + +import item_basic + +config_loader = ConfigLoader.ConfigLoader() +r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker") +config_loader = None + +get_tracker_description(tracker_uuid): + return r_serv_tracker.hget('tracker:{}'.format(term_uuid), 'description') + +def get_email_subject(tracker_uuid): + tracker_description = get_tracker_description(tracker_uuid) + if not tracker_description: + return "AIL framework: Tracker Alert" + else: + return 'AIL framework: {}'.format(tracker_description) From 9675d9543dfbc9d1311fd2c7d4a0d869296c45f8 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 10 Jul 2020 15:57:29 +0200 Subject: [PATCH 23/28] fix: typo --- bin/lib/Tracker.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index edf59198..a03645ad 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -8,14 +8,13 @@ import redis sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib/')) import ConfigLoader - -import item_basic +#import item_basic config_loader = ConfigLoader.ConfigLoader() r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker") config_loader = None -get_tracker_description(tracker_uuid): +def get_tracker_description(tracker_uuid): return r_serv_tracker.hget('tracker:{}'.format(term_uuid), 'description') def get_email_subject(tracker_uuid): From 203ecaad86c8cf70e29e71f1955ca091420414d8 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 10 Jul 2020 16:01:41 +0200 Subject: [PATCH 24/28] fix: typo --- bin/RegexTracker.py | 2 +- bin/TermTrackerMod.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/bin/RegexTracker.py b/bin/RegexTracker.py index 270df81f..938738d0 100755 --- a/bin/RegexTracker.py +++ b/bin/RegexTracker.py @@ -43,7 +43,7 @@ def new_term_found(term, term_type, item_id, item_date): mail_to_notify = Term.get_term_mails(term_uuid) if mail_to_notify: - mail_subject = Tracker.get_email_subject(tracker_uuid) + mail_subject = Tracker.get_email_subject(term_uuid) mail_body = mail_body_template.format(term, item_id, full_item_url, item_id) for mail in mail_to_notify: NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) diff --git a/bin/TermTrackerMod.py b/bin/TermTrackerMod.py index 931b5979..2b4241c0 100755 --- a/bin/TermTrackerMod.py +++ b/bin/TermTrackerMod.py @@ -50,7 +50,7 @@ def new_term_found(term, term_type, item_id, item_date): mail_to_notify = Term.get_term_mails(term_uuid) if mail_to_notify: - mail_subject = Tracker.get_email_subject(tracker_uuid) + mail_subject = Tracker.get_email_subject(term_uuid) mail_body = mail_body_template.format(term, item_id, full_item_url, item_id) for mail in mail_to_notify: NotificationHelper.sendEmailNotification(mail, mail_subject, mail_body) From 4306da666b87a6f3141a2dcc1348f8d3fd1201e7 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Fri, 10 Jul 2020 16:03:51 +0200 Subject: [PATCH 25/28] fix: typo --- bin/lib/Tracker.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/lib/Tracker.py b/bin/lib/Tracker.py index a03645ad..cfa8b511 100755 --- a/bin/lib/Tracker.py +++ b/bin/lib/Tracker.py @@ -15,7 +15,7 @@ r_serv_tracker = config_loader.get_redis_conn("ARDB_Tracker") config_loader = None def get_tracker_description(tracker_uuid): - return r_serv_tracker.hget('tracker:{}'.format(term_uuid), 'description') + return r_serv_tracker.hget('tracker:{}'.format(tracker_uuid), 'description') def get_email_subject(tracker_uuid): tracker_description = get_tracker_description(tracker_uuid) From 97905367b64cdcf866d7cdeeb7cad730e9791c69 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 14 Jul 2020 15:58:27 +0200 Subject: [PATCH 26/28] fix: [TheHive feeder] create alert --- bin/MISP_The_Hive_feeder.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/bin/MISP_The_Hive_feeder.py b/bin/MISP_The_Hive_feeder.py index 23043788..7cc312d4 100755 --- a/bin/MISP_The_Hive_feeder.py +++ b/bin/MISP_The_Hive_feeder.py @@ -58,15 +58,15 @@ from thehive4py.models import Case, CaseTask, CustomFieldHelper -def create_the_hive_alert(source, path, tag): +def create_the_hive_alert(source, item_id, tag): # # TODO: check items status (processed by all modules) # # TODO: add item metadata: decoded content, link to auto crawled content, pgp correlation, cryptocurrency correlation... # # # TODO: description, add AIL link:show items ? - tags = list( r_serv_metadata.smembers('tag:{}'.format(path)) ) + tags = list( r_serv_metadata.smembers('tag:{}'.format(item_id)) ) artifacts = [ AlertArtifact( dataType='uuid-ail', data=r_serv_db.get('ail:uuid') ), - AlertArtifact( dataType='file', data=path, tags=tags ) + AlertArtifact( dataType='file', data=item_id, tags=tags ) ] # Prepare the sample Alert @@ -115,7 +115,7 @@ def feeder(message, count=0): if HiveApi != False: if int(r_serv_db.get('hive:auto-alerts')) == 1: if r_serv_db.sismember('whitelist_hive', tag): - create_the_hive_alert(source, path, tag) + create_the_hive_alert(source, item_id, tag) else: print('hive, auto alerts creation disable') if flag_misp: From 3295b00c69748956729d6d69522bdb843586b37f Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 14 Jul 2020 16:45:27 +0200 Subject: [PATCH 27/28] fix: [TheHive feeder] create alert, get item full path --- bin/MISP_The_Hive_feeder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/MISP_The_Hive_feeder.py b/bin/MISP_The_Hive_feeder.py index 7cc312d4..41d18b43 100755 --- a/bin/MISP_The_Hive_feeder.py +++ b/bin/MISP_The_Hive_feeder.py @@ -66,7 +66,7 @@ def create_the_hive_alert(source, item_id, tag): artifacts = [ AlertArtifact( dataType='uuid-ail', data=r_serv_db.get('ail:uuid') ), - AlertArtifact( dataType='file', data=item_id, tags=tags ) + AlertArtifact( dataType='file', data=item_basic.get_item_filepath(item_id), tags=tags ) ] # Prepare the sample Alert From 4ad6b37701af6986944065fb5c0e77a72b904101 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 16 Jul 2020 09:03:59 +0200 Subject: [PATCH 28/28] fix: [install crawler] remove old python requirement --- crawler_hidden_services_install.sh | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/crawler_hidden_services_install.sh b/crawler_hidden_services_install.sh index 3fbccb74..e5831bf9 100755 --- a/crawler_hidden_services_install.sh +++ b/crawler_hidden_services_install.sh @@ -8,10 +8,10 @@ install_docker() { sudo docker pull scrapinghub/splash; } -install_python_requirement() { - . ./AILENV/bin/activate; - pip3 install -U -r crawler_requirements.txt; -} +# install_python_requirement() { +# . ./AILENV/bin/activate; +# pip3 install -U -r crawler_requirements.txt; +# } install_all() { read -p "Do you want to install docker? (use local splash server) [y/n] " -n 1 -r @@ -20,7 +20,6 @@ install_all() { then install_docker; fi - install_python_requirement; } usage() {