From 3cafc5eae7a55d4f30567a192df8f40b6a0bbb13 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 5 May 2022 16:58:05 +0200 Subject: [PATCH 01/17] chg: [telegram importer] add username correlation --- .../ail_json_importer/Ail_feeder_telegram.py | 59 +++++++++++++++++++ bin/lib/item_basic.py | 7 ++- 2 files changed, 65 insertions(+), 1 deletion(-) create mode 100755 bin/import/ail_json_importer/Ail_feeder_telegram.py diff --git a/bin/import/ail_json_importer/Ail_feeder_telegram.py b/bin/import/ail_json_importer/Ail_feeder_telegram.py new file mode 100755 index 00000000..06045a5e --- /dev/null +++ b/bin/import/ail_json_importer/Ail_feeder_telegram.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +The JSON Receiver Module +================ + +Recieve Json Items (example: Twitter feeder) + +""" +import os +import json +import sys +import datetime + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'lib')) +import item_basic +import Username + +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'import', 'ail_json_importer')) +from Default_json import Default_json + +class Ail_feeder_telegram(Default_json): + """Twitter Feeder functions""" + + def __init__(self, name, json_item): + super().__init__(name, json_item) + + def get_feeder_name(self): + return 'telegram' + + # define item id + def get_item_id(self): + # use twitter timestamp ? + item_date = datetime.date.today().strftime("%Y/%m/%d") + channel_id = str(self.json_item['meta']['channel_id']) + message_id = str(self.json_item['meta']['message_id']) + item_id = f'{channel_id}_{message_id}' + return os.path.join('telegram', item_date, item_id) + '.gz' + + def process_json_meta(self, process, item_id): + ''' + Process JSON meta filed. + ''' + channel_id = str(self.json_item['meta']['channel_id']) + message_id = str(self.json_item['meta']['message_id']) + telegram_id = f'{channel_id}_{message_id}' + item_basic.add_map_obj_id_item_id(telegram_id, item_id, 'telegram_id') + #print(self.json_item['meta']) + username = None + if self.json_item['meta'].get('user'): + username = str(self.json_item['meta']['user']) + else: + if self.json_item['meta'].get('channel'): + username = str(self.json_item['meta']['channel']['username']) + if username: + #print(username) + item_date = item_basic.get_item_date(item_id) + Username.save_item_correlation('telegram', username, item_id, item_date) + return None diff --git a/bin/lib/item_basic.py b/bin/lib/item_basic.py index 46c03a24..608a5ba6 100755 --- a/bin/lib/item_basic.py +++ b/bin/lib/item_basic.py @@ -116,6 +116,9 @@ def is_domain_root(item_id): else: return True +def get_item_url(item_id): + return r_serv_metadata.hget(f'paste_metadata:{item_id}', 'real_link') + def get_nb_children(item_id): return r_serv_metadata.scard('paste_children:{}'.format(item_id)) @@ -166,7 +169,7 @@ def add_item_parent(parent_item_id, item_id): #### UNKNOW SECTION #### def get_obj_id_item_id(parent_type, parent_id): - all_parents_type = ['twitter_id', 'jabber_id'] + all_parents_type = ['twitter_id', 'jabber_id', 'telegram_id'] if parent_type in all_parents_type: return r_serv_metadata.hget('map:{}:item_id'.format(parent_type), parent_id) else: @@ -177,6 +180,8 @@ def add_map_obj_id_item_id(obj_id, item_id, obj_type): r_serv_metadata.hset('map:twitter_id:item_id', obj_id, item_id) if obj_type == 'jabber_id': r_serv_metadata.hset('map:jabber_id:item_id', obj_id, item_id) + if obj_type == 'telegram_id': + r_serv_metadata.hset('map:telegram_id:item_id', obj_id, item_id) # delete twitter id From bbff018fdb8a8120445712bbb32843a9931b2944 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 10 May 2022 09:50:54 +0200 Subject: [PATCH 02/17] fix: [websockets] remove size limit --- bin/core/ail_2_ail_server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/core/ail_2_ail_server.py b/bin/core/ail_2_ail_server.py index 8a5bfd7a..a662e981 100755 --- a/bin/core/ail_2_ail_server.py +++ b/bin/core/ail_2_ail_server.py @@ -315,7 +315,7 @@ if __name__ == '__main__': cert_dir = os.environ['AIL_FLASK'] ssl_context.load_cert_chain(certfile=os.path.join(cert_dir, 'server.crt'), keyfile=os.path.join(cert_dir, 'server.key')) - start_server = websockets.serve(ail_to_ail_serv, host, port, ssl=ssl_context, create_protocol=AIL_2_AIL_Protocol) + start_server = websockets.serve(ail_to_ail_serv, host, port, ssl=ssl_context, create_protocol=AIL_2_AIL_Protocol, max_size=None) print(f'Server Launched: wss://{host}:{port}') redis_logger.info(f'Server Launched: wss://{host}:{port}') From 1dfa796dcae49911cceee12fb2c2456d65040bb8 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 10 May 2022 11:14:29 +0200 Subject: [PATCH 03/17] chg: [websocket server] add host and port config --- bin/core/ail_2_ail_server.py | 9 ++++++--- configs/core.cfg.sample | 4 ++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/bin/core/ail_2_ail_server.py b/bin/core/ail_2_ail_server.py index a662e981..6a49b7d9 100755 --- a/bin/core/ail_2_ail_server.py +++ b/bin/core/ail_2_ail_server.py @@ -17,6 +17,12 @@ sys.path.append(os.environ['AIL_BIN']) ################################## from pubsublogger import publisher from core import ail_2_ail +from lib.ConfigLoader import ConfigLoader + +config_loader = ConfigLoader() +host = config_loader.get_config_str('AIL_2_AIL', 'server_host') +port = config_loader.get_config_int('AIL_2_AIL', 'server_port') +config_loader = None # # TODO: refactor logging #### LOGS #### @@ -303,9 +309,6 @@ class AIL_2_AIL_Protocol(websockets.WebSocketServerProtocol): if __name__ == '__main__': - host = '0.0.0.0' - port = 4443 - print('Launching Server...') redis_logger.info('Launching Server...') diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index 705b648e..e0100533 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -57,6 +57,10 @@ minute_processed_paste = 10 #Maximum line length authorized to make a diff between duplicates DiffMaxLineLength = 10000 +[AIL_2_AIL] +server_host = 0.0.0.0 +server_port = 4443 + #### Modules #### [BankAccount] max_execution_time = 60 From 19969a02b2ea7d24016a49f85b607bae1ab12177 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 10 May 2022 13:24:32 +0200 Subject: [PATCH 04/17] chg: [websockets client] bind client ip --- bin/core/ail_2_ail_client.py | 9 +++++++++ configs/core.cfg.sample | 1 + 2 files changed, 10 insertions(+) diff --git a/bin/core/ail_2_ail_client.py b/bin/core/ail_2_ail_client.py index 810f93c6..29e6c37f 100755 --- a/bin/core/ail_2_ail_client.py +++ b/bin/core/ail_2_ail_client.py @@ -20,6 +20,14 @@ sys.path.append(os.environ['AIL_BIN']) # Import Project packages ################################## from core import ail_2_ail +from lib.ConfigLoader import ConfigLoader + +config_loader = ConfigLoader() +local_addr = config_loader.get_config_str('AIL_2_AIL', 'local_addr') +if not local_addr or local_addr == None: + local_addr = None +config_loader = None + #### LOGS #### redis_logger = publisher @@ -112,6 +120,7 @@ async def ail_to_ail_client(ail_uuid, sync_mode, api, ail_key=None, client_id=No async with websockets.connect( uri, ssl=ssl_context, + local_addr=local_addr, #open_timeout=10, websockers 10.0 /!\ python>=3.7 extra_headers={"Authorization": f"{ail_key}"} ) as websocket: diff --git a/configs/core.cfg.sample b/configs/core.cfg.sample index e0100533..0a04268f 100644 --- a/configs/core.cfg.sample +++ b/configs/core.cfg.sample @@ -60,6 +60,7 @@ DiffMaxLineLength = 10000 [AIL_2_AIL] server_host = 0.0.0.0 server_port = 4443 +local_addr = #### Modules #### [BankAccount] From 30fdc953622673603469f0a486c930f4ff9d754f Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 10 May 2022 13:36:35 +0200 Subject: [PATCH 05/17] fix: [websockets client] fix client bind --- bin/core/ail_2_ail_client.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/bin/core/ail_2_ail_client.py b/bin/core/ail_2_ail_client.py index 29e6c37f..16f108be 100755 --- a/bin/core/ail_2_ail_client.py +++ b/bin/core/ail_2_ail_client.py @@ -26,6 +26,8 @@ config_loader = ConfigLoader() local_addr = config_loader.get_config_str('AIL_2_AIL', 'local_addr') if not local_addr or local_addr == None: local_addr = None +else: + local_addr = (local_addr, 0) config_loader = None From 188747417f446777eb6e9963f329e99e4d146539 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 17 May 2022 14:26:09 +0200 Subject: [PATCH 06/17] chg: [sync client] debug --- bin/core/ail_2_ail_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/core/ail_2_ail_client.py b/bin/core/ail_2_ail_client.py index 16f108be..8f1c3633 100755 --- a/bin/core/ail_2_ail_client.py +++ b/bin/core/ail_2_ail_client.py @@ -80,7 +80,7 @@ async def push(websocket, ail_uuid): obj_ail_stream = ail_2_ail.create_ail_stream(Obj) obj_ail_stream = json.dumps(obj_ail_stream) - sys.stdout.write(obj_ail_stream) + sys.stdout.write(obj_ail_stream['meta']) # send objects await websocket.send(obj_ail_stream) From 0635bfedb654cfaccd6d4d50cedd5a0eff424f20 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 17 May 2022 14:35:28 +0200 Subject: [PATCH 07/17] chg: [sync module] debug --- bin/core/Sync_module.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/bin/core/Sync_module.py b/bin/core/Sync_module.py index d8c01d83..f68075f9 100755 --- a/bin/core/Sync_module.py +++ b/bin/core/Sync_module.py @@ -48,8 +48,6 @@ class Sync_module(AbstractModule): def compute(self, message): - print(message) - ### REFRESH DICT if self.last_refresh < ail_2_ail.get_last_updated_sync_config(): self.last_refresh = time.time() @@ -73,15 +71,14 @@ class Sync_module(AbstractModule): # check filter + tags for queue_uuid in self.dict_sync_queues: filter_tags = self.dict_sync_queues[queue_uuid]['filter'] - print(tags) - print(filter_tags) - print(tags.issubset(filter_tags)) if filter_tags and tags: + #print(message) + #print(f'tags: {tags} filter: {filter_tags}') if tags.issubset(filter_tags): obj_dict = obj.get_default_meta() # send to queue push and/or pull for dict_ail in self.dict_sync_queues[queue_uuid]['ail_instances']: - + print(f'{dict_ail['ail_uuid']} {message}') ail_2_ail.add_object_to_sync_queue(queue_uuid, dict_ail['ail_uuid'], obj_dict, push=dict_ail['push'], pull=dict_ail['pull']) From 234148f005367a6b6d968cb64f17031659de0fde Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 17 May 2022 14:39:22 +0200 Subject: [PATCH 08/17] fix: [sync module] debug --- bin/core/Sync_module.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/core/Sync_module.py b/bin/core/Sync_module.py index f68075f9..ed9388c9 100755 --- a/bin/core/Sync_module.py +++ b/bin/core/Sync_module.py @@ -78,7 +78,7 @@ class Sync_module(AbstractModule): obj_dict = obj.get_default_meta() # send to queue push and/or pull for dict_ail in self.dict_sync_queues[queue_uuid]['ail_instances']: - print(f'{dict_ail['ail_uuid']} {message}') + print(f'{dict_ail["ail_uuid"]} {message}') ail_2_ail.add_object_to_sync_queue(queue_uuid, dict_ail['ail_uuid'], obj_dict, push=dict_ail['push'], pull=dict_ail['pull']) From 7e62d0f857e9d9f53560105de2d3dab096a9064a Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 17 May 2022 14:46:19 +0200 Subject: [PATCH 09/17] fix: [sync client] debug --- bin/core/ail_2_ail_client.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/core/ail_2_ail_client.py b/bin/core/ail_2_ail_client.py index 8f1c3633..6f1f54b7 100755 --- a/bin/core/ail_2_ail_client.py +++ b/bin/core/ail_2_ail_client.py @@ -80,7 +80,7 @@ async def push(websocket, ail_uuid): obj_ail_stream = ail_2_ail.create_ail_stream(Obj) obj_ail_stream = json.dumps(obj_ail_stream) - sys.stdout.write(obj_ail_stream['meta']) + print(obj_ail_stream['meta']) # send objects await websocket.send(obj_ail_stream) From c116713dc90f0e608e82ac2bc9ccdbb9e627a6c7 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 17 May 2022 14:51:35 +0200 Subject: [PATCH 10/17] fix: [sync client] debug --- bin/core/ail_2_ail_client.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/bin/core/ail_2_ail_client.py b/bin/core/ail_2_ail_client.py index 6f1f54b7..8f23b053 100755 --- a/bin/core/ail_2_ail_client.py +++ b/bin/core/ail_2_ail_client.py @@ -78,9 +78,8 @@ async def push(websocket, ail_uuid): Obj, queue_uuid = ail_2_ail.get_sync_queue_object_and_queue_uuid(ail_uuid) if Obj: obj_ail_stream = ail_2_ail.create_ail_stream(Obj) - obj_ail_stream = json.dumps(obj_ail_stream) - print(obj_ail_stream['meta']) + obj_ail_stream = json.dumps(obj_ail_stream) # send objects await websocket.send(obj_ail_stream) From c765c867284b0f6dabd4bfaaa6091daa0d3e292f Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 18 May 2022 14:59:51 +0200 Subject: [PATCH 11/17] fix: [sync module] fix tags filter --- bin/core/Sync_module.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/bin/core/Sync_module.py b/bin/core/Sync_module.py index ed9388c9..7c1330e0 100755 --- a/bin/core/Sync_module.py +++ b/bin/core/Sync_module.py @@ -69,16 +69,16 @@ class Sync_module(AbstractModule): tags = obj.get_tags(r_set=True) # check filter + tags + #print(message) for queue_uuid in self.dict_sync_queues: filter_tags = self.dict_sync_queues[queue_uuid]['filter'] if filter_tags and tags: - #print(message) #print(f'tags: {tags} filter: {filter_tags}') - if tags.issubset(filter_tags): + if filter_tags.issubset(tags): obj_dict = obj.get_default_meta() # send to queue push and/or pull for dict_ail in self.dict_sync_queues[queue_uuid]['ail_instances']: - print(f'{dict_ail["ail_uuid"]} {message}') + print(f'ail_uuid: {dict_ail["ail_uuid"]} obj: {message}') ail_2_ail.add_object_to_sync_queue(queue_uuid, dict_ail['ail_uuid'], obj_dict, push=dict_ail['push'], pull=dict_ail['pull']) From 34ec48e58a45c3a8eb28c55184330ac94b08b0cf Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 18 May 2022 15:46:27 +0200 Subject: [PATCH 12/17] fix: [sync module] fix redis tag queue --- bin/modules/Tags.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/modules/Tags.py b/bin/modules/Tags.py index 2ed37b74..9bfc080c 100755 --- a/bin/modules/Tags.py +++ b/bin/modules/Tags.py @@ -54,7 +54,7 @@ class Tags(AbstractModule): self.send_message_to_queue(message, 'MISP_The_Hive_feeder') message = f'{item.get_type()};{item.get_subtype(r_str=True)};{item.get_id()}' - self.send_message_to_queue(message, 'Sync_module') + self.send_message_to_queue(message, 'SyncModule') else: # Malformed message From a3e576ed491a6845fc45a109c9d2efa0f0752774 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 18 May 2022 16:44:09 +0200 Subject: [PATCH 13/17] fix: [installer] remove old tor install --- installing_deps.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/installing_deps.sh b/installing_deps.sh index 977ed642..24a3b7b1 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -14,9 +14,6 @@ sudo apt-get install python3-pip virtualenv python3-dev python3-tk libfreetype6- #Needed for downloading jemalloc sudo apt-get install wget -qq -#optional tor install -sudo apt-get install tor -qq - #Needed for bloom filters sudo apt-get install libssl-dev libfreetype6-dev python3-numpy -qq From df4bb531d216828878d0b463a5c80c974721ffd7 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Thu, 19 May 2022 13:57:07 +0200 Subject: [PATCH 14/17] fix: [ui tag selector] force custom tags --- bin/core/ail_2_ail.py | 10 ++++++---- var/www/templates/tags/block_tags_selector.html | 3 ++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/bin/core/ail_2_ail.py b/bin/core/ail_2_ail.py index 027eb791..70269e82 100755 --- a/bin/core/ail_2_ail.py +++ b/bin/core/ail_2_ail.py @@ -1028,8 +1028,9 @@ def api_create_sync_queue(json_dict): tags = json_dict.get('tags') if not tags: return {"status": "error", "reason": "no tags provided"}, 400 - if not Tag.are_enabled_tags(tags): - return {"status": "error", "reason": "Invalid/Disabled tags"}, 400 + # FIXME: add custom tags + # if not Tag.are_enabled_tags(tags): + # return {"status": "error", "reason": "Invalid/Disabled tags"}, 400 max_size = json_dict.get('max_size') if not max_size: @@ -1064,8 +1065,9 @@ def api_edit_sync_queue(json_dict): tags = json_dict.get('tags') if tags: - if not Tag.are_enabled_tags(tags): - return {"status": "error", "reason": "Invalid/Disabled tags"}, 400 + # FIXME: add custom tags + # if not Tag.are_enabled_tags(tags): + # return {"status": "error", "reason": "Invalid/Disabled tags"}, 400 edit_sync_queue_filter_tags(queue_uuid, tags) max_size = json_dict.get('max_size') diff --git a/var/www/templates/tags/block_tags_selector.html b/var/www/templates/tags/block_tags_selector.html index 06b7779b..b3bae6c7 100644 --- a/var/www/templates/tags/block_tags_selector.html +++ b/var/www/templates/tags/block_tags_selector.html @@ -35,6 +35,7 @@ +