fix: [module extractor] fix tracker extractor

This commit is contained in:
Terrtia 2023-05-30 10:11:12 +02:00
parent 8252d6b69e
commit 2ebe4845a7
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0

View file

@ -1,11 +1,13 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
# -*-coding:UTF-8 -* # -*-coding:UTF-8 -*
import json import json
import logging
import os import os
import sys import sys
import yara import yara
from hashlib import sha256
from operator import itemgetter from operator import itemgetter
sys.path.append(os.environ['AIL_BIN']) sys.path.append(os.environ['AIL_BIN'])
@ -28,6 +30,8 @@ from modules.Onion import Onion
from modules.Phone import Phone from modules.Phone import Phone
from modules.Tools import Tools from modules.Tools import Tools
logger = logging.getLogger()
config_loader = ConfigLoader() config_loader = ConfigLoader()
r_cache = config_loader.get_redis_conn("Redis_Cache") r_cache = config_loader.get_redis_conn("Redis_Cache")
config_loader = None config_loader = None
@ -64,11 +68,12 @@ def get_correl_match(extract_type, obj_id, content):
if extract_type == 'title': if extract_type == 'title':
title = Title(value).get_content() title = Title(value).get_content()
to_extract.append(title) to_extract.append(title)
map_value_id[title] = value sha256_val = sha256(title.encode()).hexdigest()
else: else:
map_subtype[value] = subtype map_subtype[value] = subtype
to_extract.append(value) to_extract.append(value)
map_value_id[value] = value sha256_val = sha256(value.encode()).hexdigest()
map_value_id[sha256_val] = value
if to_extract: if to_extract:
objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj_id, content) objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj_id, content)
for obj in objs: for obj in objs:
@ -76,7 +81,12 @@ def get_correl_match(extract_type, obj_id, content):
subtype = map_subtype[obj[2]] subtype = map_subtype[obj[2]]
else: else:
subtype = '' subtype = ''
extracted.append([obj[0], obj[1], obj[2], f'{extract_type}:{subtype}:{map_value_id[obj[2]]}']) sha256_val = sha256(obj[2].encode()).hexdigest()
value_id = map_value_id.get(sha256_val)
if not value_id:
logger.critical(f'Error module extractor: {sha256_val}\n{extract_type}\n{subtype}\n{value_id}\n{map_value_id}\n{objs}')
value_id = 'ERROR'
extracted.append([obj[0], obj[1], obj[2], f'{extract_type}:{subtype}:{value_id}'])
return extracted return extracted
def _get_yara_match(data): def _get_yara_match(data):
@ -162,6 +172,7 @@ def extract(obj_id, content=None):
# CHECK CACHE # CHECK CACHE
cached = r_cache.get(f'extractor:cache:{obj_id}') cached = r_cache.get(f'extractor:cache:{obj_id}')
# cached = None
if cached: if cached:
r_cache.expire(f'extractor:cache:{obj_id}', 300) r_cache.expire(f'extractor:cache:{obj_id}', 300)
return json.loads(cached) return json.loads(cached)