mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-13 01:58:22 +00:00
fix: [module extractor] fix tracker extractor
This commit is contained in:
parent
8252d6b69e
commit
2ebe4845a7
1 changed files with 14 additions and 3 deletions
|
@ -1,11 +1,13 @@
|
||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
# -*-coding:UTF-8 -*
|
# -*-coding:UTF-8 -*
|
||||||
import json
|
import json
|
||||||
|
import logging
|
||||||
import os
|
import os
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
import yara
|
import yara
|
||||||
|
|
||||||
|
from hashlib import sha256
|
||||||
from operator import itemgetter
|
from operator import itemgetter
|
||||||
|
|
||||||
sys.path.append(os.environ['AIL_BIN'])
|
sys.path.append(os.environ['AIL_BIN'])
|
||||||
|
@ -28,6 +30,8 @@ from modules.Onion import Onion
|
||||||
from modules.Phone import Phone
|
from modules.Phone import Phone
|
||||||
from modules.Tools import Tools
|
from modules.Tools import Tools
|
||||||
|
|
||||||
|
logger = logging.getLogger()
|
||||||
|
|
||||||
config_loader = ConfigLoader()
|
config_loader = ConfigLoader()
|
||||||
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
r_cache = config_loader.get_redis_conn("Redis_Cache")
|
||||||
config_loader = None
|
config_loader = None
|
||||||
|
@ -64,11 +68,12 @@ def get_correl_match(extract_type, obj_id, content):
|
||||||
if extract_type == 'title':
|
if extract_type == 'title':
|
||||||
title = Title(value).get_content()
|
title = Title(value).get_content()
|
||||||
to_extract.append(title)
|
to_extract.append(title)
|
||||||
map_value_id[title] = value
|
sha256_val = sha256(title.encode()).hexdigest()
|
||||||
else:
|
else:
|
||||||
map_subtype[value] = subtype
|
map_subtype[value] = subtype
|
||||||
to_extract.append(value)
|
to_extract.append(value)
|
||||||
map_value_id[value] = value
|
sha256_val = sha256(value.encode()).hexdigest()
|
||||||
|
map_value_id[sha256_val] = value
|
||||||
if to_extract:
|
if to_extract:
|
||||||
objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj_id, content)
|
objs = regex_helper.regex_finditer(r_key, '|'.join(to_extract), obj_id, content)
|
||||||
for obj in objs:
|
for obj in objs:
|
||||||
|
@ -76,7 +81,12 @@ def get_correl_match(extract_type, obj_id, content):
|
||||||
subtype = map_subtype[obj[2]]
|
subtype = map_subtype[obj[2]]
|
||||||
else:
|
else:
|
||||||
subtype = ''
|
subtype = ''
|
||||||
extracted.append([obj[0], obj[1], obj[2], f'{extract_type}:{subtype}:{map_value_id[obj[2]]}'])
|
sha256_val = sha256(obj[2].encode()).hexdigest()
|
||||||
|
value_id = map_value_id.get(sha256_val)
|
||||||
|
if not value_id:
|
||||||
|
logger.critical(f'Error module extractor: {sha256_val}\n{extract_type}\n{subtype}\n{value_id}\n{map_value_id}\n{objs}')
|
||||||
|
value_id = 'ERROR'
|
||||||
|
extracted.append([obj[0], obj[1], obj[2], f'{extract_type}:{subtype}:{value_id}'])
|
||||||
return extracted
|
return extracted
|
||||||
|
|
||||||
def _get_yara_match(data):
|
def _get_yara_match(data):
|
||||||
|
@ -162,6 +172,7 @@ def extract(obj_id, content=None):
|
||||||
|
|
||||||
# CHECK CACHE
|
# CHECK CACHE
|
||||||
cached = r_cache.get(f'extractor:cache:{obj_id}')
|
cached = r_cache.get(f'extractor:cache:{obj_id}')
|
||||||
|
# cached = None
|
||||||
if cached:
|
if cached:
|
||||||
r_cache.expire(f'extractor:cache:{obj_id}', 300)
|
r_cache.expire(f'extractor:cache:{obj_id}', 300)
|
||||||
return json.loads(cached)
|
return json.loads(cached)
|
||||||
|
|
Loading…
Reference in a new issue