diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index b0942517..b8545cb0 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -229,7 +229,8 @@ function launching_scripts { sleep 0.1 screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./LibInjection.py; read x" sleep 0.1 - + screen -S "Script_AIL" -X screen -t "Zerobins" bash -c "cd ${AIL_BIN}/modules; ${ENV_PY} ./Zerobins.py; read x" + sleep 0.1 ################################## # TRACKERS MODULES # diff --git a/bin/modules/Zerobins.py b/bin/modules/Zerobins.py new file mode 100755 index 00000000..3a9b1fa7 --- /dev/null +++ b/bin/modules/Zerobins.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# -*-coding:UTF-8 -* +""" +The Zerobins Module +====================== +This module spots zerobins-like services for further processing +""" + +################################## +# Import External packages +################################## +import os +import sys +import time +import pdb +import re +sys.path.append(os.environ['AIL_BIN']) +################################## +# Import Project packages +################################## +from modules.abstract_module import AbstractModule +from lib import crawlers + + +class Zerobins(AbstractModule): + """ + Zerobins module for AIL framework + """ + + def __init__(self): + super(Zerobins, self).__init__() + + binz = [ + r'^https:\/\/(zerobin||privatebin)\..*$', # historical ones + ] + + self.regex = re.compile('|'.join(binz)) + + # Pending time between two computation (computeNone) in seconds + self.pending_seconds = 10 + + # Send module state to logs + self.redis_logger.info(f'Module {self.module_name} initialized') + + + def computeNone(self): + """ + Compute when no message in queue + """ + self.redis_logger.debug("No message in queue") + + + def compute(self, message): + """regex_helper.regex_findall(self.module_name, self.redis_cache_key + Compute a message in queue + """ + print(message) + url, id = message.split() + + # Extract zerobins addresses + matching_binz = self.regex_findall(self.regex, id, url) + + if len(matching_binz) > 0: + for bin in matching_binz: + print("send {} to crawler".format(bin)) + crawlers.create_crawler_task(bin, screenshot=False, har=False, depth_limit=1, max_pages=1, auto_crawler=False, crawler_delta=3600, crawler_type=None, cookiejar_uuid=None, user_agent=None) + + self.redis_logger.debug("Compute message in queue") + + +if __name__ == '__main__': + + module = Zerobins() + module.run() \ No newline at end of file diff --git a/bin/modules/abstract_module.py b/bin/modules/abstract_module.py index 635e1199..32c60041 100644 --- a/bin/modules/abstract_module.py +++ b/bin/modules/abstract_module.py @@ -46,6 +46,9 @@ class AbstractModule(ABC): # If provided could be a namespaced channel like script: self.redis_logger.channel = logger_channel + #Cache key + self.redis_cache_key = regex_helper.generate_redis_cache_key(self.module_name) + self.max_execution_time = 30 # Run module endlessly self.proceed = True diff --git a/bin/packages/modules.cfg b/bin/packages/modules.cfg index 4da8fd2e..0a2ae61a 100644 --- a/bin/packages/modules.cfg +++ b/bin/packages/modules.cfg @@ -165,3 +165,6 @@ publish = Redis_Mixer,Redis_Tags [IP] subscribe = Redis_Global publish = Redis_Duplicate,Redis_Tags + +[Zerobins] +subscribe = Redis_Url \ No newline at end of file