ail-framework/bin/modules/Tools.py

#!/usr/bin/env python3
# -*-coding:UTF-8 -*
"""
Tools Module
============================

Search tools outpout

"""

import os
import sys
import time

sys.path.append(os.environ['AIL_BIN'])
##################################
# Import Project packages
##################################
from modules.abstract_module import AbstractModule
from lib.objects.Items import Item


TOOLS = {
    'sqlmap': {
        'regex': r'Usage of sqlmap for attacking targets without|all tested parameters do not appear to be injectable|sqlmap identified the following injection point|Title:[^\n]*((error|time|boolean)-based|stacked queries|UNION query)',
        'tag': 'infoleak:automatic-detection="sqlmap-tool"',
    },
    'wig': {
        'regex': r'(?s)wig - WebApp Information Gatherer.+?_{10,}',
        'tag': 'infoleak:automatic-detection="wig-tool"',
    },
    'dmytry': {
        'regex': r'(?s)Gathered (TCP Port|Inet-whois|Netcraft|Subdomain|E-Mail) information for.+?-{10,}',
        'tag': 'infoleak:automatic-detection="dmitry-tool"',
    },
    'inurlbr': {
        'regex': r'Usage of INURLBR for attacking targets without prior mutual consent is illegal',
        'tag': 'infoleak:automatic-detection="inurlbr-tool"',
    },
    'wafw00f': {
        'regex': r'(?s)WAFW00F - Web Application Firewall Detection Tool.+?Checking',
        'tag': 'infoleak:automatic-detection="wafw00f-tool"',
    },
    'sslyze': {
        'regex': r'(?s)PluginSessionRenegotiation.+?SCAN RESULTS FOR',
        'tag': 'infoleak:automatic-detection="sslyze-tool"',
    },
    'nmap': {
        'regex': r'(?s)Nmap scan report for.+?Host is',
        'tag': 'infoleak:automatic-detection="nmap-tool"',
    },
    'dnsenum': {
        'regex': r'(?s)dnsenum(\.pl)? VERSION:.+?Trying Zone Transfer',
        'tag': 'infoleak:automatic-detection="dnsenum-tool"',
    },
    'knock': {
        'regex': r'I scannig with my internal wordlist',
        'tag': 'infoleak:automatic-detection="knock-tool"',
    },
    'nikto': {
        'regex': r'(?s)\+ Target IP:.+?\+ Start Time:',
        'tag': 'infoleak:automatic-detection="nikto-tool"',
    },
    'dnscan': {
        'regex': r'(?s)\[\*\] Processing domain.+?\[\+\] Getting nameservers.+?records found',
        'tag': 'infoleak:automatic-detection="dnscan-tool"',
    },
    'dnsrecon': {
        'regex': r'Performing General Enumeration of Domain:|Performing TLD Brute force Enumeration against',
        'tag': 'infoleak:automatic-detection="dnsrecon-tool"',
    },
    'striker': {
        'regex': r'Crawling the target for fuzzable URLs|Honeypot Probabilty:',
        'tag': 'infoleak:automatic-detection="striker-tool"',
    },
    'rhawk': {
        'regex': r'S U B - D O M A I N   F I N D E R',
        'tag': 'infoleak:automatic-detection="rhawk-tool"',
    },
    'uniscan': {
        'regex': r'\| \[\+\] E-mail Found:',
        'tag': 'infoleak:automatic-detection="uniscan-tool"',
    },
    'masscan': {
        'regex': r'(?s)Starting masscan [\d.]+.+?Scanning|bit.ly/14GZzcT',
        'tag': 'infoleak:automatic-detection="masscan-tool"',
    },
    'msfconsole': {
        'regex': r'=\[ metasploit v[\d.]+.+?msf >',
        'tag': 'infoleak:automatic-detection="msfconsole-tool"',
    },
    'amap': {
        'regex': r'\bamap v[\d.]+ \(www.thc.org/thc-amap\)',
        'tag': 'infoleak:automatic-detection="amap-tool"',
    },
    'automater': {
        'regex': r'(?s)\[\*\] Checking.+?_+ Results found for:',
        'tag': 'infoleak:automatic-detection="automater-tool"',
    },
    'braa': {
        'regex': r'\bbraa public@[\d.]+',
        'tag': 'infoleak:automatic-detection="braa-tool"',
    },
    'ciscotorch': {
        'regex': r'Becase we need it',
        'tag': 'infoleak:automatic-detection="ciscotorch-tool"',
    },
    'theharvester': {
        'regex': r'Starting harvesting process for domain:',
        'tag': 'infoleak:automatic-detection="theharvester-tool"',
    },
    'sslstrip': {
        'regex': r'sslstrip [\d.]+ by Moxie Marlinspike running',
        'tag': 'infoleak:automatic-detection="sslstrip-tool"',
    },
    'sslcaudit': {
        'regex': r'# filebag location:',
        'tag': 'infoleak:automatic-detection="sslcaudit-tool"',
    },
    'smbmap': {
        'regex': r'\[\+\] Finding open SMB ports\.\.\.',
        'tag': 'infoleak:automatic-detection="smbmap-tool"',
    },
    'reconng': {
        'regex': r'\[\*\] Status: unfixed|\[recon-ng\]\[default\]',
        'tag': 'infoleak:automatic-detection="reconng-tool"',
    },
    'p0f': {
        'regex': r'\bp0f [^ ]+ by Michal Zalewski',
        'tag': 'infoleak:automatic-detection="p0f-tool"',
    },
    'hping3': {
        'regex': r'\bHPING [^ ]+ \([^)]+\): [^ ]+ mode set',
        'tag': 'infoleak:automatic-detection="hping3-tool"',
    },
    'enum4linux': {
        'regex': r'Starting enum4linux v[\d.]+|\|    Target Information    \|',
        'tag': 'infoleak:automatic-detection="enum4linux-tool"',
    },
    'dnstracer': {
        'regex': r'(?s)Tracing to.+?DNS HEADER \(send\)',
        'tag': 'infoleak:automatic-detection="dnstracer-tool"',
    },
    'dnmap': {
        'regex': r'dnmap_(client|server)|Nmap output files stored in \'nmap_output\' directory',
        'tag': 'infoleak:automatic-detection="dnmap-tool"',
    },
    'arpscan': {
        'regex': r'Starting arp-scan [^ ]+ with \d+ hosts',
        'tag': 'infoleak:automatic-detection="arpscan-tool"',
    },
    'cdpsnarf': {
        'regex': r'(?s)CDPSnarf v[^ ]+.+?Waiting for a CDP packet\.\.\.',
        'tag': 'infoleak:automatic-detection="cdpsnarf-tool"',
    },
    'dnsmap': {
        'regex': r'DNS Network Mapper by pagvac',
        'tag': 'infoleak:automatic-detection="dnsmap-tool"',
    },
    'dotdotpwn': {
        'regex': r'DotDotPwn v[^ ]+|dotdotpwn@sectester.net|\[\+\] Creating Traversal patterns',
        'tag': 'infoleak:automatic-detection="dotdotpwn-tool"',
    },
    'searchsploit': {
        'regex': r'(exploits|shellcodes)/|searchsploit_rc|Exploit Title',
        'tag': 'infoleak:automatic-detection="searchsploit-tool"',
    },
    'fierce': {
        'regex': r'(?s)Trying zone transfer first.+Checking for wildcard DNS',
        'tag': 'infoleak:automatic-detection="fierce-tool"',
    },
    'firewalk': {
        'regex': r'Firewalk state initialization completed successfully|Ramping phase source port',
        'tag': 'infoleak:automatic-detection="firewalk-tool"',
    },
    'fragroute': {
        'regex': r'\bfragroute: tcp_seg -> ip_frag',
        'tag': 'infoleak:automatic-detection="fragroute-tool"',
    },
    'fragrouter': {
        'regex': r'fragrouter: frag-\d+:',
        'tag': 'infoleak:automatic-detection="fragrouter-tool"',
    },
    'goofile': {
        'regex': r'code.google.com/p/goofile\b',
        'tag': 'infoleak:automatic-detection="goofile-tool"',
    },
    'intrace': {
        'regex': r'\bInTrace [\d.]+ \-\-',
        'tag': 'infoleak:automatic-detection="intrace-tool"',
    },
    'ismtp': {
        'regex': r'Testing SMTP server \[user enumeration\]',
        'tag': 'infoleak:automatic-detection="ismtp-tool"',
    },
    'lbd': {
        'regex': r'Checking for (DNS|HTTP)-Loadbalancing',
        'tag': 'infoleak:automatic-detection="lbd-tool"',
    },
    'miranda': {
        'regex': r'Entering discovery mode for \'upnp:',
        'tag': 'infoleak:automatic-detection="miranda-tool"',
    },
    'ncat': {
        'regex': r'nmap.org/ncat',
        'tag': 'infoleak:automatic-detection="ncat-tool"',
    },
    'ohrwurm': {
        'regex': r'\bohrwurm-[\d.]+',
        'tag': 'infoleak:automatic-detection="ohrwurm-tool"',
    },
    'oscanner': {
        'regex': r'Loading services/sids from service file',
        'tag': 'infoleak:automatic-detection="oscanner-tool"',
    },
    'sfuzz': {
        'regex': r'AREALLYBADSTRING|sfuzz/sfuzz',
        'tag': 'infoleak:automatic-detection="sfuzz-tool"',
    },
    'sidguess': {
        'regex': r'SIDGuesser v[\d.]+',
        'tag': 'infoleak:automatic-detection="sidguess-tool"',
    },
    'sqlninja': {
        'regex': r'Sqlninja rel\. [\d.]+',
        'tag': 'infoleak:automatic-detection="sqlninja-tool"',
    },
    'sqlsus': {
        'regex': r'sqlsus version [\d.]+',
        'tag': 'infoleak:automatic-detection="sqlsus-tool"',
    },
    'dnsdict6': {
        'regex': r'Starting DNS enumeration work on',
        'tag': 'infoleak:automatic-detection="dnsdict6-tool"',
    },
    'unixprivesccheck': {
        'regex': r'Recording Interface IP addresses',
        'tag': 'infoleak:automatic-detection="unixprivesccheck-tool"',
    },
    'yersinia': {
        'regex': r'yersinia@yersinia.net',
        'tag': 'infoleak:automatic-detection="yersinia-tool"',
    },
    'armitage': {
        'regex': r'\[\*\] Starting msfrpcd for you',
        'tag': 'infoleak:automatic-detection="armitage-tool"',
    },
    'backdoorfactory': {
        'regex': r'\[\*\] In the backdoor module',
        'tag': 'infoleak:automatic-detection="backdoorfactory-tool"',
    },
    'beef': {
        'regex': r'Please wait as BeEF services are started',
        'tag': 'infoleak:automatic-detection="beef-tool"',
    },
    'cat': {
        'regex': r'Cisco Auditing Tool.+?g0ne',
        'tag': 'infoleak:automatic-detection="cat-tool"',
    },
    'cge': {
        'regex': r'Vulnerability successful exploited with \[',
        'tag': 'infoleak:automatic-detection="cge-tool"',
    },
    'john': {
        'regex': r'John the Ripper password cracker, ver:|Loaded \d+ password hash \(',
        'tag': 'infoleak:automatic-detection="john-tool"',
    },
    'keimpx': {
        'regex': r'\bkeimpx [\d.]+',
        'tag': 'infoleak:automatic-detection="keimpx-tool"',
    },
    'maskprocessor': {
        'regex': r'mp by atom, High-Performance word generator',
        'tag': 'infoleak:automatic-detection="maskprocessor-tool"',
    },
    'ncrack': {
        'regex': r'Starting Ncrack[^\n]+http://ncrack.org',
        'tag': 'infoleak:automatic-detection="ncrack-tool"',
    },
    'patator': {
        'regex': r'http://code.google.com/p/patator/|Starting Patator v',
        'tag': 'infoleak:automatic-detection="patator-tool"',
    },
    'phrasendrescher': {
        'regex': r'phrasen\|drescher [\d.]+',
        'tag': 'infoleak:automatic-detection="phrasendrescher-tool"',
    },
    'polenum': {
        'regex': r'\[\+\] Password Complexity Flags:',
        'tag': 'infoleak:automatic-detection="polenum-tool"',
    },
    'rainbowcrack': {
        'regex': r'Official Website: http://project-rainbowcrack.com/',
        'tag': 'infoleak:automatic-detection="rainbowcrack-tool"',
    },
    'rcracki_mt': {
        'regex': r'Found \d+ rainbowtable files\.\.\.',
        'tag': 'infoleak:automatic-detection="rcracki_mt-tool"',
    },
    'tcpdump': {
        'regex': r'tcpdump: listening on.+capture size \d+|\d+ packets received by filter',
        'tag': 'infoleak:automatic-detection="tcpdump-tool"',
    },
    'hydra': {
        'regex': r'Hydra \(http://www.thc.org/thc-hydra\)',
        'tag': 'infoleak:automatic-detection="hydra-tool"',
    },
    'netcat': {
        'regex': r'Listening on \[[\d.]+\] \(family',
        'tag': 'infoleak:automatic-detection="netcat-tool"',
    },
    'nslookup': {
        'regex': r'Non-authoritative answer:',
        'tag': 'infoleak:automatic-detection="nslookup-tool"',
    },
    'dig': {
        'regex': r'; <<>> DiG [\d.]+',
        'tag': 'infoleak:automatic-detection="dig-tool"',
    },
    'whois': {
        'regex': r'(?i)Registrar WHOIS Server:|Registrar URL: http://|DNSSEC: unsigned|information on Whois status codes|REGISTERED, DELEGATED|[Rr]egistrar:|%[^\n]+(WHOIS|2016/679)',
        'tag': 'infoleak:automatic-detection="whois-tool"',
    },
    'nessus': {
        'regex': r'nessus_(report_(get|list|exploits)|scan_(new|status))|nessuscli|nessusd|nessus-service',
        'tag': 'infoleak:automatic-detection="nessus-tool"',
    },
    'openvas': {
        'regex': r'/openvas/',
        'tag': 'infoleak:automatic-detection="openvas-tool"',
    },
    'golismero': {
        'regex': r'GoLismero[\n]+The Web Knife',
        'tag': 'infoleak:automatic-detection="golismero-tool"',
    },
    'wpscan': {
        'regex': r'WordPress Security Scanner by the WPScan Team|\[\+\] Interesting header:',
        'tag': 'infoleak:automatic-detection="wpscan-tool"',
    },
    'skipfish': {
        'regex': r'\[\+\] Sorting and annotating crawl nodes:|skipfish version [\d.]+',
        'tag': 'infoleak:automatic-detection="skipfish-tool"',
    },
    'arachni': {
        'regex': r'With the support of the community and the Arachni Team|\[\*\] Waiting for plugins to settle\.\.\.',
        'tag': 'infoleak:automatic-detection="arachni-tool"',
    },
    'dirb': {
        'regex': r'==> DIRECTORY:|\bDIRB v[\d.]+',
        'tag': 'infoleak:automatic-detection="dirb-tool"',
    },
    'joomscan': {
        'regex': r'OWASP Joomla! Vulnerability Scanner v[\d.]+',
        'tag': 'infoleak:automatic-detection="joomscan-tool"',
    },
    'jbossautopwn': {
        'regex': r'\[x\] Now creating BSH script\.\.\.|\[x\] Now deploying \.war file:',
        'tag': 'infoleak:automatic-detection="jbossautopwn-tool"',
    },
    'grabber': {
        'regex': r'runSpiderScan @',
        'tag': 'infoleak:automatic-detection="grabber-tool"',
    },
    'fimap': {
        'regex': r'Automatic LFI/RFI scanner and exploiter',
        'tag': 'infoleak:automatic-detection="fimap-tool"',
    },
    'dsxs': {
        'regex': r'Damn Small XSS Scanner \(DSXS\)',
        'tag': 'infoleak:automatic-detection="dsxs-tool"',
    },
    'dsss': {
        'regex': r'Damn Small SQLi Scanner \(DSSS\)',
        'tag': 'infoleak:automatic-detection="dsss-tool"',
    },
    'dsjs': {
        'regex': r'Damn Small JS Scanner \(DSJS\)',
        'tag': 'infoleak:automatic-detection="dsjs-tool"',
    },
    'dsfs': {
        'regex': r'Damn Small FI Scanner \(DSFS\)',
        'tag': 'infoleak:automatic-detection="dsfs-tool"',
    },
    'identywaf': {
        'regex': r'\[o\] initializing handlers\.\.\.',
        'tag': 'infoleak:automatic-detection="identywaf-tool"',
    },
    'whatwaf': {
        'regex': r'<sCRIPT>ALeRt.+?WhatWaf\?',
        'tag': 'infoleak:automatic-detection="whatwaf-tool"',
    }
}

class Tools(AbstractModule):
    """
    Tools module for AIL framework
    """

    def __init__(self):
        super(Tools, self).__init__()

        self.max_execution_time = 30
        # Waiting time in seconds between to message processed
        self.pending_seconds = 10
        # Send module state to logs
        self.redis_logger.info(f"Module {self.module_name} initialized")

    def get_tools(self):
        return TOOLS.keys()

    def extract(self, obj_id, content, tag):
        tool_name = tag.rsplit('"', 2)[1][:-5]
        return self.regex_finditer(TOOLS[tool_name]['regex'], obj_id, content)

    def compute(self, message):
        item = Item(message)
        content = item.get_content()

        for tool_name in TOOLS:
            tool = TOOLS[tool_name]
            match = self.regex_search(tool['regex'], item.id, content)
            if match:
                print(f'{item.id} found: {tool_name}')
                # Tag Item
                msg = f"{tool['tag']};{item.id}"
                self.send_message_to_queue(msg, 'Tags')
                # TODO ADD LOGS


if __name__ == '__main__':
    module = Tools()
    # module.run()
    module.compute('crawled/2022/09/15/circl.lu179c7903-5b21-452e-9f25-4b61d9934e2b')