mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
Added module Regex.py
This commit is contained in:
parent
e904fbe53a
commit
9bfb7a5fb6
3 changed files with 157 additions and 0 deletions
|
@ -170,6 +170,8 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Keys.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Regex.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ./Decoder.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ./Bitcoin.py; read x'
|
||||
|
|
138
bin/Regex.py
Normal file
138
bin/Regex.py
Normal file
|
@ -0,0 +1,138 @@
|
|||
#!/usr/bin/env python3
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
The Regex Module
|
||||
================
|
||||
|
||||
Search for regular expressions stored in a flat file + tag.
|
||||
The flat file is automatically reloaded when the MTIME changed.
|
||||
|
||||
It uses the file 'packagess/regex.cfg'. Format:
|
||||
Tag||Regex
|
||||
|
||||
Xavier Mertens <xavier@rootshell.be>
|
||||
|
||||
"""
|
||||
|
||||
import time
|
||||
import os
|
||||
import re
|
||||
from pubsublogger import publisher
|
||||
|
||||
#from bin.packages import Paste
|
||||
#from bin.Helper import Process
|
||||
|
||||
from packages import Paste
|
||||
from Helper import Process
|
||||
|
||||
# Change the path to your preferred one
|
||||
regexConfig = 'packages/regex.cfg'
|
||||
|
||||
regexes = []
|
||||
|
||||
def load_regex(force = False):
|
||||
'''
|
||||
Load regexes from the config file and validate them
|
||||
If 'True' passed as argument, force to reload
|
||||
'''
|
||||
|
||||
lregexes = regexes
|
||||
validate_regex = False
|
||||
|
||||
try:
|
||||
stats = os.stat(regexConfig)
|
||||
mtime = int(stats.st_mtime)
|
||||
if mtime > time.time()-60 or force == True:
|
||||
# Regex config changed, reload the file
|
||||
print('Loading regular expressions')
|
||||
with open(regexConfig) as f:
|
||||
lines = f.readlines()
|
||||
lines = [x.strip() for x in lines]
|
||||
validate_regex = True
|
||||
except:
|
||||
print('Cannot read {}'.format(regexConfig))
|
||||
return []
|
||||
|
||||
if validate_regex:
|
||||
# Validate regexes read from the file
|
||||
line=1
|
||||
lregexes = []
|
||||
for l in lines:
|
||||
# Skip comments and empty lines
|
||||
if len(l) > 0:
|
||||
if l[0] == '#':
|
||||
continue
|
||||
try:
|
||||
re.compile(l.split('||')[1])
|
||||
except:
|
||||
print('Ignored line {}: Syntax error in "{}"'.format(line, regexConfig))
|
||||
continue
|
||||
line += 1
|
||||
lregexes.append(l)
|
||||
print('DEBUG: regexes:')
|
||||
print(lregexes)
|
||||
return lregexes
|
||||
|
||||
def search_regex(paste):
|
||||
content = paste.get_p_content()
|
||||
find = False
|
||||
global regexes
|
||||
|
||||
regexes = load_regex(False)
|
||||
|
||||
for r in regexes:
|
||||
(tag,pattern) = r.split('||')
|
||||
|
||||
if re.findall(pattern, content, re.MULTILINE|re.IGNORECASE):
|
||||
publisher.warning('Regex match: {} ({})'.format(pattern, tag))
|
||||
# Sanitize tag to make it easy to read
|
||||
tag = tag.strip().lower().replace(' ','-')
|
||||
print('regex {} found'.format(tag))
|
||||
msg = 'infoleak:automatic-detection="regex-{}";{}'.format(tag, message)
|
||||
p.populate_set_out(msg, 'Tags')
|
||||
find = True
|
||||
|
||||
if find:
|
||||
#Send to duplicate
|
||||
p.populate_set_out(message, 'Duplicate')
|
||||
#send to Browse_warning_paste
|
||||
msg = ('regex;{}'.format(message))
|
||||
print(message)
|
||||
p.populate_set_out( msg, 'alertHandler')
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
global regexes
|
||||
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
|
||||
# Port of the redis instance used by pubsublogger
|
||||
publisher.port = 6380
|
||||
# Script is the default channel used for the modules.
|
||||
publisher.channel = 'Script'
|
||||
|
||||
# Section name in bin/packages/modules.cfg
|
||||
config_section = 'Regex'
|
||||
|
||||
# Setup the I/O queues
|
||||
p = Process(config_section)
|
||||
|
||||
# Sent to the logging a description of the module
|
||||
publisher.info("Run Regex module ")
|
||||
|
||||
# Load regular expressions from config file
|
||||
regexes = load_regex(True)
|
||||
|
||||
# Endless loop getting messages from the input queue
|
||||
while True:
|
||||
# Get one message from the input queue
|
||||
message = p.get_from_set()
|
||||
if message is None:
|
||||
publisher.debug("{} queue is empty, waiting".format(config_section))
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
# Do something with the message from the queue
|
||||
paste = Paste.Paste(message)
|
||||
search_regex(paste)
|
||||
|
||||
# (Optional) Send that thing to the next queue
|
17
bin/packages/regex.cfg
Normal file
17
bin/packages/regex.cfg
Normal file
|
@ -0,0 +1,17 @@
|
|||
#
|
||||
# Regular expressions to be search in AIL
|
||||
#
|
||||
# Format:
|
||||
# tag||regex
|
||||
#
|
||||
# tag: appened to the tag (ex: regex-tag)
|
||||
# regex: the regular expression
|
||||
# (Comments & empty lines are ignores)
|
||||
#
|
||||
|
||||
# Example1
|
||||
# Search for my name
|
||||
personal-mention||xavier\s+mertens
|
||||
|
||||
# Search for cicl.lu hostnames
|
||||
circl||\w+\.circl\.lu
|
Loading…
Reference in a new issue