mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-30 09:47:17 +00:00
commit
0263b25da8
4 changed files with 27 additions and 12 deletions
|
@ -52,6 +52,7 @@ if __name__ == "__main__":
|
||||||
config_section = 'Categ'
|
config_section = 'Categ'
|
||||||
|
|
||||||
p = Process(config_section)
|
p = Process(config_section)
|
||||||
|
matchingThreshold = p.config.getint("Categ", "matchingThreshold")
|
||||||
|
|
||||||
# SCRIPT PARSER #
|
# SCRIPT PARSER #
|
||||||
parser = argparse.ArgumentParser(description='Start Categ module on files.')
|
parser = argparse.ArgumentParser(description='Start Categ module on files.')
|
||||||
|
@ -90,7 +91,7 @@ if __name__ == "__main__":
|
||||||
|
|
||||||
for categ, pattern in tmp_dict.items():
|
for categ, pattern in tmp_dict.items():
|
||||||
found = set(re.findall(pattern, content))
|
found = set(re.findall(pattern, content))
|
||||||
if len(found) > 0:
|
if len(found) >= matchingThreshold:
|
||||||
msg = '{} {}'.format(paste.p_path, len(found))
|
msg = '{} {}'.format(paste.p_path, len(found))
|
||||||
print msg, categ
|
print msg, categ
|
||||||
p.populate_set_out(msg, categ)
|
p.populate_set_out(msg, categ)
|
||||||
|
|
|
@ -41,7 +41,6 @@ REDIS_KEY_ALL_CRED_SET_REV = 'AllCredentialsRev'
|
||||||
REDIS_KEY_ALL_PATH_SET = 'AllPath'
|
REDIS_KEY_ALL_PATH_SET = 'AllPath'
|
||||||
REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev'
|
REDIS_KEY_ALL_PATH_SET_REV = 'AllPathRev'
|
||||||
REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
|
REDIS_KEY_MAP_CRED_TO_PATH = 'CredToPathMapping'
|
||||||
MINIMUMSIZETHRESHOLD = 3
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
if __name__ == "__main__":
|
||||||
publisher.port = 6380
|
publisher.port = 6380
|
||||||
|
@ -50,13 +49,16 @@ if __name__ == "__main__":
|
||||||
p = Process(config_section)
|
p = Process(config_section)
|
||||||
publisher.info("Find credentials")
|
publisher.info("Find credentials")
|
||||||
|
|
||||||
|
minimumLengthThreshold = p.config.getint("Credential", "minimumLengthThreshold")
|
||||||
|
|
||||||
faup = Faup()
|
faup = Faup()
|
||||||
server_cred = redis.StrictRedis(
|
server_cred = redis.StrictRedis(
|
||||||
host=p.config.get("Redis_Level_DB_TermCred", "host"),
|
host=p.config.get("Redis_Level_DB_TermCred", "host"),
|
||||||
port=p.config.get("Redis_Level_DB_TermCred", "port"),
|
port=p.config.get("Redis_Level_DB_TermCred", "port"),
|
||||||
db=p.config.get("Redis_Level_DB_TermCred", "db"))
|
db=p.config.get("Redis_Level_DB_TermCred", "db"))
|
||||||
|
|
||||||
critical = 8
|
criticalNumberToAlert = p.config.getint("Credential", "criticalNumberToAlert")
|
||||||
|
minTopPassList = p.config.getint("Credential", "minTopPassList")
|
||||||
|
|
||||||
regex_web = "((?:https?:\/\/)[-_0-9a-zA-Z]+\.[0-9a-zA-Z]+)"
|
regex_web = "((?:https?:\/\/)[-_0-9a-zA-Z]+\.[0-9a-zA-Z]+)"
|
||||||
regex_cred = "[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}:[a-zA-Z0-9\_\-]+"
|
regex_cred = "[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}:[a-zA-Z0-9\_\-]+"
|
||||||
|
@ -71,7 +73,7 @@ if __name__ == "__main__":
|
||||||
|
|
||||||
filepath, count = message.split()
|
filepath, count = message.split()
|
||||||
|
|
||||||
if count < 5:
|
if count < minTopPassList:
|
||||||
# Less than 5 matches from the top password list, false positive.
|
# Less than 5 matches from the top password list, false positive.
|
||||||
print("false positive:", count)
|
print("false positive:", count)
|
||||||
continue
|
continue
|
||||||
|
@ -94,7 +96,7 @@ if __name__ == "__main__":
|
||||||
print('\n '.join(creds))
|
print('\n '.join(creds))
|
||||||
|
|
||||||
#num of creds above tresh, publish an alert
|
#num of creds above tresh, publish an alert
|
||||||
if len(creds) > critical:
|
if len(creds) > criticalNumberToAlert:
|
||||||
print("========> Found more than 10 credentials in this file : {}".format(filepath))
|
print("========> Found more than 10 credentials in this file : {}".format(filepath))
|
||||||
publisher.warning(to_print)
|
publisher.warning(to_print)
|
||||||
#Send to duplicate
|
#Send to duplicate
|
||||||
|
@ -154,6 +156,6 @@ if __name__ == "__main__":
|
||||||
#Add the split to redis, each split point towards its initial credential unique number
|
#Add the split to redis, each split point towards its initial credential unique number
|
||||||
splitedCred = re.findall(REGEX_CRED, cred)
|
splitedCred = re.findall(REGEX_CRED, cred)
|
||||||
for partCred in splitedCred:
|
for partCred in splitedCred:
|
||||||
if len(partCred) > MINIMUMSIZETHRESHOLD:
|
if len(partCred) > minimumLengthThreshold:
|
||||||
server_cred.sadd(partCred, uniq_num_cred)
|
server_cred.sadd(partCred, uniq_num_cred)
|
||||||
|
|
||||||
|
|
|
@ -19,8 +19,8 @@ Depending on the configuration, this module will process the feed as follow:
|
||||||
- Elseif, the saved content associated with the paste is not the same, process it
|
- Elseif, the saved content associated with the paste is not the same, process it
|
||||||
- Else, do not process it but keep track for statistics on duplicate
|
- Else, do not process it but keep track for statistics on duplicate
|
||||||
|
|
||||||
operation_mode 3: "Don't look if duplicate"
|
operation_mode 3: "Don't look if duplicated content"
|
||||||
- SImply do not bother to check if it is a duplicate
|
- Simply do not bother to check if it is a duplicate
|
||||||
|
|
||||||
Note that the hash of the content is defined as the sha1(gzip64encoded).
|
Note that the hash of the content is defined as the sha1(gzip64encoded).
|
||||||
|
|
||||||
|
|
|
@ -30,6 +30,18 @@ default_display = 10
|
||||||
minute_processed_paste = 10
|
minute_processed_paste = 10
|
||||||
|
|
||||||
#### Modules ####
|
#### Modules ####
|
||||||
|
[Categ]
|
||||||
|
#Minimum number of match between the paste and the category file
|
||||||
|
matchingThreshold=1
|
||||||
|
|
||||||
|
[Credential]
|
||||||
|
#Minimum length that a credential must have to be considered as such
|
||||||
|
minimumLengthThreshold=3
|
||||||
|
#Will be pushed as alert if the number of credentials is greater to that number
|
||||||
|
criticalNumberToAlert=8
|
||||||
|
#Will be considered as false positive if less that X matches from the top password list
|
||||||
|
minTopPassList=5
|
||||||
|
|
||||||
[Modules_Duplicates]
|
[Modules_Duplicates]
|
||||||
#Number of month to look back
|
#Number of month to look back
|
||||||
maximum_month_range = 3
|
maximum_month_range = 3
|
||||||
|
@ -45,8 +57,8 @@ min_paste_size = 0.3
|
||||||
threshold_stucked_module=600
|
threshold_stucked_module=600
|
||||||
|
|
||||||
[Module_Mixer]
|
[Module_Mixer]
|
||||||
#Define the configuration of the mixer, possible value: 1 or 2
|
#Define the configuration of the mixer, possible value: 1, 2 or 3
|
||||||
operation_mode = 1
|
operation_mode = 3
|
||||||
#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
|
#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
|
||||||
ttl_duplicate = 86400
|
ttl_duplicate = 86400
|
||||||
|
|
||||||
|
@ -139,7 +151,7 @@ maxDuplicateToPushToMISP=10
|
||||||
# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
|
# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
|
||||||
[ZMQ_Global]
|
[ZMQ_Global]
|
||||||
#address = tcp://crf.circl.lu:5556
|
#address = tcp://crf.circl.lu:5556
|
||||||
address = tcp://127.0.0.1:5556
|
address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
|
||||||
channel = 102
|
channel = 102
|
||||||
bind = tcp://127.0.0.1:5556
|
bind = tcp://127.0.0.1:5556
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue