AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-23 14:37:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' of https://github.com/CIRCL/AIL-framework

Browse source
This commit is contained in:
Terrtia 2018-09-17 15:42:48 +02:00
commit e25d517d06
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
19 changed files with 803 additions and 120 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
.gitignore vendored
View file

@ -31,6 +31,7 @@ var/www/submitted
# Local config
bin/packages/config.cfg
bin/packages/config.cfg.backup
configs/keys
# installed files
@ -39,3 +40,4 @@ doc/all_modules.txt
# auto generated
doc/module-data-flow.png
doc/data-flow.png
doc/statistics

5
Dockerfile
View file

@ -23,4 +23,7 @@ WORKDIR /opt/AIL
# Default to UTF-8 file.encoding
ENV LANG C.UTF-8
CMD bash docker_start.sh
RUN ./pystemon/install.sh
COPY docker_start.sh /docker_start.sh
ENTRYPOINT ["/bin/bash", "docker_start.sh"]

10
OVERVIEW.md
View file

@ -23,6 +23,15 @@ Redis and ARDB overview
ARDB overview
---------------------------
ARDB_DB
* DB 1 - Curve
* DB 2 - TermFreq
* DB 3 - Trending
* DB 4 - Sentiment
* DB 5 - TermCred
* DB 6 - Tags
* DB 7 - Metadata
* DB 8 - Statistics
* DB 7 - Metadata:
----------------------------------------- BASE64 ----------------------------------------
@ -62,4 +71,3 @@ ARDB overview
GET - 'base64_decoded:'+date nd_decoded
GET - 'binary_decoded:'+date nd_decoded

11
bin/BankAccount.py
View file

@ -11,6 +11,8 @@ It apply IBAN regexes on paste content and warn if above a threshold.
import redis
import time
import redis
import datetime
import re
import string
from itertools import chain
@ -54,11 +56,13 @@ def check_all_iban(l_iban, paste, filename):
iban = ''.join(e for e in iban if e.isalnum())
#iban = iban.upper()
res = iban_regex_verify.findall(iban)
date = datetime.datetime.now().strftime("%Y%m")
if res:
print('checking '+iban)
if is_valid_iban(iban):
print('------')
nb_valid_iban = nb_valid_iban + 1
server_statistics.hincrby('iban_by_country:'+date, iban[0:2], 1)
if(nb_valid_iban > 0):
to_print = 'Iban;{};{};{};'.format(paste.p_source, paste.p_date, paste.p_name)
@ -79,6 +83,13 @@ if __name__ == "__main__":
p = Process(config_section)
max_execution_time = p.config.getint("BankAccount", "max_execution_time")
# ARDB #
server_statistics = redis.StrictRedis(
host=p.config.get("ARDB_Statistics", "host"),
port=p.config.getint("ARDB_Statistics", "port"),
db=p.config.getint("ARDB_Statistics", "db"),
decode_responses=True)
publisher.info("BankAccount started")
message = p.get_from_set()

15
bin/Credential.py
View file

@ -28,6 +28,7 @@ import sys
from packages import Paste
from pubsublogger import publisher
from Helper import Process
import datetime
import re
import redis
from pyfaup.faup import Faup
@ -58,6 +59,12 @@ if __name__ == "__main__":
db=p.config.get("ARDB_TermCred", "db"),
decode_responses=True)
server_statistics = redis.StrictRedis(
host=p.config.get("ARDB_Statistics", "host"),
port=p.config.getint("ARDB_Statistics", "port"),
db=p.config.getint("ARDB_Statistics", "db"),
decode_responses=True)
criticalNumberToAlert = p.config.getint("Credential", "criticalNumberToAlert")
minTopPassList = p.config.getint("Credential", "minTopPassList")
@ -65,6 +72,7 @@ if __name__ == "__main__":
#regex_cred = "[a-zA-Z0-9._-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}:[a-zA-Z0-9\_\-]+"
regex_cred = "[a-zA-Z0-9\\._-]+@[a-zA-Z0-9\\.-]+\.[a-zA-Z]{2,6}[\\rn :\_\-]{1,10}[a-zA-Z0-9\_\-]+"
regex_site_for_stats = "@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,6}:"
while True:
message = p.get_from_set()
if message is None:
@ -132,6 +140,13 @@ if __name__ == "__main__":
if sites_set:
print("=======> Probably on : {}".format(', '.join(sites_set)))
date = datetime.datetime.now().strftime("%Y%m")
for cred in creds:
maildomains = re.findall("@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,20}", cred.lower())[0]
faup.decode(maildomains)
tld = faup.get()['tld']
server_statistics.hincrby('credential_by_tld:'+date, tld, 1)
else:
publisher.info(to_print)
print('found {} credentials'.format(len(creds)))

33
bin/Helper.py
View file

@ -111,7 +111,7 @@ class PubSub(object):
class Process(object):
def __init__(self, conf_section):
def __init__(self, conf_section, module=True):
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
@ -125,23 +125,24 @@ class Process(object):
self.subscriber_name = conf_section
self.pubsub = None
if self.modules.has_section(conf_section):
self.pubsub = PubSub()
else:
raise Exception('Your process has to listen to at least one feed.')
self.r_temp = redis.StrictRedis(
host=self.config.get('RedisPubSub', 'host'),
port=self.config.get('RedisPubSub', 'port'),
db=self.config.get('RedisPubSub', 'db'),
decode_responses=True)
if module:
if self.modules.has_section(conf_section):
self.pubsub = PubSub()
else:
raise Exception('Your process has to listen to at least one feed.')
self.r_temp = redis.StrictRedis(
host=self.config.get('RedisPubSub', 'host'),
port=self.config.get('RedisPubSub', 'port'),
db=self.config.get('RedisPubSub', 'db'),
decode_responses=True)
self.serv_statistics = redis.StrictRedis(
host=self.config.get('ARDB_Statistics', 'host'),
port=self.config.get('ARDB_Statistics', 'port'),
db=self.config.get('ARDB_Statistics', 'db'),
decode_responses=True)
self.serv_statistics = redis.StrictRedis(
host=self.config.get('ARDB_Statistics', 'host'),
port=self.config.get('ARDB_Statistics', 'port'),
db=self.config.get('ARDB_Statistics', 'db'),
decode_responses=True)
self.moduleNum = os.getpid()
self.moduleNum = os.getpid()
def populate_set_in(self):
# monoproc

19
bin/LAUNCH.sh
View file

@ -27,6 +27,7 @@ islogged=`screen -ls | egrep '[0-9]+.Logging_AIL' | cut -d. -f1`
isqueued=`screen -ls | egrep '[0-9]+.Queue_AIL' | cut -d. -f1`
isscripted=`screen -ls | egrep '[0-9]+.Script_AIL' | cut -d. -f1`
isflasked=`screen -ls | egrep '[0-9]+.Flask_AIL' | cut -d. -f1`
isfeeded=`screen -ls | egrep '[0-9]+.Feeder' | cut -d. -f1`
function helptext {
echo -e $YELLOW"
@ -168,7 +169,7 @@ function launching_scripts {
sleep 0.1
screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Keys.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ./Decoder.py; read x'
screen -S "Script_AIL" -X screen -t "Base64" bash -c 'cd '${AIL_BIN}'; ./Base64.py; read x'
sleep 0.1
screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ./Bitcoin.py; read x'
sleep 0.1
@ -317,6 +318,19 @@ function launch_flask {
fi
}
function launch_feeder {
if [[ ! $isfeeded ]]; then
screen -dmS "Feeder"
sleep 0.1
echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT
screen -S "Feeder" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ./feeder/pystemon-feeder.py; read x'
sleep 0.1
screen -S "Feeder" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; python2 pystemon.py; read x'
else
echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT
fi
}
function killall {
if [[ $isredis || $isardb || $islogged || $isqueued || $isscripted || $isflasked ]]; then
echo -e $GREEN"Gracefully closing redis servers"$DEFAULT
@ -325,7 +339,7 @@ function killall {
echo -e $GREEN"Gracefully closing ardb servers"$DEFAULT
shutting_down_ardb;
echo -e $GREEN"Killing all"$DEFAULT
kill $isredis $isardb $islogged $isqueued $isscripted $isflasked
kill $isredis $isardb $islogged $isqueued $isscripted $isflasked $isfeeded
sleep 0.2
echo -e $ROSE`screen -ls`$DEFAULT
echo -e $GREEN"\t* $isredis $isardb $islogged $isqueued $isscripted killed."$DEFAULT
@ -357,6 +371,7 @@ function launch_all {
launch_queues;
launch_scripts $1;
launch_flask;
launch_feeder;
}
#If no params, display the menu

14
bin/LibInjection.py
View file

@ -12,6 +12,8 @@ It tries to identify SQL Injections with libinjection.
"""
import time
import datetime
import redis
import string
import urllib.request
import re
@ -54,6 +56,12 @@ def analyse(url, path):
msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
p.populate_set_out(msg, 'Tags')
#statistics
tld = url_parsed['tld']
if tld is not None:
date = datetime.datetime.now().strftime("%Y%m")
server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1)
if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
# Port of the redis instance used by pubsublogger
@ -70,6 +78,12 @@ if __name__ == '__main__':
# Sent to the logging a description of the module
publisher.info("Try to detect SQL injection with LibInjection")
server_statistics = redis.StrictRedis(
host=p.config.get("ARDB_Statistics", "host"),
port=p.config.getint("ARDB_Statistics", "port"),
db=p.config.getint("ARDB_Statistics", "db"),
decode_responses=True)
faup = Faup()
# Endless loop getting messages from the input queue

31
bin/Mail.py
View file

@ -2,7 +2,7 @@
# -*-coding:UTF-8 -*
"""
The CreditCards Module
The Mail Module
======================
This module is consuming the Redis-list created by the Categ module.
@ -12,13 +12,15 @@ It apply mail regexes on paste content and warn if above a threshold.
"""
import redis
import pprint
import time
import datetime
import dns.exception
from packages import Paste
from packages import lib_refine
from pubsublogger import publisher
from pyfaup.faup import Faup
from Helper import Process
if __name__ == "__main__":
@ -27,6 +29,8 @@ if __name__ == "__main__":
config_section = 'Mail'
faup = Faup()
p = Process(config_section)
addr_dns = p.config.get("Mail", "dns")
@ -36,6 +40,12 @@ if __name__ == "__main__":
port=p.config.getint("Redis_Cache", "port"),
db=p.config.getint("Redis_Cache", "db"),
decode_responses=True)
# ARDB #
server_statistics = redis.StrictRedis(
host=p.config.get("ARDB_Statistics", "host"),
port=p.config.getint("ARDB_Statistics", "port"),
db=p.config.getint("ARDB_Statistics", "db"),
decode_responses=True)
# FUNCTIONS #
publisher.info("Suscribed to channel mails_categ")
@ -66,7 +76,6 @@ if __name__ == "__main__":
PST.save_attribute_redis(channel, (MX_values[0],
list(MX_values[1])))
pprint.pprint(MX_values)
to_print = 'Mails;{};{};{};Checked {} e-mail(s);{}'.\
format(PST.p_source, PST.p_date, PST.p_name,
MX_values[0], PST.p_path)
@ -79,12 +88,22 @@ if __name__ == "__main__":
msg = 'infoleak:automatic-detection="mail";{}'.format(filename)
p.populate_set_out(msg, 'Tags')
#create country statistics
date = datetime.datetime.now().strftime("%Y%m")
for mail in MX_values[1]:
print('mail;{};{};{}'.format(MX_values[1][mail], mail, PST.p_date))
p.populate_set_out('mail;{};{};{}'.format(MX_values[1][mail], mail, PST.p_date), 'ModuleStats')
faup.decode(mail)
tld = faup.get()['tld']
server_statistics.hincrby('mail_by_tld:'+date, tld, MX_values[1][mail])
else:
publisher.info(to_print)
#Send to ModuleStats
#create country statistics
for mail in MX_values[1]:
print('mail;{};{};{}'.format(1, mail, PST.p_date))
p.populate_set_out('mail;{};{};{}'.format(1, mail, PST.p_date), 'ModuleStats')
print('mail;{};{};{}'.format(MX_values[1][mail], mail, PST.p_date))
p.populate_set_out('mail;{};{};{}'.format(MX_values[1][mail], mail, PST.p_date), 'ModuleStats')
prec_filename = filename

15
bin/SQLInjectionDetection.py
View file

@ -12,6 +12,8 @@ It test different possibility to makes some sqlInjection.
"""
import time
import datetime
import redis
import string
import urllib.request
import re
@ -85,6 +87,13 @@ def analyse(url, path):
msg = 'infoleak:automatic-detection="sql-injection";{}'.format(path)
p.populate_set_out(msg, 'Tags')
#statistics
tld = url_parsed['tld']
if tld is not None:
date = datetime.datetime.now().strftime("%Y%m")
server_statistics.hincrby('SQLInjection_by_tld:'+date, tld, 1)
else:
print("Potential SQL injection:")
print(urllib.request.unquote(url))
@ -143,6 +152,12 @@ if __name__ == '__main__':
# Sent to the logging a description of the module
publisher.info("Try to detect SQL injection")
server_statistics = redis.StrictRedis(
host=p.config.get("ARDB_Statistics", "host"),
port=p.config.getint("ARDB_Statistics", "port"),
db=p.config.getint("ARDB_Statistics", "db"),
decode_responses=True)
faup = Faup()
# Endless loop getting messages from the input queue

21
bin/packages/lib_refine.py
View file

@ -38,6 +38,7 @@ def checking_MX_record(r_serv, adress_set, addr_dns):
score = 0
num = len(adress_set)
WalidMX = set([])
validMX = {}
# Transforming the set into a string
MXdomains = re.findall("@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,20}", str(adress_set).lower())
resolver = dns.resolver.Resolver()
@ -46,20 +47,23 @@ def checking_MX_record(r_serv, adress_set, addr_dns):
resolver.lifetime = 2
if MXdomains != []:
for MXdomain in set(MXdomains):
for MXdomain in MXdomains:
try:
MXdomain = MXdomain[1:]
# Already in Redis living.
if r_serv.exists(MXdomain[1:]):
if r_serv.exists(MXdomain):
score += 1
WalidMX.add(MXdomain[1:])
WalidMX.add(MXdomain)
validMX[MXdomain] = validMX.get(MXdomain, 0) + 1
# Not already in Redis
else:
# If I'm Walid MX domain
if resolver.query(MXdomain[1:], rdtype=dns.rdatatype.MX):
if resolver.query(MXdomain, rdtype=dns.rdatatype.MX):
# Gonna be added in redis.
r_serv.setex(MXdomain[1:], 1, timedelta(days=1))
r_serv.setex(MXdomain, 1, timedelta(days=1))
score += 1
WalidMX.add(MXdomain[1:])
WalidMX.add(MXdomain)
validMX[MXdomain] = validMX.get(MXdomain, 0) + 1
else:
pass
@ -86,13 +90,14 @@ def checking_MX_record(r_serv, adress_set, addr_dns):
except dns.resolver.Timeout:
print('timeout')
r_serv.setex(MXdomain[1:], 1, timedelta(days=1))
r_serv.setex(MXdomain, 1, timedelta(days=1))
except Exception as e:
print(e)
publisher.debug("emails before: {0} after: {1} (valid)".format(num, score))
return (num, WalidMX)
#return (num, WalidMX)
return (num, validMX)
def checking_A_record(r_serv, domains_set):

BIN
doc/presentation/ail-training-luxembourg-2018.pdf Normal file
View file

Binary file not shown.

184
doc/statistics/create_graph_by_tld.py Executable file
View file

@ -0,0 +1,184 @@
#!/usr/bin/env python3
# -*-coding:UTF-8 -*
'''
Create statistics pie charts by tld
Default tld: lu
'''
import matplotlib
matplotlib.use('Agg')
import os
import sys
import redis
import argparse
import datetime
import heapq
import operator
import matplotlib.pyplot as plt
import numpy as np
sys.path.append(os.environ['AIL_BIN'])
from Helper import Process
def create_pie_chart(country ,db_key, date, pie_title, path, save_name):
monthly_credential_by_tld = server_statistics.hkeys(db_key + date)
l_tld = []
for tld in monthly_credential_by_tld:
nb_tld = server_statistics.hget(db_key + date, tld)
if nb_tld is not None:
nb_tld = int(nb_tld)
else:
nb_tld = 0
l_tld.append( (tld, nb_tld) )
mail_tld_top5 = heapq.nlargest(5, l_tld, key=operator.itemgetter(1))
# Pie chart, where the slices will be ordered and plotted counter-clockwise:
labels = []
sizes = []
explode = [] # only "explode" the 2nd slice (i.e. 'Hogs')
explode_value = 0
for tld in mail_tld_top5:
labels.append(tld[0] +' ('+str(tld[1])+')')
sizes.append(tld[1])
explode.append(explode_value)
explode_value = explode_value +0.1
nb_tld = server_statistics.hget(db_key + date, country)
if nb_tld is not None:
nb_tld = int(nb_tld)
else:
nb_tld = 0
country_label = country + ' ('+str(nb_tld)+')'
if country_label not in labels:
labels.append(country_label)
sizes.append(nb_tld)
explode.append(explode_value)
explode = tuple(explode)
fig1, ax1 = plt.subplots()
ax1.pie(sizes, explode=explode, labels=labels, autopct='%1.1f%%',
shadow=True, startangle=90)
ax1.axis('equal') # Equal aspect ratio ensures that pie is drawn as a circle.
ax1.set_title(pie_title)
#plt.show()
plt.savefig(os.path.join(path,save_name))
plt.close(fig1)
def create_donut_chart(db_key, date, pie_title, path, save_name):
monthly_credential_by_tld = server_statistics.hkeys(db_key + date)
print()
l_tld = []
for tld in monthly_credential_by_tld:
nb_tld = server_statistics.hget(db_key + date, tld)
if nb_tld is not None:
nb_tld = int(nb_tld)
else:
nb_tld = 0
l_tld.append( (tld, nb_tld) )
mail_tld_top5 = heapq.nlargest(5, l_tld, key=operator.itemgetter(1))
# Pie chart, where the slices will be ordered and plotted counter-clockwise:
recipe = []
data = []
for tld in mail_tld_top5:
recipe.append(tld[0])
data.append(tld[1])
nb_tld = server_statistics.hget(db_key + date, country)
if nb_tld is not None:
nb_tld = int(nb_tld)
else:
nb_tld = 0
if country not in recipe:
recipe.append(country)
data.append(nb_tld)
fig1, ax1 = plt.subplots(figsize=(6, 3), subplot_kw=dict(aspect="equal"))
wedges, texts = ax1.pie(data, wedgeprops=dict(width=0.5), startangle=-40)
bbox_props = dict(boxstyle="square,pad=0.3", fc="w", ec="k", lw=0.72)
kw = dict(xycoords='data', textcoords='data', arrowprops=dict(arrowstyle="-"),
bbox=bbox_props, zorder=0, va="center")
for i, p in enumerate(wedges):
ang = (p.theta2 - p.theta1)/2. + p.theta1
y = np.sin(np.deg2rad(ang))
x = np.cos(np.deg2rad(ang))
horizontalalignment = {-1: "right", 1: "left"}[int(np.sign(x))]
connectionstyle = "angle,angleA=0,angleB={}".format(ang)
kw["arrowprops"].update({"connectionstyle": connectionstyle})
ax1.annotate(recipe[i], xy=(x, y), xytext=(1.35*np.sign(x), 1.4*y),
horizontalalignment=horizontalalignment, **kw)
ax1.set_title(pie_title)
#plt.show()
plt.savefig(os.path.join(path, save_name))
plt.close(fig1)
if __name__ == '__main__':
parser = argparse.ArgumentParser(
description='''This script is a part of the Analysis Information Leak
framework. Create statistics pie charts".''',
epilog='Example: ./create_lu_graph.py 0 lu now, create_lu_graph.py 0 lu 201807')
parser.add_argument('type', type=int, default=0,
help='''The graph type (default 0),
0: all,
1: credential_pie,
2: mail_pie
3: sqlinjection_pie,
4: iban_pie,''',
choices=[0, 1, 2, 3, 4], action='store')
parser.add_argument('country', type=str, default="lu",
help='''The country code, lu:default''',
action='store')
parser.add_argument('date', type=str, default="now",
help='''month %Y%m, example: 201810''', action='store')
args = parser.parse_args()
path = os.path.join(os.environ['AIL_HOME'], 'doc', 'statistics') # save path
config_section = 'ARDB_Statistics'
p = Process(config_section, False)
# ARDB #
server_statistics = redis.StrictRedis(
host=p.config.get("ARDB_Statistics", "host"),
port=p.config.getint("ARDB_Statistics", "port"),
db=p.config.getint("ARDB_Statistics", "db"),
decode_responses=True)
if args.date == 'now' or len(args.date) != 6:
date = datetime.datetime.now().strftime("%Y%m")
else:
date = args.date
if args.type == 0:
create_pie_chart(args.country, 'credential_by_tld:', date, "AIL: Credential leak by tld", path, 'AIL_credential_by_tld.png')
create_pie_chart(args.country, 'mail_by_tld:', date, "AIL: mail leak by tld", path, 'AIL_mail_by_tld.png')
create_pie_chart(args.country, 'SQLInjection_by_tld:', date, "AIL: SQLInjection by tld", path, 'AIL_SQLInjection_by_tld.png')
create_pie_chart(args.country.upper(), 'iban_by_country:', date, "AIL: Iban by country", path, 'AIL_iban_by_country.png')
elif args.type == 1:
create_pie_chart(args.country, 'credential_by_tld:', date, "AIL: Credential leak by tld", path, 'AIL_credential_by_tld.png')
elif args.type == 2:
create_pie_chart(args.country, 'mail_by_tld:', date, "AIL: mail leak by tld", path, 'AIL_mail_by_tld.png')
elif args.type == 3:
create_pie_chart(args.country, 'SQLInjection_by_tld:', date, "AIL: sqlInjection by tld", path, 'AIL_sqlInjectionl_by_tld.png')
elif args.type == 4:
create_pie_chart(args.country.upper(), 'iban_by_country:', date, "AIL: Iban by country", path, 'AIL_iban_by_country.png')

98
docker_start.sh
View file

@ -1,89 +1,19 @@
echo "Currently unmaintained, continue at your own risk of not having a working AIL at the end :("
exit 1
#!/bin/bash
signalListener() {
"$@" &
pid="$!"
trap "echo 'Stopping'; kill -SIGTERM $pid" SIGINT SIGTERM
while kill -0 $pid > /dev/null 2>&1; do
wait
done
}
source ./AILENV/bin/activate
cd bin
./LAUNCH.sh -l
export PATH=$AIL_HOME:$PATH
export PATH=$AIL_REDIS:$PATH
export PATH=$AIL_LEVELDB:$PATH
export PATH=$AIL_ARDB:$PATH
if [ -z $1 ]; then
export AILENV=/opt/AIL
else
export AILENV=$1
fi
signalListener tail -f /dev/null $@
conf_dir="${AIL_HOME}/configs/"
screen -dmS "Redis"
screen -S "Redis" -X screen -t "6379" bash -c 'redis-server '$conf_dir'6379.conf ; read x'
screen -S "Redis" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x'
screen -S "Redis" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x'
# For Words and curves
sleep 0.1
screen -dmS "ARDB_AIL"
screen -S "ARDB_AIL" -X screen -t "6382" bash -c 'ardb-server '$conf_dir'6382.conf ; read x'
#Want to launch more level_db?
lvdbhost='127.0.0.1'
lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
db1_y='2013'
db2_y='2014'
db3_y='2016'
db4_y='2017'
dbC_y='3016'
nb_db=13
screen -dmS "LevelDB"
#Add lines here with appropriates options.
screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2017/ -P '$db4_y' -M '$nb_db'; read x'
# For Curve
screen -S "LevelDB" -X screen -t "3016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'3016/ -P '$dbC_y' -M '$nb_db'; read x'
screen -dmS "Logging"
screen -S "Logging" -X screen -t "LogQueue" bash -c 'log_subscriber -p 6380 -c Queuing -l ../logs/; read x'
screen -S "Logging" -X screen -t "LogScript" bash -c 'log_subscriber -p 6380 -c Script -l ../logs/; read x'
screen -dmS "Queue"
screen -S "Queue" -X screen -t "Queues" bash -c './launch_queues.py; read x'
screen -dmS "Script"
screen -S "Script" -X screen -t "ModuleInformation" bash -c './ModuleInformation.py -k 0 -c 1; read x'
screen -S "Script" -X screen -t "Mixer" bash -c './Mixer.py; read x'
screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
screen -S "Script" -X screen -t "Duplicates" bash -c './Duplicates.py; read x'
screen -S "Script" -X screen -t "Attributes" bash -c './Attributes.py; read x'
screen -S "Script" -X screen -t "Lines" bash -c './Lines.py; read x'
screen -S "Script" -X screen -t "DomClassifier" bash -c './DomClassifier.py; read x'
screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x'
screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x'
screen -S "Script" -X screen -t "CreditCards" bash -c './CreditCards.py; read x'
screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x'
screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x'
screen -S "Script" -X screen -t "Web" bash -c './Web.py; read x'
screen -S "Script" -X screen -t "Credential" bash -c './Credential.py; read x'
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
screen -S "Script" -X screen -t "CurveManageTopSets" bash -c './CurveManageTopSets.py; read x'
screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x'
screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x'
screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
screen -S "Script" -X screen -t "ModuleStats" bash -c './ModuleStats.py; read x'
screen -S "Script" -X screen -t "SQLInjectionDetection" bash -c './SQLInjectionDetection.py; read x'
screen -S "Script" -X screen -t "BrowseWarningPaste" bash -c './BrowseWarningPaste.py; read x'
screen -S "Script" -X screen -t "SentimentAnalysis" bash -c './SentimentAnalysis.py; read x'
cd $AILENV
cd var/www/
python Flask_server.py
./LAUNCH.sh -k

2
installing_deps.sh
View file

@ -5,7 +5,7 @@ set -x
sudo apt-get update
sudo apt-get install python3-pip python-virtualenv python3-dev libfreetype6-dev \
sudo apt-get install python3-pip python-virtualenv python3-dev python3-tk libfreetype6-dev \
screen g++ python-tk unzip libsnappy-dev cmake -y
#optional tor install

3
pip3_packages_requirement.txt
View file

@ -70,3 +70,6 @@ https://github.com/saffsd/langid.py/archive/master.zip
#LibInjection bindings
pylibinjection
# Graph
matplotlib

219
pystemon/config.cfg Normal file
View file

@ -0,0 +1,219 @@
[Directories]
bloomfilters = Blooms
dicofilters = Dicos
pastes = PASTES
base64 = BASE64
wordtrending_csv = var/www/static/csv/wordstrendingdata
wordsfile = files/wordfile
protocolstrending_csv = var/www/static/csv/protocolstrendingdata
protocolsfile = files/protocolsfile
tldstrending_csv = var/www/static/csv/tldstrendingdata
tldsfile = faup/src/data/mozilla.tlds
domainstrending_csv = var/www/static/csv/domainstrendingdata
pystemonpath = /opt/pystemon/
sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt
##### Notifications ######
[Notifications]
ail_domain = http://localhost:7000
sender = sender@example.com
sender_host = smtp.example.com
sender_port = 1337
# optional for using with authenticated SMTP over SSL
# sender_pw = securepassword
##### Flask #####
[Flask]
#Number of logs to display in the dashboard
max_dashboard_logs = 15
#Maximum number of character to display in the toolip
max_preview_char = 250
#Maximum number of character to display in the modal
max_preview_modal = 800
#Default number of header to display in trending graphs
default_display = 10
#Number of minutes displayed for the number of processed pastes.
minute_processed_paste = 10
#Maximum line length authorized to make a diff between duplicates
DiffMaxLineLength = 10000
#### Modules ####
[BankAccount]
max_execution_time = 60
[Categ]
#Minimum number of match between the paste and the category file
matchingThreshold=1
[Credential]
#Minimum length that a credential must have to be considered as such
minimumLengthThreshold=3
#Will be pushed as alert if the number of credentials is greater to that number
criticalNumberToAlert=8
#Will be considered as false positive if less that X matches from the top password list
minTopPassList=5
[Curve]
max_execution_time = 90
[Base64]
path = Base64/
max_execution_time = 60
[Modules_Duplicates]
#Number of month to look back
maximum_month_range = 3
#The value where two pastes are considerate duplicate for ssdeep.
threshold_duplicate_ssdeep = 50
#The value where two pastes are considerate duplicate for tlsh.
threshold_duplicate_tlsh = 52
#Minimum size of the paste considered
min_paste_size = 0.3
[Module_ModuleInformation]
#Threshold to deduce if a module is stuck or not, in seconds.
threshold_stucked_module=600
[Module_Mixer]
#Define the configuration of the mixer, possible value: 1, 2 or 3
operation_mode = 3
#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
ttl_duplicate = 86400
##### Redis #####
[Redis_Cache]
host = localhost
port = 6379
db = 0
[Redis_Log]
host = localhost
port = 6380
db = 0
[Redis_Log_submit]
host = localhost
port = 6380
db = 1
[Redis_Queues]
host = localhost
port = 6381
db = 0
[Redis_Data_Merging]
host = localhost
port = 6379
db = 1
[Redis_Paste_Name]
host = localhost
port = 6379
db = 2
[Redis_Mixer_Cache]
host = localhost
port = 6381
db = 1
##### ARDB #####
[ARDB_Curve]
host = localhost
port = 6382
db = 1
[ARDB_Sentiment]
host = localhost
port = 6382
db = 4
[ARDB_TermFreq]
host = localhost
port = 6382
db = 2
[ARDB_TermCred]
host = localhost
port = 6382
db = 5
[ARDB_DB]
host = localhost
port = 6382
db = 0
[ARDB_Trending]
host = localhost
port = 6382
db = 3
[ARDB_Hashs]
host = localhost
db = 1
[ARDB_Tags]
host = localhost
port = 6382
db = 6
[ARDB_Metadata]
host = localhost
port = 6382
db = 7
[ARDB_Statistics]
host = localhost
port = 6382
db = 8
[Url]
cc_critical = DE
[DomClassifier]
cc = DE
cc_tld = r'\.de$'
dns = 8.8.8.8
[Mail]
dns = 8.8.8.8
# Indexer configuration
[Indexer]
type = whoosh
path = indexdir
register = indexdir/all_index.txt
#size in Mb
index_max_size = 2000
[ailleakObject]
maxDuplicateToPushToMISP=10
###############################################################################
# For multiple feed, add them with "," without space
# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
[ZMQ_Global]
#address = tcp://crf.circl.lu:5556
address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
channel = 102
bind = tcp://127.0.0.1:5556
[ZMQ_Url]
address = tcp://127.0.0.1:5004
channel = urls
[ZMQ_FetchedOnion]
address = tcp://127.0.0.1:5005
channel = FetchedOnion
[RedisPubSub]
host = localhost
port = 6381
db = 0

9
pystemon/install.sh Executable file
View file

@ -0,0 +1,9 @@
git clone https://github.com/cvandeplas/pystemon.git /opt/pystemon
apt-get install -y python-pip python-requests python-yaml python-redis
pip install beautifulsoup4
BASEDIR=$(dirname "$0")
cp $BASEDIR/config.cfg /opt/AIL/bin/packages/
cp $BASEDIR/pystemon.yaml /opt/pystemon/

230
pystemon/pystemon.yaml Normal file
View file

@ -0,0 +1,230 @@
#network: # Network settings
# ip: '1.1.1.1' # Specify source IP address if you want to bind on a specific one
archive:
save: yes # Keep a copy of pasties that triggered alerts
save-all: yes # Keep a copy of all pasties
dir: "alerts" # Directory where matching pasties should be kept
dir-all: "archive" # Directory where all pasties should be kept (if save-all is set to yes)
compress: yes # Store the pasties compressed
engine: re # Only re (default) or regex (pip install regex) are supported.
strict_regex: no # when compiling regex, hard fail or not on error
save-thread: no # Use a separate thread to save pasties
db:
sqlite3: # Store information about the pastie in a database
enable: no # Activate this DB engine # NOT FULLY IMPLEMENTED
file: 'db.sqlite3' # The filename of the database
lookup: no # lookup sqlite for already seen pasties
mongo:
save: no # Keep a copy of pasties that triggered alerts
save-all: no # Keep a copy of all pasties
save-profile: # configure which data to save
content-on-miss: no # save the content even on miss
timestamp: no # add the timestamp (UTC)
url: no # add the public URL
site: no # add the site
id: no # add the per-site id
matched: no # add the matched status (usefull if content-on-miss = yes)
filename: no # add the local filename (to no store content in mongodb)
lookup: no # lookup mongodb for already seen pasties
database: "paste"
collection: "paste"
url: "mongodb://localhost"
user:
password:
redis:
queue: yes # Keep a copy of pasties that triggered alerts
queue-all: yes # Keep a copy of all pasties
server: "localhost"
port: 6379
database: 10
lookup: no # lookup redisdb for already seen pasties (NOT IMPLEMENTED)
email:
alert: no # Enable/disable email alerts
from: alert@example.com
to: alert@example.com
server: 127.0.0.1 # Address of the server (hostname or IP)
port: 25 # Outgoing SMTP port: 25, 587, ...
tls: no # Enable/disable tls support
username: '' # (optional) Username for authentication. Leave blank for no authentication.
password: '' # (optional) Password for authentication. Leave blank for no authentication.
subject: '[pystemon] - {subject}'
#####
# Definition of regular expressions to search for in the pasties
#
search:
# - description: '' # (optional) A human readable description used in alerts.
# # If left unspecified the search regular expression
# # will be used as description.
# search: '' # The regular expression to search for.
# count: '' # (optional) How many hits should it have to be interesting?
# exclude: '' # (optional) Do not alert if this regular expression matches
# regex-flags: '' # (optional) Regular expression flags to give to the find function.
# # Default = re.IGNORECASE
# # Set to 0 to have no flags set
# # See http://docs.python.org/2/library/re.html#re.DEBUG for more info.
# # Warning: when setting this the default is overridden
# # example: 're.MULTILINE + re.DOTALL + re.IGNORECASE'
# to: '' # (optional) Additional recipients for email alert, comma separated list
- search: '[^a-zA-Z0-9]example\.com'
- search: '[^a-zA-Z0-9]foobar\.com'
- description: 'Download (non-porn)'
search: 'download'
exclude: 'porn|sex|teen'
count: 4
#####
# Configuration section for the paste sites
#
threads: 1 # number of download threads per site
site:
# example.com:
# archive-url: # the url where the list of last pasties is present
# # example: 'http://pastebin.com/archive'
# archive-regex: # a regular expression to extract the pastie-id from the page.
# # do not forget the () to extract the pastie-id
# # example: '<a href="/(\w{8})">.+</a></td>'
# download-url: # url for the raw pastie.
# # Should contain {id} on the place where the ID of the pastie needs to be placed
# # example: 'http://pastebin.com/raw.php?i={id}'
# public-url: # optional, defaults to be the same as download-url, so it should meet the same requirements
# # is used for display in logging and e-mail notifications
# update-max: 40 # every X seconds check for new updates to see if new pasties are available
# update-min: 30 # a random number will be chosen between these two numbers
# pastie-classname: # OPTIONAL: The name of a custom Class that inherits from Pastie
# # This is practical for sites that require custom fetchPastie() functions
pastebin.com:
archive-url: 'https://pastebin.com/archive'
archive-regex: '<a href="/(\w{8})">.+</a></td>'
download-url: 'https://pastebin.com/raw/{id}'
update-max: 50
update-min: 40
# See https://pastebin.com/api_scraping_faq , you will need a pro account on pastebin
pastebin.com_pro:
archive-url: 'https://scrape.pastebin.com/api_scraping.php?limit=500'
archive-regex: '"key": "(.+)",'
download-url: 'https://scrape.pastebin.com/api_scrape_item.php?i={id}'
public-url: 'https://pastebin.com/raw/{id}'
update-max: 50
update-min: 40
slexy.org:
archive-url: 'https://slexy.org/recent'
archive-regex: '<a href="/view/([a-zA-Z0-9]+)">View paste</a>'
download-url: 'https://slexy.org/view/{id}'
pastie-classname: PastieSlexyOrg
gist.github.com:
archive-url: 'https://gist.github.com/discover'
archive-regex: '<a href="/([A-Za-z0-9]+/[A-Za-z0-9]+)">'
download-url: 'https://gist.githubusercontent.com/{id}/raw/'
codepad.org:
archive-url: 'http://codepad.org/recent'
archive-regex: '<a href="http://codepad.org/([a-zA-Z0-9]+)">view'
download-url: 'http://codepad.org/{id}/raw.txt'
kpaste.net:
archive-url: 'http://kpaste.net/'
archive-regex: '" href="/(\w+)">'
download-url: 'http://kpaste.net/{id}?raw'
ideone.com:
archive-url: 'http://ideone.com/recent'
archive-regex: '<a href="/([a-zA-Z0-9]+)">#'
download-url: 'http://ideone.com/plain/{id}'
pastebin.ru:
archive-url: 'http://pastebin.ru/'
archive-regex: '<a href="/(\w+)">'
download-url: 'http://pastebin.ru/{id}'
pastebin.fr:
archive-url: 'http://pastebin.fr'
archive-regex: '<a href="http://pastebin.fr/(\d+)'
download-url: 'http://pastebin.fr/{id}'
# Sites that are offline:
# pastesite.com:
# pastie-classname: PastiePasteSiteCom
# archive-url: 'http://pastesite.com/recent'
# archive-regex: '<a href="(\d+)" title="View this Paste'
# download-url: 'http://pastesite.com/plain/{id}.txt'
# pastie.org:
# archive-url: 'http://pastie.org/pastes'
# archive-regex: '<a href="http://pastie.org/pastes/(\d+)">'
# download-url: 'http://pastie.org/pastes/{id}/text'
# pastebin.ca:
# archive-url: 'http://pastebin.ca'
# archive-regex: 'rel="/preview.php\?id=(\d+)'
# download-url: 'http://pastebin.ca/{id}'
# nopaste.me:
# archive-url: 'http://nopaste.me/recent'
# archive-regex: '<a href="http://nopaste.me/paste/([a-zA-Z0-9]+)">'
# download-url: 'http://nopaste.me/download/{id}.txt'
# cdv.lt:
# pastie-classname: PastieCdvLt
# archive-url: 'http://cdv.lt/snippets'
# archive-regex: '<a href="/([a-zA-Z0-9]+)">[0-9]'
# download-url: 'http://cdv.lt/api/snippet/{id}'
# snipt.net:
# pastie-classname: PastieSniptNet
# archive-url: 'https://snipt.net/public/?rss'
# archive-regex: '<link>https://snipt.net/(.+)/</link>'
# download-url: 'https://snipt.net/{id}/'
# quickleak.se:
# archive-url: 'http://www.quickleak.se/last-pastes.html'
# archive-regex: '<td><a href="([A-Za-z0-9]+)">'
# download-url: 'http://www.quickleak.se/{id}'
# safebin.net: # FIXME not finished
# archive-url: 'http://safebin.net/?archive'
# archive-regex: '<a title="[a-zA-Z0-9 :,]+" href="/([0-9]+)">'
# download-url: 'http://safebin.net/{id}'
# update-max: 60
# update-min: 50
# TODO
# http://www.safebin.net/ # more complex site
# http://www.heypasteit.com/ # http://www.heypasteit.com/clip/0IZA => incremental
# http://hastebin.com/ # no list of last pastes
# http://sebsauvage.net/paste/ # no list of last pastes
# http://tny.cz/ # no list of last pastes
# https://pastee.org/ # no list of last pastes
# http://paste2.org/ # no list of last pastes
# http://0bin.net/ # no list of last pastes
# http://markable.in/ # no list of last pastes
#####
# Configuration section to configure proxies
# Currently only HTTP proxies are permitted
#
proxy:
random: no
file: 'proxies.txt'
#####
# Configuration section for User-Agents
#
user-agent:
random: no
file: 'user-agents.txt'