Merge pull request #335 from 5thColumn/docker

Add docker-compose support
This commit is contained in:
Alexandre Dulaunoy 2019-03-13 21:38:12 +01:00 committed by GitHub
commit 85d2ea578c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 788 additions and 0 deletions

3
.gitignore vendored
View file

@ -36,6 +36,9 @@ bin/packages/config.cfg.backup
configs/keys
files
# Pystemon archives
pystemon/archives
# installed files
nltk_data/
doc/all_modules.txt

View file

@ -27,8 +27,18 @@ WORKDIR /opt/AIL
# Default to UTF-8 file.encoding
ENV LANG C.UTF-8
ENV AIL_HOME /opt/AIL
ENV AIL_BIN ${AIL_HOME}/bin
ENV AIL_FLASK ${AIL_HOME}/var/www
ENV AIL_REDIS ${AIL_HOME}/redis/src
ENV AIL_ARDB ${AIL_HOME}/ardb/src
ENV AIL_VENV ${AIL_HOME}/AILENV
ENV PATH ${AIL_VENV}/bin:${AIL_HOME}:${AIL_REDIS}:${AIL_ARDB}:${AIL_BIN}:${AIL_FLASK}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
RUN ./pystemon/install.sh
RUN pip install -r /opt/pystemon/requirements.txt
RUN pip install -r /opt/AIL/crawler_requirements.txt
COPY docker_start.sh /docker_start.sh
ENTRYPOINT ["/bin/bash", "docker_start.sh"]

View file

@ -0,0 +1,253 @@
[Directories]
bloomfilters = Blooms
dicofilters = Dicos
pastes = PASTES
hash = HASHS
crawled = crawled
crawled_screenshot = CRAWLED_SCREENSHOT
wordtrending_csv = var/www/static/csv/wordstrendingdata
wordsfile = files/wordfile
protocolstrending_csv = var/www/static/csv/protocolstrendingdata
protocolsfile = files/protocolsfile
tldstrending_csv = var/www/static/csv/tldstrendingdata
tldsfile = faup/src/data/mozilla.tlds
domainstrending_csv = var/www/static/csv/domainstrendingdata
pystemonpath = /opt/pystemon/
sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt
##### Notifications ######
[Notifications]
ail_domain = http://localhost:7000
sender = sender@example.com
sender_host = smtp.example.com
sender_port = 1337
sender_pw = None
# optional for using with authenticated SMTP over SSL
# sender_pw = securepassword
##### Flask #####
[Flask]
#Proxying requests to the app
baseUrl = /
#Number of logs to display in the dashboard
max_dashboard_logs = 15
#Maximum number of character to display in the toolip
max_preview_char = 250
#Maximum number of character to display in the modal
max_preview_modal = 800
#Default number of header to display in trending graphs
default_display = 10
#Number of minutes displayed for the number of processed pastes.
minute_processed_paste = 10
#Maximum line length authorized to make a diff between duplicates
DiffMaxLineLength = 10000
#### Modules ####
[BankAccount]
max_execution_time = 60
[Categ]
#Minimum number of match between the paste and the category file
matchingThreshold=1
[Credential]
#Minimum length that a credential must have to be considered as such
minimumLengthThreshold=3
#Will be pushed as alert if the number of credentials is greater to that number
criticalNumberToAlert=8
#Will be considered as false positive if less that X matches from the top password list
minTopPassList=5
[Curve]
max_execution_time = 90
[Onion]
max_execution_time = 180
[Base64]
path = Base64/
max_execution_time = 60
[Binary]
path = Base64/
max_execution_time = 60
[Hex]
path = Base64/
max_execution_time = 60
[Modules_Duplicates]
#Number of month to look back
maximum_month_range = 3
#The value where two pastes are considerate duplicate for ssdeep.
threshold_duplicate_ssdeep = 50
#The value where two pastes are considerate duplicate for tlsh.
threshold_duplicate_tlsh = 52
#Minimum size of the paste considered
min_paste_size = 0.3
[Module_ModuleInformation]
#Threshold to deduce if a module is stuck or not, in seconds.
threshold_stucked_module=600
[Module_Mixer]
#Define the configuration of the mixer, possible value: 1, 2 or 3
operation_mode = 3
#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
ttl_duplicate = 86400
default_unnamed_feed_name = unnamed_feeder
[RegexForTermsFrequency]
max_execution_time = 60
##### Redis #####
[Redis_Cache]
host = localhost
port = 6379
db = 0
[Redis_Log]
host = localhost
port = 6380
db = 0
[Redis_Log_submit]
host = localhost
port = 6380
db = 1
[Redis_Queues]
host = localhost
port = 6381
db = 0
[Redis_Data_Merging]
host = localhost
port = 6379
db = 1
[Redis_Paste_Name]
host = localhost
port = 6379
db = 2
[Redis_Mixer_Cache]
host = localhost
port = 6381
db = 1
##### ARDB #####
[ARDB_Curve]
host = localhost
port = 6382
db = 1
[ARDB_Sentiment]
host = localhost
port = 6382
db = 4
[ARDB_TermFreq]
host = localhost
port = 6382
db = 2
[ARDB_TermCred]
host = localhost
port = 6382
db = 5
[ARDB_DB]
host = localhost
port = 6382
db = 0
[ARDB_Trending]
host = localhost
port = 6382
db = 3
[ARDB_Hashs]
host = localhost
db = 1
[ARDB_Tags]
host = localhost
port = 6382
db = 6
[ARDB_Metadata]
host = localhost
port = 6382
db = 7
[ARDB_Statistics]
host = localhost
port = 6382
db = 8
[ARDB_Onion]
host = localhost
port = 6382
db = 9
[Url]
cc_critical = DE
[DomClassifier]
cc = DE
cc_tld = r'\.de$'
dns = 8.8.8.8
[Mail]
dns = 8.8.8.8
[Web]
dns = 149.13.33.69
# Indexer configuration
[Indexer]
type = whoosh
path = indexdir
register = indexdir/all_index.txt
#size in Mb
index_max_size = 2000
[ailleakObject]
maxDuplicateToPushToMISP=10
###############################################################################
# For multiple feed, add them with "," without space
# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
[ZMQ_Global]
#address = tcp://crf.circl.lu:5556
address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
channel = 102
bind = tcp://127.0.0.1:5556
[ZMQ_Url]
address = tcp://127.0.0.1:5004
channel = urls
[ZMQ_FetchedOnion]
address = tcp://127.0.0.1:5005
channel = FetchedOnion
[RedisPubSub]
host = localhost
port = 6381
db = 0
[Crawler]
activate_crawler = False
crawler_depth_limit = 1
splash_url_onion = http://172.17.0.1
splash_onion_port = 8050

521
docker-compose.yml Normal file
View file

@ -0,0 +1,521 @@
version: '3'
services:
ardb:
entrypoint:
- ardb-server
- /opt/AIL/configs/6382.conf
healthcheck:
test: ["CMD", "redis-cli", "-p", "6382", "ping"]
interval: 30s
timeout: 10s
retries: 5
network_mode: service:flask
image: ail-framework
volumes:
- ./configs:/opt/AIL/configs:ro
crawler:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Crawler.py
- onion
- "8050"
network_mode: service:flask
image: ail-framework
volumes:
- ./CRAWLED_SCREENSHOT/:/opt/AIL/CRAWLED_SCREENSHOT
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
flask:
build: .
entrypoint:
- /opt/AIL/var/www/Flask_server.py
ports:
- "7000:7000"
image: ail-framework
volumes:
- ./CRAWLED_SCREENSHOT/:/opt/AIL/CRAWLED_SCREENSHOT
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/var/www
log-queue:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/AILENV/bin/log_subscriber
- -p
- "6380"
- -c
- Queing
- -l
- /opt/AIL/logs/
network_mode: service:flask
image: ail-framework
volumes:
- ./configs:/opt/AIL/configs:ro
log-script:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/AILENV/bin/log_subscriber
- -p
- "6380"
- -c
- Script
- -l
- /opt/AIL/logs/
network_mode: service:flask
image: ail-framework
volumes:
- ./configs:/opt/AIL/configs:ro
pystemon:
depends_on:
- redis-log
entrypoint:
- /opt/pystemon/pystemon.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./pystemon/archives:/opt/pystemon/archive
- ./pystemon/proxies.txt:/opt/pystemon/proxies.txt:ro
- ./pystemon/pystemon.yaml:/opt/pystemon/pystemon.yaml:ro
working_dir: /opt/pystemon
pystemon-feeder:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/feeder/pystemon-feeder.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./pystemon/archives:/opt/pystemon/archive
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
queues:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/launch_queues.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
redis-cache:
entrypoint:
- redis-server
- /opt/AIL/configs/6379.conf
healthcheck:
test: ["CMD", "redis-cli", "-p", "6379", "ping"]
interval: 30s
timeout: 10s
retries: 5
image: ail-framework
network_mode: service:flask
volumes:
- ./configs:/opt/AIL/configs:ro
redis-log:
entrypoint:
- redis-server
- /opt/AIL/configs/6380.conf
healthcheck:
test: ["CMD", "redis-cli", "-p", "6380", "ping"]
interval: 30s
timeout: 10s
retries: 5
network_mode: service:flask
image: ail-framework
volumes:
- ./configs:/opt/AIL/configs:ro
redis-mixer-cache:
entrypoint:
- redis-server
- /opt/AIL/configs/6381.conf
healthcheck:
test: ["CMD", "redis-cli", "-p", "6381", "ping"]
interval: 30s
timeout: 10s
retries: 5
image: ail-framework
network_mode: service:flask
volumes:
- ./configs:/opt/AIL/configs:ro
script-alerthandler:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/alertHandler.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-apikey:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/ApiKey.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-bankaccount:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/BankAccount.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-bitcoin:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Bitcoin.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-categ:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Categ.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-credential:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Credential.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-creditcards:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/CreditCards.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-curve:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Curve.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-curvemanagetopsets:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/CurveManageTopSets.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-cve:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Cve.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-decoder:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Decoder.py
network_mode: service:flask
image: ail-framework
volumes:
- ./HASHS:/opt/AIL/HASHS
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-domclassifier:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/DomClassifier.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-duplicates:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Duplicates.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-global:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Global.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-indexer:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Indexer.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-keys:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Keys.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-liblinjection:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/LibInjection.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-lines:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Lines.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-mail:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Mail.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-mispthehivefeeder:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/MISP_The_Hive_feeder.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-mixer:
depends_on:
- redis-mixer-cache
entrypoint:
- /opt/AIL/bin/Mixer.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-modulestats:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/ModuleStats.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-onion:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Onion.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-phone:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Phone.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-regexfortermsfrequency:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/RegexForTermsFrequency.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-release:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Release.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-sentimentanalysis:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/SentimentAnalysis.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-setfortermsfrequency:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/SetForTermsFrequency.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-sqlinjectiondetection:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/SQLInjectionDetection.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-submitpaste:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/submit_paste.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-tags:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Tags.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-tokenize:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Tokenize.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-web:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/Web.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin
script-webstats:
depends_on:
- redis-log
entrypoint:
- /opt/AIL/bin/WebStats.py
network_mode: service:flask
image: ail-framework
volumes:
- ./PASTES/:/opt/AIL/PASTES
- ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
working_dir: /opt/AIL/bin

1
pystemon/proxies.txt Normal file
View file

@ -0,0 +1 @@
http://127.0.0.1:8080