Merge branch 'master' into advanced_crawler

.gitignore

@@ -37,6 +37,9 @@ configs/keys
 update/current_version
 files
 
+# Pystemon archives
+pystemon/archives
+
 # installed files
 nltk_data/
 doc/all_modules.txt

Dockerfile

@@ -27,8 +27,18 @@ WORKDIR /opt/AIL
 
 # Default to UTF-8 file.encoding
 ENV LANG C.UTF-8
+ENV AIL_HOME /opt/AIL
+ENV AIL_BIN ${AIL_HOME}/bin
+ENV AIL_FLASK ${AIL_HOME}/var/www
+ENV AIL_REDIS ${AIL_HOME}/redis/src
+ENV AIL_ARDB ${AIL_HOME}/ardb/src
+ENV AIL_VENV ${AIL_HOME}/AILENV
+
+ENV PATH ${AIL_VENV}/bin:${AIL_HOME}:${AIL_REDIS}:${AIL_ARDB}:${AIL_BIN}:${AIL_FLASK}:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
 
 RUN ./pystemon/install.sh
+RUN pip install -r /opt/pystemon/requirements.txt
+RUN pip install -r /opt/AIL/crawler_requirements.txt
 
 COPY docker_start.sh /docker_start.sh
 ENTRYPOINT ["/bin/bash", "docker_start.sh"]

bin/ModulesInformationV2.py

@@ -31,7 +31,7 @@ lastTimeKillCommand = {}
 current_selected_value = 0
 current_selected_queue = ""
 current_selected_action = ""
-current_selected_action = 0
+current_selected_amount = 0
 
 # Map PID to Queue name (For restart and killing)
 PID_NAME_DICO = {}
@@ -480,7 +480,10 @@ class Show_paste(Frame):
                 self.label_list[i]._text = ""
 
         except Exception as e:
+            if current_selected_value in COMPLETE_PASTE_PATH_PER_PID:
                 self.label_list[0]._text = "Error while displaying the paste: " + COMPLETE_PASTE_PATH_PER_PID[current_selected_value]
+            else:
+                self.label_list[0]._text = "Error Generic exception caught"
             self.label_list[1]._text = str(e)
             for i in range(2,self.num_label):
                 self.label_list[i]._text = ""

bin/packages/config.cfg.docker-compose-sample

@@ -0,0 +1,253 @@
+[Directories]
+bloomfilters = Blooms
+dicofilters = Dicos
+pastes = PASTES
+hash = HASHS
+crawled = crawled
+crawled_screenshot = CRAWLED_SCREENSHOT
+
+wordtrending_csv = var/www/static/csv/wordstrendingdata
+wordsfile = files/wordfile
+
+protocolstrending_csv = var/www/static/csv/protocolstrendingdata
+protocolsfile = files/protocolsfile
+
+tldstrending_csv = var/www/static/csv/tldstrendingdata
+tldsfile = faup/src/data/mozilla.tlds
+
+domainstrending_csv = var/www/static/csv/domainstrendingdata
+
+pystemonpath = /opt/pystemon/
+
+sentiment_lexicon_file = sentiment/vader_lexicon.zip/vader_lexicon/vader_lexicon.txt
+
+##### Notifications ######
+[Notifications]
+ail_domain = http://localhost:7000
+sender = sender@example.com
+sender_host = smtp.example.com
+sender_port = 1337
+sender_pw = None
+
+# optional for using with authenticated SMTP over SSL
+# sender_pw = securepassword
+
+##### Flask #####
+[Flask]
+#Proxying requests to the app
+baseUrl = /
+#Number of logs to display in the dashboard
+max_dashboard_logs = 15
+#Maximum number of character to display in the toolip
+max_preview_char = 250
+#Maximum number of character to display in the modal
+max_preview_modal = 800
+#Default number of header to display in trending graphs
+default_display = 10
+#Number of minutes displayed for the number of processed pastes.
+minute_processed_paste = 10
+#Maximum line length authorized to make a diff between duplicates
+DiffMaxLineLength = 10000
+
+#### Modules ####
+[BankAccount]
+max_execution_time = 60
+
+[Categ]
+#Minimum number of match between the paste and the category file
+matchingThreshold=1
+
+[Credential]
+#Minimum length that a credential must have to be considered as such
+minimumLengthThreshold=3
+#Will be pushed as alert if the number of credentials is greater to that number
+criticalNumberToAlert=8
+#Will be considered as false positive if less that X matches from the top password list
+minTopPassList=5
+
+[Curve]
+max_execution_time = 90
+
+[Onion]
+max_execution_time = 180
+
+[Base64]
+path = Base64/
+max_execution_time = 60
+
+[Binary]
+path = Base64/
+max_execution_time = 60
+
+[Hex]
+path = Base64/
+max_execution_time = 60
+
+[Modules_Duplicates]
+#Number of month to look back
+maximum_month_range = 3
+#The value where two pastes are considerate duplicate for ssdeep.
+threshold_duplicate_ssdeep = 50
+#The value where two pastes are considerate duplicate for tlsh.
+threshold_duplicate_tlsh = 52
+#Minimum size of the paste considered
+min_paste_size = 0.3
+
+[Module_ModuleInformation]
+#Threshold to deduce if a module is stuck or not, in seconds.
+threshold_stucked_module=600
+
+[Module_Mixer]
+#Define the configuration of the mixer, possible value: 1, 2 or 3
+operation_mode = 3
+#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
+ttl_duplicate = 86400
+default_unnamed_feed_name = unnamed_feeder
+
+[RegexForTermsFrequency]
+max_execution_time = 60
+
+##### Redis #####
+[Redis_Cache]
+host = localhost
+port = 6379
+db = 0
+
+[Redis_Log]
+host = localhost
+port = 6380
+db = 0
+
+[Redis_Log_submit]
+host = localhost
+port = 6380
+db = 1
+
+[Redis_Queues]
+host = localhost
+port = 6381
+db = 0
+
+[Redis_Data_Merging]
+host = localhost
+port = 6379
+db = 1
+
+[Redis_Paste_Name]
+host = localhost
+port = 6379
+db = 2
+
+[Redis_Mixer_Cache]
+host = localhost
+port = 6381
+db = 1
+
+##### ARDB #####
+[ARDB_Curve]
+host = localhost
+port = 6382
+db = 1
+
+[ARDB_Sentiment]
+host = localhost
+port = 6382
+db = 4
+
+[ARDB_TermFreq]
+host = localhost
+port = 6382
+db = 2
+
+[ARDB_TermCred]
+host = localhost
+port = 6382
+db = 5
+
+[ARDB_DB]
+host = localhost
+port = 6382
+db = 0
+
+[ARDB_Trending]
+host = localhost
+port = 6382
+db = 3
+
+[ARDB_Hashs]
+host = localhost
+db = 1
+
+[ARDB_Tags]
+host = localhost
+port = 6382
+db = 6
+
+[ARDB_Metadata]
+host = localhost
+port = 6382
+db = 7
+
+[ARDB_Statistics]
+host = localhost
+port = 6382
+db = 8
+
+[ARDB_Onion]
+host = localhost
+port = 6382
+db = 9
+
+[Url]
+cc_critical = DE
+
+[DomClassifier]
+cc = DE
+cc_tld = r'\.de$'
+dns = 8.8.8.8
+
+[Mail]
+dns = 8.8.8.8
+
+[Web]
+dns = 149.13.33.69
+
+# Indexer configuration
+[Indexer]
+type = whoosh
+path = indexdir
+register = indexdir/all_index.txt
+#size in Mb
+index_max_size = 2000
+
+[ailleakObject]
+maxDuplicateToPushToMISP=10
+
+###############################################################################
+
+# For multiple feed, add them with "," without space
+# e.g.: tcp://127.0.0.1:5556,tcp://127.0.0.1:5557
+[ZMQ_Global]
+#address = tcp://crf.circl.lu:5556
+address = tcp://127.0.0.1:5556,tcp://crf.circl.lu:5556
+channel = 102
+bind = tcp://127.0.0.1:5556
+
+[ZMQ_Url]
+address = tcp://127.0.0.1:5004
+channel = urls
+
+[ZMQ_FetchedOnion]
+address = tcp://127.0.0.1:5005
+channel = FetchedOnion
+
+[RedisPubSub]
+host = localhost
+port = 6381
+db = 0
+
+[Crawler]
+activate_crawler = False
+crawler_depth_limit = 1
+splash_url_onion = http://172.17.0.1
+splash_onion_port = 8050

docker-compose.yml

@@ -0,0 +1,523 @@
+version: '3'
+services:
+  ardb:
+    entrypoint:
+      - ardb-server
+      - /opt/AIL/configs/6382.conf
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "6382", "ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./configs:/opt/AIL/configs:ro
+  crawler:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Crawler.py
+      - onion
+      - "8050"
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./CRAWLED_SCREENSHOT/:/opt/AIL/CRAWLED_SCREENSHOT
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  flask:
+    build: .
+    entrypoint:
+      - /opt/AIL/var/www/Flask_server.py
+    ports:
+      - "7000:7000"
+    image: ail-framework
+    volumes:
+      - ./CRAWLED_SCREENSHOT/:/opt/AIL/CRAWLED_SCREENSHOT
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./indexdir:/opt/AIL/indexdir
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/var/www
+  log-queue:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/AILENV/bin/log_subscriber
+      - -p
+      - "6380"
+      - -c
+      - Queing
+      - -l
+      - /opt/AIL/logs/
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./configs:/opt/AIL/configs:ro
+  log-script:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/AILENV/bin/log_subscriber
+      - -p
+      - "6380"
+      - -c
+      - Script
+      - -l
+      - /opt/AIL/logs/
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./configs:/opt/AIL/configs:ro
+  pystemon:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/pystemon/pystemon.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./pystemon/archives:/opt/pystemon/archive
+      - ./pystemon/proxies.txt:/opt/pystemon/proxies.txt:ro
+      - ./pystemon/pystemon.yaml:/opt/pystemon/pystemon.yaml:ro
+    working_dir: /opt/pystemon
+  pystemon-feeder:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/feeder/pystemon-feeder.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./pystemon/archives:/opt/pystemon/archive
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  queues:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/launch_queues.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  redis-cache:
+    entrypoint:
+      - redis-server
+      - /opt/AIL/configs/6379.conf
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "6379", "ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    image: ail-framework
+    network_mode: service:flask
+    volumes:
+      - ./configs:/opt/AIL/configs:ro
+  redis-log:
+    entrypoint:
+      - redis-server
+      - /opt/AIL/configs/6380.conf
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "6380", "ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./configs:/opt/AIL/configs:ro
+  redis-mixer-cache:
+    entrypoint:
+      - redis-server
+      - /opt/AIL/configs/6381.conf
+    healthcheck:
+      test: ["CMD", "redis-cli", "-p", "6381", "ping"]
+      interval: 30s
+      timeout: 10s
+      retries: 5
+    image: ail-framework
+    network_mode: service:flask
+    volumes:
+      - ./configs:/opt/AIL/configs:ro
+  script-alerthandler:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/alertHandler.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-apikey:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/ApiKey.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-bankaccount:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/BankAccount.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-bitcoin:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Bitcoin.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-categ:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Categ.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-credential:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Credential.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-creditcards:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/CreditCards.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-curve:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Curve.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-curvemanagetopsets:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/CurveManageTopSets.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-cve:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Cve.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-decoder:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Decoder.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./HASHS:/opt/AIL/HASHS
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-domclassifier:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/DomClassifier.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-duplicates:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Duplicates.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-global:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Global.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-indexer:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Indexer.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./indexdir:/opt/AIL/indexdir
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-keys:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Keys.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-libinjection:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/LibInjection.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-lines:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Lines.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-mail:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Mail.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-mispthehivefeeder:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/MISP_The_Hive_feeder.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-mixer:
+    depends_on:
+      - redis-mixer-cache
+    entrypoint:
+      - /opt/AIL/bin/Mixer.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-modulestats:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/ModuleStats.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-onion:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Onion.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-phone:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Phone.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-regexfortermsfrequency:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/RegexForTermsFrequency.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-release:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Release.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-sentimentanalysis:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/SentimentAnalysis.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-setfortermsfrequency:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/SetForTermsFrequency.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-sqlinjectiondetection:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/SQLInjectionDetection.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-submitpaste:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/submit_paste.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-tags:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Tags.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-tokenize:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Tokenize.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-web:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/Web.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin
+  script-webstats:
+    depends_on:
+      - redis-log
+    entrypoint:
+      - /opt/AIL/bin/WebStats.py
+    network_mode: service:flask
+    image: ail-framework
+    volumes:
+      - ./PASTES/:/opt/AIL/PASTES
+      - ./bin/packages/config.cfg:/opt/AIL/bin/packages/config.cfg:ro
+    working_dir: /opt/AIL/bin

installing_deps.sh

@@ -95,6 +95,7 @@ popd
 mkdir -p $AIL_HOME/PASTES
 
 pip3 install -U pip
+pip3 install 'git+https://github.com/D4-project/BGP-Ranking.git/@7e698f87366e6f99b4d0d11852737db28e3ddc62#egg=pybgpranking&subdirectory=client'
 pip3 install -U -r pip3_packages_requirement.txt
 
 # Pyfaup

pystemon/proxies.txt

@@ -0,0 +1 @@
+http://127.0.0.1:8080

var/www/update_thirdparty.sh

@@ -104,8 +104,12 @@ pushd static/image
 wget https://www.circl.lu/assets/images/logos/AIL.png -O AIL.png
 popd
 
-#active virtualenv
+if ! [[ -n "$AIL_HOME" ]]
-source ./../../AILENV/bin/activate
+then
+    #active virtualenv
+    source ./../../AILENV/bin/activate
+fi
+
 #Update MISP Taxonomies and Galaxies
 python3 -m pip install git+https://github.com/MISP/PyTaxonomies --upgrade
 python3 -m pip install git+https://github.com/MISP/PyMISPGalaxies --upgrade