mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 14:17:16 +00:00
commit
e3820d94ca
6 changed files with 183 additions and 0 deletions
83
.travis.yml
Normal file
83
.travis.yml
Normal file
|
@ -0,0 +1,83 @@
|
|||
language: python
|
||||
|
||||
python:
|
||||
- "2.7"
|
||||
|
||||
sudo: required
|
||||
|
||||
dist: trusty
|
||||
|
||||
group: edge
|
||||
|
||||
addons:
|
||||
apt:
|
||||
packages:
|
||||
# General dependencies
|
||||
- python-pip
|
||||
- python-virtualenv
|
||||
- python-dev
|
||||
- g++
|
||||
- python-tk
|
||||
- unzip
|
||||
- libsnappy-dev
|
||||
# Needed for bloomfilters
|
||||
- libssl-dev
|
||||
- python-numpy
|
||||
- libfreetype6-dev
|
||||
# Leveldb
|
||||
- libgmp-dev
|
||||
- libev-dev
|
||||
|
||||
env:
|
||||
- AIL_HOME=$TRAVIS_BUILD_DIR AIL_BIN=$TRAVIS_BUILD_DIR/bin/ \
|
||||
AIL_FLASK=$TRAVIS_BUILD_DIR/var/www/ AIL_REDIS=$TRAVIS_BUILD_DIR/redis/src/ \
|
||||
AIL_LEVELDB=$TRAVIS_BUILD_DIR/redis-leveldb/ PATH=$AIL_HOME:$AIL_REDIS:$AIL_LEVELDB:$PATH
|
||||
|
||||
|
||||
|
||||
install:
|
||||
- pip install -U pip
|
||||
# DNS
|
||||
- sudo apt-get install libadns1 libadns1-dev screen
|
||||
# required for mathplotlib
|
||||
- test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/
|
||||
- pip install distribute
|
||||
# Redis
|
||||
- test ! -d redis/ && git clone https://github.com/antirez/redis.git
|
||||
- pushd redis
|
||||
- git checkout 3.0
|
||||
- make
|
||||
- popd
|
||||
# Redis leveldb
|
||||
- test ! -d redis-leveldb/ && git clone https://github.com/KDr2/redis-leveldb.git
|
||||
- pushd redis-leveldb/
|
||||
- git submodule init
|
||||
- git submodule update
|
||||
- make
|
||||
- popd
|
||||
# Set config
|
||||
- cp bin/packages/config.cfg.sample bin/packages/config.cfg
|
||||
- mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}
|
||||
- mkdir -p $AIL_HOME/LEVEL_DB_DATA/{2016,2015,2014,2013}
|
||||
- pip install -r pip_packages_requirement.txt
|
||||
- python -m textblob.download_corpora
|
||||
- pushd var/www/
|
||||
- ./update_thirdparty.sh
|
||||
- popd
|
||||
|
||||
|
||||
script:
|
||||
- pushd bin
|
||||
- ./launch_redis.sh
|
||||
- ./launch_lvldb.sh
|
||||
- ./launch_logs.sh
|
||||
- ./launch_queues.sh
|
||||
- ./launch_scripts.sh
|
||||
- sleep 120
|
||||
- ./Shutdown.py
|
||||
- popd
|
||||
|
||||
notifications:
|
||||
email:
|
||||
on_success: change
|
||||
on_failure: change
|
12
bin/launch_logs.sh
Executable file
12
bin/launch_logs.sh
Executable file
|
@ -0,0 +1,12 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
screen -dmS "Logging"
|
||||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching logging process"$DEFAULT
|
||||
screen -S "Logging" -X screen -t "LogQueue" bash -c 'log_subscriber -p 6380 -c Queuing -l ../logs/; read x'
|
||||
sleep 0.1
|
||||
screen -S "Logging" -X screen -t "LogScript" bash -c 'log_subscriber -p 6380 -c Script -l ../logs/; read x'
|
||||
|
25
bin/launch_lvldb.sh
Executable file
25
bin/launch_lvldb.sh
Executable file
|
@ -0,0 +1,25 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
lvdbhost='127.0.0.1'
|
||||
lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
|
||||
db1_y='2013'
|
||||
db2_y='2014'
|
||||
db2_y='2015'
|
||||
db2_y='2016'
|
||||
nb_db=13
|
||||
|
||||
screen -dmS "LevelDB"
|
||||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT
|
||||
#Add lines here with appropriates options.
|
||||
screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
|
||||
sleep 0.1
|
||||
screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
|
||||
sleep 0.1
|
||||
screen -S "LevelDB" -X screen -t "2015" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2015/ -P '$db2_y' -M '$nb_db'; read x'
|
||||
sleep 0.1
|
||||
screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db2_y' -M '$nb_db'; read x'
|
||||
|
11
bin/launch_queues.sh
Executable file
11
bin/launch_queues.sh
Executable file
|
@ -0,0 +1,11 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
screen -dmS "Queue"
|
||||
sleep 0.1
|
||||
|
||||
echo -e $GREEN"\t* Launching all the queues"$DEFAULT
|
||||
screen -S "Queue" -X screen -t "Queues" bash -c './launch_queues.py; read x'
|
||||
|
16
bin/launch_redis.sh
Executable file
16
bin/launch_redis.sh
Executable file
|
@ -0,0 +1,16 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
conf_dir="${AIL_HOME}/configs/"
|
||||
|
||||
screen -dmS "Redis"
|
||||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching Redis servers"$DEFAULT
|
||||
screen -S "Redis" -X screen -t "6379" bash -c 'redis-server '$conf_dir'6379.conf ; read x'
|
||||
sleep 0.1
|
||||
screen -S "Redis" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x'
|
||||
sleep 0.1
|
||||
screen -S "Redis" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x'
|
||||
|
36
bin/launch_scripts.sh
Executable file
36
bin/launch_scripts.sh
Executable file
|
@ -0,0 +1,36 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -e
|
||||
set -x
|
||||
|
||||
screen -dmS "Script"
|
||||
sleep 0.1
|
||||
|
||||
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
|
||||
|
||||
screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Line" bash -c './Line.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "CreditCard" bash -c './CreditCard.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Url" bash -c './Url.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
|
||||
|
Loading…
Reference in a new issue