Merge pull request #44 from Rafiot/travis

Add initial Travis file
This commit is contained in:
Raphaël Vinot 2016-01-19 11:51:46 +01:00
commit e3820d94ca
6 changed files with 183 additions and 0 deletions

83
.travis.yml Normal file
View file

@ -0,0 +1,83 @@
language: python
python:
- "2.7"
sudo: required
dist: trusty
group: edge
addons:
apt:
packages:
# General dependencies
- python-pip
- python-virtualenv
- python-dev
- g++
- python-tk
- unzip
- libsnappy-dev
# Needed for bloomfilters
- libssl-dev
- python-numpy
- libfreetype6-dev
# Leveldb
- libgmp-dev
- libev-dev
env:
- AIL_HOME=$TRAVIS_BUILD_DIR AIL_BIN=$TRAVIS_BUILD_DIR/bin/ \
AIL_FLASK=$TRAVIS_BUILD_DIR/var/www/ AIL_REDIS=$TRAVIS_BUILD_DIR/redis/src/ \
AIL_LEVELDB=$TRAVIS_BUILD_DIR/redis-leveldb/ PATH=$AIL_HOME:$AIL_REDIS:$AIL_LEVELDB:$PATH
install:
- pip install -U pip
# DNS
- sudo apt-get install libadns1 libadns1-dev screen
# required for mathplotlib
- test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/
- pip install distribute
# Redis
- test ! -d redis/ && git clone https://github.com/antirez/redis.git
- pushd redis
- git checkout 3.0
- make
- popd
# Redis leveldb
- test ! -d redis-leveldb/ && git clone https://github.com/KDr2/redis-leveldb.git
- pushd redis-leveldb/
- git submodule init
- git submodule update
- make
- popd
# Set config
- cp bin/packages/config.cfg.sample bin/packages/config.cfg
- mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}
- mkdir -p $AIL_HOME/LEVEL_DB_DATA/{2016,2015,2014,2013}
- pip install -r pip_packages_requirement.txt
- python -m textblob.download_corpora
- pushd var/www/
- ./update_thirdparty.sh
- popd
script:
- pushd bin
- ./launch_redis.sh
- ./launch_lvldb.sh
- ./launch_logs.sh
- ./launch_queues.sh
- ./launch_scripts.sh
- sleep 120
- ./Shutdown.py
- popd
notifications:
email:
on_success: change
on_failure: change

12
bin/launch_logs.sh Executable file
View file

@ -0,0 +1,12 @@
#!/bin/bash
set -e
set -x
screen -dmS "Logging"
sleep 0.1
echo -e $GREEN"\t* Launching logging process"$DEFAULT
screen -S "Logging" -X screen -t "LogQueue" bash -c 'log_subscriber -p 6380 -c Queuing -l ../logs/; read x'
sleep 0.1
screen -S "Logging" -X screen -t "LogScript" bash -c 'log_subscriber -p 6380 -c Script -l ../logs/; read x'

25
bin/launch_lvldb.sh Executable file
View file

@ -0,0 +1,25 @@
#!/bin/bash
set -e
set -x
lvdbhost='127.0.0.1'
lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
db1_y='2013'
db2_y='2014'
db2_y='2015'
db2_y='2016'
nb_db=13
screen -dmS "LevelDB"
sleep 0.1
echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT
#Add lines here with appropriates options.
screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
sleep 0.1
screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
sleep 0.1
screen -S "LevelDB" -X screen -t "2015" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2015/ -P '$db2_y' -M '$nb_db'; read x'
sleep 0.1
screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db2_y' -M '$nb_db'; read x'

11
bin/launch_queues.sh Executable file
View file

@ -0,0 +1,11 @@
#!/bin/bash
set -e
set -x
screen -dmS "Queue"
sleep 0.1
echo -e $GREEN"\t* Launching all the queues"$DEFAULT
screen -S "Queue" -X screen -t "Queues" bash -c './launch_queues.py; read x'

16
bin/launch_redis.sh Executable file
View file

@ -0,0 +1,16 @@
#!/bin/bash
set -e
set -x
conf_dir="${AIL_HOME}/configs/"
screen -dmS "Redis"
sleep 0.1
echo -e $GREEN"\t* Launching Redis servers"$DEFAULT
screen -S "Redis" -X screen -t "6379" bash -c 'redis-server '$conf_dir'6379.conf ; read x'
sleep 0.1
screen -S "Redis" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x'
sleep 0.1
screen -S "Redis" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x'

36
bin/launch_scripts.sh Executable file
View file

@ -0,0 +1,36 @@
#!/bin/bash
set -e
set -x
screen -dmS "Script"
sleep 0.1
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Line" bash -c './Line.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "CreditCard" bash -c './CreditCard.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Url" bash -c './Url.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'