mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
Add initial Travis file
This commit is contained in:
parent
52ec3e5083
commit
0d6adc2063
6 changed files with 183 additions and 0 deletions
83
.travis.yml
Normal file
83
.travis.yml
Normal file
|
@ -0,0 +1,83 @@
|
||||||
|
language: python
|
||||||
|
|
||||||
|
python:
|
||||||
|
- "2.7"
|
||||||
|
|
||||||
|
sudo: required
|
||||||
|
|
||||||
|
dist: trusty
|
||||||
|
|
||||||
|
group: edge
|
||||||
|
|
||||||
|
addons:
|
||||||
|
apt:
|
||||||
|
packages:
|
||||||
|
# General dependencies
|
||||||
|
- python-pip
|
||||||
|
- python-virtualenv
|
||||||
|
- python-dev
|
||||||
|
- g++
|
||||||
|
- python-tk
|
||||||
|
- unzip
|
||||||
|
- libsnappy-dev
|
||||||
|
# Needed for bloomfilters
|
||||||
|
- libssl-dev
|
||||||
|
- python-numpy
|
||||||
|
- libfreetype6-dev
|
||||||
|
# Leveldb
|
||||||
|
- libgmp-dev
|
||||||
|
- libev-dev
|
||||||
|
|
||||||
|
env:
|
||||||
|
- AIL_HOME=$TRAVIS_BUILD_DIR AIL_BIN=$TRAVIS_BUILD_DIR/bin/ \
|
||||||
|
AIL_FLASK=$TRAVIS_BUILD_DIR/var/www/ AIL_REDIS=$TRAVIS_BUILD_DIR/redis/src/ \
|
||||||
|
AIL_LEVELDB=$TRAVIS_BUILD_DIR/redis-leveldb/ PATH=$AIL_HOME:$AIL_REDIS:$AIL_LEVELDB:$PATH
|
||||||
|
|
||||||
|
|
||||||
|
|
||||||
|
install:
|
||||||
|
- pip install -U pip
|
||||||
|
# DNS
|
||||||
|
- sudo apt-get install libadns1 libadns1-dev screen
|
||||||
|
# required for mathplotlib
|
||||||
|
- test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/
|
||||||
|
- pip install distribute
|
||||||
|
# Redis
|
||||||
|
- test ! -d redis/ && git clone https://github.com/antirez/redis.git
|
||||||
|
- pushd redis
|
||||||
|
- git checkout 3.0
|
||||||
|
- make
|
||||||
|
- popd
|
||||||
|
# Redis leveldb
|
||||||
|
- test ! -d redis-leveldb/ && git clone https://github.com/KDr2/redis-leveldb.git
|
||||||
|
- pushd redis-leveldb/
|
||||||
|
- git submodule init
|
||||||
|
- git submodule update
|
||||||
|
- make
|
||||||
|
- popd
|
||||||
|
# Set config
|
||||||
|
- cp bin/packages/config.cfg.sample bin/packages/config.cfg
|
||||||
|
- mkdir -p $AIL_HOME/{PASTES,Blooms,dumps}
|
||||||
|
- mkdir -p $AIL_HOME/LEVEL_DB_DATA/{2016,2015,2014,2013}
|
||||||
|
- pip install -r pip_packages_requirement.txt
|
||||||
|
- python -m textblob.download_corpora
|
||||||
|
- pushd var/www/
|
||||||
|
- ./update_thirdparty.sh
|
||||||
|
- popd
|
||||||
|
|
||||||
|
|
||||||
|
script:
|
||||||
|
- pushd bin
|
||||||
|
- ./launch_redis.sh
|
||||||
|
- ./launch_lvldb.sh
|
||||||
|
- ./launch_logs.sh
|
||||||
|
- ./launch_queues.sh
|
||||||
|
- ./launch_scripts.sh
|
||||||
|
- sleep 120
|
||||||
|
- ./Shutdown.py
|
||||||
|
- popd
|
||||||
|
|
||||||
|
notifications:
|
||||||
|
email:
|
||||||
|
on_success: change
|
||||||
|
on_failure: change
|
12
bin/launch_logs.sh
Executable file
12
bin/launch_logs.sh
Executable file
|
@ -0,0 +1,12 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
set -x
|
||||||
|
|
||||||
|
screen -dmS "Logging"
|
||||||
|
sleep 0.1
|
||||||
|
echo -e $GREEN"\t* Launching logging process"$DEFAULT
|
||||||
|
screen -S "Logging" -X screen -t "LogQueue" bash -c 'log_subscriber -p 6380 -c Queuing -l ../logs/; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "Logging" -X screen -t "LogScript" bash -c 'log_subscriber -p 6380 -c Script -l ../logs/; read x'
|
||||||
|
|
25
bin/launch_lvldb.sh
Executable file
25
bin/launch_lvldb.sh
Executable file
|
@ -0,0 +1,25 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
set -x
|
||||||
|
|
||||||
|
lvdbhost='127.0.0.1'
|
||||||
|
lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
|
||||||
|
db1_y='2013'
|
||||||
|
db2_y='2014'
|
||||||
|
db2_y='2015'
|
||||||
|
db2_y='2016'
|
||||||
|
nb_db=13
|
||||||
|
|
||||||
|
screen -dmS "LevelDB"
|
||||||
|
sleep 0.1
|
||||||
|
echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT
|
||||||
|
#Add lines here with appropriates options.
|
||||||
|
screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "LevelDB" -X screen -t "2015" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2015/ -P '$db2_y' -M '$nb_db'; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db2_y' -M '$nb_db'; read x'
|
||||||
|
|
11
bin/launch_queues.sh
Executable file
11
bin/launch_queues.sh
Executable file
|
@ -0,0 +1,11 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
set -x
|
||||||
|
|
||||||
|
screen -dmS "Queue"
|
||||||
|
sleep 0.1
|
||||||
|
|
||||||
|
echo -e $GREEN"\t* Launching all the queues"$DEFAULT
|
||||||
|
screen -S "Queue" -X screen -t "Queues" bash -c './launch_queues.py; read x'
|
||||||
|
|
16
bin/launch_redis.sh
Executable file
16
bin/launch_redis.sh
Executable file
|
@ -0,0 +1,16 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
set -x
|
||||||
|
|
||||||
|
conf_dir="${AIL_HOME}/configs/"
|
||||||
|
|
||||||
|
screen -dmS "Redis"
|
||||||
|
sleep 0.1
|
||||||
|
echo -e $GREEN"\t* Launching Redis servers"$DEFAULT
|
||||||
|
screen -S "Redis" -X screen -t "6379" bash -c 'redis-server '$conf_dir'6379.conf ; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "Redis" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "Redis" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x'
|
||||||
|
|
36
bin/launch_scripts.sh
Executable file
36
bin/launch_scripts.sh
Executable file
|
@ -0,0 +1,36 @@
|
||||||
|
#!/bin/bash
|
||||||
|
|
||||||
|
set -e
|
||||||
|
set -x
|
||||||
|
|
||||||
|
screen -dmS "Script"
|
||||||
|
sleep 0.1
|
||||||
|
|
||||||
|
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
|
||||||
|
|
||||||
|
screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate.py; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "Script" -X screen -t "Line" bash -c './Line.py; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "Script" -X screen -t "CreditCard" bash -c './CreditCard.py; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "Script" -X screen -t "Url" bash -c './Url.py; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
|
||||||
|
sleep 0.1
|
||||||
|
screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
|
||||||
|
|
Loading…
Reference in a new issue