diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 00000000..292c42f6 --- /dev/null +++ b/.travis.yml @@ -0,0 +1,83 @@ +language: python + +python: + - "2.7" + +sudo: required + +dist: trusty + +group: edge + +addons: + apt: + packages: + # General dependencies + - python-pip + - python-virtualenv + - python-dev + - g++ + - python-tk + - unzip + - libsnappy-dev + # Needed for bloomfilters + - libssl-dev + - python-numpy + - libfreetype6-dev + # Leveldb + - libgmp-dev + - libev-dev + +env: + - AIL_HOME=$TRAVIS_BUILD_DIR AIL_BIN=$TRAVIS_BUILD_DIR/bin/ \ + AIL_FLASK=$TRAVIS_BUILD_DIR/var/www/ AIL_REDIS=$TRAVIS_BUILD_DIR/redis/src/ \ + AIL_LEVELDB=$TRAVIS_BUILD_DIR/redis-leveldb/ PATH=$AIL_HOME:$AIL_REDIS:$AIL_LEVELDB:$PATH + + + +install: + - pip install -U pip + # DNS + - sudo apt-get install libadns1 libadns1-dev screen + # required for mathplotlib + - test ! -L /usr/include/ft2build.h && sudo ln -s freetype2/ft2build.h /usr/include/ + - pip install distribute + # Redis + - test ! -d redis/ && git clone https://github.com/antirez/redis.git + - pushd redis + - git checkout 3.0 + - make + - popd + # Redis leveldb + - test ! -d redis-leveldb/ && git clone https://github.com/KDr2/redis-leveldb.git + - pushd redis-leveldb/ + - git submodule init + - git submodule update + - make + - popd + # Set config + - cp bin/packages/config.cfg.sample bin/packages/config.cfg + - mkdir -p $AIL_HOME/{PASTES,Blooms,dumps} + - mkdir -p $AIL_HOME/LEVEL_DB_DATA/{2016,2015,2014,2013} + - pip install -r pip_packages_requirement.txt + - python -m textblob.download_corpora + - pushd var/www/ + - ./update_thirdparty.sh + - popd + + +script: + - pushd bin + - ./launch_redis.sh + - ./launch_lvldb.sh + - ./launch_logs.sh + - ./launch_queues.sh + - ./launch_scripts.sh + - sleep 120 + - ./Shutdown.py + - popd + +notifications: + email: + on_success: change + on_failure: change diff --git a/bin/launch_logs.sh b/bin/launch_logs.sh new file mode 100755 index 00000000..107a2450 --- /dev/null +++ b/bin/launch_logs.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -e +set -x + +screen -dmS "Logging" +sleep 0.1 +echo -e $GREEN"\t* Launching logging process"$DEFAULT +screen -S "Logging" -X screen -t "LogQueue" bash -c 'log_subscriber -p 6380 -c Queuing -l ../logs/; read x' +sleep 0.1 +screen -S "Logging" -X screen -t "LogScript" bash -c 'log_subscriber -p 6380 -c Script -l ../logs/; read x' + diff --git a/bin/launch_lvldb.sh b/bin/launch_lvldb.sh new file mode 100755 index 00000000..3eb68acf --- /dev/null +++ b/bin/launch_lvldb.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +set -e +set -x + +lvdbhost='127.0.0.1' +lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/" +db1_y='2013' +db2_y='2014' +db2_y='2015' +db2_y='2016' +nb_db=13 + +screen -dmS "LevelDB" +sleep 0.1 +echo -e $GREEN"\t* Launching Levels DB servers"$DEFAULT +#Add lines here with appropriates options. +screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x' +sleep 0.1 +screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x' +sleep 0.1 +screen -S "LevelDB" -X screen -t "2015" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2015/ -P '$db2_y' -M '$nb_db'; read x' +sleep 0.1 +screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db2_y' -M '$nb_db'; read x' + diff --git a/bin/launch_queues.sh b/bin/launch_queues.sh new file mode 100755 index 00000000..8c2699ea --- /dev/null +++ b/bin/launch_queues.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -e +set -x + +screen -dmS "Queue" +sleep 0.1 + +echo -e $GREEN"\t* Launching all the queues"$DEFAULT +screen -S "Queue" -X screen -t "Queues" bash -c './launch_queues.py; read x' + diff --git a/bin/launch_redis.sh b/bin/launch_redis.sh new file mode 100755 index 00000000..8ee410a0 --- /dev/null +++ b/bin/launch_redis.sh @@ -0,0 +1,16 @@ +#!/bin/bash + +set -e +set -x + +conf_dir="${AIL_HOME}/configs/" + +screen -dmS "Redis" +sleep 0.1 +echo -e $GREEN"\t* Launching Redis servers"$DEFAULT +screen -S "Redis" -X screen -t "6379" bash -c 'redis-server '$conf_dir'6379.conf ; read x' +sleep 0.1 +screen -S "Redis" -X screen -t "6380" bash -c 'redis-server '$conf_dir'6380.conf ; read x' +sleep 0.1 +screen -S "Redis" -X screen -t "6381" bash -c 'redis-server '$conf_dir'6381.conf ; read x' + diff --git a/bin/launch_scripts.sh b/bin/launch_scripts.sh new file mode 100755 index 00000000..0e494bcf --- /dev/null +++ b/bin/launch_scripts.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +set -e +set -x + +screen -dmS "Script" +sleep 0.1 + +echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT + +screen -S "Script" -X screen -t "Global" bash -c './Global.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Duplicate" bash -c './Duplicate.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Attribute" bash -c './Attribute.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Line" bash -c './Line.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "DomainClassifier" bash -c './DomClassifier.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Categ" bash -c './Categ.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Tokenize" bash -c './Tokenize.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "CreditCard" bash -c './CreditCard.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Onion" bash -c './Onion.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Mail" bash -c './Mail.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Url" bash -c './Url.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x' +sleep 0.1 +screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x' +