AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-14 10:38:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

chg: [python] Added necessary checks in LAUNCH.sh to be able to launch the script without doing bin/activate

Browse source 
chg: [doc] Added correct pystemon path
fix: [zmq] Small ZMQ test fix.
fix: [doc] Moved the print statement in feeder so it is less verbose but still useful
This commit is contained in:
Steve Clement 2018-11-22 14:57:24 +09:00
parent cb41c54135
commit e4ced241eb
5 changed files with 65 additions and 52 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
HOWTO.md
View file

@ -27,7 +27,7 @@ Feed data to AIL:
4. Edit your configuration file ```bin/packages/config.cfg``` and modify the pystemonpath path accordingly 4. Edit your configuration file ```bin/packages/config.cfg``` and modify the pystemonpath path accordingly
5. Launch pystemon-feeder ``` ./pystemon-feeder.py ``` 5. Launch pystemon-feeder ``` ./bin/feeder/pystemon-feeder.py ```
How to create a new module How to create a new module

2
README.md
View file

@ -70,8 +70,6 @@ Type these command lines for a fully automated installation and start AIL framew
git clone https://github.com/CIRCL/AIL-framework.git git clone https://github.com/CIRCL/AIL-framework.git
cd AIL-framework cd AIL-framework
./installing_deps.sh ./installing_deps.sh
cd var/www/
./update_thirdparty.sh
cd ~/AIL-framework/ cd ~/AIL-framework/
. ./AILENV/bin/activate . ./AILENV/bin/activate
cd bin/ cd bin/

109
bin/LAUNCH.sh
View file

@ -9,11 +9,26 @@ WHITE="\\033[0;02m"
YELLOW="\\033[1;33m" YELLOW="\\033[1;33m"
CYAN="\\033[1;36m" CYAN="\\033[1;36m"
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; # Getting CWD where bash script resides
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; AIL_HOME="${DIR}"
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; cd ${AIL_HOME}
if [ -e "${DIR}/AILENV/bin/python" ]; then
echo "AIL-framework virtualenv seems to exist, good"
ENV_PY="${DIR}/AILENV/bin/python"
else
echo "Please make sure you have a AIL-framework environment, au revoir"
exit 1
fi
[ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1
export AIL_BIN=${AIL_HOME}/bin/
export AIL_FLASK=${AIL_HOME}/var/www/
export AIL_REDIS=${AIL_HOME}/redis/src/
export AIL_ARDB=${AIL_HOME}/ardb/src/
export PATH=$AIL_HOME:$PATH export PATH=$AIL_HOME:$PATH
export PATH=$AIL_REDIS:$PATH export PATH=$AIL_REDIS:$PATH
@ -101,16 +116,16 @@ function launching_queues {
sleep 0.1 sleep 0.1
echo -e $GREEN"\t* Launching all the queues"$DEFAULT echo -e $GREEN"\t* Launching all the queues"$DEFAULT
screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; python3 launch_queues.py; read x' screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} launch_queues.py; read x'
} }
function checking_configuration { function checking_configuration {
bin_dir=${AIL_HOME}/bin bin_dir=${AIL_HOME}/bin
echo -e "\t* Checking configuration" echo -e "\t* Checking configuration"
if [ "$1" == "automatic" ]; then if [ "$1" == "automatic" ]; then
bash -c "python3 $bin_dir/Update-conf.py True" bash -c "${ENV_PY} $bin_dir/Update-conf.py True"
else else
bash -c "python3 $bin_dir/Update-conf.py False" bash -c "${ENV_PY} $bin_dir/Update-conf.py False"
fi fi
exitStatus=$? exitStatus=$?
@ -128,75 +143,75 @@ function launching_scripts {
sleep 0.1 sleep 0.1
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ./ModulesInformationV2.py -k 0 -c 1; read x' screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ./Mixer.py; read x' screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Mixer.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ./Global.py; read x' screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Global.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ./Duplicates.py; read x' screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Duplicates.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ./Lines.py; read x' screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Lines.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ./DomClassifier.py; read x' screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./DomClassifier.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ./Categ.py; read x' screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Categ.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ./Tokenize.py; read x' screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Tokenize.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ./CreditCards.py; read x' screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./CreditCards.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ./BankAccount.py; read x' screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./BankAccount.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ./Onion.py; read x' screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Onion.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ./Mail.py; read x' screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Mail.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ./ApiKey.py; read x' screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ApiKey.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ./Web.py; read x' screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Web.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ./Credential.py; read x' screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Credential.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ./Curve.py; read x' screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Curve.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ./CurveManageTopSets.py; read x' screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./CurveManageTopSets.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./RegexForTermsFrequency.py; read x' screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./RegexForTermsFrequency.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./SetForTermsFrequency.py; read x' screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SetForTermsFrequency.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ./Indexer.py; read x' screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Indexer.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Keys.py; read x' screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Keys.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ./Decoder.py; read x' screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Decoder.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ./Bitcoin.py; read x' screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Bitcoin.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ./Phone.py; read x' screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Phone.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ./Release.py; read x' screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Release.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ./Cve.py; read x' screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Cve.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ./WebStats.py; read x' screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./WebStats.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ./ModuleStats.py; read x' screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ModuleStats.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ./SQLInjectionDetection.py; read x' screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SQLInjectionDetection.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ./LibInjection.py; read x' screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./LibInjection.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ./alertHandler.py; read x' screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./alertHandler.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ./MISP_The_Hive_feeder.py; read x' screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ./Tags.py; read x' screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Tags.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ./SentimentAnalysis.py; read x' screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SentimentAnalysis.py; read x'
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ./submit_paste.py; read x' screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./submit_paste.py; read x'
} }
@ -219,7 +234,7 @@ function launching_crawler {
sleep 0.1 sleep 0.1
for ((i=first_port;i<=last_port;i++)); do for ((i=first_port;i<=last_port;i++)); do
screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ./Crawler.py onion '$i'; read x' screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Crawler.py onion '$i'; read x'
sleep 0.1 sleep 0.1
done done
@ -342,7 +357,7 @@ function launch_flask {
screen -dmS "Flask_AIL" screen -dmS "Flask_AIL"
sleep 0.1 sleep 0.1
echo -e $GREEN"\t* Launching Flask server"$DEFAULT echo -e $GREEN"\t* Launching Flask server"$DEFAULT
screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ./Flask_server.py; read x" screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ${ENV_PY} ./Flask_server.py; read x"
else else
echo -e $RED"\t* A Flask screen is already launched"$DEFAULT echo -e $RED"\t* A Flask screen is already launched"$DEFAULT
fi fi
@ -353,9 +368,9 @@ function launch_feeder {
screen -dmS "Feeder_Pystemon" screen -dmS "Feeder_Pystemon"
sleep 0.1 sleep 0.1
echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT
screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ./feeder/pystemon-feeder.py; read x' screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./feeder/pystemon-feeder.py; read x'
sleep 0.1 sleep 0.1
screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; python2 pystemon.py; read x' screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; ${ENV_PY} ./pystemon.py; read x'
else else
echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT
fi fi

2
bin/feeder/pystemon-feeder.py
View file

@ -61,10 +61,10 @@ topic = '102'
while True: while True:
time.sleep(base_sleeptime + sleep_inc) time.sleep(base_sleeptime + sleep_inc)
paste = r.lpop("pastes") paste = r.lpop("pastes")
print(paste)
if paste is None: if paste is None:
continue continue
try: try:
print(paste)
with open(pystemonpath+paste, 'rb') as f: #.read() with open(pystemonpath+paste, 'rb') as f: #.read()
messagedata = f.read() messagedata = f.read()
path_to_send = pastes_directory+paste path_to_send = pastes_directory+paste

2
bin/feeder/test-zmq.py
View file

@ -20,7 +20,7 @@ socket.connect ("tcp://crf.circl.lu:%s" % port)
# 102 Full pastes in raw base64(gz) # 102 Full pastes in raw base64(gz)
topicfilter = "102" topicfilter = "102"
socket.setsockopt(zmq.SUBSCRIBE, topicfilter) socket.setsockopt_string(zmq.SUBSCRIBE, topicfilter)
while True: while True:
message = socket.recv() message = socket.recv()