mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 00:28:22 +00:00
chg: [python] Added necessary checks in LAUNCH.sh to be able to launch the script without doing bin/activate
chg: [doc] Added correct pystemon path fix: [zmq] Small ZMQ test fix. fix: [doc] Moved the print statement in feeder so it is less verbose but still useful
This commit is contained in:
parent
cb41c54135
commit
e4ced241eb
5 changed files with 65 additions and 52 deletions
2
HOWTO.md
2
HOWTO.md
|
@ -27,7 +27,7 @@ Feed data to AIL:
|
|||
|
||||
4. Edit your configuration file ```bin/packages/config.cfg``` and modify the pystemonpath path accordingly
|
||||
|
||||
5. Launch pystemon-feeder ``` ./pystemon-feeder.py ```
|
||||
5. Launch pystemon-feeder ``` ./bin/feeder/pystemon-feeder.py ```
|
||||
|
||||
|
||||
How to create a new module
|
||||
|
|
|
@ -70,8 +70,6 @@ Type these command lines for a fully automated installation and start AIL framew
|
|||
git clone https://github.com/CIRCL/AIL-framework.git
|
||||
cd AIL-framework
|
||||
./installing_deps.sh
|
||||
cd var/www/
|
||||
./update_thirdparty.sh
|
||||
cd ~/AIL-framework/
|
||||
. ./AILENV/bin/activate
|
||||
cd bin/
|
||||
|
|
109
bin/LAUNCH.sh
109
bin/LAUNCH.sh
|
@ -9,11 +9,26 @@ WHITE="\\033[0;02m"
|
|||
YELLOW="\\033[1;33m"
|
||||
CYAN="\\033[1;36m"
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
# Getting CWD where bash script resides
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
|
||||
AIL_HOME="${DIR}"
|
||||
|
||||
cd ${AIL_HOME}
|
||||
|
||||
if [ -e "${DIR}/AILENV/bin/python" ]; then
|
||||
echo "AIL-framework virtualenv seems to exist, good"
|
||||
ENV_PY="${DIR}/AILENV/bin/python"
|
||||
else
|
||||
echo "Please make sure you have a AIL-framework environment, au revoir"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
[ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1
|
||||
|
||||
export AIL_BIN=${AIL_HOME}/bin/
|
||||
export AIL_FLASK=${AIL_HOME}/var/www/
|
||||
export AIL_REDIS=${AIL_HOME}/redis/src/
|
||||
export AIL_ARDB=${AIL_HOME}/ardb/src/
|
||||
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
|
@ -101,16 +116,16 @@ function launching_queues {
|
|||
sleep 0.1
|
||||
|
||||
echo -e $GREEN"\t* Launching all the queues"$DEFAULT
|
||||
screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; python3 launch_queues.py; read x'
|
||||
screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} launch_queues.py; read x'
|
||||
}
|
||||
|
||||
function checking_configuration {
|
||||
bin_dir=${AIL_HOME}/bin
|
||||
echo -e "\t* Checking configuration"
|
||||
if [ "$1" == "automatic" ]; then
|
||||
bash -c "python3 $bin_dir/Update-conf.py True"
|
||||
bash -c "${ENV_PY} $bin_dir/Update-conf.py True"
|
||||
else
|
||||
bash -c "python3 $bin_dir/Update-conf.py False"
|
||||
bash -c "${ENV_PY} $bin_dir/Update-conf.py False"
|
||||
fi
|
||||
|
||||
exitStatus=$?
|
||||
|
@ -128,75 +143,75 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
|
||||
|
||||
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ./ModulesInformationV2.py -k 0 -c 1; read x'
|
||||
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ./Mixer.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Mixer.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ./Global.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Global.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ./Duplicates.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Duplicates.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ./Lines.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Lines.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ./DomClassifier.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./DomClassifier.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ./Categ.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Categ.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ./Tokenize.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Tokenize.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ./CreditCards.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./CreditCards.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ./BankAccount.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./BankAccount.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ./Onion.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Onion.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ./Mail.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Mail.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ./ApiKey.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ApiKey.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ./Web.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Web.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ./Credential.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Credential.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ./Curve.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Curve.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ./CurveManageTopSets.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./CurveManageTopSets.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./RegexForTermsFrequency.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./RegexForTermsFrequency.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./SetForTermsFrequency.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SetForTermsFrequency.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ./Indexer.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Indexer.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Keys.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Keys.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ./Decoder.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Decoder.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ./Bitcoin.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Bitcoin.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ./Phone.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Phone.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ./Release.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Release.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ./Cve.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Cve.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ./WebStats.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./WebStats.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ./ModuleStats.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ModuleStats.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ./SQLInjectionDetection.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SQLInjectionDetection.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ./LibInjection.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./LibInjection.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ./alertHandler.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./alertHandler.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ./MISP_The_Hive_feeder.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ./Tags.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Tags.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ./SentimentAnalysis.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SentimentAnalysis.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ./submit_paste.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./submit_paste.py; read x'
|
||||
|
||||
}
|
||||
|
||||
|
@ -219,7 +234,7 @@ function launching_crawler {
|
|||
sleep 0.1
|
||||
|
||||
for ((i=first_port;i<=last_port;i++)); do
|
||||
screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ./Crawler.py onion '$i'; read x'
|
||||
screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Crawler.py onion '$i'; read x'
|
||||
sleep 0.1
|
||||
done
|
||||
|
||||
|
@ -342,7 +357,7 @@ function launch_flask {
|
|||
screen -dmS "Flask_AIL"
|
||||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching Flask server"$DEFAULT
|
||||
screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ./Flask_server.py; read x"
|
||||
screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ${ENV_PY} ./Flask_server.py; read x"
|
||||
else
|
||||
echo -e $RED"\t* A Flask screen is already launched"$DEFAULT
|
||||
fi
|
||||
|
@ -353,9 +368,9 @@ function launch_feeder {
|
|||
screen -dmS "Feeder_Pystemon"
|
||||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT
|
||||
screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ./feeder/pystemon-feeder.py; read x'
|
||||
screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./feeder/pystemon-feeder.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; python2 pystemon.py; read x'
|
||||
screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; ${ENV_PY} ./pystemon.py; read x'
|
||||
else
|
||||
echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT
|
||||
fi
|
||||
|
|
|
@ -61,10 +61,10 @@ topic = '102'
|
|||
while True:
|
||||
time.sleep(base_sleeptime + sleep_inc)
|
||||
paste = r.lpop("pastes")
|
||||
print(paste)
|
||||
if paste is None:
|
||||
continue
|
||||
try:
|
||||
print(paste)
|
||||
with open(pystemonpath+paste, 'rb') as f: #.read()
|
||||
messagedata = f.read()
|
||||
path_to_send = pastes_directory+paste
|
||||
|
|
|
@ -20,7 +20,7 @@ socket.connect ("tcp://crf.circl.lu:%s" % port)
|
|||
# 102 Full pastes in raw base64(gz)
|
||||
|
||||
topicfilter = "102"
|
||||
socket.setsockopt(zmq.SUBSCRIBE, topicfilter)
|
||||
socket.setsockopt_string(zmq.SUBSCRIBE, topicfilter)
|
||||
|
||||
while True:
|
||||
message = socket.recv()
|
||||
|
|
Loading…
Reference in a new issue