From cb41c54135e8aa487a728de943ba243e9bea9043 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Wed, 21 Nov 2018 08:49:58 +0100 Subject: [PATCH 01/16] fix: [lib_refine] add dns configuration --- bin/packages/config.cfg.sample | 3 +++ bin/packages/lib_refine.py | 13 ++++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/bin/packages/config.cfg.sample b/bin/packages/config.cfg.sample index 6efb4be4..b5980766 100644 --- a/bin/packages/config.cfg.sample +++ b/bin/packages/config.cfg.sample @@ -206,6 +206,9 @@ dns = 8.8.8.8 [Mail] dns = 8.8.8.8 +[Web] +dns = 149.13.33.69 + # Indexer configuration [Indexer] type = whoosh diff --git a/bin/packages/lib_refine.py b/bin/packages/lib_refine.py index 5d2af0a9..32f56900 100644 --- a/bin/packages/lib_refine.py +++ b/bin/packages/lib_refine.py @@ -1,6 +1,8 @@ #!/usr/bin/python3 import re +import os +import configparser import dns.resolver from pubsublogger import publisher @@ -101,11 +103,20 @@ def checking_MX_record(r_serv, adress_set, addr_dns): def checking_A_record(r_serv, domains_set): + configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') + if not os.path.exists(configfile): + raise Exception('Unable to find the configuration file. \ + Did you set environment variables? \ + Or activate the virtualenv.') + cfg = configparser.ConfigParser() + cfg.read(configfile) + dns_server = cfg.get("Web", "dns") + score = 0 num = len(domains_set) WalidA = set([]) resolver = dns.resolver.Resolver() - resolver.nameservers = ['149.13.33.69'] + resolver.nameservers = [dns_server] resolver.timeout = 5 resolver.lifetime = 2 From e4ced241ebbaa34ce638207c1271ca3d0c1d855d Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 22 Nov 2018 14:57:24 +0900 Subject: [PATCH 02/16] chg: [python] Added necessary checks in LAUNCH.sh to be able to launch the script without doing bin/activate chg: [doc] Added correct pystemon path fix: [zmq] Small ZMQ test fix. fix: [doc] Moved the print statement in feeder so it is less verbose but still useful --- HOWTO.md | 2 +- README.md | 2 - bin/LAUNCH.sh | 109 +++++++++++++++++++--------------- bin/feeder/pystemon-feeder.py | 2 +- bin/feeder/test-zmq.py | 2 +- 5 files changed, 65 insertions(+), 52 deletions(-) diff --git a/HOWTO.md b/HOWTO.md index 2228d3a6..c21a970f 100644 --- a/HOWTO.md +++ b/HOWTO.md @@ -27,7 +27,7 @@ Feed data to AIL: 4. Edit your configuration file ```bin/packages/config.cfg``` and modify the pystemonpath path accordingly -5. Launch pystemon-feeder ``` ./pystemon-feeder.py ``` +5. Launch pystemon-feeder ``` ./bin/feeder/pystemon-feeder.py ``` How to create a new module diff --git a/README.md b/README.md index 72166c58..5e014852 100644 --- a/README.md +++ b/README.md @@ -70,8 +70,6 @@ Type these command lines for a fully automated installation and start AIL framew git clone https://github.com/CIRCL/AIL-framework.git cd AIL-framework ./installing_deps.sh -cd var/www/ -./update_thirdparty.sh cd ~/AIL-framework/ . ./AILENV/bin/activate cd bin/ diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 684af83b..7b217f88 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -9,11 +9,26 @@ WHITE="\\033[0;02m" YELLOW="\\033[1;33m" CYAN="\\033[1;36m" -[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; -[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1; +# Getting CWD where bash script resides +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +AIL_HOME="${DIR}" + +cd ${AIL_HOME} + +if [ -e "${DIR}/AILENV/bin/python" ]; then + echo "AIL-framework virtualenv seems to exist, good" + ENV_PY="${DIR}/AILENV/bin/python" +else + echo "Please make sure you have a AIL-framework environment, au revoir" + exit 1 +fi + +[ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1 + +export AIL_BIN=${AIL_HOME}/bin/ +export AIL_FLASK=${AIL_HOME}/var/www/ +export AIL_REDIS=${AIL_HOME}/redis/src/ +export AIL_ARDB=${AIL_HOME}/ardb/src/ export PATH=$AIL_HOME:$PATH export PATH=$AIL_REDIS:$PATH @@ -101,16 +116,16 @@ function launching_queues { sleep 0.1 echo -e $GREEN"\t* Launching all the queues"$DEFAULT - screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; python3 launch_queues.py; read x' + screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} launch_queues.py; read x' } function checking_configuration { bin_dir=${AIL_HOME}/bin echo -e "\t* Checking configuration" if [ "$1" == "automatic" ]; then - bash -c "python3 $bin_dir/Update-conf.py True" + bash -c "${ENV_PY} $bin_dir/Update-conf.py True" else - bash -c "python3 $bin_dir/Update-conf.py False" + bash -c "${ENV_PY} $bin_dir/Update-conf.py False" fi exitStatus=$? @@ -128,75 +143,75 @@ function launching_scripts { sleep 0.1 echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT - screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ./ModulesInformationV2.py -k 0 -c 1; read x' + screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ./Mixer.py; read x' + screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Mixer.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ./Global.py; read x' + screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Global.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ./Duplicates.py; read x' + screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Duplicates.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ./Lines.py; read x' + screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Lines.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ./DomClassifier.py; read x' + screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./DomClassifier.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ./Categ.py; read x' + screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Categ.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ./Tokenize.py; read x' + screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Tokenize.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ./CreditCards.py; read x' + screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./CreditCards.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ./BankAccount.py; read x' + screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./BankAccount.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ./Onion.py; read x' + screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Onion.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ./Mail.py; read x' + screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Mail.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ./ApiKey.py; read x' + screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ApiKey.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ./Web.py; read x' + screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Web.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ./Credential.py; read x' + screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Credential.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ./Curve.py; read x' + screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Curve.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ./CurveManageTopSets.py; read x' + screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./CurveManageTopSets.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./RegexForTermsFrequency.py; read x' + screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./RegexForTermsFrequency.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./SetForTermsFrequency.py; read x' + screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SetForTermsFrequency.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ./Indexer.py; read x' + screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Indexer.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Keys.py; read x' + screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Keys.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ./Decoder.py; read x' + screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Decoder.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ./Bitcoin.py; read x' + screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Bitcoin.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ./Phone.py; read x' + screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Phone.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ./Release.py; read x' + screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Release.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ./Cve.py; read x' + screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Cve.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ./WebStats.py; read x' + screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./WebStats.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ./ModuleStats.py; read x' + screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ModuleStats.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ./SQLInjectionDetection.py; read x' + screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SQLInjectionDetection.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ./LibInjection.py; read x' + screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./LibInjection.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ./alertHandler.py; read x' + screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./alertHandler.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ./MISP_The_Hive_feeder.py; read x' + screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ./Tags.py; read x' + screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Tags.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ./SentimentAnalysis.py; read x' + screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SentimentAnalysis.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ./submit_paste.py; read x' + screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./submit_paste.py; read x' } @@ -219,7 +234,7 @@ function launching_crawler { sleep 0.1 for ((i=first_port;i<=last_port;i++)); do - screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ./Crawler.py onion '$i'; read x' + screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Crawler.py onion '$i'; read x' sleep 0.1 done @@ -342,7 +357,7 @@ function launch_flask { screen -dmS "Flask_AIL" sleep 0.1 echo -e $GREEN"\t* Launching Flask server"$DEFAULT - screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ./Flask_server.py; read x" + screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ${ENV_PY} ./Flask_server.py; read x" else echo -e $RED"\t* A Flask screen is already launched"$DEFAULT fi @@ -353,9 +368,9 @@ function launch_feeder { screen -dmS "Feeder_Pystemon" sleep 0.1 echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT - screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ./feeder/pystemon-feeder.py; read x' + screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./feeder/pystemon-feeder.py; read x' sleep 0.1 - screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; python2 pystemon.py; read x' + screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; ${ENV_PY} ./pystemon.py; read x' else echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT fi diff --git a/bin/feeder/pystemon-feeder.py b/bin/feeder/pystemon-feeder.py index b6680ee9..280849ba 100755 --- a/bin/feeder/pystemon-feeder.py +++ b/bin/feeder/pystemon-feeder.py @@ -61,10 +61,10 @@ topic = '102' while True: time.sleep(base_sleeptime + sleep_inc) paste = r.lpop("pastes") - print(paste) if paste is None: continue try: + print(paste) with open(pystemonpath+paste, 'rb') as f: #.read() messagedata = f.read() path_to_send = pastes_directory+paste diff --git a/bin/feeder/test-zmq.py b/bin/feeder/test-zmq.py index f6f28aa1..110c5de2 100644 --- a/bin/feeder/test-zmq.py +++ b/bin/feeder/test-zmq.py @@ -20,7 +20,7 @@ socket.connect ("tcp://crf.circl.lu:%s" % port) # 102 Full pastes in raw base64(gz) topicfilter = "102" -socket.setsockopt(zmq.SUBSCRIBE, topicfilter) +socket.setsockopt_string(zmq.SUBSCRIBE, topicfilter) while True: message = socket.recv() From 41a34b3519d1a3a7cc0e06da5f81d79a6d772e55 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 22 Nov 2018 15:08:59 +0900 Subject: [PATCH 03/16] fix: [bash] As we instantiate from a subdirectory, we need to take this into account. --- bin/LAUNCH.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 7b217f88..c8571866 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -10,7 +10,7 @@ YELLOW="\\033[1;33m" CYAN="\\033[1;36m" # Getting CWD where bash script resides -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )" +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd |sed 's/bin//' )" AIL_HOME="${DIR}" cd ${AIL_HOME} From 6522cffd57a10b662da1351314e79cf1422bafbe Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 22 Nov 2018 15:10:57 +0900 Subject: [PATCH 04/16] chg: [doc] Added note that redis-server is bundled during install --- bin/LAUNCH.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index c8571866..abc49d92 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -23,7 +23,8 @@ else exit 1 fi -[ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1 +# redis-server is bundled during install +## [ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1 export AIL_BIN=${AIL_HOME}/bin/ export AIL_FLASK=${AIL_HOME}/var/www/ From 33bc23a297a4288f4ab394da414536532a60b629 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 22 Nov 2018 15:22:35 +0900 Subject: [PATCH 05/16] fix: [bash] evoking bash -c and splitting vars in this fashion wants to be changed in future revisions. --- bin/LAUNCH.sh | 78 +++++++++++++++++++++++++-------------------------- 1 file changed, 39 insertions(+), 39 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index abc49d92..b46e73fc 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -117,7 +117,7 @@ function launching_queues { sleep 0.1 echo -e $GREEN"\t* Launching all the queues"$DEFAULT - screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} launch_queues.py; read x' + screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' launch_queues.py; read x' } function checking_configuration { @@ -144,75 +144,75 @@ function launching_scripts { sleep 0.1 echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT - screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x' + screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./ModulesInformationV2.py -k 0 -c 1; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Mixer.py; read x' + screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Mixer.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Global.py; read x' + screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Global.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Duplicates.py; read x' + screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Duplicates.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Lines.py; read x' + screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Lines.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./DomClassifier.py; read x' + screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./DomClassifier.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Categ.py; read x' + screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Categ.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Tokenize.py; read x' + screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Tokenize.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./CreditCards.py; read x' + screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./CreditCards.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./BankAccount.py; read x' + screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./BankAccount.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Onion.py; read x' + screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Onion.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Mail.py; read x' + screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Mail.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ApiKey.py; read x' + screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./ApiKey.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Web.py; read x' + screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Web.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Credential.py; read x' + screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Credential.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Curve.py; read x' + screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Curve.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./CurveManageTopSets.py; read x' + screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./CurveManageTopSets.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./RegexForTermsFrequency.py; read x' + screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./RegexForTermsFrequency.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SetForTermsFrequency.py; read x' + screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./SetForTermsFrequency.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Indexer.py; read x' + screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Indexer.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Keys.py; read x' + screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Keys.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Decoder.py; read x' + screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Decoder.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Bitcoin.py; read x' + screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Bitcoin.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Phone.py; read x' + screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Phone.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Release.py; read x' + screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Release.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Cve.py; read x' + screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Cve.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./WebStats.py; read x' + screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./WebStats.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./ModuleStats.py; read x' + screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./ModuleStats.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SQLInjectionDetection.py; read x' + screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./SQLInjectionDetection.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./LibInjection.py; read x' + screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./LibInjection.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./alertHandler.py; read x' + screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./alertHandler.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x' + screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./MISP_The_Hive_feeder.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Tags.py; read x' + screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Tags.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./SentimentAnalysis.py; read x' + screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./SentimentAnalysis.py; read x' sleep 0.1 - screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./submit_paste.py; read x' + screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./submit_paste.py; read x' } @@ -235,7 +235,7 @@ function launching_crawler { sleep 0.1 for ((i=first_port;i<=last_port;i++)); do - screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./Crawler.py onion '$i'; read x' + screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Crawler.py onion '$i'; read x' sleep 0.1 done @@ -369,9 +369,9 @@ function launch_feeder { screen -dmS "Feeder_Pystemon" sleep 0.1 echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT - screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ${ENV_PY} ./feeder/pystemon-feeder.py; read x' + screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./feeder/pystemon-feeder.py; read x' sleep 0.1 - screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; ${ENV_PY} ./pystemon.py; read x' + screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; '${ENV_PY} ' ./pystemon.py; read x' else echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT fi From 88e561aa20705a060cacaf0b3316e58fada0d281 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 22 Nov 2018 15:41:45 +0900 Subject: [PATCH 06/16] chg: [bash] Replace '' with "" --- bin/LAUNCH.sh | 82 +++++++++++++++++++++++++-------------------------- 1 file changed, 41 insertions(+), 41 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index b46e73fc..19a96cc0 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -107,9 +107,9 @@ function launching_logs { screen -dmS "Logging_AIL" sleep 0.1 echo -e $GREEN"\t* Launching logging process"$DEFAULT - screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Queuing -l ../logs/; read x' + screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c "cd ${AIL_BIN}; log_subscriber -p 6380 -c Queuing -l ../logs/; read x" sleep 0.1 - screen -S "Logging_AIL" -X screen -t "LogScript" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Script -l ../logs/; read x' + screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; log_subscriber -p 6380 -c Script -l ../logs/; read x" } function launching_queues { @@ -117,7 +117,7 @@ function launching_queues { sleep 0.1 echo -e $GREEN"\t* Launching all the queues"$DEFAULT - screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' launch_queues.py; read x' + screen -S "Queue_AIL" -X screen -t "Queues" bash -c "cd ${AIL_BIN}; ${ENV_PY} launch_queues.py; read x" } function checking_configuration { @@ -144,75 +144,75 @@ function launching_scripts { sleep 0.1 echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT - screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./ModulesInformationV2.py -k 0 -c 1; read x' + screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Mixer.py; read x' + screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Global.py; read x' + screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Global.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Duplicates.py; read x' + screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Lines.py; read x' + screen -S "Script_AIL" -X screen -t "Lines" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Lines.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./DomClassifier.py; read x' + screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./DomClassifier.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Categ.py; read x' + screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Categ.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Tokenize.py; read x' + screen -S "Script_AIL" -X screen -t "Tokenize" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tokenize.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./CreditCards.py; read x' + screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./BankAccount.py; read x' + screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Onion.py; read x' + screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Onion.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Mail.py; read x' + screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./ApiKey.py; read x' + screen -S "Script_AIL" -X screen -t "ApiKey" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ApiKey.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Web.py; read x' + screen -S "Script_AIL" -X screen -t "Web" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Web.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Credential.py; read x' + screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Credential.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Curve.py; read x' + screen -S "Script_AIL" -X screen -t "Curve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Curve.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./CurveManageTopSets.py; read x' + screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CurveManageTopSets.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./RegexForTermsFrequency.py; read x' + screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexForTermsFrequency.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./SetForTermsFrequency.py; read x' + screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SetForTermsFrequency.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Indexer.py; read x' + screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Keys.py; read x' + screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Decoder.py; read x' + screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Decoder.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Bitcoin.py; read x' + screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Bitcoin.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Phone.py; read x' + screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Phone.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Release.py; read x' + screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Cve.py; read x' + screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cve.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./WebStats.py; read x' + screen -S "Script_AIL" -X screen -t "WebStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./WebStats.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./ModuleStats.py; read x' + screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModuleStats.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./SQLInjectionDetection.py; read x' + screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SQLInjectionDetection.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./LibInjection.py; read x' + screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./LibInjection.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./alertHandler.py; read x' + screen -S "Script_AIL" -X screen -t "alertHandler" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./alertHandler.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./MISP_The_Hive_feeder.py; read x' + screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Tags.py; read x' + screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tags.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./SentimentAnalysis.py; read x' + screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SentimentAnalysis.py; read x" sleep 0.1 - screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./submit_paste.py; read x' + screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./submit_paste.py; read x" } @@ -235,7 +235,7 @@ function launching_crawler { sleep 0.1 for ((i=first_port;i<=last_port;i++)); do - screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./Crawler.py onion '$i'; read x' + screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Crawler.py onion $i; read x" sleep 0.1 done @@ -369,9 +369,9 @@ function launch_feeder { screen -dmS "Feeder_Pystemon" sleep 0.1 echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT - screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; '${ENV_PY} ' ./feeder/pystemon-feeder.py; read x' + screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./feeder/pystemon-feeder.py; read x" sleep 0.1 - screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; '${ENV_PY} ' ./pystemon.py; read x' + screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c "cd ${AIL_HOME}/../pystemon; ${ENV_PY} ./pystemon.py; read x" else echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT fi From f82c899e7bed6581e8048f672ab587dc14514c23 Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 22 Nov 2018 15:48:12 +0900 Subject: [PATCH 07/16] fix: [bash] fixed log_subscriber path --- bin/LAUNCH.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index 19a96cc0..c4aba1cc 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -30,6 +30,7 @@ export AIL_BIN=${AIL_HOME}/bin/ export AIL_FLASK=${AIL_HOME}/var/www/ export AIL_REDIS=${AIL_HOME}/redis/src/ export AIL_ARDB=${AIL_HOME}/ardb/src/ +export AIL_VENV=${AIL_HOME}/AILENV/ export PATH=$AIL_HOME:$PATH export PATH=$AIL_REDIS:$PATH @@ -107,9 +108,9 @@ function launching_logs { screen -dmS "Logging_AIL" sleep 0.1 echo -e $GREEN"\t* Launching logging process"$DEFAULT - screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c "cd ${AIL_BIN}; log_subscriber -p 6380 -c Queuing -l ../logs/; read x" + screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Queuing -l ../logs/; read x" sleep 0.1 - screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; log_subscriber -p 6380 -c Script -l ../logs/; read x" + screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Script -l ../logs/; read x" } function launching_queues { From b60fd55efed595026ce892aced821415c7daed7c Mon Sep 17 00:00:00 2001 From: Steve Clement Date: Thu, 22 Nov 2018 16:16:31 +0900 Subject: [PATCH 08/16] chg: [bash] Small PATH precedence issue. --- bin/LAUNCH.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/bin/LAUNCH.sh b/bin/LAUNCH.sh index c4aba1cc..549c0425 100755 --- a/bin/LAUNCH.sh +++ b/bin/LAUNCH.sh @@ -11,7 +11,7 @@ CYAN="\\033[1;36m" # Getting CWD where bash script resides DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd |sed 's/bin//' )" -AIL_HOME="${DIR}" +export AIL_HOME="${DIR}" cd ${AIL_HOME} @@ -32,6 +32,7 @@ export AIL_REDIS=${AIL_HOME}/redis/src/ export AIL_ARDB=${AIL_HOME}/ardb/src/ export AIL_VENV=${AIL_HOME}/AILENV/ +export PATH=$AIL_VENV/bin:$PATH export PATH=$AIL_HOME:$PATH export PATH=$AIL_REDIS:$PATH export PATH=$AIL_ARDB:$PATH From 9c0a65cad87819fa171d0f6f4313d43fe420f686 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thirion=20Aur=C3=A9lien?= Date: Thu, 29 Nov 2018 08:58:24 +0100 Subject: [PATCH 09/16] fix: [requirement] force redis version --- pip3_packages_requirement.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pip3_packages_requirement.txt b/pip3_packages_requirement.txt index dd447d5c..12ef09ff 100644 --- a/pip3_packages_requirement.txt +++ b/pip3_packages_requirement.txt @@ -2,7 +2,7 @@ pymisp thehive4py -redis +redis==2.10.6 #filemagic conflict with magic crcmod mmh3 From 423ddb6a3429fd623e9cc6f892fa15c147dc7b4b Mon Sep 17 00:00:00 2001 From: Alexandre Dulaunoy Date: Mon, 3 Dec 2018 17:34:11 +0100 Subject: [PATCH 10/16] training added --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 9b447bac..fbcb5363 100644 --- a/README.md +++ b/README.md @@ -155,6 +155,11 @@ Eventually you can browse the status of the AIL framework website at the followi http://localhost:7000/ ``` +Training +-------- + +CIRCL organises training on how to use or extend the AIL framework. The next training will be [Thursday, 20 Dec](https://en.xing-events.com/ZEQWMLJ.html) in Luxembourg. + HOWTO ----- From 08661eda4e3c34c285d5d53ee95ffdf54cd1cc78 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Thirion=20Aur=C3=A9lien?= Date: Wed, 5 Dec 2018 08:57:17 +0100 Subject: [PATCH 11/16] fix: [requirements] remove duplicate entry, #296 --- pip3_packages_requirement.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/pip3_packages_requirement.txt b/pip3_packages_requirement.txt index 12ef09ff..3991e158 100644 --- a/pip3_packages_requirement.txt +++ b/pip3_packages_requirement.txt @@ -13,7 +13,6 @@ zmq langid #Essential -redis pyzmq dnspython logbook From 3ff991995e61689e15900abc3e944cc3e0146fb9 Mon Sep 17 00:00:00 2001 From: Mike Eriksson Date: Sun, 9 Dec 2018 12:29:31 +0000 Subject: [PATCH 12/16] Changed the Dockerfile so that it starts with installing any and all outstanding updates for Ubuntu --- Dockerfile | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 71318ba4..340e5014 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,14 +1,19 @@ FROM ubuntu:16.04 -RUN mkdir /opt/AIL && apt-get update -y \ - && apt-get install git python-dev build-essential \ - libffi-dev libssl-dev libfuzzy-dev wget sudo -y +# Make sure that all updates are in place +RUN apt-get clean && apt-get update -y && apt-get upgrade -y \ + && apt-get dist-upgrade -y && apt-get autoremove -y + +# Install needed packages +RUN apt-get install git python-dev build-essential \ + libffi-dev libssl-dev libfuzzy-dev wget sudo -y # Adding sudo command RUN useradd -m docker && echo "docker:docker" | chpasswd && adduser docker sudo RUN echo "root ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers # Installing AIL dependencies +RUN mkdir /opt/AIL ADD . /opt/AIL WORKDIR /opt/AIL RUN ./installing_deps.sh From 3051c7280f234034f2e9791660edf9df389ed2ca Mon Sep 17 00:00:00 2001 From: Terrtia Date: Tue, 11 Dec 2018 16:55:47 +0100 Subject: [PATCH 13/16] fix: [install script] use virtualenv package --- installing_deps.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/installing_deps.sh b/installing_deps.sh index 6110d534..484ca770 100755 --- a/installing_deps.sh +++ b/installing_deps.sh @@ -5,7 +5,7 @@ set -x sudo apt-get update -sudo apt-get install python3-pip python-virtualenv python3-dev python3-tk libfreetype6-dev \ +sudo apt-get install python3-pip virtualenv python3-dev python3-tk libfreetype6-dev \ screen g++ python-tk unzip libsnappy-dev cmake -y #optional tor install From 4e08aaa80f3d7364f67bdfd7c3dab4d1c2711a78 Mon Sep 17 00:00:00 2001 From: Terrtia Date: Mon, 17 Dec 2018 15:58:48 +0100 Subject: [PATCH 14/16] chg: [UI Crawled Domains] show crawled domains by date range (options: domains tags, up domains, down domains) todo: Shows domain history --- bin/packages/HiddenServices.py | 8 +- .../hiddenServices/Flask_hiddenServices.py | 100 +++++- .../hiddenServices/templates/domains.html | 291 ++++++++++++++++++ .../templates/hiddenServices.html | 177 ++++++++--- 4 files changed, 530 insertions(+), 46 deletions(-) create mode 100644 var/www/modules/hiddenServices/templates/domains.html diff --git a/bin/packages/HiddenServices.py b/bin/packages/HiddenServices.py index d515c955..92e2e57c 100755 --- a/bin/packages/HiddenServices.py +++ b/bin/packages/HiddenServices.py @@ -81,8 +81,12 @@ class HiddenServices(object): return '' return origin_paste.replace(self.paste_directory+'/', '') - def get_domain_tags(self): - return self.tags + def get_domain_tags(self, update=False): + if not update: + return self.tags + else: + self.get_last_crawled_pastes() + return self.tags def update_domain_tags(self, children): p_tags = self.r_serv_metadata.smembers('tag:'+children) diff --git a/var/www/modules/hiddenServices/Flask_hiddenServices.py b/var/www/modules/hiddenServices/Flask_hiddenServices.py index 47ea56f1..4421505d 100644 --- a/var/www/modules/hiddenServices/Flask_hiddenServices.py +++ b/var/www/modules/hiddenServices/Flask_hiddenServices.py @@ -8,7 +8,7 @@ import redis import datetime import sys import os -from flask import Flask, render_template, jsonify, request, Blueprint +from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for from Date import Date from HiddenServices import HiddenServices @@ -40,6 +40,16 @@ def get_date_range(num_day): return list(reversed(date_list)) +def substract_date(date_from, date_to): + date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8])) + date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8])) + delta = date_to - date_from # timedelta + l_date = [] + for i in range(delta.days + 1): + date = date_from + datetime.timedelta(i) + l_date.append( date.strftime('%Y%m%d') ) + return l_date + def unpack_paste_tags(p_tags): l_tags = [] for tag in p_tags: @@ -94,6 +104,94 @@ def hiddenServices_page(): return render_template("hiddenServices.html", last_onions=list_onion, statDomains=statDomains) +@hiddenServices.route("/hiddenServices/get_onions_by_daterange", methods=['POST']) +def get_onions_by_daterange(): + date_from = request.form.get('date_from') + date_to = request.form.get('date_to') + domains_up = request.form.get('domains_up') + domains_down = request.form.get('domains_down') + domains_tags = request.form.get('domains_tags') + + return redirect(url_for('hiddenServices.show_domains_by_daterange', date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down, domains_tags=domains_tags)) + +@hiddenServices.route("/hiddenServices/show_domains_by_daterange", methods=['GET']) +def show_domains_by_daterange(): + date_from = request.args.get('date_from') + date_to = request.args.get('date_to') + domains_up = request.args.get('domains_up') + domains_down = request.args.get('domains_down') + domains_tags = request.args.get('domains_tags') + + date_range = [] + if date_from is not None and date_to is not None: + #change format + try: + if len(date_from) != 8: + date_from = date_from[0:4] + date_from[5:7] + date_from[8:10] + date_to = date_to[0:4] + date_to[5:7] + date_to[8:10] + date_range = substract_date(date_from, date_to) + except: + pass + + if not date_range: + date_range.append(datetime.date.today().strftime("%Y%m%d")) + date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8] + date_to = date_from + + else: + date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8] + date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8] + + domains_by_day = {} + domain_metadata = {} + for date in date_range: + if domains_up: + domains_up = True + domains_by_day[date] = list(r_serv_onion.smembers('onion_up:{}'.format(date))) + for domain in domains_by_day[date]: + h = HiddenServices(domain, 'onion') + domain_metadata[domain] = {} + if domains_tags: + domains_tags = True + domain_metadata[domain]['tags'] = h.get_domain_tags(update=True) + + domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check') + if domain_metadata[domain]['last_check'] is None: + domain_metadata[domain]['last_check'] = '********' + domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen') + if domain_metadata[domain]['first_seen'] is None: + domain_metadata[domain]['first_seen'] = '********' + domain_metadata[domain]['status_text'] = 'UP' + domain_metadata[domain]['status_color'] = 'Green' + domain_metadata[domain]['status_icon'] = 'fa-check-circle' + + if domains_down: + domains_down = True + domains_by_day_down = list(r_serv_onion.smembers('onion_down:{}'.format(date))) + if domains_up: + domains_by_day[date].extend(domains_by_day_down) + else: + domains_by_day[date] = domains_by_day_down + for domain in domains_by_day_down: + #h = HiddenServices(onion_domain, 'onion') + domain_metadata[domain] = {} + #domain_metadata[domain]['tags'] = h.get_domain_tags() + + domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check') + if domain_metadata[domain]['last_check'] is None: + domain_metadata[domain]['last_check'] = '********' + domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen') + if domain_metadata[domain]['first_seen'] is None: + domain_metadata[domain]['first_seen'] = '********' + + domain_metadata[domain]['status_text'] = 'DOWN' + domain_metadata[domain]['status_color'] = 'Red' + domain_metadata[domain]['status_icon'] = 'fa-times-circle' + + return render_template("domains.html", date_range=date_range, domains_by_day=domains_by_day, domain_metadata=domain_metadata, + date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down, + domains_tags=domains_tags, bootstrap_label=bootstrap_label) + @hiddenServices.route("/hiddenServices/onion_domain", methods=['GET']) def onion_domain(): onion_domain = request.args.get('onion_domain') diff --git a/var/www/modules/hiddenServices/templates/domains.html b/var/www/modules/hiddenServices/templates/domains.html new file mode 100644 index 00000000..136291b1 --- /dev/null +++ b/var/www/modules/hiddenServices/templates/domains.html @@ -0,0 +1,291 @@ + + + + + + + + Hidden Service - AIL + + + + + + + + + + + + + + + + + + + + + + {% include 'navbar.html' %} + +
+ +
+
+ + {% for date in date_range %} + {% if domains_by_day[date]%} +
+
+

{{'{}/{}/{}'.format(date[0:4], date[4:6], date[6:8])}}

+
+
+ + + + + + + + + + + + {% for domain in domains_by_day[date] %} + + + + + + + {% endfor %} + +
DomainFirst SeenLast CheckStatus
+ {{ domain }} +
+ {% for tag in domain_metadata[domain]['tags'] %} + + {{ tag }} {{ domain_metadata[domain]['tags'][tag] }} + + {% endfor %} +
+ 		{{'{}/{}/{}'.format(domain_metadata[domain]['first_seen'][0:4], domain_metadata[domain]['first_seen'][4:6], domain_metadata[domain]['first_seen'][6:8])}}{{'{}/{}/{}'.format(domain_metadata[domain]['last_check'][0:4], domain_metadata[domain]['last_check'][4:6], domain_metadata[domain]['last_check'][6:8])}}
+ + {{domain_metadata[domain]['status_text']}} +
+
+
+
+
+ {% endif %} + {% endfor %} + + +
+ +
+
+
+ Select domains by date range : +
+
+
+
+
+
+ + +
+
+ + +
+
+ +
+
+
+
+ +
+
+ +
+
+
+ + +
+
+ +
+ +
+ +
+ + + + + + + + diff --git a/var/www/modules/hiddenServices/templates/hiddenServices.html b/var/www/modules/hiddenServices/templates/hiddenServices.html index 59aeb2ae..b781a26a 100644 --- a/var/www/modules/hiddenServices/templates/hiddenServices.html +++ b/var/www/modules/hiddenServices/templates/hiddenServices.html @@ -12,11 +12,17 @@ + + +