AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-30 01:37:17 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' into pre_update

Browse source
This commit is contained in:
Terrtia 2018-12-19 10:01:52 +01:00
commit 2560260ce4
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
16 changed files with 666 additions and 118 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
Dockerfile
View file

@ -1,14 +1,19 @@
FROM ubuntu:16.04 FROM ubuntu:16.04
RUN mkdir /opt/AIL && apt-get update -y \ # Make sure that all updates are in place
&& apt-get install git python-dev build-essential \ RUN apt-get clean && apt-get update -y && apt-get upgrade -y \
libffi-dev libssl-dev libfuzzy-dev wget sudo -y && apt-get dist-upgrade -y && apt-get autoremove -y
# Install needed packages
RUN apt-get install git python-dev build-essential \
libffi-dev libssl-dev libfuzzy-dev wget sudo -y
# Adding sudo command # Adding sudo command
RUN useradd -m docker && echo "docker:docker" | chpasswd && adduser docker sudo RUN useradd -m docker && echo "docker:docker" | chpasswd && adduser docker sudo
RUN echo "root ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers RUN echo "root ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
# Installing AIL dependencies # Installing AIL dependencies
RUN mkdir /opt/AIL
ADD . /opt/AIL ADD . /opt/AIL
WORKDIR /opt/AIL WORKDIR /opt/AIL
RUN ./installing_deps.sh RUN ./installing_deps.sh

2
HOWTO.md
View file

@ -27,7 +27,7 @@ Feed data to AIL:
4. Edit your configuration file ```bin/packages/config.cfg``` and modify the pystemonpath path accordingly 4. Edit your configuration file ```bin/packages/config.cfg``` and modify the pystemonpath path accordingly
5. Launch pystemon-feeder ``` ./pystemon-feeder.py ``` 5. Launch pystemon-feeder ``` ./bin/feeder/pystemon-feeder.py ```
How to create a new module How to create a new module

7
README.md
View file

@ -70,8 +70,6 @@ Type these command lines for a fully automated installation and start AIL framew
git clone https://github.com/CIRCL/AIL-framework.git git clone https://github.com/CIRCL/AIL-framework.git
cd AIL-framework cd AIL-framework
./installing_deps.sh ./installing_deps.sh
cd var/www/
./update_thirdparty.sh
cd ~/AIL-framework/ cd ~/AIL-framework/
. ./AILENV/bin/activate . ./AILENV/bin/activate
cd bin/ cd bin/
@ -155,6 +153,11 @@ Eventually you can browse the status of the AIL framework website at the followi
http://localhost:7000/ http://localhost:7000/
``` ```
Training
--------
CIRCL organises training on how to use or extend the AIL framework. The next training will be [Thursday, 20 Dec](https://en.xing-events.com/ZEQWMLJ.html) in Luxembourg.
HOWTO HOWTO
----- -----

30
bin/Crawler.py
View file

@ -27,17 +27,27 @@ def crawl_onion(url, domain, date, date_month, message):
if super_father is None: if super_father is None:
super_father=paste super_father=paste
try: retry = True
r = requests.get(splash_url , timeout=30.0) nb_retry = 0
except Exception: while retry:
# TODO: relaunch docker or send error message try:
r = requests.get(splash_url , timeout=30.0)
retry = False
except Exception:
# TODO: relaunch docker or send error message
nb_retry += 1
on_error_send_message_back_in_queue(type_hidden_service, domain, message) if nb_retry == 30:
publisher.error('{} SPASH DOWN'.format(splash_url)) on_error_send_message_back_in_queue(type_hidden_service, domain, message)
print('--------------------------------------') publisher.error('{} SPASH DOWN'.format(splash_url))
print(' \033[91m DOCKER SPLASH DOWN\033[0m') print('--------------------------------------')
print(' {} DOWN'.format(splash_url)) print(' \033[91m DOCKER SPLASH DOWN\033[0m')
exit(1) print(' {} DOWN'.format(splash_url))
exit(1)
print(' \033[91m DOCKER SPLASH NOT AVAILABLE\033[0m')
print(' Retry({}) in 10 seconds'.format(nb_retry))
time.sleep(10)
if r.status_code == 200: if r.status_code == 200:
process = subprocess.Popen(["python", './torcrawler/tor_crawler.py', splash_url, type_hidden_service, url, domain, paste, super_father], process = subprocess.Popen(["python", './torcrawler/tor_crawler.py', splash_url, type_hidden_service, url, domain, paste, super_father],

116
bin/LAUNCH.sh
View file

@ -9,12 +9,30 @@ WHITE="\\033[0;02m"
YELLOW="\\033[1;33m" YELLOW="\\033[1;33m"
CYAN="\\033[1;36m" CYAN="\\033[1;36m"
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; # Getting CWD where bash script resides
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd |sed 's/bin//' )"
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; export AIL_HOME="${DIR}"
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
cd ${AIL_HOME}
if [ -e "${DIR}/AILENV/bin/python" ]; then
echo "AIL-framework virtualenv seems to exist, good"
ENV_PY="${DIR}/AILENV/bin/python"
else
echo "Please make sure you have a AIL-framework environment, au revoir"
exit 1
fi
# redis-server is bundled during install
## [ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1
export AIL_BIN=${AIL_HOME}/bin/
export AIL_FLASK=${AIL_HOME}/var/www/
export AIL_REDIS=${AIL_HOME}/redis/src/
export AIL_ARDB=${AIL_HOME}/ardb/src/
export AIL_VENV=${AIL_HOME}/AILENV/
export PATH=$AIL_VENV/bin:$PATH
export PATH=$AIL_HOME:$PATH export PATH=$AIL_HOME:$PATH
export PATH=$AIL_REDIS:$PATH export PATH=$AIL_REDIS:$PATH
export PATH=$AIL_ARDB:$PATH export PATH=$AIL_ARDB:$PATH
@ -76,9 +94,9 @@ function launching_logs {
screen -dmS "Logging_AIL" screen -dmS "Logging_AIL"
sleep 0.1 sleep 0.1
echo -e $GREEN"\t* Launching logging process"$DEFAULT echo -e $GREEN"\t* Launching logging process"$DEFAULT
screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Queuing -l ../logs/; read x' screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Queuing -l ../logs/; read x"
sleep 0.1 sleep 0.1
screen -S "Logging_AIL" -X screen -t "LogScript" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Script -l ../logs/; read x' screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Script -l ../logs/; read x"
} }
function launching_queues { function launching_queues {
@ -86,16 +104,16 @@ function launching_queues {
sleep 0.1 sleep 0.1
echo -e $GREEN"\t* Launching all the queues"$DEFAULT echo -e $GREEN"\t* Launching all the queues"$DEFAULT
screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; python3 launch_queues.py; read x' screen -S "Queue_AIL" -X screen -t "Queues" bash -c "cd ${AIL_BIN}; ${ENV_PY} launch_queues.py; read x"
} }
function checking_configuration { function checking_configuration {
bin_dir=${AIL_HOME}/bin bin_dir=${AIL_HOME}/bin
echo -e "\t* Checking configuration" echo -e "\t* Checking configuration"
if [ "$1" == "automatic" ]; then if [ "$1" == "automatic" ]; then
bash -c "python3 $bin_dir/Update-conf.py True" bash -c "${ENV_PY} $bin_dir/Update-conf.py True"
else else
bash -c "python3 $bin_dir/Update-conf.py False" bash -c "${ENV_PY} $bin_dir/Update-conf.py False"
fi fi
exitStatus=$? exitStatus=$?
@ -113,75 +131,75 @@ function launching_scripts {
sleep 0.1 sleep 0.1
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ./ModulesInformationV2.py -k 0 -c 1; read x' screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ./Mixer.py; read x' screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ./Global.py; read x' screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Global.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ./Duplicates.py; read x' screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ./Lines.py; read x' screen -S "Script_AIL" -X screen -t "Lines" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Lines.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ./DomClassifier.py; read x' screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./DomClassifier.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ./Categ.py; read x' screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Categ.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ./Tokenize.py; read x' screen -S "Script_AIL" -X screen -t "Tokenize" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tokenize.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ./CreditCards.py; read x' screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ./BankAccount.py; read x' screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ./Onion.py; read x' screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Onion.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ./Mail.py; read x' screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ./ApiKey.py; read x' screen -S "Script_AIL" -X screen -t "ApiKey" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ApiKey.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ./Web.py; read x' screen -S "Script_AIL" -X screen -t "Web" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Web.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ./Credential.py; read x' screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Credential.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ./Curve.py; read x' screen -S "Script_AIL" -X screen -t "Curve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Curve.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ./CurveManageTopSets.py; read x' screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CurveManageTopSets.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./RegexForTermsFrequency.py; read x' screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexForTermsFrequency.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./SetForTermsFrequency.py; read x' screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SetForTermsFrequency.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ./Indexer.py; read x' screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Keys.py; read x' screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ./Decoder.py; read x' screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Decoder.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ./Bitcoin.py; read x' screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Bitcoin.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ./Phone.py; read x' screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Phone.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ./Release.py; read x' screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ./Cve.py; read x' screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cve.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ./WebStats.py; read x' screen -S "Script_AIL" -X screen -t "WebStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./WebStats.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ./ModuleStats.py; read x' screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModuleStats.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ./SQLInjectionDetection.py; read x' screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SQLInjectionDetection.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ./LibInjection.py; read x' screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./LibInjection.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ./alertHandler.py; read x' screen -S "Script_AIL" -X screen -t "alertHandler" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./alertHandler.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ./MISP_The_Hive_feeder.py; read x' screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ./Tags.py; read x' screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tags.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ./SentimentAnalysis.py; read x' screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SentimentAnalysis.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ./submit_paste.py; read x' screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./submit_paste.py; read x"
} }
@ -204,7 +222,7 @@ function launching_crawler {
sleep 0.1 sleep 0.1
for ((i=first_port;i<=last_port;i++)); do for ((i=first_port;i<=last_port;i++)); do
screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ./Crawler.py onion '$i'; read x' screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Crawler.py onion $i; read x"
sleep 0.1 sleep 0.1
done done
@ -309,7 +327,7 @@ function launch_flask {
screen -dmS "Flask_AIL" screen -dmS "Flask_AIL"
sleep 0.1 sleep 0.1
echo -e $GREEN"\t* Launching Flask server"$DEFAULT echo -e $GREEN"\t* Launching Flask server"$DEFAULT
screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ./Flask_server.py; read x" screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ${ENV_PY} ./Flask_server.py; read x"
else else
echo -e $RED"\t* A Flask screen is already launched"$DEFAULT echo -e $RED"\t* A Flask screen is already launched"$DEFAULT
fi fi
@ -320,9 +338,9 @@ function launch_feeder {
screen -dmS "Feeder_Pystemon" screen -dmS "Feeder_Pystemon"
sleep 0.1 sleep 0.1
echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT
screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ./feeder/pystemon-feeder.py; read x' screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./feeder/pystemon-feeder.py; read x"
sleep 0.1 sleep 0.1
screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; python2 pystemon.py; read x' screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c "cd ${AIL_HOME}/../pystemon; ${ENV_PY} ./pystemon.py; read x"
else else
echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT
fi fi

2
bin/feeder/pystemon-feeder.py
View file

@ -61,10 +61,10 @@ topic = '102'
while True: while True:
time.sleep(base_sleeptime + sleep_inc) time.sleep(base_sleeptime + sleep_inc)
paste = r.lpop("pastes") paste = r.lpop("pastes")
print(paste)
if paste is None: if paste is None:
continue continue
try: try:
print(paste)
with open(pystemonpath+paste, 'rb') as f: #.read() with open(pystemonpath+paste, 'rb') as f: #.read()
messagedata = f.read() messagedata = f.read()
path_to_send = pastes_directory+paste path_to_send = pastes_directory+paste

2
bin/feeder/test-zmq.py
View file

@ -20,7 +20,7 @@ socket.connect ("tcp://crf.circl.lu:%s" % port)
# 102 Full pastes in raw base64(gz) # 102 Full pastes in raw base64(gz)
topicfilter = "102" topicfilter = "102"
socket.setsockopt(zmq.SUBSCRIBE, topicfilter) socket.setsockopt_string(zmq.SUBSCRIBE, topicfilter)
while True: while True:
message = socket.recv() message = socket.recv()

8
bin/packages/HiddenServices.py
View file

@ -81,8 +81,12 @@ class HiddenServices(object):
return '' return ''
return origin_paste.replace(self.paste_directory+'/', '') return origin_paste.replace(self.paste_directory+'/', '')
def get_domain_tags(self): def get_domain_tags(self, update=False):
return self.tags if not update:
return self.tags
else:
self.get_last_crawled_pastes()
return self.tags
def update_domain_tags(self, children): def update_domain_tags(self, children):
p_tags = self.r_serv_metadata.smembers('tag:'+children) p_tags = self.r_serv_metadata.smembers('tag:'+children)

3
bin/packages/config.cfg.sample
View file

@ -206,6 +206,9 @@ dns = 8.8.8.8
[Mail] [Mail]
dns = 8.8.8.8 dns = 8.8.8.8
[Web]
dns = 149.13.33.69
# Indexer configuration # Indexer configuration
[Indexer] [Indexer]
type = whoosh type = whoosh

13
bin/packages/lib_refine.py
View file

@ -1,6 +1,8 @@
#!/usr/bin/python3 #!/usr/bin/python3
import re import re
import os
import configparser
import dns.resolver import dns.resolver
from pubsublogger import publisher from pubsublogger import publisher
@ -101,11 +103,20 @@ def checking_MX_record(r_serv, adress_set, addr_dns):
def checking_A_record(r_serv, domains_set): def checking_A_record(r_serv, domains_set):
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
Or activate the virtualenv.')
cfg = configparser.ConfigParser()
cfg.read(configfile)
dns_server = cfg.get("Web", "dns")
score = 0 score = 0
num = len(domains_set) num = len(domains_set)
WalidA = set([]) WalidA = set([])
resolver = dns.resolver.Resolver() resolver = dns.resolver.Resolver()
resolver.nameservers = ['149.13.33.69'] resolver.nameservers = [dns_server]
resolver.timeout = 5 resolver.timeout = 5
resolver.lifetime = 2 resolver.lifetime = 2

17
bin/torcrawler/launch_splash_crawler.sh
View file

@ -32,6 +32,21 @@ if [ -z "${p}" ] || [ -z "${f}" ] || [ -z "${n}" ]; then
usage; usage;
fi fi
RED="\\033[1;31m"
DEFAULT="\\033[0;39m"
GREEN="\\033[1;32m"
WHITE="\\033[0;02m"
if [ ! -d "${f}" ]; then
printf "$RED\n Error -f, proxy-profiles directory: $WHITE${f}$RED not found\n$DEFAULT Please check if you enter the correct path\n"
exit 1
fi
if [ ! -f "${f}default.ini" ]; then
printf "$RED\n Error -f, proxy configuration file:$WHITE default.ini$RED not found\n$DEFAULT Please check if you enter the correct path\n"
exit 1
fi
screen -dmS "Docker_Splash" screen -dmS "Docker_Splash"
sleep 0.1 sleep 0.1
@ -39,5 +54,5 @@ for ((i=0;i<=$((${n} - 1));i++)); do
port_number=$((${p} + $i)) port_number=$((${p} + $i))
screen -S "Docker_Splash" -X screen -t "docker_splash:$port_number" bash -c 'sudo docker run -p '$port_number':8050 --cpus=1 --memory=4.5G -v '$f':/etc/splash/proxy-profiles/ --net="bridge" scrapinghub/splash; read x' screen -S "Docker_Splash" -X screen -t "docker_splash:$port_number" bash -c 'sudo docker run -p '$port_number':8050 --cpus=1 --memory=4.5G -v '$f':/etc/splash/proxy-profiles/ --net="bridge" scrapinghub/splash; read x'
sleep 0.1 sleep 0.1
echo " Splash server launched on port $port_number" printf "$GREEN Splash server launched on port $port_number$DEFAULT\n"
done done

2
installing_deps.sh
View file

@ -5,7 +5,7 @@ set -x
sudo apt-get update sudo apt-get update
sudo apt-get install python3-pip python-virtualenv python3-dev python3-tk libfreetype6-dev \ sudo apt-get install python3-pip virtualenv python3-dev python3-tk libfreetype6-dev \
screen g++ python-tk unzip libsnappy-dev cmake -y screen g++ python-tk unzip libsnappy-dev cmake -y
#optional tor install #optional tor install

3
pip3_packages_requirement.txt
View file

@ -2,7 +2,7 @@ pymisp
thehive4py thehive4py
redis redis==2.10.6
#filemagic conflict with magic #filemagic conflict with magic
crcmod crcmod
mmh3 mmh3
@ -13,7 +13,6 @@ zmq
langid langid
#Essential #Essential
redis
pyzmq pyzmq
dnspython dnspython
logbook logbook

100
var/www/modules/hiddenServices/Flask_hiddenServices.py
View file

@ -8,7 +8,7 @@ import redis
import datetime import datetime
import sys import sys
import os import os
from flask import Flask, render_template, jsonify, request, Blueprint from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
from Date import Date from Date import Date
from HiddenServices import HiddenServices from HiddenServices import HiddenServices
@ -40,6 +40,16 @@ def get_date_range(num_day):
return list(reversed(date_list)) return list(reversed(date_list))
def substract_date(date_from, date_to):
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
delta = date_to - date_from # timedelta
l_date = []
for i in range(delta.days + 1):
date = date_from + datetime.timedelta(i)
l_date.append( date.strftime('%Y%m%d') )
return l_date
def unpack_paste_tags(p_tags): def unpack_paste_tags(p_tags):
l_tags = [] l_tags = []
for tag in p_tags: for tag in p_tags:
@ -94,6 +104,94 @@ def hiddenServices_page():
return render_template("hiddenServices.html", last_onions=list_onion, statDomains=statDomains) return render_template("hiddenServices.html", last_onions=list_onion, statDomains=statDomains)
@hiddenServices.route("/hiddenServices/get_onions_by_daterange", methods=['POST'])
def get_onions_by_daterange():
date_from = request.form.get('date_from')
date_to = request.form.get('date_to')
domains_up = request.form.get('domains_up')
domains_down = request.form.get('domains_down')
domains_tags = request.form.get('domains_tags')
return redirect(url_for('hiddenServices.show_domains_by_daterange', date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down, domains_tags=domains_tags))
@hiddenServices.route("/hiddenServices/show_domains_by_daterange", methods=['GET'])
def show_domains_by_daterange():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
domains_up = request.args.get('domains_up')
domains_down = request.args.get('domains_down')
domains_tags = request.args.get('domains_tags')
date_range = []
if date_from is not None and date_to is not None:
#change format
try:
if len(date_from) != 8:
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
date_range = substract_date(date_from, date_to)
except:
pass
if not date_range:
date_range.append(datetime.date.today().strftime("%Y%m%d"))
date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8]
date_to = date_from
else:
date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
domains_by_day = {}
domain_metadata = {}
for date in date_range:
if domains_up:
domains_up = True
domains_by_day[date] = list(r_serv_onion.smembers('onion_up:{}'.format(date)))
for domain in domains_by_day[date]:
h = HiddenServices(domain, 'onion')
domain_metadata[domain] = {}
if domains_tags:
domains_tags = True
domain_metadata[domain]['tags'] = h.get_domain_tags(update=True)
domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check')
if domain_metadata[domain]['last_check'] is None:
domain_metadata[domain]['last_check'] = '********'
domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen')
if domain_metadata[domain]['first_seen'] is None:
domain_metadata[domain]['first_seen'] = '********'
domain_metadata[domain]['status_text'] = 'UP'
domain_metadata[domain]['status_color'] = 'Green'
domain_metadata[domain]['status_icon'] = 'fa-check-circle'
if domains_down:
domains_down = True
domains_by_day_down = list(r_serv_onion.smembers('onion_down:{}'.format(date)))
if domains_up:
domains_by_day[date].extend(domains_by_day_down)
else:
domains_by_day[date] = domains_by_day_down
for domain in domains_by_day_down:
#h = HiddenServices(onion_domain, 'onion')
domain_metadata[domain] = {}
#domain_metadata[domain]['tags'] = h.get_domain_tags()
domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check')
if domain_metadata[domain]['last_check'] is None:
domain_metadata[domain]['last_check'] = '********'
domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen')
if domain_metadata[domain]['first_seen'] is None:
domain_metadata[domain]['first_seen'] = '********'
domain_metadata[domain]['status_text'] = 'DOWN'
domain_metadata[domain]['status_color'] = 'Red'
domain_metadata[domain]['status_icon'] = 'fa-times-circle'
return render_template("domains.html", date_range=date_range, domains_by_day=domains_by_day, domain_metadata=domain_metadata,
date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down,
domains_tags=domains_tags, bootstrap_label=bootstrap_label)
@hiddenServices.route("/hiddenServices/onion_domain", methods=['GET']) @hiddenServices.route("/hiddenServices/onion_domain", methods=['GET'])
def onion_domain(): def onion_domain():
onion_domain = request.args.get('onion_domain') onion_domain = request.args.get('onion_domain')

291
var/www/modules/hiddenServices/templates/domains.html Normal file
View file

@ -0,0 +1,291 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Hidden Service - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<style>
.red_table thead{
background: #d91f2d;
color: #fff;
}
.input-group .form-control {
position: unset;
}
</style>
</head>
<body>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-md-8">
{% for date in date_range %}
{% if domains_by_day[date]%}
<div class="panel panel-info" style="text-align:center;">
<div class="panel-heading">
<h3 style="text-align:center;">{{'{}/{}/{}'.format(date[0:4], date[4:6], date[6:8])}}</h3>
</div>
</div>
<table id="table_{{date}}" class="red_table table table-striped table-bordered">
<thead>
<tr>
<th>Domain</th>
<th>First Seen</th>
<th>Last Check</th>
<th>Status</th>
</tr>
</thead>
<tbody>
{% for domain in domains_by_day[date] %}
<tr>
<td>
<a target="_blank" href="{{ url_for('hiddenServices.onion_domain') }}?onion_domain={{ domain }}">{{ domain }}</a>
<div>
{% for tag in domain_metadata[domain]['tags'] %}
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }} <i>{{ domain_metadata[domain]['tags'][tag] }}</i></span>
</a>
{% endfor %}
</div>
</td>
<td>{{'{}/{}/{}'.format(domain_metadata[domain]['first_seen'][0:4], domain_metadata[domain]['first_seen'][4:6], domain_metadata[domain]['first_seen'][6:8])}}</td>
<td>{{'{}/{}/{}'.format(domain_metadata[domain]['last_check'][0:4], domain_metadata[domain]['last_check'][4:6], domain_metadata[domain]['last_check'][6:8])}}</td>
<td><div style="color:{{domain_metadata[domain]['status_color']}}; display:inline-block">
<i class="fa {{domain_metadata[domain]['status_icon']}} fa-2x"></i>
{{domain_metadata[domain]['status_text']}}
</div>
</td>
</tr>
{% endfor %}
</tbody>
</table>
<br>
<br>
<br>
{% endif %}
{% endfor %}
</div>
<div class="col-md-4">
<div class="panel panel-info" style="text-align:center;">
<div class="panel-heading">
Select domains by date range :
<br>
<br>
<form action="{{ url_for('hiddenServices.get_onions_by_daterange') }}" id="hash_selector_form" method='post'>
<div class="row">
<div class="col-md-6">
<div class="input-group">
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
<input class="form-control" id="date-range-from" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from">
</div>
<div class="input-group">
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
<input class="form-control" id="date-range-to" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to">
</div>
<div class="checkbox">
<label>
<input type="checkbox" name="domains_tags" value="True" {% if domains_tags %}checked{% endif %}>
<div style="color:#286090; display:inline-block">
Domains Tags <i class="fa fa-tags"></i>
</div>
</label>
</div>
</div>
<div class="col-md-6">
<div class="checkbox">
<label>
<input type="checkbox" name="domains_up" value="True" {% if domains_up %}checked{% endif %}>
<div style="color:Green; display:inline-block">
Domains UP <i class="fa fa-check-circle"></i>
</div>
</label>
</div>
<div class="checkbox">
<label>
<input type="checkbox" name="domains_down" value="True" {% if domains_down %}checked{% endif %}>
<div style="color:Red; display:inline-block">
Domains DOWN <i class="fa fa-times-circle"></i>
</div>
</label>
</div>
</div>
</div>
<button class="btn btn-primary" style="text-align:center;">
<i class="fa fa-eye-slash"></i> Show Onions
</button>
<form>
</div>
</div>
</div>
</div>
</div>
<!-- /#page-wrapper -->
<script>
var all_graph = {};
$(document).ready(function(){
activePage = "page-hiddenServices"
$("#"+activePage).addClass("active");
$('#date-range-from').dateRangePicker({
separator : ' to ',
getValue: function()
{
if ($('#date-range-from').val() && $('#date-range-to').val() )
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
else
return '';
},
setValue: function(s,s1,s2)
{
$('#date-range-from').val(s1);
$('#date-range-to').val(s2);
}
});
$('#date-range-to').dateRangePicker({
separator : ' to ',
getValue: function()
{
if ($('#date-range-from').val() && $('#date-range-to').val() )
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
else
return '';
},
setValue: function(s,s1,s2)
{
$('#date-range-from').val(s1);
$('#date-range-to').val(s2);
}
});
{% for date in date_range %}
$('#table_{{date}}').DataTable({
"aLengthMenu": [[5, 15, 30, -1], [5, 15, 30, "All"]],
"iDisplayLength": 15,
"order": [[ 0, "asc" ]]
});
{% endfor %}
//all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hiddenServices.domain_crawled_7days_json') }}?type=onion");
});
/*$(window).on("resize", function() {
all_graph.onResize();
});*/
</script>
<script>
/*
all_graph.onResize = function () {
var aspect = 1000 / 500, all_graph = $("#graph_div");
var targetWidth = all_graph.parent().width();
all_graph.attr("width", targetWidth);
all_graph.attr("height", targetWidth / aspect);
}
window.all_graph = all_graph;
function create_line_chart(id, url){
var width = 650;
var height = Math.round(width / 6);
var margin = {top: 20, right: 55, bottom: 50, left: 40};
var x = d3.scaleTime().range([0, width]);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x);
var yAxis = d3.axisLeft(y);
var parseTime = d3.timeParse("%Y-%m-%d");
var line = d3.line()
.x(function(d) {
return x(d.date);
})
.y(function(d) {
return y(d.value);
});
var svg_line = d3.select('#'+id).append('svg')
.attr("id", "graph_div")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append('g')
.attr('transform', "translate("+ margin.left +","+ margin.top +")");
var div = d3.select('body').append('div')
.attr('class', 'tooltip')
.style('opacity', 0);
//add div tooltip
d3.json(url)
.then(function(data){
data.forEach(function(d) {
d.date_label = d.date;
d.date = parseTime(d.date);
d.value = +d.value;
});
// fit the data
x.domain(d3.extent(data, function(d) { return d.date; }));
//x.domain(data.map(function (d) { return d.date; })); //E
y.domain([0, d3.max(data, function(d){ return d.value ; })]);
//line
svg_line.append("path")
.data([data])
.attr("class", "line_graph")
.attr("d", line);
// add X axis
svg_line.append("g")
.attr("transform", "translate(0," + height + ")")
.call(d3.axisBottom(x))
.selectAll("text")
.style("text-anchor", "end")
.attr("transform", "rotate(-45)" );
// Add the Y Axis
svg_line.append("g")
.call(d3.axisLeft(y));
//add a dot circle
svg_line.selectAll('dot')
.data(data).enter()
.append('circle')
.attr('r', 2)
.attr('cx', function(d) { return x(d.date); })
.attr('cy', function(d) { return y(d.value); })
.on('mouseover', function(d) {
div.transition().style('opacity', .9);
div.html('' + d.date_label+ '<br/>' + d.value).style('left', (d3.event.pageX) + 'px')
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
})
.on('mouseout', function(d)
{
div.transition().style('opacity', 0);
});
});
}
*/
</script>
</body>
</html>

177
var/www/modules/hiddenServices/templates/hiddenServices.html
View file

@ -12,11 +12,17 @@
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet"> <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet"> <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet"> <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet" type="text/css" />
<!-- JS --> <!-- JS -->
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script> <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script> <script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<style> <style>
.input-group .form-control {
position: unset;
}
div.tooltip { div.tooltip {
position: absolute; position: absolute;
text-align: center; text-align: center;
@ -84,50 +90,103 @@
</div> </div>
<div class="col-md-6"> <div class="col-md-6">
<div class="panel panel-info"> <div class="panel panel-info" style="text-align:center;">
<div class="panel-heading"> <div class="panel-heading">
<i class="fa fa-eye-slash"></i> Domains Crawled Today Select domains by date range :
</div> <br>
<br>
<form action="{{ url_for('hiddenServices.get_onions_by_daterange') }}" id="hash_selector_form" method='post'>
<div class="row">
<div class="col-md-6">
<div class="input-group">
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
<input class="form-control" id="date-range-from" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from">
</div>
<div class="input-group">
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
<input class="form-control" id="date-range-to" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to">
</div>
<div class="checkbox">
<label>
<input type="checkbox" name="domains_tags" value="True" checked>
<div style="color:#286090; display:inline-block">
Domains Tags <i class="fa fa-tags"></i>
</div>
</label>
</div>
</div>
<div class="col-md-6">
<div class="checkbox">
<label>
<input type="checkbox" name="domains_up" value="True" checked>
<div style="color:Green; display:inline-block">
Domains UP <i class="fa fa-check-circle"></i>
</div>
</label>
</div>
<div class="checkbox">
<label>
<input type="checkbox" name="domains_down" value="True">
<div style="color:Red; display:inline-block">
Domains DOWN <i class="fa fa-times-circle"></i>
</div>
</label>
</div>
<table class="table table-hover table-striped"> </div>
<tbody> </div>
<tr> <button class="btn btn-primary" style="text-align:center;">
<td> <i class="fa fa-eye-slash"></i> Show Onions
<div style="color:Green; display:inline-block"> </button>
<i class="fa fa-check-circle fa-2x"></i> <form>
Domains UP </div>
</div> </div>
</td>
<td>
<div style="color:Green; display:inline-block"> <div class="panel panel-info">
{{ statDomains['domains_up'] }} <div class="panel-heading">
</div> <i class="fa fa-eye-slash"></i> Domains Crawled Today
</td> </div>
</tr>
<tr> <table class="table table-hover table-striped">
<td> <tbody>
<div style="color:Red; display:inline-block"> <tr>
<i class="fa fa-times-circle fa-2x"></i> <td>
Domains DOWN <div style="color:Green; display:inline-block">
</div> <i class="fa fa-check-circle fa-2x"></i>
</td> Domains UP
<td> </div>
<div style="color:Red; display:inline-block"> </td>
{{ statDomains['domains_down'] }} <td>
</div> <div style="color:Green; display:inline-block">
</td> {{ statDomains['domains_up'] }}
</tr> </div>
<tr> </td>
<td>Crawled Domains</td> </tr>
<td>{{ statDomains['total'] }}</td> <tr>
</tr> <td>
<tr> <div style="color:Red; display:inline-block">
<td>Domains in Queue</td> <i class="fa fa-times-circle fa-2x"></i>
<td>{{ statDomains['domains_queue'] }}</td> Domains DOWN
</tr> </div>
</tbody> </td>
</table> <td>
</div> <div style="color:Red; display:inline-block">
{{ statDomains['domains_down'] }}
</div>
</td>
</tr>
<tr>
<td>Crawled Domains</td>
<td>{{ statDomains['total'] }}</td>
</tr>
<tr>
<td>Domains in Queue</td>
<td>{{ statDomains['domains_queue'] }}</td>
</tr>
</tbody>
</table>
</div>
</div> </div>
</div> </div>
@ -140,6 +199,38 @@
$(document).ready(function(){ $(document).ready(function(){
activePage = "page-hiddenServices" activePage = "page-hiddenServices"
$("#"+activePage).addClass("active"); $("#"+activePage).addClass("active");
$('#date-range-from').dateRangePicker({
separator : ' to ',
getValue: function()
{
if ($('#date-range-from').val() && $('#date-range-to').val() )
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
else
return '';
},
setValue: function(s,s1,s2)
{
$('#date-range-from').val(s1);
$('#date-range-to').val(s2);
}
});
$('#date-range-to').dateRangePicker({
separator : ' to ',
getValue: function()
{
if ($('#date-range-from').val() && $('#date-range-to').val() )
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
else
return '';
},
setValue: function(s,s1,s2)
{
$('#date-range-from').val(s1);
$('#date-range-to').val(s2);
}
});
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hiddenServices.domain_crawled_7days_json') }}?type=onion"); all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hiddenServices.domain_crawled_7days_json') }}?type=onion");
}); });
$(window).on("resize", function() { $(window).on("resize", function() {