mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-22 22:27:17 +00:00
Merge branch 'master' into pre_update
This commit is contained in:
Terrtia
commit
2560260ce4
16 changed files with 666 additions and 118 deletions
11
Dockerfile
11
Dockerfile
|
|
@ -1,14 +1,19 @@
|
|||
FROM ubuntu:16.04
|
||||
|
||||
RUN mkdir /opt/AIL && apt-get update -y \
|
||||
&& apt-get install git python-dev build-essential \
|
||||
libffi-dev libssl-dev libfuzzy-dev wget sudo -y
|
||||
# Make sure that all updates are in place
|
||||
RUN apt-get clean && apt-get update -y && apt-get upgrade -y \
|
||||
&& apt-get dist-upgrade -y && apt-get autoremove -y
|
||||
|
||||
# Install needed packages
|
||||
RUN apt-get install git python-dev build-essential \
|
||||
libffi-dev libssl-dev libfuzzy-dev wget sudo -y
|
||||
|
||||
# Adding sudo command
|
||||
RUN useradd -m docker && echo "docker:docker" | chpasswd && adduser docker sudo
|
||||
RUN echo "root ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
|
||||
|
||||
# Installing AIL dependencies
|
||||
RUN mkdir /opt/AIL
|
||||
ADD . /opt/AIL
|
||||
WORKDIR /opt/AIL
|
||||
RUN ./installing_deps.sh
|
||||
|
|
|
|||
2
HOWTO.md
2
HOWTO.md
|
|
@ -27,7 +27,7 @@ Feed data to AIL:
|
|||
|
||||
4. Edit your configuration file ```bin/packages/config.cfg``` and modify the pystemonpath path accordingly
|
||||
|
||||
5. Launch pystemon-feeder ``` ./pystemon-feeder.py ```
|
||||
5. Launch pystemon-feeder ``` ./bin/feeder/pystemon-feeder.py ```
|
||||
|
||||
|
||||
How to create a new module
|
||||
|
|
|
|||
|
|
@ -70,8 +70,6 @@ Type these command lines for a fully automated installation and start AIL framew
|
|||
git clone https://github.com/CIRCL/AIL-framework.git
|
||||
cd AIL-framework
|
||||
./installing_deps.sh
|
||||
cd var/www/
|
||||
./update_thirdparty.sh
|
||||
cd ~/AIL-framework/
|
||||
. ./AILENV/bin/activate
|
||||
cd bin/
|
||||
|
|
@ -155,6 +153,11 @@ Eventually you can browse the status of the AIL framework website at the followi
|
|||
http://localhost:7000/
|
||||
```
|
||||
|
||||
Training
|
||||
--------
|
||||
|
||||
CIRCL organises training on how to use or extend the AIL framework. The next training will be [Thursday, 20 Dec](https://en.xing-events.com/ZEQWMLJ.html) in Luxembourg.
|
||||
|
||||
HOWTO
|
||||
-----
|
||||
|
||||
|
|
|
|||
|
|
@ -27,17 +27,27 @@ def crawl_onion(url, domain, date, date_month, message):
|
|||
if super_father is None:
|
||||
super_father=paste
|
||||
|
||||
try:
|
||||
r = requests.get(splash_url , timeout=30.0)
|
||||
except Exception:
|
||||
# TODO: relaunch docker or send error message
|
||||
retry = True
|
||||
nb_retry = 0
|
||||
while retry:
|
||||
try:
|
||||
r = requests.get(splash_url , timeout=30.0)
|
||||
retry = False
|
||||
except Exception:
|
||||
# TODO: relaunch docker or send error message
|
||||
nb_retry += 1
|
||||
|
||||
on_error_send_message_back_in_queue(type_hidden_service, domain, message)
|
||||
publisher.error('{} SPASH DOWN'.format(splash_url))
|
||||
print('--------------------------------------')
|
||||
print(' \033[91m DOCKER SPLASH DOWN\033[0m')
|
||||
print(' {} DOWN'.format(splash_url))
|
||||
exit(1)
|
||||
if nb_retry == 30:
|
||||
on_error_send_message_back_in_queue(type_hidden_service, domain, message)
|
||||
publisher.error('{} SPASH DOWN'.format(splash_url))
|
||||
print('--------------------------------------')
|
||||
print(' \033[91m DOCKER SPLASH DOWN\033[0m')
|
||||
print(' {} DOWN'.format(splash_url))
|
||||
exit(1)
|
||||
|
||||
print(' \033[91m DOCKER SPLASH NOT AVAILABLE\033[0m')
|
||||
print(' Retry({}) in 10 seconds'.format(nb_retry))
|
||||
time.sleep(10)
|
||||
|
||||
if r.status_code == 200:
|
||||
process = subprocess.Popen(["python", './torcrawler/tor_crawler.py', splash_url, type_hidden_service, url, domain, paste, super_father],
|
||||
|
|
|
|||
116
bin/LAUNCH.sh
116
bin/LAUNCH.sh
|
|
@ -9,12 +9,30 @@ WHITE="\\033[0;02m"
|
|||
YELLOW="\\033[1;33m"
|
||||
CYAN="\\033[1;36m"
|
||||
|
||||
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
|
||||
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
|
||||
# Getting CWD where bash script resides
|
||||
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd |sed 's/bin//' )"
|
||||
export AIL_HOME="${DIR}"
|
||||
|
||||
cd ${AIL_HOME}
|
||||
|
||||
if [ -e "${DIR}/AILENV/bin/python" ]; then
|
||||
echo "AIL-framework virtualenv seems to exist, good"
|
||||
ENV_PY="${DIR}/AILENV/bin/python"
|
||||
else
|
||||
echo "Please make sure you have a AIL-framework environment, au revoir"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# redis-server is bundled during install
|
||||
## [ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1
|
||||
|
||||
export AIL_BIN=${AIL_HOME}/bin/
|
||||
export AIL_FLASK=${AIL_HOME}/var/www/
|
||||
export AIL_REDIS=${AIL_HOME}/redis/src/
|
||||
export AIL_ARDB=${AIL_HOME}/ardb/src/
|
||||
export AIL_VENV=${AIL_HOME}/AILENV/
|
||||
|
||||
export PATH=$AIL_VENV/bin:$PATH
|
||||
export PATH=$AIL_HOME:$PATH
|
||||
export PATH=$AIL_REDIS:$PATH
|
||||
export PATH=$AIL_ARDB:$PATH
|
||||
|
|
@ -76,9 +94,9 @@ function launching_logs {
|
|||
screen -dmS "Logging_AIL"
|
||||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching logging process"$DEFAULT
|
||||
screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Queuing -l ../logs/; read x'
|
||||
screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Queuing -l ../logs/; read x"
|
||||
sleep 0.1
|
||||
screen -S "Logging_AIL" -X screen -t "LogScript" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Script -l ../logs/; read x'
|
||||
screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Script -l ../logs/; read x"
|
||||
}
|
||||
|
||||
function launching_queues {
|
||||
|
|
@ -86,16 +104,16 @@ function launching_queues {
|
|||
sleep 0.1
|
||||
|
||||
echo -e $GREEN"\t* Launching all the queues"$DEFAULT
|
||||
screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; python3 launch_queues.py; read x'
|
||||
screen -S "Queue_AIL" -X screen -t "Queues" bash -c "cd ${AIL_BIN}; ${ENV_PY} launch_queues.py; read x"
|
||||
}
|
||||
|
||||
function checking_configuration {
|
||||
bin_dir=${AIL_HOME}/bin
|
||||
echo -e "\t* Checking configuration"
|
||||
if [ "$1" == "automatic" ]; then
|
||||
bash -c "python3 $bin_dir/Update-conf.py True"
|
||||
bash -c "${ENV_PY} $bin_dir/Update-conf.py True"
|
||||
else
|
||||
bash -c "python3 $bin_dir/Update-conf.py False"
|
||||
bash -c "${ENV_PY} $bin_dir/Update-conf.py False"
|
||||
fi
|
||||
|
||||
exitStatus=$?
|
||||
|
|
@ -113,75 +131,75 @@ function launching_scripts {
|
|||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
|
||||
|
||||
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ./ModulesInformationV2.py -k 0 -c 1; read x'
|
||||
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ./Mixer.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ./Global.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Global.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ./Duplicates.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ./Lines.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Lines" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Lines.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ./DomClassifier.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./DomClassifier.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ./Categ.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Categ.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ./Tokenize.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Tokenize" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tokenize.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ./CreditCards.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ./BankAccount.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ./Onion.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Onion.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ./Mail.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ./ApiKey.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "ApiKey" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ApiKey.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ./Web.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Web" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Web.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ./Credential.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Credential.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ./Curve.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Curve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Curve.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ./CurveManageTopSets.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CurveManageTopSets.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./RegexForTermsFrequency.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexForTermsFrequency.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./SetForTermsFrequency.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SetForTermsFrequency.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ./Indexer.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Keys.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ./Decoder.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Decoder.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ./Bitcoin.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Bitcoin.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ./Phone.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Phone.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ./Release.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ./Cve.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cve.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ./WebStats.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "WebStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./WebStats.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ./ModuleStats.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModuleStats.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ./SQLInjectionDetection.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SQLInjectionDetection.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ./LibInjection.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./LibInjection.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ./alertHandler.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "alertHandler" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./alertHandler.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ./MISP_The_Hive_feeder.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ./Tags.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tags.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ./SentimentAnalysis.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SentimentAnalysis.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ./submit_paste.py; read x'
|
||||
screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./submit_paste.py; read x"
|
||||
|
||||
}
|
||||
|
||||
|
|
@ -204,7 +222,7 @@ function launching_crawler {
|
|||
sleep 0.1
|
||||
|
||||
for ((i=first_port;i<=last_port;i++)); do
|
||||
screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ./Crawler.py onion '$i'; read x'
|
||||
screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Crawler.py onion $i; read x"
|
||||
sleep 0.1
|
||||
done
|
||||
|
||||
|
|
@ -309,7 +327,7 @@ function launch_flask {
|
|||
screen -dmS "Flask_AIL"
|
||||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching Flask server"$DEFAULT
|
||||
screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ./Flask_server.py; read x"
|
||||
screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ${ENV_PY} ./Flask_server.py; read x"
|
||||
else
|
||||
echo -e $RED"\t* A Flask screen is already launched"$DEFAULT
|
||||
fi
|
||||
|
|
@ -320,9 +338,9 @@ function launch_feeder {
|
|||
screen -dmS "Feeder_Pystemon"
|
||||
sleep 0.1
|
||||
echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT
|
||||
screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ./feeder/pystemon-feeder.py; read x'
|
||||
screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./feeder/pystemon-feeder.py; read x"
|
||||
sleep 0.1
|
||||
screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; python2 pystemon.py; read x'
|
||||
screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c "cd ${AIL_HOME}/../pystemon; ${ENV_PY} ./pystemon.py; read x"
|
||||
else
|
||||
echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT
|
||||
fi
|
||||
|
|
|
|||
|
|
@ -61,10 +61,10 @@ topic = '102'
|
|||
while True:
|
||||
time.sleep(base_sleeptime + sleep_inc)
|
||||
paste = r.lpop("pastes")
|
||||
print(paste)
|
||||
if paste is None:
|
||||
continue
|
||||
try:
|
||||
print(paste)
|
||||
with open(pystemonpath+paste, 'rb') as f: #.read()
|
||||
messagedata = f.read()
|
||||
path_to_send = pastes_directory+paste
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ socket.connect ("tcp://crf.circl.lu:%s" % port)
|
|||
# 102 Full pastes in raw base64(gz)
|
||||
|
||||
topicfilter = "102"
|
||||
socket.setsockopt(zmq.SUBSCRIBE, topicfilter)
|
||||
socket.setsockopt_string(zmq.SUBSCRIBE, topicfilter)
|
||||
|
||||
while True:
|
||||
message = socket.recv()
|
||||
|
|
|
|||
|
|
@ -81,8 +81,12 @@ class HiddenServices(object):
|
|||
return ''
|
||||
return origin_paste.replace(self.paste_directory+'/', '')
|
||||
|
||||
def get_domain_tags(self):
|
||||
return self.tags
|
||||
def get_domain_tags(self, update=False):
|
||||
if not update:
|
||||
return self.tags
|
||||
else:
|
||||
self.get_last_crawled_pastes()
|
||||
return self.tags
|
||||
|
||||
def update_domain_tags(self, children):
|
||||
p_tags = self.r_serv_metadata.smembers('tag:'+children)
|
||||
|
|
|
|||
|
|
@ -206,6 +206,9 @@ dns = 8.8.8.8
|
|||
[Mail]
|
||||
dns = 8.8.8.8
|
||||
|
||||
[Web]
|
||||
dns = 149.13.33.69
|
||||
|
||||
# Indexer configuration
|
||||
[Indexer]
|
||||
type = whoosh
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
#!/usr/bin/python3
|
||||
|
||||
import re
|
||||
import os
|
||||
import configparser
|
||||
import dns.resolver
|
||||
|
||||
from pubsublogger import publisher
|
||||
|
|
@ -101,11 +103,20 @@ def checking_MX_record(r_serv, adress_set, addr_dns):
|
|||
|
||||
|
||||
def checking_A_record(r_serv, domains_set):
|
||||
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
|
||||
if not os.path.exists(configfile):
|
||||
raise Exception('Unable to find the configuration file. \
|
||||
Did you set environment variables? \
|
||||
Or activate the virtualenv.')
|
||||
cfg = configparser.ConfigParser()
|
||||
cfg.read(configfile)
|
||||
dns_server = cfg.get("Web", "dns")
|
||||
|
||||
score = 0
|
||||
num = len(domains_set)
|
||||
WalidA = set([])
|
||||
resolver = dns.resolver.Resolver()
|
||||
resolver.nameservers = ['149.13.33.69']
|
||||
resolver.nameservers = [dns_server]
|
||||
resolver.timeout = 5
|
||||
resolver.lifetime = 2
|
||||
|
||||
|
|
|
|||
|
|
@ -32,6 +32,21 @@ if [ -z "${p}" ] || [ -z "${f}" ] || [ -z "${n}" ]; then
|
|||
usage;
|
||||
fi
|
||||
|
||||
RED="\\033[1;31m"
|
||||
DEFAULT="\\033[0;39m"
|
||||
GREEN="\\033[1;32m"
|
||||
WHITE="\\033[0;02m"
|
||||
|
||||
if [ ! -d "${f}" ]; then
|
||||
printf "$RED\n Error -f, proxy-profiles directory: $WHITE${f}$RED not found\n$DEFAULT Please check if you enter the correct path\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -f "${f}default.ini" ]; then
|
||||
printf "$RED\n Error -f, proxy configuration file:$WHITE default.ini$RED not found\n$DEFAULT Please check if you enter the correct path\n"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
screen -dmS "Docker_Splash"
|
||||
sleep 0.1
|
||||
|
||||
|
|
@ -39,5 +54,5 @@ for ((i=0;i<=$((${n} - 1));i++)); do
|
|||
port_number=$((${p} + $i))
|
||||
screen -S "Docker_Splash" -X screen -t "docker_splash:$port_number" bash -c 'sudo docker run -p '$port_number':8050 --cpus=1 --memory=4.5G -v '$f':/etc/splash/proxy-profiles/ --net="bridge" scrapinghub/splash; read x'
|
||||
sleep 0.1
|
||||
echo " Splash server launched on port $port_number"
|
||||
printf "$GREEN Splash server launched on port $port_number$DEFAULT\n"
|
||||
done
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ set -x
|
|||
|
||||
sudo apt-get update
|
||||
|
||||
sudo apt-get install python3-pip python-virtualenv python3-dev python3-tk libfreetype6-dev \
|
||||
sudo apt-get install python3-pip virtualenv python3-dev python3-tk libfreetype6-dev \
|
||||
screen g++ python-tk unzip libsnappy-dev cmake -y
|
||||
|
||||
#optional tor install
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ pymisp
|
|||
|
||||
thehive4py
|
||||
|
||||
redis
|
||||
redis==2.10.6
|
||||
#filemagic conflict with magic
|
||||
crcmod
|
||||
mmh3
|
||||
|
|
@ -13,7 +13,6 @@ zmq
|
|||
langid
|
||||
|
||||
#Essential
|
||||
redis
|
||||
pyzmq
|
||||
dnspython
|
||||
logbook
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import redis
|
|||
import datetime
|
||||
import sys
|
||||
import os
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint
|
||||
from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
|
||||
|
||||
from Date import Date
|
||||
from HiddenServices import HiddenServices
|
||||
|
|
@ -40,6 +40,16 @@ def get_date_range(num_day):
|
|||
|
||||
return list(reversed(date_list))
|
||||
|
||||
def substract_date(date_from, date_to):
|
||||
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
|
||||
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
|
||||
delta = date_to - date_from # timedelta
|
||||
l_date = []
|
||||
for i in range(delta.days + 1):
|
||||
date = date_from + datetime.timedelta(i)
|
||||
l_date.append( date.strftime('%Y%m%d') )
|
||||
return l_date
|
||||
|
||||
def unpack_paste_tags(p_tags):
|
||||
l_tags = []
|
||||
for tag in p_tags:
|
||||
|
|
@ -94,6 +104,94 @@ def hiddenServices_page():
|
|||
|
||||
return render_template("hiddenServices.html", last_onions=list_onion, statDomains=statDomains)
|
||||
|
||||
@hiddenServices.route("/hiddenServices/get_onions_by_daterange", methods=['POST'])
|
||||
def get_onions_by_daterange():
|
||||
date_from = request.form.get('date_from')
|
||||
date_to = request.form.get('date_to')
|
||||
domains_up = request.form.get('domains_up')
|
||||
domains_down = request.form.get('domains_down')
|
||||
domains_tags = request.form.get('domains_tags')
|
||||
|
||||
return redirect(url_for('hiddenServices.show_domains_by_daterange', date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down, domains_tags=domains_tags))
|
||||
|
||||
@hiddenServices.route("/hiddenServices/show_domains_by_daterange", methods=['GET'])
|
||||
def show_domains_by_daterange():
|
||||
date_from = request.args.get('date_from')
|
||||
date_to = request.args.get('date_to')
|
||||
domains_up = request.args.get('domains_up')
|
||||
domains_down = request.args.get('domains_down')
|
||||
domains_tags = request.args.get('domains_tags')
|
||||
|
||||
date_range = []
|
||||
if date_from is not None and date_to is not None:
|
||||
#change format
|
||||
try:
|
||||
if len(date_from) != 8:
|
||||
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
|
||||
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
|
||||
date_range = substract_date(date_from, date_to)
|
||||
except:
|
||||
pass
|
||||
|
||||
if not date_range:
|
||||
date_range.append(datetime.date.today().strftime("%Y%m%d"))
|
||||
date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8]
|
||||
date_to = date_from
|
||||
|
||||
else:
|
||||
date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
|
||||
date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
|
||||
|
||||
domains_by_day = {}
|
||||
domain_metadata = {}
|
||||
for date in date_range:
|
||||
if domains_up:
|
||||
domains_up = True
|
||||
domains_by_day[date] = list(r_serv_onion.smembers('onion_up:{}'.format(date)))
|
||||
for domain in domains_by_day[date]:
|
||||
h = HiddenServices(domain, 'onion')
|
||||
domain_metadata[domain] = {}
|
||||
if domains_tags:
|
||||
domains_tags = True
|
||||
domain_metadata[domain]['tags'] = h.get_domain_tags(update=True)
|
||||
|
||||
domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check')
|
||||
if domain_metadata[domain]['last_check'] is None:
|
||||
domain_metadata[domain]['last_check'] = '********'
|
||||
domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen')
|
||||
if domain_metadata[domain]['first_seen'] is None:
|
||||
domain_metadata[domain]['first_seen'] = '********'
|
||||
domain_metadata[domain]['status_text'] = 'UP'
|
||||
domain_metadata[domain]['status_color'] = 'Green'
|
||||
domain_metadata[domain]['status_icon'] = 'fa-check-circle'
|
||||
|
||||
if domains_down:
|
||||
domains_down = True
|
||||
domains_by_day_down = list(r_serv_onion.smembers('onion_down:{}'.format(date)))
|
||||
if domains_up:
|
||||
domains_by_day[date].extend(domains_by_day_down)
|
||||
else:
|
||||
domains_by_day[date] = domains_by_day_down
|
||||
for domain in domains_by_day_down:
|
||||
#h = HiddenServices(onion_domain, 'onion')
|
||||
domain_metadata[domain] = {}
|
||||
#domain_metadata[domain]['tags'] = h.get_domain_tags()
|
||||
|
||||
domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check')
|
||||
if domain_metadata[domain]['last_check'] is None:
|
||||
domain_metadata[domain]['last_check'] = '********'
|
||||
domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen')
|
||||
if domain_metadata[domain]['first_seen'] is None:
|
||||
domain_metadata[domain]['first_seen'] = '********'
|
||||
|
||||
domain_metadata[domain]['status_text'] = 'DOWN'
|
||||
domain_metadata[domain]['status_color'] = 'Red'
|
||||
domain_metadata[domain]['status_icon'] = 'fa-times-circle'
|
||||
|
||||
return render_template("domains.html", date_range=date_range, domains_by_day=domains_by_day, domain_metadata=domain_metadata,
|
||||
date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down,
|
||||
domains_tags=domains_tags, bootstrap_label=bootstrap_label)
|
||||
|
||||
@hiddenServices.route("/hiddenServices/onion_domain", methods=['GET'])
|
||||
def onion_domain():
|
||||
onion_domain = request.args.get('onion_domain')
|
||||
|
|
|
|||
291
var/www/modules/hiddenServices/templates/domains.html
Normal file
291
var/www/modules/hiddenServices/templates/domains.html
Normal file
|
|
@ -0,0 +1,291 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Hidden Service - AIL</title>
|
||||
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet" type="text/css" />
|
||||
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
|
||||
<!-- JS -->
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
|
||||
<style>
|
||||
.red_table thead{
|
||||
background: #d91f2d;
|
||||
color: #fff;
|
||||
}
|
||||
.input-group .form-control {
|
||||
position: unset;
|
||||
}
|
||||
</style>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
{% include 'navbar.html' %}
|
||||
|
||||
<div id="page-wrapper">
|
||||
|
||||
<div class="row">
|
||||
<div class="col-md-8">
|
||||
|
||||
{% for date in date_range %}
|
||||
{% if domains_by_day[date]%}
|
||||
<div class="panel panel-info" style="text-align:center;">
|
||||
<div class="panel-heading">
|
||||
<h3 style="text-align:center;">{{'{}/{}/{}'.format(date[0:4], date[4:6], date[6:8])}}</h3>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<table id="table_{{date}}" class="red_table table table-striped table-bordered">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>Domain</th>
|
||||
<th>First Seen</th>
|
||||
<th>Last Check</th>
|
||||
<th>Status</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{% for domain in domains_by_day[date] %}
|
||||
<tr>
|
||||
<td>
|
||||
<a target="_blank" href="{{ url_for('hiddenServices.onion_domain') }}?onion_domain={{ domain }}">{{ domain }}</a>
|
||||
<div>
|
||||
{% for tag in domain_metadata[domain]['tags'] %}
|
||||
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
|
||||
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }} <i>{{ domain_metadata[domain]['tags'][tag] }}</i></span>
|
||||
</a>
|
||||
{% endfor %}
|
||||
</div>
|
||||
</td>
|
||||
<td>{{'{}/{}/{}'.format(domain_metadata[domain]['first_seen'][0:4], domain_metadata[domain]['first_seen'][4:6], domain_metadata[domain]['first_seen'][6:8])}}</td>
|
||||
<td>{{'{}/{}/{}'.format(domain_metadata[domain]['last_check'][0:4], domain_metadata[domain]['last_check'][4:6], domain_metadata[domain]['last_check'][6:8])}}</td>
|
||||
<td><div style="color:{{domain_metadata[domain]['status_color']}}; display:inline-block">
|
||||
<i class="fa {{domain_metadata[domain]['status_icon']}} fa-2x"></i>
|
||||
{{domain_metadata[domain]['status_text']}}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
{% endfor %}
|
||||
</tbody>
|
||||
</table>
|
||||
<br>
|
||||
<br>
|
||||
<br>
|
||||
{% endif %}
|
||||
{% endfor %}
|
||||
|
||||
|
||||
</div>
|
||||
|
||||
<div class="col-md-4">
|
||||
<div class="panel panel-info" style="text-align:center;">
|
||||
<div class="panel-heading">
|
||||
Select domains by date range :
|
||||
<br>
|
||||
<br>
|
||||
<form action="{{ url_for('hiddenServices.get_onions_by_daterange') }}" id="hash_selector_form" method='post'>
|
||||
<div class="row">
|
||||
<div class="col-md-6">
|
||||
<div class="input-group">
|
||||
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
|
||||
<input class="form-control" id="date-range-from" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from">
|
||||
</div>
|
||||
<div class="input-group">
|
||||
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
|
||||
<input class="form-control" id="date-range-to" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to">
|
||||
</div>
|
||||
<div class="checkbox">
|
||||
<label>
|
||||
<input type="checkbox" name="domains_tags" value="True" {% if domains_tags %}checked{% endif %}>
|
||||
<div style="color:#286090; display:inline-block">
|
||||
Domains Tags <i class="fa fa-tags"></i>
|
||||
</div>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="checkbox">
|
||||
<label>
|
||||
<input type="checkbox" name="domains_up" value="True" {% if domains_up %}checked{% endif %}>
|
||||
<div style="color:Green; display:inline-block">
|
||||
Domains UP <i class="fa fa-check-circle"></i>
|
||||
</div>
|
||||
</label>
|
||||
</div>
|
||||
<div class="checkbox">
|
||||
<label>
|
||||
<input type="checkbox" name="domains_down" value="True" {% if domains_down %}checked{% endif %}>
|
||||
<div style="color:Red; display:inline-block">
|
||||
Domains DOWN <i class="fa fa-times-circle"></i>
|
||||
</div>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<button class="btn btn-primary" style="text-align:center;">
|
||||
<i class="fa fa-eye-slash"></i> Show Onions
|
||||
</button>
|
||||
<form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
<!-- /#page-wrapper -->
|
||||
|
||||
<script>
|
||||
var all_graph = {};
|
||||
$(document).ready(function(){
|
||||
activePage = "page-hiddenServices"
|
||||
$("#"+activePage).addClass("active");
|
||||
|
||||
$('#date-range-from').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function()
|
||||
{
|
||||
if ($('#date-range-from').val() && $('#date-range-to').val() )
|
||||
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2)
|
||||
{
|
||||
$('#date-range-from').val(s1);
|
||||
$('#date-range-to').val(s2);
|
||||
}
|
||||
});
|
||||
$('#date-range-to').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function()
|
||||
{
|
||||
if ($('#date-range-from').val() && $('#date-range-to').val() )
|
||||
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2)
|
||||
{
|
||||
$('#date-range-from').val(s1);
|
||||
$('#date-range-to').val(s2);
|
||||
}
|
||||
});
|
||||
|
||||
{% for date in date_range %}
|
||||
$('#table_{{date}}').DataTable({
|
||||
"aLengthMenu": [[5, 15, 30, -1], [5, 15, 30, "All"]],
|
||||
"iDisplayLength": 15,
|
||||
"order": [[ 0, "asc" ]]
|
||||
});
|
||||
{% endfor %}
|
||||
|
||||
//all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hiddenServices.domain_crawled_7days_json') }}?type=onion");
|
||||
});
|
||||
/*$(window).on("resize", function() {
|
||||
all_graph.onResize();
|
||||
});*/
|
||||
</script>
|
||||
|
||||
<script>
|
||||
/*
|
||||
all_graph.onResize = function () {
|
||||
var aspect = 1000 / 500, all_graph = $("#graph_div");
|
||||
var targetWidth = all_graph.parent().width();
|
||||
all_graph.attr("width", targetWidth);
|
||||
all_graph.attr("height", targetWidth / aspect);
|
||||
}
|
||||
window.all_graph = all_graph;
|
||||
|
||||
function create_line_chart(id, url){
|
||||
var width = 650;
|
||||
var height = Math.round(width / 6);
|
||||
var margin = {top: 20, right: 55, bottom: 50, left: 40};
|
||||
var x = d3.scaleTime().range([0, width]);
|
||||
var y = d3.scaleLinear().rangeRound([height, 0]);
|
||||
var xAxis = d3.axisBottom(x);
|
||||
var yAxis = d3.axisLeft(y);
|
||||
var parseTime = d3.timeParse("%Y-%m-%d");
|
||||
var line = d3.line()
|
||||
.x(function(d) {
|
||||
return x(d.date);
|
||||
})
|
||||
.y(function(d) {
|
||||
return y(d.value);
|
||||
});
|
||||
var svg_line = d3.select('#'+id).append('svg')
|
||||
.attr("id", "graph_div")
|
||||
.attr("width", width + margin.left + margin.right)
|
||||
.attr("height", height + margin.top + margin.bottom)
|
||||
.append('g')
|
||||
.attr('transform', "translate("+ margin.left +","+ margin.top +")");
|
||||
var div = d3.select('body').append('div')
|
||||
.attr('class', 'tooltip')
|
||||
.style('opacity', 0);
|
||||
//add div tooltip
|
||||
d3.json(url)
|
||||
.then(function(data){
|
||||
data.forEach(function(d) {
|
||||
d.date_label = d.date;
|
||||
d.date = parseTime(d.date);
|
||||
d.value = +d.value;
|
||||
});
|
||||
// fit the data
|
||||
x.domain(d3.extent(data, function(d) { return d.date; }));
|
||||
//x.domain(data.map(function (d) { return d.date; })); //E
|
||||
y.domain([0, d3.max(data, function(d){ return d.value ; })]);
|
||||
//line
|
||||
svg_line.append("path")
|
||||
.data([data])
|
||||
.attr("class", "line_graph")
|
||||
.attr("d", line);
|
||||
// add X axis
|
||||
svg_line.append("g")
|
||||
.attr("transform", "translate(0," + height + ")")
|
||||
.call(d3.axisBottom(x))
|
||||
.selectAll("text")
|
||||
.style("text-anchor", "end")
|
||||
.attr("transform", "rotate(-45)" );
|
||||
// Add the Y Axis
|
||||
svg_line.append("g")
|
||||
.call(d3.axisLeft(y));
|
||||
//add a dot circle
|
||||
svg_line.selectAll('dot')
|
||||
.data(data).enter()
|
||||
.append('circle')
|
||||
.attr('r', 2)
|
||||
.attr('cx', function(d) { return x(d.date); })
|
||||
.attr('cy', function(d) { return y(d.value); })
|
||||
.on('mouseover', function(d) {
|
||||
div.transition().style('opacity', .9);
|
||||
div.html('' + d.date_label+ '<br/>' + d.value).style('left', (d3.event.pageX) + 'px')
|
||||
.style("left", (d3.event.pageX) + "px")
|
||||
.style("top", (d3.event.pageY - 28) + "px");
|
||||
})
|
||||
.on('mouseout', function(d)
|
||||
{
|
||||
div.transition().style('opacity', 0);
|
||||
});
|
||||
});
|
||||
}
|
||||
*/
|
||||
</script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -12,11 +12,17 @@
|
|||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet" type="text/css" />
|
||||
<!-- JS -->
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
|
||||
|
||||
<style>
|
||||
.input-group .form-control {
|
||||
position: unset;
|
||||
}
|
||||
div.tooltip {
|
||||
position: absolute;
|
||||
text-align: center;
|
||||
|
|
@ -84,50 +90,103 @@
|
|||
</div>
|
||||
|
||||
<div class="col-md-6">
|
||||
<div class="panel panel-info">
|
||||
<div class="panel-heading">
|
||||
<i class="fa fa-eye-slash"></i> Domains Crawled Today
|
||||
</div>
|
||||
<div class="panel panel-info" style="text-align:center;">
|
||||
<div class="panel-heading">
|
||||
Select domains by date range :
|
||||
<br>
|
||||
<br>
|
||||
<form action="{{ url_for('hiddenServices.get_onions_by_daterange') }}" id="hash_selector_form" method='post'>
|
||||
<div class="row">
|
||||
<div class="col-md-6">
|
||||
<div class="input-group">
|
||||
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
|
||||
<input class="form-control" id="date-range-from" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from">
|
||||
</div>
|
||||
<div class="input-group">
|
||||
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
|
||||
<input class="form-control" id="date-range-to" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to">
|
||||
</div>
|
||||
<div class="checkbox">
|
||||
<label>
|
||||
<input type="checkbox" name="domains_tags" value="True" checked>
|
||||
<div style="color:#286090; display:inline-block">
|
||||
Domains Tags <i class="fa fa-tags"></i>
|
||||
</div>
|
||||
</label>
|
||||
</div>
|
||||
</div>
|
||||
<div class="col-md-6">
|
||||
<div class="checkbox">
|
||||
<label>
|
||||
<input type="checkbox" name="domains_up" value="True" checked>
|
||||
<div style="color:Green; display:inline-block">
|
||||
Domains UP <i class="fa fa-check-circle"></i>
|
||||
</div>
|
||||
</label>
|
||||
</div>
|
||||
<div class="checkbox">
|
||||
<label>
|
||||
<input type="checkbox" name="domains_down" value="True">
|
||||
<div style="color:Red; display:inline-block">
|
||||
Domains DOWN <i class="fa fa-times-circle"></i>
|
||||
</div>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<table class="table table-hover table-striped">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<div style="color:Green; display:inline-block">
|
||||
<i class="fa fa-check-circle fa-2x"></i>
|
||||
Domains UP
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<div style="color:Green; display:inline-block">
|
||||
{{ statDomains['domains_up'] }}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<div style="color:Red; display:inline-block">
|
||||
<i class="fa fa-times-circle fa-2x"></i>
|
||||
Domains DOWN
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<div style="color:Red; display:inline-block">
|
||||
{{ statDomains['domains_down'] }}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Crawled Domains</td>
|
||||
<td>{{ statDomains['total'] }}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Domains in Queue</td>
|
||||
<td>{{ statDomains['domains_queue'] }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<button class="btn btn-primary" style="text-align:center;">
|
||||
<i class="fa fa-eye-slash"></i> Show Onions
|
||||
</button>
|
||||
<form>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
|
||||
<div class="panel panel-info">
|
||||
<div class="panel-heading">
|
||||
<i class="fa fa-eye-slash"></i> Domains Crawled Today
|
||||
</div>
|
||||
|
||||
<table class="table table-hover table-striped">
|
||||
<tbody>
|
||||
<tr>
|
||||
<td>
|
||||
<div style="color:Green; display:inline-block">
|
||||
<i class="fa fa-check-circle fa-2x"></i>
|
||||
Domains UP
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<div style="color:Green; display:inline-block">
|
||||
{{ statDomains['domains_up'] }}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>
|
||||
<div style="color:Red; display:inline-block">
|
||||
<i class="fa fa-times-circle fa-2x"></i>
|
||||
Domains DOWN
|
||||
</div>
|
||||
</td>
|
||||
<td>
|
||||
<div style="color:Red; display:inline-block">
|
||||
{{ statDomains['domains_down'] }}
|
||||
</div>
|
||||
</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Crawled Domains</td>
|
||||
<td>{{ statDomains['total'] }}</td>
|
||||
</tr>
|
||||
<tr>
|
||||
<td>Domains in Queue</td>
|
||||
<td>{{ statDomains['domains_queue'] }}</td>
|
||||
</tr>
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
|
@ -140,6 +199,38 @@
|
|||
$(document).ready(function(){
|
||||
activePage = "page-hiddenServices"
|
||||
$("#"+activePage).addClass("active");
|
||||
|
||||
$('#date-range-from').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function()
|
||||
{
|
||||
if ($('#date-range-from').val() && $('#date-range-to').val() )
|
||||
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2)
|
||||
{
|
||||
$('#date-range-from').val(s1);
|
||||
$('#date-range-to').val(s2);
|
||||
}
|
||||
});
|
||||
$('#date-range-to').dateRangePicker({
|
||||
separator : ' to ',
|
||||
getValue: function()
|
||||
{
|
||||
if ($('#date-range-from').val() && $('#date-range-to').val() )
|
||||
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
|
||||
else
|
||||
return '';
|
||||
},
|
||||
setValue: function(s,s1,s2)
|
||||
{
|
||||
$('#date-range-from').val(s1);
|
||||
$('#date-range-to').val(s2);
|
||||
}
|
||||
});
|
||||
|
||||
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hiddenServices.domain_crawled_7days_json') }}?type=onion");
|
||||
});
|
||||
$(window).on("resize", function() {
|
||||
|
|
|
|||
Loading…
Reference in a new issue