AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-09-20 07:58:29 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

Merge remote-tracking branch 'upstream/master'

Browse source
This commit is contained in:
Xavier Mertens 2019-01-30 22:27:44 +01:00
commit 0372a78704
41 changed files with 1681 additions and 327 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
Dockerfile
View file

@ -1,7 +1,11 @@
FROM ubuntu:16.04 FROM ubuntu:16.04
RUN mkdir /opt/AIL && apt-get update -y \ # Make sure that all updates are in place
&& apt-get install git python-dev build-essential \ RUN apt-get clean && apt-get update -y && apt-get upgrade -y \
&& apt-get dist-upgrade -y && apt-get autoremove -y
# Install needed packages
RUN apt-get install git python-dev build-essential \
libffi-dev libssl-dev libfuzzy-dev wget sudo -y libffi-dev libssl-dev libfuzzy-dev wget sudo -y
# Adding sudo command # Adding sudo command
@ -9,6 +13,7 @@ RUN useradd -m docker && echo "docker:docker" | chpasswd && adduser docker sudo
RUN echo "root ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers RUN echo "root ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
# Installing AIL dependencies # Installing AIL dependencies
RUN mkdir /opt/AIL
ADD . /opt/AIL ADD . /opt/AIL
WORKDIR /opt/AIL WORKDIR /opt/AIL
RUN ./installing_deps.sh RUN ./installing_deps.sh

2
HOWTO.md
View file

@ -27,7 +27,7 @@ Feed data to AIL:
4. Edit your configuration file ```bin/packages/config.cfg``` and modify the pystemonpath path accordingly 4. Edit your configuration file ```bin/packages/config.cfg``` and modify the pystemonpath path accordingly
5. Launch pystemon-feeder ``` ./pystemon-feeder.py ``` 5. Launch pystemon-feeder ``` ./bin/feeder/pystemon-feeder.py ```
How to create a new module How to create a new module

19
OVERVIEW.md
View file

@ -26,6 +26,24 @@ ARDB overview
ARDB_DB ARDB_DB
* DB 1 - Curve * DB 1 - Curve
* DB 2 - TermFreq * DB 2 - TermFreq
----------------------------------------- TERM ----------------------------------------
SET - 'TrackedRegexSet' term
HSET - 'TrackedRegexDate' tracked_regex today_timestamp
SET - 'TrackedSetSet' set_to_add
HSET - 'TrackedSetDate' set_to_add today_timestamp
SET - 'TrackedSetTermSet' term
HSET - 'TrackedTermDate' tracked_regex today_timestamp
SET - 'TrackedNotificationEmails_'+term/set email
SET - 'TrackedNotifications' term/set
* DB 3 - Trending * DB 3 - Trending
* DB 4 - Sentiment * DB 4 - Sentiment
* DB 5 - TermCred * DB 5 - TermCred
@ -65,7 +83,6 @@ ARDB_DB
ZADD - 'base64_hash:'+hash paste * nb_seen_in_paste ZADD - 'base64_hash:'+hash paste * nb_seen_in_paste
ZADD - 'binary_hash:'+hash paste * nb_seen_in_paste ZADD - 'binary_hash:'+hash paste * nb_seen_in_paste
ZADD - 'hash_type:'+type date nb_seen
ZADD - 'base64_type:'+type date nb_seen ZADD - 'base64_type:'+type date nb_seen
ZADD - 'binary_type:'+type date nb_seen ZADD - 'binary_type:'+type date nb_seen

7
README.md
View file

@ -70,8 +70,6 @@ Type these command lines for a fully automated installation and start AIL framew
git clone https://github.com/CIRCL/AIL-framework.git git clone https://github.com/CIRCL/AIL-framework.git
cd AIL-framework cd AIL-framework
./installing_deps.sh ./installing_deps.sh
cd var/www/
./update_thirdparty.sh
cd ~/AIL-framework/ cd ~/AIL-framework/
. ./AILENV/bin/activate . ./AILENV/bin/activate
cd bin/ cd bin/
@ -155,6 +153,11 @@ Eventually you can browse the status of the AIL framework website at the followi
http://localhost:7000/ http://localhost:7000/
``` ```
Training
--------
CIRCL organises training on how to use or extend the AIL framework. The next training will be [Thursday, 20 Dec](https://en.xing-events.com/ZEQWMLJ.html) in Luxembourg.
HOWTO HOWTO
----- -----

65
bin/Crawler.py
View file

@ -10,6 +10,8 @@ import time
import subprocess import subprocess
import requests import requests
from pyfaup.faup import Faup
sys.path.append(os.environ['AIL_BIN']) sys.path.append(os.environ['AIL_BIN'])
from Helper import Process from Helper import Process
from pubsublogger import publisher from pubsublogger import publisher
@ -18,28 +20,43 @@ def on_error_send_message_back_in_queue(type_hidden_service, domain, message):
# send this msg back in the queue # send this msg back in the queue
if not r_onion.sismember('{}_domain_crawler_queue'.format(type_hidden_service), domain): if not r_onion.sismember('{}_domain_crawler_queue'.format(type_hidden_service), domain):
r_onion.sadd('{}_domain_crawler_queue'.format(type_hidden_service), domain) r_onion.sadd('{}_domain_crawler_queue'.format(type_hidden_service), domain)
r_onion.sadd('{}_crawler_queue'.format(type_hidden_service), message) r_onion.sadd('{}_crawler_priority_queue'.format(type_hidden_service), message)
def crawl_onion(url, domain, date, date_month, message): def crawl_onion(url, domain, date, date_month, message):
r_cache.hset('metadata_crawler:{}'.format(splash_port), 'crawling_domain', domain)
r_cache.hset('metadata_crawler:{}'.format(splash_port), 'started_time', datetime.datetime.now().strftime("%Y/%m/%d - %H:%M.%S"))
#if not r_onion.sismember('full_onion_up', domain) and not r_onion.sismember('onion_down:'+date , domain): #if not r_onion.sismember('full_onion_up', domain) and not r_onion.sismember('onion_down:'+date , domain):
super_father = r_serv_metadata.hget('paste_metadata:'+paste, 'super_father') super_father = r_serv_metadata.hget('paste_metadata:'+paste, 'super_father')
if super_father is None: if super_father is None:
super_father=paste super_father=paste
retry = True
nb_retry = 0
while retry:
try: try:
r = requests.get(splash_url , timeout=30.0) r = requests.get(splash_url , timeout=30.0)
retry = False
except Exception: except Exception:
# TODO: relaunch docker or send error message # TODO: relaunch docker or send error message
nb_retry += 1
if nb_retry == 6:
on_error_send_message_back_in_queue(type_hidden_service, domain, message) on_error_send_message_back_in_queue(type_hidden_service, domain, message)
publisher.error('{} SPASH DOWN'.format(splash_url)) publisher.error('{} SPASH DOWN'.format(splash_url))
print('--------------------------------------') print('--------------------------------------')
print(' \033[91m DOCKER SPLASH DOWN\033[0m') print(' \033[91m DOCKER SPLASH DOWN\033[0m')
print(' {} DOWN'.format(splash_url)) print(' {} DOWN'.format(splash_url))
exit(1) r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'SPLASH DOWN')
nb_retry == 0
print(' \033[91m DOCKER SPLASH NOT AVAILABLE\033[0m')
print(' Retry({}) in 10 seconds'.format(nb_retry))
time.sleep(10)
if r.status_code == 200: if r.status_code == 200:
r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Crawling')
process = subprocess.Popen(["python", './torcrawler/tor_crawler.py', splash_url, type_hidden_service, url, domain, paste, super_father], process = subprocess.Popen(["python", './torcrawler/tor_crawler.py', splash_url, type_hidden_service, url, domain, paste, super_father],
stdout=subprocess.PIPE) stdout=subprocess.PIPE)
while process.poll() is None: while process.poll() is None:
@ -57,6 +74,7 @@ def crawl_onion(url, domain, date, date_month, message):
print('') print('')
print(' PROXY DOWN OR BAD CONFIGURATION\033[0m'.format(splash_url)) print(' PROXY DOWN OR BAD CONFIGURATION\033[0m'.format(splash_url))
print('------------------------------------------------------------------------') print('------------------------------------------------------------------------')
r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Error')
exit(-2) exit(-2)
else: else:
print(process.stdout.read()) print(process.stdout.read())
@ -66,6 +84,7 @@ def crawl_onion(url, domain, date, date_month, message):
print('--------------------------------------') print('--------------------------------------')
print(' \033[91m DOCKER SPLASH DOWN\033[0m') print(' \033[91m DOCKER SPLASH DOWN\033[0m')
print(' {} DOWN'.format(splash_url)) print(' {} DOWN'.format(splash_url))
r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Crawling')
exit(1) exit(1)
@ -109,6 +128,7 @@ if __name__ == '__main__':
print('splash url: {}'.format(splash_url)) print('splash url: {}'.format(splash_url))
crawler_depth_limit = p.config.getint("Crawler", "crawler_depth_limit") crawler_depth_limit = p.config.getint("Crawler", "crawler_depth_limit")
faup = Faup()
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes")) PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes"))
@ -130,6 +150,10 @@ if __name__ == '__main__':
db=p.config.getint("ARDB_Onion", "db"), db=p.config.getint("ARDB_Onion", "db"),
decode_responses=True) decode_responses=True)
r_cache.sadd('all_crawler:{}'.format(type_hidden_service), splash_port)
r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Waiting')
r_cache.hset('metadata_crawler:{}'.format(splash_port), 'started_time', datetime.datetime.now().strftime("%Y/%m/%d - %H:%M.%S"))
# load domains blacklist # load domains blacklist
try: try:
with open(os.environ['AIL_BIN']+'/torcrawler/blacklist_onion.txt', 'r') as f: with open(os.environ['AIL_BIN']+'/torcrawler/blacklist_onion.txt', 'r') as f:
@ -142,6 +166,10 @@ if __name__ == '__main__':
while True: while True:
# Priority Queue - Recovering the streamed message informations.
message = r_onion.spop('{}_crawler_priority_queue'.format(type_hidden_service))
if message is None:
# Recovering the streamed message informations. # Recovering the streamed message informations.
message = r_onion.spop('{}_crawler_queue'.format(type_hidden_service)) message = r_onion.spop('{}_crawler_queue'.format(type_hidden_service))
@ -163,6 +191,8 @@ if __name__ == '__main__':
domain_url = 'http://{}'.format(domain) domain_url = 'http://{}'.format(domain)
print()
print()
print('\033[92m------------------START CRAWLER------------------\033[0m') print('\033[92m------------------START CRAWLER------------------\033[0m')
print('crawler type: {}'.format(type_hidden_service)) print('crawler type: {}'.format(type_hidden_service))
print('\033[92m-------------------------------------------------\033[0m') print('\033[92m-------------------------------------------------\033[0m')
@ -170,12 +200,24 @@ if __name__ == '__main__':
print('domain: {}'.format(domain)) print('domain: {}'.format(domain))
print('domain_url: {}'.format(domain_url)) print('domain_url: {}'.format(domain_url))
if not r_onion.sismember('blacklist_{}'.format(type_hidden_service), domain): faup.decode(domain)
onion_domain=faup.get()['domain'].decode()
if not r_onion.sismember('blacklist_{}'.format(type_hidden_service), domain) and not r_onion.sismember('blacklist_{}'.format(type_hidden_service), onion_domain):
date = datetime.datetime.now().strftime("%Y%m%d") date = datetime.datetime.now().strftime("%Y%m%d")
date_month = datetime.datetime.now().strftime("%Y%m") date_month = datetime.datetime.now().strftime("%Y%m")
if not r_onion.sismember('month_{}_up:{}'.format(type_hidden_service, date_month), domain) and not r_onion.sismember('{}_down:{}'.format(type_hidden_service, date), domain): if not r_onion.sismember('month_{}_up:{}'.format(type_hidden_service, date_month), domain) and not r_onion.sismember('{}_down:{}'.format(type_hidden_service, date), domain):
# first seen
if not r_onion.hexists('{}_metadata:{}'.format(type_hidden_service, domain), 'first_seen'):
r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'first_seen', date)
# last_father
r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'paste_parent', paste)
# last check
r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'last_check', date)
crawl_onion(url, domain, date, date_month, message) crawl_onion(url, domain, date, date_month, message)
if url != domain_url: if url != domain_url:
@ -188,21 +230,12 @@ if __name__ == '__main__':
r_onion.sadd('{}_down:{}'.format(type_hidden_service, date), domain) r_onion.sadd('{}_down:{}'.format(type_hidden_service, date), domain)
#r_onion.sadd('{}_down_link:{}'.format(type_hidden_service, date), url) #r_onion.sadd('{}_down_link:{}'.format(type_hidden_service, date), url)
#r_onion.hincrby('{}_link_down'.format(type_hidden_service), url, 1) #r_onion.hincrby('{}_link_down'.format(type_hidden_service), url, 1)
if not r_onion.exists('{}_metadata:{}'.format(type_hidden_service, domain)):
r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'first_seen', date)
r_onion.hset('{}_metadata:{}'.format(type_hidden_service,domain), 'last_seen', date)
else: else:
#r_onion.hincrby('{}_link_up'.format(type_hidden_service), url, 1) #r_onion.hincrby('{}_link_up'.format(type_hidden_service), url, 1)
if r_onion.sismember('month_{}_up:{}'.format(type_hidden_service, date_month), domain) and r_serv_metadata.exists('paste_children:'+paste): if r_onion.sismember('month_{}_up:{}'.format(type_hidden_service, date_month), domain) and r_serv_metadata.exists('paste_children:'+paste):
msg = 'infoleak:automatic-detection="{}";{}'.format(type_hidden_service, paste) msg = 'infoleak:automatic-detection="{}";{}'.format(type_hidden_service, paste)
p.populate_set_out(msg, 'Tags') p.populate_set_out(msg, 'Tags')
# last check
r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'last_check', date)
# last_father
r_onion.hset('{}_metadata:{}'.format(type_hidden_service, domain), 'paste_parent', paste)
# add onion screenshot history # add onion screenshot history
# add crawled days # add crawled days
if r_onion.lindex('{}_history:{}'.format(type_hidden_service, domain), 0) != date: if r_onion.lindex('{}_history:{}'.format(type_hidden_service, domain), 0) != date:
@ -233,6 +266,14 @@ if __name__ == '__main__':
r_onion.lpush('last_{}'.format(type_hidden_service), domain) r_onion.lpush('last_{}'.format(type_hidden_service), domain)
r_onion.ltrim('last_{}'.format(type_hidden_service), 0, 15) r_onion.ltrim('last_{}'.format(type_hidden_service), 0, 15)
#update crawler status
r_cache.hset('metadata_crawler:{}'.format(splash_port), 'status', 'Waiting')
r_cache.hdel('metadata_crawler:{}'.format(splash_port), 'crawling_domain')
else:
print(' Blacklisted Onion')
print()
print()
else: else:
continue continue
else: else:

7
bin/Curve.py
View file

@ -48,6 +48,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
# create direct link in mail # create direct link in mail
full_paste_url = "/showsavedpaste/?paste=" full_paste_url = "/showsavedpaste/?paste="
@ -71,6 +73,11 @@ def check_if_tracked_term(term, path):
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term): for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + term):
sendEmailNotification(email, 'Term', mail_body) sendEmailNotification(email, 'Term', mail_body)
# tag paste
for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + term):
msg = '{};{}'.format(tag, path)
p.populate_set_out(msg, 'Tags')
def getValueOverRange(word, startDate, num_day): def getValueOverRange(word, startDate, num_day):
to_return = 0 to_return = 0

116
bin/LAUNCH.sh
View file

@ -9,12 +9,30 @@ WHITE="\\033[0;02m"
YELLOW="\\033[1;33m" YELLOW="\\033[1;33m"
CYAN="\\033[1;36m" CYAN="\\033[1;36m"
[ -z "$AIL_HOME" ] && echo "Needs the env var AIL_HOME. Run the script from the virtual environment." && exit 1; # Getting CWD where bash script resides
[ -z "$AIL_REDIS" ] && echo "Needs the env var AIL_REDIS. Run the script from the virtual environment." && exit 1; DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd |sed 's/bin//' )"
[ -z "$AIL_ARDB" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1; export AIL_HOME="${DIR}"
[ -z "$AIL_BIN" ] && echo "Needs the env var AIL_ARDB. Run the script from the virtual environment." && exit 1;
[ -z "$AIL_FLASK" ] && echo "Needs the env var AIL_FLASK. Run the script from the virtual environment." && exit 1;
cd ${AIL_HOME}
if [ -e "${DIR}/AILENV/bin/python" ]; then
echo "AIL-framework virtualenv seems to exist, good"
ENV_PY="${DIR}/AILENV/bin/python"
else
echo "Please make sure you have a AIL-framework environment, au revoir"
exit 1
fi
# redis-server is bundled during install
## [ ! -f "`which redis-server`" ] && echo "'redis-server' is not installed/not on PATH. Please fix and run again." && exit 1
export AIL_BIN=${AIL_HOME}/bin/
export AIL_FLASK=${AIL_HOME}/var/www/
export AIL_REDIS=${AIL_HOME}/redis/src/
export AIL_ARDB=${AIL_HOME}/ardb/src/
export AIL_VENV=${AIL_HOME}/AILENV/
export PATH=$AIL_VENV/bin:$PATH
export PATH=$AIL_HOME:$PATH export PATH=$AIL_HOME:$PATH
export PATH=$AIL_REDIS:$PATH export PATH=$AIL_REDIS:$PATH
export PATH=$AIL_ARDB:$PATH export PATH=$AIL_ARDB:$PATH
@ -91,9 +109,9 @@ function launching_logs {
screen -dmS "Logging_AIL" screen -dmS "Logging_AIL"
sleep 0.1 sleep 0.1
echo -e $GREEN"\t* Launching logging process"$DEFAULT echo -e $GREEN"\t* Launching logging process"$DEFAULT
screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Queuing -l ../logs/; read x' screen -S "Logging_AIL" -X screen -t "LogQueue" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Queuing -l ../logs/; read x"
sleep 0.1 sleep 0.1
screen -S "Logging_AIL" -X screen -t "LogScript" bash -c 'cd '${AIL_BIN}'; log_subscriber -p 6380 -c Script -l ../logs/; read x' screen -S "Logging_AIL" -X screen -t "LogScript" bash -c "cd ${AIL_BIN}; ${AIL_VENV}/bin/log_subscriber -p 6380 -c Script -l ../logs/; read x"
} }
function launching_queues { function launching_queues {
@ -101,16 +119,16 @@ function launching_queues {
sleep 0.1 sleep 0.1
echo -e $GREEN"\t* Launching all the queues"$DEFAULT echo -e $GREEN"\t* Launching all the queues"$DEFAULT
screen -S "Queue_AIL" -X screen -t "Queues" bash -c 'cd '${AIL_BIN}'; python3 launch_queues.py; read x' screen -S "Queue_AIL" -X screen -t "Queues" bash -c "cd ${AIL_BIN}; ${ENV_PY} launch_queues.py; read x"
} }
function checking_configuration { function checking_configuration {
bin_dir=${AIL_HOME}/bin bin_dir=${AIL_HOME}/bin
echo -e "\t* Checking configuration" echo -e "\t* Checking configuration"
if [ "$1" == "automatic" ]; then if [ "$1" == "automatic" ]; then
bash -c "python3 $bin_dir/Update-conf.py True" bash -c "${ENV_PY} $bin_dir/Update-conf.py True"
else else
bash -c "python3 $bin_dir/Update-conf.py False" bash -c "${ENV_PY} $bin_dir/Update-conf.py False"
fi fi
exitStatus=$? exitStatus=$?
@ -128,75 +146,75 @@ function launching_scripts {
sleep 0.1 sleep 0.1
echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT echo -e $GREEN"\t* Launching ZMQ scripts"$DEFAULT
screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c 'cd '${AIL_BIN}'; ./ModulesInformationV2.py -k 0 -c 1; read x' screen -S "Script_AIL" -X screen -t "ModuleInformation" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModulesInformationV2.py -k 0 -c 1; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Mixer" bash -c 'cd '${AIL_BIN}'; ./Mixer.py; read x' screen -S "Script_AIL" -X screen -t "Mixer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mixer.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Global" bash -c 'cd '${AIL_BIN}'; ./Global.py; read x' screen -S "Script_AIL" -X screen -t "Global" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Global.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Duplicates" bash -c 'cd '${AIL_BIN}'; ./Duplicates.py; read x' screen -S "Script_AIL" -X screen -t "Duplicates" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Duplicates.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Lines" bash -c 'cd '${AIL_BIN}'; ./Lines.py; read x' screen -S "Script_AIL" -X screen -t "Lines" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Lines.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c 'cd '${AIL_BIN}'; ./DomClassifier.py; read x' screen -S "Script_AIL" -X screen -t "DomClassifier" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./DomClassifier.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Categ" bash -c 'cd '${AIL_BIN}'; ./Categ.py; read x' screen -S "Script_AIL" -X screen -t "Categ" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Categ.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Tokenize" bash -c 'cd '${AIL_BIN}'; ./Tokenize.py; read x' screen -S "Script_AIL" -X screen -t "Tokenize" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tokenize.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "CreditCards" bash -c 'cd '${AIL_BIN}'; ./CreditCards.py; read x' screen -S "Script_AIL" -X screen -t "CreditCards" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CreditCards.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "BankAccount" bash -c 'cd '${AIL_BIN}'; ./BankAccount.py; read x' screen -S "Script_AIL" -X screen -t "BankAccount" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./BankAccount.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Onion" bash -c 'cd '${AIL_BIN}'; ./Onion.py; read x' screen -S "Script_AIL" -X screen -t "Onion" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Onion.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Mail" bash -c 'cd '${AIL_BIN}'; ./Mail.py; read x' screen -S "Script_AIL" -X screen -t "Mail" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Mail.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "ApiKey" bash -c 'cd '${AIL_BIN}'; ./ApiKey.py; read x' screen -S "Script_AIL" -X screen -t "ApiKey" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ApiKey.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Web" bash -c 'cd '${AIL_BIN}'; ./Web.py; read x' screen -S "Script_AIL" -X screen -t "Web" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Web.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Credential" bash -c 'cd '${AIL_BIN}'; ./Credential.py; read x' screen -S "Script_AIL" -X screen -t "Credential" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Credential.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Curve" bash -c 'cd '${AIL_BIN}'; ./Curve.py; read x' screen -S "Script_AIL" -X screen -t "Curve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Curve.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c 'cd '${AIL_BIN}'; ./CurveManageTopSets.py; read x' screen -S "Script_AIL" -X screen -t "CurveManageTopSets" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./CurveManageTopSets.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./RegexForTermsFrequency.py; read x' screen -S "Script_AIL" -X screen -t "RegexForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./RegexForTermsFrequency.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c 'cd '${AIL_BIN}'; ./SetForTermsFrequency.py; read x' screen -S "Script_AIL" -X screen -t "SetForTermsFrequency" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SetForTermsFrequency.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Indexer" bash -c 'cd '${AIL_BIN}'; ./Indexer.py; read x' screen -S "Script_AIL" -X screen -t "Indexer" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Indexer.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Keys" bash -c 'cd '${AIL_BIN}'; ./Keys.py; read x' screen -S "Script_AIL" -X screen -t "Keys" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Keys.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Decoder" bash -c 'cd '${AIL_BIN}'; ./Decoder.py; read x' screen -S "Script_AIL" -X screen -t "Decoder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Decoder.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c 'cd '${AIL_BIN}'; ./Bitcoin.py; read x' screen -S "Script_AIL" -X screen -t "Bitcoin" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Bitcoin.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Phone" bash -c 'cd '${AIL_BIN}'; ./Phone.py; read x' screen -S "Script_AIL" -X screen -t "Phone" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Phone.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Release" bash -c 'cd '${AIL_BIN}'; ./Release.py; read x' screen -S "Script_AIL" -X screen -t "Release" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Release.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Cve" bash -c 'cd '${AIL_BIN}'; ./Cve.py; read x' screen -S "Script_AIL" -X screen -t "Cve" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Cve.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "WebStats" bash -c 'cd '${AIL_BIN}'; ./WebStats.py; read x' screen -S "Script_AIL" -X screen -t "WebStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./WebStats.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c 'cd '${AIL_BIN}'; ./ModuleStats.py; read x' screen -S "Script_AIL" -X screen -t "ModuleStats" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./ModuleStats.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c 'cd '${AIL_BIN}'; ./SQLInjectionDetection.py; read x' screen -S "Script_AIL" -X screen -t "SQLInjectionDetection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SQLInjectionDetection.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "LibInjection" bash -c 'cd '${AIL_BIN}'; ./LibInjection.py; read x' screen -S "Script_AIL" -X screen -t "LibInjection" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./LibInjection.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "alertHandler" bash -c 'cd '${AIL_BIN}'; ./alertHandler.py; read x' screen -S "Script_AIL" -X screen -t "alertHandler" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./alertHandler.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c 'cd '${AIL_BIN}'; ./MISP_The_Hive_feeder.py; read x' screen -S "Script_AIL" -X screen -t "MISPtheHIVEfeeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./MISP_The_Hive_feeder.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "Tags" bash -c 'cd '${AIL_BIN}'; ./Tags.py; read x' screen -S "Script_AIL" -X screen -t "Tags" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Tags.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c 'cd '${AIL_BIN}'; ./SentimentAnalysis.py; read x' screen -S "Script_AIL" -X screen -t "SentimentAnalysis" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./SentimentAnalysis.py; read x"
sleep 0.1 sleep 0.1
screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c 'cd '${AIL_BIN}'; ./submit_paste.py; read x' screen -S "Script_AIL" -X screen -t "SubmitPaste" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./submit_paste.py; read x"
} }
@ -219,7 +237,7 @@ function launching_crawler {
sleep 0.1 sleep 0.1
for ((i=first_port;i<=last_port;i++)); do for ((i=first_port;i<=last_port;i++)); do
screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c 'cd '${AIL_BIN}'; ./Crawler.py onion '$i'; read x' screen -S "Crawler_AIL" -X screen -t "onion_crawler:$i" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./Crawler.py onion $i; read x"
sleep 0.1 sleep 0.1
done done
@ -342,7 +360,7 @@ function launch_flask {
screen -dmS "Flask_AIL" screen -dmS "Flask_AIL"
sleep 0.1 sleep 0.1
echo -e $GREEN"\t* Launching Flask server"$DEFAULT echo -e $GREEN"\t* Launching Flask server"$DEFAULT
screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ./Flask_server.py; read x" screen -S "Flask_AIL" -X screen -t "Flask_server" bash -c "cd $flask_dir; ls; ${ENV_PY} ./Flask_server.py; read x"
else else
echo -e $RED"\t* A Flask screen is already launched"$DEFAULT echo -e $RED"\t* A Flask screen is already launched"$DEFAULT
fi fi
@ -353,9 +371,9 @@ function launch_feeder {
screen -dmS "Feeder_Pystemon" screen -dmS "Feeder_Pystemon"
sleep 0.1 sleep 0.1
echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT echo -e $GREEN"\t* Launching Pystemon feeder"$DEFAULT
screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c 'cd '${AIL_BIN}'; ./feeder/pystemon-feeder.py; read x' screen -S "Feeder_Pystemon" -X screen -t "Pystemon_feeder" bash -c "cd ${AIL_BIN}; ${ENV_PY} ./feeder/pystemon-feeder.py; read x"
sleep 0.1 sleep 0.1
screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c 'cd '${AIL_HOME}/../pystemon'; python2 pystemon.py; read x' screen -S "Feeder_Pystemon" -X screen -t "Pystemon" bash -c "cd ${AIL_HOME}/../pystemon; ${ENV_PY} ./pystemon.py; read x"
else else
echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT echo -e $RED"\t* A Feeder screen is already launched"$DEFAULT
fi fi

90
bin/MISP_The_Hive_feeder.py
View file

@ -54,7 +54,7 @@ from thehive4py.models import Case, CaseTask, CustomFieldHelper
def create_the_hive_alert(source, path, content, tag): def create_the_hive_alert(source, path, tag):
tags = list(r_serv_metadata.smembers('tag:'+path)) tags = list(r_serv_metadata.smembers('tag:'+path))
artifacts = [ artifacts = [
@ -63,7 +63,6 @@ def create_the_hive_alert(source, path, content, tag):
] ]
l_tags = tag.split(',') l_tags = tag.split(',')
print(tag)
# Prepare the sample Alert # Prepare the sample Alert
sourceRef = str(uuid.uuid4())[0:6] sourceRef = str(uuid.uuid4())[0:6]
@ -91,6 +90,41 @@ def create_the_hive_alert(source, path, content, tag):
except: except:
print('hive connection error') print('hive connection error')
def feeder(message, count=0):
if flag_the_hive or flag_misp:
tag, path = message.split(';')
## FIXME: remove it
if PASTES_FOLDER not in path:
path = os.path.join(PASTES_FOLDER, path)
try:
paste = Paste.Paste(path)
except FileNotFoundError:
if count < 10:
r_serv_db.zincrby('mess_not_saved_export', message, 1)
return 0
else:
r_serv_db.zrem('mess_not_saved_export', message)
print('Error: {} do not exist, tag= {}'.format(path, tag))
return 0
source = '/'.join(paste.p_path.split('/')[-6:])
if HiveApi != False:
if int(r_serv_db.get('hive:auto-alerts')) == 1:
whitelist_hive = r_serv_db.scard('whitelist_hive')
if r_serv_db.sismember('whitelist_hive', tag):
create_the_hive_alert(source, path, tag)
else:
print('hive, auto alerts creation disable')
if flag_misp:
if int(r_serv_db.get('misp:auto-events')) == 1:
if r_serv_db.sismember('whitelist_misp', tag):
misp_wrapper.pushToMISP(uuid_ail, path, tag)
else:
print('misp, auto events creation disable')
if __name__ == "__main__": if __name__ == "__main__":
publisher.port = 6380 publisher.port = 6380
@ -119,10 +153,18 @@ if __name__ == "__main__":
db=cfg.getint("ARDB_Metadata", "db"), db=cfg.getint("ARDB_Metadata", "db"),
decode_responses=True) decode_responses=True)
# set sensor uuid
uuid_ail = r_serv_db.get('ail:uuid') uuid_ail = r_serv_db.get('ail:uuid')
if uuid_ail is None: if uuid_ail is None:
uuid_ail = r_serv_db.set('ail:uuid', uuid.uuid4() ) uuid_ail = r_serv_db.set('ail:uuid', uuid.uuid4() )
# set default
if r_serv_db.get('hive:auto-alerts') is None:
r_serv_db.set('hive:auto-alerts', 0)
if r_serv_db.get('misp:auto-events') is None:
r_serv_db.set('misp:auto-events', 0)
p = Process(config_section) p = Process(config_section)
# create MISP connection # create MISP connection
if flag_misp: if flag_misp:
@ -167,36 +209,30 @@ if __name__ == "__main__":
r_serv_db.set('ail:thehive', False) r_serv_db.set('ail:thehive', False)
print('Not connected to The HIVE') print('Not connected to The HIVE')
refresh_time = 3
## FIXME: remove it
PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes"))
time_1 = time.time()
while True: while True:
# Get one message from the input queue # Get one message from the input queue
message = p.get_from_set() message = p.get_from_set()
if message is None: if message is None:
# handle not saved pastes
if int(time.time() - time_1) > refresh_time:
num_queu = r_serv_db.zcard('mess_not_saved_export')
list_queu = r_serv_db.zrange('mess_not_saved_export', 0, -1, withscores=True)
if num_queu and list_queu:
for i in range(0, num_queu):
feeder(list_queu[i][0],list_queu[i][1])
time_1 = time.time()
else:
publisher.debug("{} queue is empty, waiting 1s".format(config_section)) publisher.debug("{} queue is empty, waiting 1s".format(config_section))
time.sleep(1) time.sleep(1)
continue
else: else:
feeder(message)
if flag_the_hive or flag_misp:
tag, path = message.split(';')
paste = Paste.Paste(path)
source = '/'.join(paste.p_path.split('/')[-6:])
full_path = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "pastes"), path)
if HiveApi != False:
if int(r_serv_db.get('hive:auto-alerts')) == 1:
whitelist_hive = r_serv_db.scard('whitelist_hive')
if r_serv_db.sismember('whitelist_hive', tag):
create_the_hive_alert(source, path, full_path, tag)
else:
print('hive, auto alerts creation disable')
if flag_misp:
if int(r_serv_db.get('misp:auto-events')) == 1:
if r_serv_db.sismember('whitelist_misp', tag):
misp_wrapper.pushToMISP(uuid_ail, path, tag)
else:
print('misp, auto events creation disable')

3
bin/Mixer.py
View file

@ -80,6 +80,7 @@ if __name__ == '__main__':
# OTHER CONFIG # # OTHER CONFIG #
operation_mode = cfg.getint("Module_Mixer", "operation_mode") operation_mode = cfg.getint("Module_Mixer", "operation_mode")
ttl_key = cfg.getint("Module_Mixer", "ttl_duplicate") ttl_key = cfg.getint("Module_Mixer", "ttl_duplicate")
default_unnamed_feed_name = cfg.get("Module_Mixer", "default_unnamed_feed_name")
# STATS # # STATS #
processed_paste = 0 processed_paste = 0
@ -106,7 +107,7 @@ if __name__ == '__main__':
paste_name = complete_paste paste_name = complete_paste
except ValueError as e: except ValueError as e:
feeder_name = "unnamed_feeder" feeder_name = default_unnamed_feed_name
paste_name = complete_paste paste_name = complete_paste
# Processed paste # Processed paste

34
bin/NotificationHelper.py
View file

@ -3,8 +3,10 @@
import argparse import argparse
import configparser import configparser
import traceback
import os import os
import smtplib import smtplib
from pubsublogger import publisher
from email.mime.multipart import MIMEMultipart from email.mime.multipart import MIMEMultipart
from email.mime.text import MIMEText from email.mime.text import MIMEText
@ -15,6 +17,9 @@ This module allows the global configuration and management of notification setti
# CONFIG # # CONFIG #
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
publisher.port = 6380
publisher.channel = "Script"
# notifications enabled/disabled # notifications enabled/disabled
TrackedTermsNotificationEnabled_Name = "TrackedNotifications" TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
@ -22,7 +27,6 @@ TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
# Keys will be e.g. TrackedNotificationEmails<TERMNAME> # Keys will be e.g. TrackedNotificationEmails<TERMNAME>
TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_" TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
def sendEmailNotification(recipient, alert_name, content): def sendEmailNotification(recipient, alert_name, content):
if not os.path.exists(configfile): if not os.path.exists(configfile):
@ -33,22 +37,12 @@ def sendEmailNotification(recipient, alert_name, content):
cfg = configparser.ConfigParser() cfg = configparser.ConfigParser()
cfg.read(configfile) cfg.read(configfile)
sender = cfg.get("Notifications", "sender"), sender = cfg.get("Notifications", "sender")
sender_host = cfg.get("Notifications", "sender_host"), sender_host = cfg.get("Notifications", "sender_host")
sender_port = cfg.getint("Notifications", "sender_port"), sender_port = cfg.getint("Notifications", "sender_port")
sender_pw = cfg.get("Notifications", "sender_pw"), sender_pw = cfg.get("Notifications", "sender_pw")
if sender_pw == 'None':
if isinstance(sender, tuple): sender_pw = None
sender = sender[0]
if isinstance(sender_host, tuple):
sender_host = sender_host[0]
if isinstance(sender_port, tuple):
sender_port = sender_port[0]
if isinstance(sender_pw, tuple):
sender_pw = sender_pw[0]
# raise an exception if any of these is None # raise an exception if any of these is None
if (sender is None or if (sender is None or
@ -83,9 +77,9 @@ def sendEmailNotification(recipient, alert_name, content):
smtp_server.quit() smtp_server.quit()
print('Send notification ' + alert_name + ' to '+recipient) print('Send notification ' + alert_name + ' to '+recipient)
except Exception as e: except Exception as err:
print(str(e)) traceback.print_tb(err.__traceback__)
# raise e publisher.warning(err)
if __name__ == '__main__': if __name__ == '__main__':
parser = argparse.ArgumentParser(description='Test notification sender.') parser = argparse.ArgumentParser(description='Test notification sender.')

24
bin/Onion.py
View file

@ -29,10 +29,18 @@ import os
import base64 import base64
import subprocess import subprocess
import redis import redis
import signal
import re import re
from Helper import Process from Helper import Process
class TimeoutException(Exception):
pass
def timeout_handler(signum, frame):
raise TimeoutException
signal.signal(signal.SIGALRM, timeout_handler)
def fetch(p, r_cache, urls, domains, path): def fetch(p, r_cache, urls, domains, path):
failed = [] failed = []
@ -113,6 +121,8 @@ if __name__ == "__main__":
message = p.get_from_set() message = p.get_from_set()
prec_filename = None prec_filename = None
max_execution_time = p.config.getint("Onion", "max_execution_time")
# send to crawler: # send to crawler:
activate_crawler = p.config.get("Crawler", "activate_crawler") activate_crawler = p.config.get("Crawler", "activate_crawler")
if activate_crawler == 'True': if activate_crawler == 'True':
@ -130,6 +140,7 @@ if __name__ == "__main__":
while True: while True:
message = p.get_from_set()
if message is not None: if message is not None:
print(message) print(message)
filename, score = message.split() filename, score = message.split()
@ -140,6 +151,9 @@ if __name__ == "__main__":
urls = [] urls = []
PST = Paste.Paste(filename) PST = Paste.Paste(filename)
# max execution time on regex
signal.alarm(max_execution_time)
try:
for x in PST.get_regex(url_regex): for x in PST.get_regex(url_regex):
print(x) print(x)
# Extracting url with regex # Extracting url with regex
@ -150,6 +164,11 @@ if __name__ == "__main__":
print(url) print(url)
domains_list.append(domain) domains_list.append(domain)
urls.append(url) urls.append(url)
except TimeoutException:
encoded_list = []
p.incr_module_timeout_statistic()
print ("{0} processing timeout".format(PST.p_path))
continue
''' '''
for x in PST.get_regex(i2p_regex): for x in PST.get_regex(i2p_regex):
@ -204,6 +223,10 @@ if __name__ == "__main__":
print('send to onion crawler') print('send to onion crawler')
r_onion.sadd('onion_domain_crawler_queue', domain) r_onion.sadd('onion_domain_crawler_queue', domain)
msg = '{};{}'.format(url,PST.p_path) msg = '{};{}'.format(url,PST.p_path)
if not r_onion.hexists('onion_metadata:{}'.format(domain), 'first_seen'):
r_onion.sadd('onion_crawler_priority_queue', msg)
print('send to priority queue')
else:
r_onion.sadd('onion_crawler_queue', msg) r_onion.sadd('onion_crawler_queue', msg)
#p.populate_set_out(msg, 'Crawler') #p.populate_set_out(msg, 'Crawler')
@ -222,4 +245,3 @@ if __name__ == "__main__":
publisher.debug("Script url is Idling 10s") publisher.debug("Script url is Idling 10s")
#print('Sleeping') #print('Sleeping')
time.sleep(10) time.sleep(10)
message = p.get_from_set()

7
bin/RegexForTermsFrequency.py
View file

@ -42,6 +42,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
top_termFreq_set_array = [top_termFreq_setName_day, top_termFreq_setName_week, top_termFreq_setName_month] top_termFreq_set_array = [top_termFreq_setName_day, top_termFreq_setName_week, top_termFreq_setName_month]
TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
# create direct link in mail # create direct link in mail
full_paste_url = "/showsavedpaste/?paste=" full_paste_url = "/showsavedpaste/?paste="
@ -129,6 +131,11 @@ if __name__ == "__main__":
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + regex_str_complete): for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + regex_str_complete):
sendEmailNotification(email, 'Term', mail_body) sendEmailNotification(email, 'Term', mail_body)
# tag paste
for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + regex_str_complete):
msg = '{};{}'.format(tag, filename)
p.populate_set_out(msg, 'Tags')
set_name = 'regex_' + dico_regexname_to_redis[regex_str] set_name = 'regex_' + dico_regexname_to_redis[regex_str]
new_to_the_set = server_term.sadd(set_name, filename) new_to_the_set = server_term.sadd(set_name, filename)
new_to_the_set = True if new_to_the_set == 1 else False new_to_the_set = True if new_to_the_set == 1 else False

7
bin/SetForTermsFrequency.py
View file

@ -34,6 +34,8 @@ top_termFreq_setName_week = ["TopTermFreq_set_week", 7]
top_termFreq_setName_month = ["TopTermFreq_set_month", 31] top_termFreq_setName_month = ["TopTermFreq_set_month", 31]
top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month] top_termFreq_set_array = [top_termFreq_setName_day,top_termFreq_setName_week, top_termFreq_setName_month]
TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
# create direct link in mail # create direct link in mail
full_paste_url = "/showsavedpaste/?paste=" full_paste_url = "/showsavedpaste/?paste="
@ -121,6 +123,11 @@ if __name__ == "__main__":
for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + dico_setname_to_redis[str(the_set)]): for email in server_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + dico_setname_to_redis[str(the_set)]):
sendEmailNotification(email, 'Term', mail_body) sendEmailNotification(email, 'Term', mail_body)
# tag paste
for tag in server_term.smembers(TrackedTermsNotificationTagsPrefix_Name + dico_setname_to_redis[str(the_set)]):
msg = '{};{}'.format(tag, filename)
p.populate_set_out(msg, 'Tags')
print(the_set, "matched in", filename) print(the_set, "matched in", filename)
set_name = 'set_' + dico_setname_to_redis[the_set] set_name = 'set_' + dico_setname_to_redis[the_set]
new_to_the_set = server_term.sadd(set_name, filename) new_to_the_set = server_term.sadd(set_name, filename)

4
bin/feeder/pystemon-feeder.py
View file

@ -61,13 +61,13 @@ topic = '102'
while True: while True:
time.sleep(base_sleeptime + sleep_inc) time.sleep(base_sleeptime + sleep_inc)
paste = r.lpop("pastes") paste = r.lpop("pastes")
print(paste)
if paste is None: if paste is None:
continue continue
try: try:
print(paste)
with open(pystemonpath+paste, 'rb') as f: #.read() with open(pystemonpath+paste, 'rb') as f: #.read()
messagedata = f.read() messagedata = f.read()
path_to_send = pastes_directory+paste path_to_send = os.path.join(pastes_directory,paste)
s = b' '.join( [ topic.encode(), path_to_send.encode(), base64.b64encode(messagedata) ] ) s = b' '.join( [ topic.encode(), path_to_send.encode(), base64.b64encode(messagedata) ] )
socket.send(s) socket.send(s)

2
bin/feeder/test-zmq.py
View file

@ -20,7 +20,7 @@ socket.connect ("tcp://crf.circl.lu:%s" % port)
# 102 Full pastes in raw base64(gz) # 102 Full pastes in raw base64(gz)
topicfilter = "102" topicfilter = "102"
socket.setsockopt(zmq.SUBSCRIBE, topicfilter) socket.setsockopt_string(zmq.SUBSCRIBE, topicfilter)
while True: while True:
message = socket.recv() message = socket.recv()

6
bin/packages/HiddenServices.py
View file

@ -81,7 +81,11 @@ class HiddenServices(object):
return '' return ''
return origin_paste.replace(self.paste_directory+'/', '') return origin_paste.replace(self.paste_directory+'/', '')
def get_domain_tags(self): def get_domain_tags(self, update=False):
if not update:
return self.tags
else:
self.get_last_crawled_pastes()
return self.tags return self.tags
def update_domain_tags(self, children): def update_domain_tags(self, children):

8
bin/packages/config.cfg.sample
View file

@ -27,6 +27,7 @@ ail_domain = http://localhost:7000
sender = sender@example.com sender = sender@example.com
sender_host = smtp.example.com sender_host = smtp.example.com
sender_port = 1337 sender_port = 1337
sender_pw = None
# optional for using with authenticated SMTP over SSL # optional for using with authenticated SMTP over SSL
# sender_pw = securepassword # sender_pw = securepassword
@ -67,6 +68,9 @@ minTopPassList=5
[Curve] [Curve]
max_execution_time = 90 max_execution_time = 90
[Onion]
max_execution_time = 180
[Base64] [Base64]
path = Base64/ path = Base64/
max_execution_time = 60 max_execution_time = 60
@ -98,6 +102,7 @@ threshold_stucked_module=600
operation_mode = 3 operation_mode = 3
#Define the time that a paste will be considerate duplicate. in seconds (1day = 86400) #Define the time that a paste will be considerate duplicate. in seconds (1day = 86400)
ttl_duplicate = 86400 ttl_duplicate = 86400
default_unnamed_feed_name = unnamed_feeder
[RegexForTermsFrequency] [RegexForTermsFrequency]
max_execution_time = 60 max_execution_time = 60
@ -207,6 +212,9 @@ dns = 8.8.8.8
[Mail] [Mail]
dns = 8.8.8.8 dns = 8.8.8.8
[Web]
dns = 149.13.33.69
# Indexer configuration # Indexer configuration
[Indexer] [Indexer]
type = whoosh type = whoosh

13
bin/packages/lib_refine.py
View file

@ -1,6 +1,8 @@
#!/usr/bin/python3 #!/usr/bin/python3
import re import re
import os
import configparser
import dns.resolver import dns.resolver
from pubsublogger import publisher from pubsublogger import publisher
@ -101,11 +103,20 @@ def checking_MX_record(r_serv, adress_set, addr_dns):
def checking_A_record(r_serv, domains_set): def checking_A_record(r_serv, domains_set):
configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg')
if not os.path.exists(configfile):
raise Exception('Unable to find the configuration file. \
Did you set environment variables? \
Or activate the virtualenv.')
cfg = configparser.ConfigParser()
cfg.read(configfile)
dns_server = cfg.get("Web", "dns")
score = 0 score = 0
num = len(domains_set) num = len(domains_set)
WalidA = set([]) WalidA = set([])
resolver = dns.resolver.Resolver() resolver = dns.resolver.Resolver()
resolver.nameservers = ['149.13.33.69'] resolver.nameservers = [dns_server]
resolver.timeout = 5 resolver.timeout = 5
resolver.lifetime = 2 resolver.lifetime = 2

4
bin/packages/modules.cfg
View file

@ -32,13 +32,15 @@ publish = Redis_Words
[Curve] [Curve]
subscribe = Redis_Words subscribe = Redis_Words
publish = Redis_CurveManageTopSets publish = Redis_CurveManageTopSets,Redis_Tags
[RegexForTermsFrequency] [RegexForTermsFrequency]
subscribe = Redis_Global subscribe = Redis_Global
publish = Redis_Tags
[SetForTermsFrequency] [SetForTermsFrequency]
subscribe = Redis_Global subscribe = Redis_Global
publish = Redis_Tags
[CurveManageTopSets] [CurveManageTopSets]
subscribe = Redis_CurveManageTopSets subscribe = Redis_CurveManageTopSets

34
bin/torcrawler/TorSplashCrawler.py
View file

@ -10,10 +10,12 @@ import datetime
import base64 import base64
import redis import redis
import json import json
import time
from scrapy.spidermiddlewares.httperror import HttpError from scrapy.spidermiddlewares.httperror import HttpError
from twisted.internet.error import DNSLookupError from twisted.internet.error import DNSLookupError
from twisted.internet.error import TimeoutError from twisted.internet.error import TimeoutError
from twisted.web._newclient import ResponseNeverReceived
from scrapy import Spider from scrapy import Spider
from scrapy.linkextractors import LinkExtractor from scrapy.linkextractors import LinkExtractor
@ -39,6 +41,8 @@ class TorSplashCrawler():
'SPIDER_MIDDLEWARES': {'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,}, 'SPIDER_MIDDLEWARES': {'scrapy_splash.SplashDeduplicateArgsMiddleware': 100,},
'DUPEFILTER_CLASS': 'scrapy_splash.SplashAwareDupeFilter', 'DUPEFILTER_CLASS': 'scrapy_splash.SplashAwareDupeFilter',
'HTTPERROR_ALLOW_ALL': True, 'HTTPERROR_ALLOW_ALL': True,
'RETRY_TIMES': 2,
'CLOSESPIDER_PAGECOUNT': 50,
'DEPTH_LIMIT': crawler_depth_limit 'DEPTH_LIMIT': crawler_depth_limit
}) })
@ -97,7 +101,7 @@ class TorSplashCrawler():
yield SplashRequest( yield SplashRequest(
self.start_urls, self.start_urls,
self.parse, self.parse,
#errback=self.errback_catcher, errback=self.errback_catcher,
endpoint='render.json', endpoint='render.json',
meta={'father': self.original_paste}, meta={'father': self.original_paste},
args={ 'html': 1, args={ 'html': 1,
@ -174,7 +178,7 @@ class TorSplashCrawler():
yield SplashRequest( yield SplashRequest(
link.url, link.url,
self.parse, self.parse,
#errback=self.errback_catcher, errback=self.errback_catcher,
endpoint='render.json', endpoint='render.json',
meta={'father': relative_filename_paste}, meta={'father': relative_filename_paste},
args={ 'html': 1, args={ 'html': 1,
@ -184,17 +188,39 @@ class TorSplashCrawler():
'wait': 10} 'wait': 10}
) )
'''
def errback_catcher(self, failure): def errback_catcher(self, failure):
# catch all errback failures, # catch all errback failures,
self.logger.error(repr(failure)) self.logger.error(repr(failure))
if failure.check(ResponseNeverReceived):
request = failure.request
url = request.meta['splash']['args']['url']
father = request.meta['father']
self.logger.error('Splash, ResponseNeverReceived for %s, retry in 10s ...', url)
time.sleep(10)
yield SplashRequest(
url,
self.parse,
errback=self.errback_catcher,
endpoint='render.json',
meta={'father': father},
args={ 'html': 1,
'png': 1,
'render_all': 1,
'har': 1,
'wait': 10}
)
else:
print('failure') print('failure')
#print(failure) #print(failure)
print(failure.type) print(failure.type)
#print(failure.request.meta['item']) #print(failure.request.meta['item'])
'''
#if isinstance(failure.value, HttpError): #if isinstance(failure.value, HttpError):
if failure.check(HttpError): elif failure.check(HttpError):
# you can get the response # you can get the response
response = failure.value.response response = failure.value.response
print('HttpError') print('HttpError')

1
bin/torcrawler/blacklist_onion.txt
View file

@ -3,3 +3,4 @@ facebookcorewwwi.onion
graylady3jvrrxbe.onion graylady3jvrrxbe.onion
expyuzz4wqqyqhjn.onion expyuzz4wqqyqhjn.onion
dccbbv6cooddgcrq.onion dccbbv6cooddgcrq.onion
pugljpwjhbiagkrn.onion

31
bin/torcrawler/launch_splash_crawler.sh
View file

@ -5,12 +5,15 @@ usage() { echo "Usage: sudo $0 [-f <config_absolute_path>] [-p <port_start>] [-n
echo " -p: number of the first splash server port number. This number is incremented for the others splash server"; echo " -p: number of the first splash server port number. This number is incremented for the others splash server";
echo " -n: number of splash servers to start"; echo " -n: number of splash servers to start";
echo ""; echo "";
echo " -options:";
echo " -u: max unbound in-memory cache (Mb, Restart Splash when full, default=3000 Mb)";
echo "";
echo "example:"; echo "example:";
echo "sudo ./launch_splash_crawler.sh -f /home/my_user/AIL-framework/configs/docker/splash_onion/etc/splash/proxy-profiles/ -p 8050 -n 3"; echo "sudo ./launch_splash_crawler.sh -f /home/my_user/AIL-framework/configs/docker/splash_onion/etc/splash/proxy-profiles/ -p 8050 -n 3";
exit 1; exit 1;
} }
while getopts ":p:f:n:" o; do while getopts ":p:f:n:u:" o; do
case "${o}" in case "${o}" in
p) p)
p=${OPTARG} p=${OPTARG}
@ -21,6 +24,9 @@ while getopts ":p:f:n:" o; do
n) n)
n=${OPTARG} n=${OPTARG}
;; ;;
u)
u=${OPTARG}
;;
*) *)
usage usage
;; ;;
@ -28,16 +34,35 @@ while getopts ":p:f:n:" o; do
done done
shift $((OPTIND-1)) shift $((OPTIND-1))
if [ -z "${u}" ]; then
u=3000;
fi
if [ -z "${p}" ] || [ -z "${f}" ] || [ -z "${n}" ]; then if [ -z "${p}" ] || [ -z "${f}" ] || [ -z "${n}" ]; then
usage; usage;
fi fi
RED="\\033[1;31m"
DEFAULT="\\033[0;39m"
GREEN="\\033[1;32m"
WHITE="\\033[0;02m"
if [ ! -d "${f}" ]; then
printf "$RED\n Error -f, proxy-profiles directory: $WHITE${f}$RED not found\n$DEFAULT Please check if you enter the correct path\n"
exit 1
fi
if [ ! -f "${f}default.ini" ]; then
printf "$RED\n Error -f, proxy configuration file:$WHITE default.ini$RED not found\n$DEFAULT Please check if you enter the correct path\n"
exit 1
fi
screen -dmS "Docker_Splash" screen -dmS "Docker_Splash"
sleep 0.1 sleep 0.1
for ((i=0;i<=$((${n} - 1));i++)); do for ((i=0;i<=$((${n} - 1));i++)); do
port_number=$((${p} + $i)) port_number=$((${p} + $i))
screen -S "Docker_Splash" -X screen -t "docker_splash:$port_number" bash -c 'sudo docker run -p '$port_number':8050 --cpus=1 --memory=4.5G -v '$f':/etc/splash/proxy-profiles/ --net="bridge" scrapinghub/splash; read x' screen -S "Docker_Splash" -X screen -t "docker_splash:$port_number" bash -c 'sudo docker run -d -p '$port_number':8050 --restart=always --cpus=1 --memory=4.5G -v '$f':/etc/splash/proxy-profiles/ --net="bridge" scrapinghub/splash --maxrss '$u'; read x'
sleep 0.1 sleep 0.1
echo " Splash server launched on port $port_number" printf "$GREEN Splash server launched on port $port_number$DEFAULT\n"
done done

2
configs/6379.conf
View file

@ -61,7 +61,7 @@ tcp-backlog 511
# Examples: # Examples:
# #
# bind 192.168.1.100 10.0.0.1 # bind 192.168.1.100 10.0.0.1
# bind 127.0.0.1 bind 127.0.0.1
# Specify the path for the Unix socket that will be used to listen for # Specify the path for the Unix socket that will be used to listen for
# incoming connections. There is no default, so Redis will not listen # incoming connections. There is no default, so Redis will not listen

2
configs/6380.conf
View file

@ -61,7 +61,7 @@ tcp-backlog 511
# Examples: # Examples:
# #
# bind 192.168.1.100 10.0.0.1 # bind 192.168.1.100 10.0.0.1
# bind 127.0.0.1 bind 127.0.0.1
# Specify the path for the Unix socket that will be used to listen for # Specify the path for the Unix socket that will be used to listen for
# incoming connections. There is no default, so Redis will not listen # incoming connections. There is no default, so Redis will not listen

2
configs/6381.conf
View file

@ -61,7 +61,7 @@ tcp-backlog 511
# Examples: # Examples:
# #
# bind 192.168.1.100 10.0.0.1 # bind 192.168.1.100 10.0.0.1
# bind 127.0.0.1 bind 127.0.0.1
# Specify the path for the Unix socket that will be used to listen for # Specify the path for the Unix socket that will be used to listen for
# incoming connections. There is no default, so Redis will not listen # incoming connections. There is no default, so Redis will not listen

BIN
doc/presentation/ail-training-december-2018.pdf Normal file
View file

Binary file not shown.

2
installing_deps.sh
View file

@ -5,7 +5,7 @@ set -x
sudo apt-get update sudo apt-get update
sudo apt-get install python3-pip python-virtualenv python3-dev python3-tk libfreetype6-dev \ sudo apt-get install python3-pip virtualenv python3-dev python3-tk libfreetype6-dev \
screen g++ python-tk unzip libsnappy-dev cmake -y screen g++ python-tk unzip libsnappy-dev cmake -y
#optional tor install #optional tor install

3
pip3_packages_requirement.txt
View file

@ -2,7 +2,7 @@ pymisp
thehive4py thehive4py
redis redis==2.10.6
#filemagic conflict with magic #filemagic conflict with magic
crcmod crcmod
mmh3 mmh3
@ -13,7 +13,6 @@ zmq
langid langid
#Essential #Essential
redis
pyzmq pyzmq
dnspython dnspython
logbook logbook

7
var/www/modules/Flask_config.py
View file

@ -30,6 +30,12 @@ r_serv = redis.StrictRedis(
db=cfg.getint("Redis_Queues", "db"), db=cfg.getint("Redis_Queues", "db"),
decode_responses=True) decode_responses=True)
r_cache = redis.StrictRedis(
host=cfg.get("Redis_Cache", "host"),
port=cfg.getint("Redis_Cache", "port"),
db=cfg.getint("Redis_Cache", "db"),
decode_responses=True)
r_serv_log = redis.StrictRedis( r_serv_log = redis.StrictRedis(
host=cfg.get("Redis_Log", "host"), host=cfg.get("Redis_Log", "host"),
port=cfg.getint("Redis_Log", "port"), port=cfg.getint("Redis_Log", "port"),
@ -102,7 +108,6 @@ r_serv_onion = redis.StrictRedis(
db=cfg.getint("ARDB_Onion", "db"), db=cfg.getint("ARDB_Onion", "db"),
decode_responses=True) decode_responses=True)
sys.path.append('../../configs/keys') sys.path.append('../../configs/keys')
# MISP # # MISP #
try: try:

40
var/www/modules/PasteSubmit/Flask_PasteSubmit.py
View file

@ -506,6 +506,8 @@ def edit_tag_export():
status_misp = [] status_misp = []
status_hive = [] status_hive = []
infoleak_tags = Taxonomies().get('infoleak').machinetags()
is_infoleak_tag = []
for tag in list_export_tags: for tag in list_export_tags:
if r_serv_db.sismember('whitelist_misp', tag): if r_serv_db.sismember('whitelist_misp', tag):
@ -519,6 +521,11 @@ def edit_tag_export():
else: else:
status_hive.append(False) status_hive.append(False)
if tag in infoleak_tags:
is_infoleak_tag.append(True)
else:
is_infoleak_tag.append(False)
if misp_auto_events is not None: if misp_auto_events is not None:
if int(misp_auto_events) == 1: if int(misp_auto_events) == 1:
misp_active = True misp_active = True
@ -543,6 +550,7 @@ def edit_tag_export():
misp_active=misp_active, misp_active=misp_active,
hive_active=hive_active, hive_active=hive_active,
list_export_tags=list_export_tags, list_export_tags=list_export_tags,
is_infoleak_tag=is_infoleak_tag,
status_misp=status_misp, status_misp=status_misp,
status_hive=status_hive, status_hive=status_hive,
nb_tags_whitelist_misp=nb_tags_whitelist_misp, nb_tags_whitelist_misp=nb_tags_whitelist_misp,
@ -594,5 +602,37 @@ def disable_hive_auto_alert():
r_serv_db.set('hive:auto-alerts', 0) r_serv_db.set('hive:auto-alerts', 0)
return edit_tag_export() return edit_tag_export()
@PasteSubmit.route("/PasteSubmit/add_push_tag")
def add_push_tag():
tag = request.args.get('tag')
if tag is not None:
#limit tag length
if len(tag) > 49:
tag = tag[0:48]
r_serv_db.sadd('list_export_tags', tag)
to_return = {}
to_return["tag"] = tag
return jsonify(to_return)
else:
return 'None args', 400
@PasteSubmit.route("/PasteSubmit/delete_push_tag")
def delete_push_tag():
tag = request.args.get('tag')
infoleak_tags = Taxonomies().get('infoleak').machinetags()
if tag not in infoleak_tags and r_serv_db.sismember('list_export_tags', tag):
r_serv_db.srem('list_export_tags', tag)
r_serv_db.srem('whitelist_misp', tag)
r_serv_db.srem('whitelist_hive', tag)
to_return = {}
to_return["tag"] = tag
return jsonify(to_return)
else:
return 'this tag can\'t be removed', 400
# ========= REGISTRATION ========= # ========= REGISTRATION =========
app.register_blueprint(PasteSubmit, url_prefix=baseUrl) app.register_blueprint(PasteSubmit, url_prefix=baseUrl)

76
var/www/modules/PasteSubmit/templates/edit_tag_export.html
View file

@ -37,6 +37,9 @@
background: #d91f2d; background: #d91f2d;
color: #fff; color: #fff;
} }
.mouse_pointer{
cursor: pointer;
}
</style> </style>
</head> </head>
@ -169,7 +172,14 @@
<input type="checkbox" value="{{ tag }}" name="tag_enabled_misp" > <input type="checkbox" value="{{ tag }}" name="tag_enabled_misp" >
{% endif %} {% endif %}
</td> </td>
<td>{{ tag }}</td> <td>
{{ tag }}
{% if not is_infoleak_tag[loop.index0] %}
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" title="Delete this tag" onclick="delete_push_tag('{{ tag }}')">
<span class="glyphicon glyphicon-trash"></span>
</div>
{% endif %}
</td>
</tr> </tr>
{% endfor %} {% endfor %}
@ -209,7 +219,14 @@
<input type="checkbox" value="{{ tag }}" name="tag_enabled_hive" > <input type="checkbox" value="{{ tag }}" name="tag_enabled_hive" >
{% endif %} {% endif %}
</td> </td>
<td>{{ tag }}</td> <td>
{{ tag }}
{% if not is_infoleak_tag[loop.index0] %}
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" title="Delete this tag" onclick="delete_push_tag('{{ tag }}')">
<span class="glyphicon glyphicon-trash"></span>
</div>
{% endif %}
</td>
</tr> </tr>
{% endfor %} {% endfor %}
@ -232,6 +249,42 @@
</form> </form>
<div>
<div id="add_custom_tag_modal" class="modal fade" role="dialog">
<div class="modal-dialog">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
<h2 class="text-center">Add Custom Tag</h2>
</div>
<div class="modal-body">
<div class="form-group input-group" style="margin-bottom: 0px;">
<span class="input-group-addon"><i class="fa fa-tag fa"></i></span>
<input id="new_custom_tag" class="form-control" placeholder="Add a new custom tag to the tag export" type="text">
</div>
</div>
<div class="modal-footer">
<button class="btn btn-primary btn-tags" onclick="add_custom_tag()">
<span class="glyphicon glyphicon-plus"></span>
<span class="label-icon">Add Custom Tag</span>
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
<button type="button" class="btn btn-primary pull-right" data-toggle="modal" data-target="#add_custom_tag_modal" data-url="{{ url_for('Tags.taxonomies') }}">
<span class="glyphicon glyphicon-plus "></span> Add Custom Tag
</button>
</div>
</div> </div>
<!-- /#page-wrapper --> <!-- /#page-wrapper -->
@ -277,6 +330,25 @@ $(document).ready(function(){
} }
); );
} }
function delete_push_tag(tag){
//var row_tr = $(this).closest("tr");
$.get("{{ url_for('PasteSubmit.delete_push_tag') }}", { tag: tag }, function(data, status){
if(status == "success") {
//row_tr.remove();
window.location.reload(false);
}
});
}
function add_custom_tag(){
$.get("{{ url_for('PasteSubmit.add_push_tag') }}", { tag: document.getElementById('new_custom_tag').value }, function(data, status){
if(status == "success") {
//row_tr.remove();
window.location.reload(false);
}
});
}
</script> </script>
</html> </html>

65
var/www/modules/hashDecoded/Flask_hashDecoded.py
View file

@ -101,7 +101,8 @@ def all_hash_search():
date_to = request.form.get('date_to') date_to = request.form.get('date_to')
type = request.form.get('type') type = request.form.get('type')
encoding = request.form.get('encoding') encoding = request.form.get('encoding')
return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type, encoding=encoding)) show_decoded_files = request.form.get('show_decoded_files')
return redirect(url_for('hashDecoded.hashDecoded_page', date_from=date_from, date_to=date_to, type=type, encoding=encoding, show_decoded_files=show_decoded_files))
@hashDecoded.route("/hashDecoded/", methods=['GET']) @hashDecoded.route("/hashDecoded/", methods=['GET'])
def hashDecoded_page(): def hashDecoded_page():
@ -109,6 +110,7 @@ def hashDecoded_page():
date_to = request.args.get('date_to') date_to = request.args.get('date_to')
type = request.args.get('type') type = request.args.get('type')
encoding = request.args.get('encoding') encoding = request.args.get('encoding')
show_decoded_files = request.args.get('show_decoded_files')
if type == 'All types': if type == 'All types':
type = None type = None
@ -161,6 +163,8 @@ def hashDecoded_page():
daily_date = None daily_date = None
l_64 = set() l_64 = set()
if show_decoded_files:
show_decoded_files = True
for date in date_range: for date in date_range:
if encoding is None: if encoding is None:
l_hash = r_serv_metadata.zrange('hash_date:' +date, 0, -1) l_hash = r_serv_metadata.zrange('hash_date:' +date, 0, -1)
@ -214,7 +218,7 @@ def hashDecoded_page():
l_type = r_serv_metadata.smembers('hash_all_type') l_type = r_serv_metadata.smembers('hash_all_type')
return render_template("hashDecoded.html", l_64=b64_metadata, vt_enabled=vt_enabled, l_type=l_type, type=type, daily_type_chart=daily_type_chart, daily_date=daily_date, return render_template("hashDecoded.html", l_64=b64_metadata, vt_enabled=vt_enabled, l_type=l_type, type=type, daily_type_chart=daily_type_chart, daily_date=daily_date,
encoding=encoding, all_encoding=all_encoding, date_from=date_from, date_to=date_to) encoding=encoding, all_encoding=all_encoding, date_from=date_from, date_to=date_to, show_decoded_files=show_decoded_files)
@hashDecoded.route('/hashDecoded/hash_by_type') @hashDecoded.route('/hashDecoded/hash_by_type')
def hash_by_type(): def hash_by_type():
@ -400,6 +404,63 @@ def decoder_type_json():
to_json.append({'name': decoder, 'value': nb_decoded[decoder]}) to_json.append({'name': decoder, 'value': nb_decoded[decoder]})
return jsonify(to_json) return jsonify(to_json)
@hashDecoded.route('/hashDecoded/top5_type_json')
def top5_type_json():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
typ = request.args.get('type')
decoder = request.args.get('encoding')
if decoder == 'All encoding' or decoder is None:
all_decoder = r_serv_metadata.smembers('all_decoder')
else:
if not r_serv_metadata.sismember('all_decoder', decoder):
return jsonify({'Error': 'This decoder do not exist'})
else:
all_decoder = [decoder]
if typ == 'All types' or typ is None or typ=='None':
all_type = r_serv_metadata.smembers('hash_all_type')
else:
typ = typ.replace(' ', '+')
if not r_serv_metadata.sismember('hash_all_type', typ):
return jsonify({'Error': 'This type do not exist'})
else:
all_type = [typ]
date_range = []
if date_from is not None and date_to is not None:
#change format
try:
if len(date_from) != 8:
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
date_range = substract_date(date_from, date_to)
except:
pass
if not date_range:
date_range.append(datetime.date.today().strftime("%Y%m%d"))
# TODO replace with ZUNIONSTORE
nb_types_decoded = {}
for date in date_range:
for typ in all_type:
for decoder in all_decoder:
nb_decoded = r_serv_metadata.zscore('{}_type:{}'.format(decoder, typ), date)
if nb_decoded is not None:
if typ in nb_types_decoded:
nb_types_decoded[typ] = nb_types_decoded[typ] + int(nb_decoded)
else:
nb_types_decoded[typ] = int(nb_decoded)
to_json = []
top5_types = sorted(nb_types_decoded, key=nb_types_decoded.get, reverse=True)[:5]
for typ in top5_types:
to_json.append({'name': typ, 'value': nb_types_decoded[typ]})
return jsonify(to_json)
@hashDecoded.route('/hashDecoded/daily_type_json') @hashDecoded.route('/hashDecoded/daily_type_json')
def daily_type_json(): def daily_type_json():

54
var/www/modules/hashDecoded/templates/hashDecoded.html
View file

@ -121,7 +121,14 @@
{% endif %} {% endif %}
{% endfor %} {% endfor %}
</select> </select>
<br> <div class="checkbox">
<label>
<input type="checkbox" name="show_decoded_files" value="True" {% if show_decoded_files %}checked{% endif %}>
<div style="color:#286090; display:inline-block">
Show decoded files <i class="fa fa-file"></i>
</div>
</label>
</div>
<button class="btn btn-primary" style="text-align:center;"> <button class="btn btn-primary" style="text-align:center;">
<i class="fa fa-files-o"></i> Search <i class="fa fa-files-o"></i> Search
</button> </button>
@ -129,6 +136,8 @@
</div> </div>
</div> </div>
<div id="pie_chart_encoded"> <div id="pie_chart_encoded">
</div>
<div id="pie_chart_top5_types">
</div> </div>
</div> </div>
</div> </div>
@ -189,12 +198,14 @@
</tbody> </tbody>
</table> </table>
{% else %} {% else %}
{% if show_decoded_files %}
{% if date_from|string == date_to|string %} {% if date_from|string == date_to|string %}
<h3> {{ date_from }}, No Hashes</h3> <h3> {{ date_from }}, No Hashes</h3>
{% else %} {% else %}
<h3> {{ date_from }} to {{ date_to }}, No Hashes</h3> <h3> {{ date_from }} to {{ date_to }}, No Hashes</h3>
{% endif %} {% endif %}
{% endif %} {% endif %}
{% endif %}
</div> </div>
</div> </div>
@ -248,9 +259,12 @@
{% elif daily_type_chart %} {% elif daily_type_chart %}
chart.stackBarChart =barchart_type_stack("{{ url_for('hashDecoded.range_type_json') }}?date_from={{daily_date}}&date_to={{daily_date}}", 'id'); chart.stackBarChart =barchart_type_stack("{{ url_for('hashDecoded.range_type_json') }}?date_from={{daily_date}}&date_to={{daily_date}}", 'id');
{% else %} {% else %}
chart.stackBarChart = barchart_type_stack("{{ url_for('hashDecoded.range_type_json') }}?date_from={{date_from}}&date_to={{date_to}}", 'id') chart.stackBarChart = barchart_type_stack("{{ url_for('hashDecoded.range_type_json') }}?date_from={{date_from}}&date_to={{date_to}}", 'id');
{% endif %} {% endif %}
draw_pie_chart("pie_chart_encoded" ,"{{ url_for('hashDecoded.decoder_type_json') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}", "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}&encoding=");
draw_pie_chart("pie_chart_top5_types" ,"{{ url_for('hashDecoded.top5_type_json') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}", "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type=");
chart.onResize(); chart.onResize();
$(window).on("resize", function() { $(window).on("resize", function() {
chart.onResize(); chart.onResize();
@ -498,21 +512,22 @@ window.chart = chart;
</script> </script>
<script> <script>
function draw_pie_chart(id, url_json, pie_on_click_url) {
var width_pie = 200; var width_pie = 200;
var height_pie = 200; var height_pie = 200;
var padding_pie = 10; var padding_pie = 10;
var opacity_pie = .8; var opacity_pie = .8;
var radius_pie = Math.min(width_pie - padding_pie, height_pie - padding_pie) / 2; var radius_pie = Math.min(width_pie - padding_pie, height_pie - padding_pie) / 2;
//var color_pie = d3.scaleOrdinal(d3.schemeCategory10); //var color_pie = d3.scaleOrdinal(d3.schemeCategory10);
var color_pie = d3.scaleOrdinal(d3.schemeSet3); var color_pie = d3.scaleOrdinal(d3.schemeSet3);
var div_pie = d3.select("body").append("div") var div_pie = d3.select("body").append("div")
.attr("class", "tooltip") .attr("class", "tooltip")
.style("opacity", 0); .style("opacity", 0);
var svg_pie = d3.select("#pie_chart_encoded") var svg_pie = d3.select("#"+id)
.append('svg') .append('svg')
.attr("width", '100%') .attr("width", '100%')
.attr("height", '100%') .attr("height", '100%')
@ -520,14 +535,14 @@ var svg_pie = d3.select("#pie_chart_encoded")
.attr('preserveAspectRatio','xMinYMin') .attr('preserveAspectRatio','xMinYMin')
var g_pie = svg_pie.append('g') var g_pie = svg_pie.append('g')
.attr('transform', 'translate(' + (width_pie/2) + ',' + (height_pie/2) + ')'); .attr('transform', 'translate(' + (width_pie/2) + ',' + (height_pie/2) + ')');
var arc_pie = d3.arc() var arc_pie = d3.arc()
.innerRadius(0) .innerRadius(0)
.outerRadius(radius_pie); .outerRadius(radius_pie);
d3.json("{{ url_for('hashDecoded.decoder_type_json') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}") d3.json(url_json)
.then(function(data){ .then(function(data){
var pie_pie = d3.pie() var pie_pie = d3.pie()
@ -544,12 +559,13 @@ d3.json("{{ url_for('hashDecoded.decoder_type_json') }}?date_from={{date_from}}&
.attr('class', 'pie_path') .attr('class', 'pie_path')
.on("mouseover", mouseovered_pie) .on("mouseover", mouseovered_pie)
.on("mouseout", mouseouted_pie) .on("mouseout", mouseouted_pie)
.on("click", function (d) {window.location.href = "{{ url_for('hashDecoded.hashDecoded_page') }}?date_from={{date_from}}&date_to={{date_to}}&type={{type}}&encoding="+d.data.name }) .on("click", function (d) {window.location.href = pie_on_click_url+d.data.name })
.style('opacity', opacity_pie) .style('opacity', opacity_pie)
.style('stroke', 'white'); .style('stroke', 'white');
}); });
function mouseovered_pie(d) {
function mouseovered_pie(d) {
// tooltip // tooltip
var content; var content;
@ -564,13 +580,15 @@ function mouseovered_pie(d) {
div_pie.html(content) div_pie.html(content)
.style("left", (d3.event.pageX) + "px") .style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px"); .style("top", (d3.event.pageY - 28) + "px");
} }
function mouseouted_pie() { function mouseouted_pie() {
div_pie.transition() div_pie.transition()
.duration(500) .duration(500)
.style("opacity", 0); .style("opacity", 0);
}
} }
</script> </script>

175
var/www/modules/hiddenServices/Flask_hiddenServices.py
View file

@ -8,7 +8,7 @@ import redis
import datetime import datetime
import sys import sys
import os import os
from flask import Flask, render_template, jsonify, request, Blueprint from flask import Flask, render_template, jsonify, request, Blueprint, redirect, url_for
from Date import Date from Date import Date
from HiddenServices import HiddenServices from HiddenServices import HiddenServices
@ -19,6 +19,7 @@ import Flask_config
app = Flask_config.app app = Flask_config.app
cfg = Flask_config.cfg cfg = Flask_config.cfg
baseUrl = Flask_config.baseUrl baseUrl = Flask_config.baseUrl
r_cache = Flask_config.r_cache
r_serv_onion = Flask_config.r_serv_onion r_serv_onion = Flask_config.r_serv_onion
r_serv_metadata = Flask_config.r_serv_metadata r_serv_metadata = Flask_config.r_serv_metadata
bootstrap_label = Flask_config.bootstrap_label bootstrap_label = Flask_config.bootstrap_label
@ -40,6 +41,16 @@ def get_date_range(num_day):
return list(reversed(date_list)) return list(reversed(date_list))
def substract_date(date_from, date_to):
date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
delta = date_to - date_from # timedelta
l_date = []
for i in range(delta.days + 1):
date = date_from + datetime.timedelta(i)
l_date.append( date.strftime('%Y%m%d') )
return l_date
def unpack_paste_tags(p_tags): def unpack_paste_tags(p_tags):
l_tags = [] l_tags = []
for tag in p_tags: for tag in p_tags:
@ -81,7 +92,11 @@ def hiddenServices_page():
metadata_onion = {} metadata_onion = {}
metadata_onion['domain'] = onion metadata_onion['domain'] = onion
metadata_onion['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(onion), 'last_check') metadata_onion['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(onion), 'last_check')
if metadata_onion['last_check'] is None:
metadata_onion['last_check'] = '********'
metadata_onion['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(onion), 'first_seen') metadata_onion['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(onion), 'first_seen')
if metadata_onion['first_seen'] is None:
metadata_onion['first_seen'] = '********'
if get_onion_status(onion, metadata_onion['last_check']): if get_onion_status(onion, metadata_onion['last_check']):
metadata_onion['status_text'] = 'UP' metadata_onion['status_text'] = 'UP'
metadata_onion['status_color'] = 'Green' metadata_onion['status_color'] = 'Green'
@ -92,7 +107,159 @@ def hiddenServices_page():
metadata_onion['status_icon'] = 'fa-times-circle' metadata_onion['status_icon'] = 'fa-times-circle'
list_onion.append(metadata_onion) list_onion.append(metadata_onion)
return render_template("hiddenServices.html", last_onions=list_onion, statDomains=statDomains) crawler_metadata=[]
all_onion_crawler = r_cache.smembers('all_crawler:onion')
for crawler in all_onion_crawler:
crawling_domain = r_cache.hget('metadata_crawler:{}'.format(crawler), 'crawling_domain')
started_time = r_cache.hget('metadata_crawler:{}'.format(crawler), 'started_time')
status_info = r_cache.hget('metadata_crawler:{}'.format(crawler), 'status')
crawler_info = '{} - {}'.format(crawler, started_time)
if status_info=='Waiting' or status_info=='Crawling':
status=True
else:
status=False
crawler_metadata.append({'crawler_info': crawler_info, 'crawling_domain': crawling_domain, 'status_info': status_info, 'status': status})
date_string = '{}-{}-{}'.format(date[0:4], date[4:6], date[6:8])
return render_template("hiddenServices.html", last_onions=list_onion, statDomains=statDomains,
crawler_metadata=crawler_metadata, date_from=date_string, date_to=date_string)
@hiddenServices.route("/hiddenServices/last_crawled_domains_with_stats_json", methods=['GET'])
def last_crawled_domains_with_stats_json():
last_onions = r_serv_onion.lrange('last_onion', 0 ,-1)
list_onion = []
now = datetime.datetime.now()
date = '{}{}{}'.format(now.strftime("%Y"), now.strftime("%m"), now.strftime("%d"))
statDomains = {}
statDomains['domains_up'] = r_serv_onion.scard('onion_up:{}'.format(date))
statDomains['domains_down'] = r_serv_onion.scard('onion_down:{}'.format(date))
statDomains['total'] = statDomains['domains_up'] + statDomains['domains_down']
statDomains['domains_queue'] = r_serv_onion.scard('onion_domain_crawler_queue')
for onion in last_onions:
metadata_onion = {}
metadata_onion['domain'] = onion
metadata_onion['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(onion), 'last_check')
if metadata_onion['last_check'] is None:
metadata_onion['last_check'] = '********'
metadata_onion['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(onion), 'first_seen')
if metadata_onion['first_seen'] is None:
metadata_onion['first_seen'] = '********'
if get_onion_status(onion, metadata_onion['last_check']):
metadata_onion['status_text'] = 'UP'
metadata_onion['status_color'] = 'Green'
metadata_onion['status_icon'] = 'fa-check-circle'
else:
metadata_onion['status_text'] = 'DOWN'
metadata_onion['status_color'] = 'Red'
metadata_onion['status_icon'] = 'fa-times-circle'
list_onion.append(metadata_onion)
crawler_metadata=[]
all_onion_crawler = r_cache.smembers('all_crawler:onion')
for crawler in all_onion_crawler:
crawling_domain = r_cache.hget('metadata_crawler:{}'.format(crawler), 'crawling_domain')
started_time = r_cache.hget('metadata_crawler:{}'.format(crawler), 'started_time')
status_info = r_cache.hget('metadata_crawler:{}'.format(crawler), 'status')
crawler_info = '{} - {}'.format(crawler, started_time)
if status_info=='Waiting' or status_info=='Crawling':
status=True
else:
status=False
crawler_metadata.append({'crawler_info': crawler_info, 'crawling_domain': crawling_domain, 'status_info': status_info, 'status': status})
date_string = '{}-{}-{}'.format(date[0:4], date[4:6], date[6:8])
return jsonify({'last_onions': list_onion, 'statDomains': statDomains, 'crawler_metadata':crawler_metadata})
@hiddenServices.route("/hiddenServices/get_onions_by_daterange", methods=['POST'])
def get_onions_by_daterange():
date_from = request.form.get('date_from')
date_to = request.form.get('date_to')
domains_up = request.form.get('domains_up')
domains_down = request.form.get('domains_down')
domains_tags = request.form.get('domains_tags')
return redirect(url_for('hiddenServices.show_domains_by_daterange', date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down, domains_tags=domains_tags))
@hiddenServices.route("/hiddenServices/show_domains_by_daterange", methods=['GET'])
def show_domains_by_daterange():
date_from = request.args.get('date_from')
date_to = request.args.get('date_to')
domains_up = request.args.get('domains_up')
domains_down = request.args.get('domains_down')
domains_tags = request.args.get('domains_tags')
date_range = []
if date_from is not None and date_to is not None:
#change format
try:
if len(date_from) != 8:
date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
date_range = substract_date(date_from, date_to)
except:
pass
if not date_range:
date_range.append(datetime.date.today().strftime("%Y%m%d"))
date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8]
date_to = date_from
else:
date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
domains_by_day = {}
domain_metadata = {}
for date in date_range:
if domains_up:
domains_up = True
domains_by_day[date] = list(r_serv_onion.smembers('onion_up:{}'.format(date)))
for domain in domains_by_day[date]:
h = HiddenServices(domain, 'onion')
domain_metadata[domain] = {}
if domains_tags:
domains_tags = True
domain_metadata[domain]['tags'] = h.get_domain_tags(update=True)
domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check')
if domain_metadata[domain]['last_check'] is None:
domain_metadata[domain]['last_check'] = '********'
domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen')
if domain_metadata[domain]['first_seen'] is None:
domain_metadata[domain]['first_seen'] = '********'
domain_metadata[domain]['status_text'] = 'UP'
domain_metadata[domain]['status_color'] = 'Green'
domain_metadata[domain]['status_icon'] = 'fa-check-circle'
if domains_down:
domains_down = True
domains_by_day_down = list(r_serv_onion.smembers('onion_down:{}'.format(date)))
if domains_up:
domains_by_day[date].extend(domains_by_day_down)
else:
domains_by_day[date] = domains_by_day_down
for domain in domains_by_day_down:
#h = HiddenServices(onion_domain, 'onion')
domain_metadata[domain] = {}
#domain_metadata[domain]['tags'] = h.get_domain_tags()
domain_metadata[domain]['last_check'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'last_check')
if domain_metadata[domain]['last_check'] is None:
domain_metadata[domain]['last_check'] = '********'
domain_metadata[domain]['first_seen'] = r_serv_onion.hget('onion_metadata:{}'.format(domain), 'first_seen')
if domain_metadata[domain]['first_seen'] is None:
domain_metadata[domain]['first_seen'] = '********'
domain_metadata[domain]['status_text'] = 'DOWN'
domain_metadata[domain]['status_color'] = 'Red'
domain_metadata[domain]['status_icon'] = 'fa-times-circle'
return render_template("domains.html", date_range=date_range, domains_by_day=domains_by_day, domain_metadata=domain_metadata,
date_from=date_from, date_to=date_to, domains_up=domains_up, domains_down=domains_down,
domains_tags=domains_tags, bootstrap_label=bootstrap_label)
@hiddenServices.route("/hiddenServices/onion_domain", methods=['GET']) @hiddenServices.route("/hiddenServices/onion_domain", methods=['GET'])
def onion_domain(): def onion_domain():
@ -102,8 +269,12 @@ def onion_domain():
# # TODO: FIXME return 404 # # TODO: FIXME return 404
last_check = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'last_check') last_check = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'last_check')
if last_check is None:
last_check = '********'
last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8]) last_check = '{}/{}/{}'.format(last_check[0:4], last_check[4:6], last_check[6:8])
first_seen = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'first_seen') first_seen = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'first_seen')
if first_seen is None:
first_seen = '********'
first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8]) first_seen = '{}/{}/{}'.format(first_seen[0:4], first_seen[4:6], first_seen[6:8])
origin_paste = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'paste_parent') origin_paste = r_serv_onion.hget('onion_metadata:{}'.format(onion_domain), 'paste_parent')

291
var/www/modules/hiddenServices/templates/domains.html Normal file
View file

@ -0,0 +1,291 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Hidden Service - AIL</title>
<link rel="icon" href="{{ url_for('static', filename='image/ail-icon.png') }}">
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet" type="text/css" />
<link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
<script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<style>
.red_table thead{
background: #d91f2d;
color: #fff;
}
.input-group .form-control {
position: unset;
}
</style>
</head>
<body>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-md-8">
{% for date in date_range %}
{% if domains_by_day[date]%}
<div class="panel panel-info" style="text-align:center;">
<div class="panel-heading">
<h3 style="text-align:center;">{{'{}/{}/{}'.format(date[0:4], date[4:6], date[6:8])}}</h3>
</div>
</div>
<table id="table_{{date}}" class="red_table table table-striped table-bordered">
<thead>
<tr>
<th>Domain</th>
<th>First Seen</th>
<th>Last Check</th>
<th>Status</th>
</tr>
</thead>
<tbody>
{% for domain in domains_by_day[date] %}
<tr>
<td>
<a target="_blank" href="{{ url_for('hiddenServices.onion_domain') }}?onion_domain={{ domain }}">{{ domain }}</a>
<div>
{% for tag in domain_metadata[domain]['tags'] %}
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }} <i>{{ domain_metadata[domain]['tags'][tag] }}</i></span>
</a>
{% endfor %}
</div>
</td>
<td>{{'{}/{}/{}'.format(domain_metadata[domain]['first_seen'][0:4], domain_metadata[domain]['first_seen'][4:6], domain_metadata[domain]['first_seen'][6:8])}}</td>
<td>{{'{}/{}/{}'.format(domain_metadata[domain]['last_check'][0:4], domain_metadata[domain]['last_check'][4:6], domain_metadata[domain]['last_check'][6:8])}}</td>
<td><div style="color:{{domain_metadata[domain]['status_color']}}; display:inline-block">
<i class="fa {{domain_metadata[domain]['status_icon']}} fa-2x"></i>
{{domain_metadata[domain]['status_text']}}
</div>
</td>
</tr>
{% endfor %}
</tbody>
</table>
<br>
<br>
<br>
{% endif %}
{% endfor %}
</div>
<div class="col-md-4">
<div class="panel panel-info" style="text-align:center;">
<div class="panel-heading">
Select domains by date range :
<br>
<br>
<form action="{{ url_for('hiddenServices.get_onions_by_daterange') }}" id="hash_selector_form" method='post'>
<div class="row">
<div class="col-md-6">
<div class="input-group">
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
<input class="form-control" id="date-range-from" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from">
</div>
<div class="input-group">
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
<input class="form-control" id="date-range-to" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to">
</div>
<div class="checkbox">
<label>
<input type="checkbox" name="domains_tags" value="True" {% if domains_tags %}checked{% endif %}>
<div style="color:#286090; display:inline-block">
Domains Tags <i class="fa fa-tags"></i>
</div>
</label>
</div>
</div>
<div class="col-md-6">
<div class="checkbox">
<label>
<input type="checkbox" name="domains_up" value="True" {% if domains_up %}checked{% endif %}>
<div style="color:Green; display:inline-block">
Domains UP <i class="fa fa-check-circle"></i>
</div>
</label>
</div>
<div class="checkbox">
<label>
<input type="checkbox" name="domains_down" value="True" {% if domains_down %}checked{% endif %}>
<div style="color:Red; display:inline-block">
Domains DOWN <i class="fa fa-times-circle"></i>
</div>
</label>
</div>
</div>
</div>
<button class="btn btn-primary" style="text-align:center;">
<i class="fa fa-eye-slash"></i> Show Onions
</button>
<form>
</div>
</div>
</div>
</div>
</div>
<!-- /#page-wrapper -->
<script>
var all_graph = {};
$(document).ready(function(){
activePage = "page-hiddenServices"
$("#"+activePage).addClass("active");
$('#date-range-from').dateRangePicker({
separator : ' to ',
getValue: function()
{
if ($('#date-range-from').val() && $('#date-range-to').val() )
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
else
return '';
},
setValue: function(s,s1,s2)
{
$('#date-range-from').val(s1);
$('#date-range-to').val(s2);
}
});
$('#date-range-to').dateRangePicker({
separator : ' to ',
getValue: function()
{
if ($('#date-range-from').val() && $('#date-range-to').val() )
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
else
return '';
},
setValue: function(s,s1,s2)
{
$('#date-range-from').val(s1);
$('#date-range-to').val(s2);
}
});
{% for date in date_range %}
$('#table_{{date}}').DataTable({
"aLengthMenu": [[5, 15, 30, -1], [5, 15, 30, "All"]],
"iDisplayLength": 15,
"order": [[ 0, "asc" ]]
});
{% endfor %}
//all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hiddenServices.domain_crawled_7days_json') }}?type=onion");
});
/*$(window).on("resize", function() {
all_graph.onResize();
});*/
</script>
<script>
/*
all_graph.onResize = function () {
var aspect = 1000 / 500, all_graph = $("#graph_div");
var targetWidth = all_graph.parent().width();
all_graph.attr("width", targetWidth);
all_graph.attr("height", targetWidth / aspect);
}
window.all_graph = all_graph;
function create_line_chart(id, url){
var width = 650;
var height = Math.round(width / 6);
var margin = {top: 20, right: 55, bottom: 50, left: 40};
var x = d3.scaleTime().range([0, width]);
var y = d3.scaleLinear().rangeRound([height, 0]);
var xAxis = d3.axisBottom(x);
var yAxis = d3.axisLeft(y);
var parseTime = d3.timeParse("%Y-%m-%d");
var line = d3.line()
.x(function(d) {
return x(d.date);
})
.y(function(d) {
return y(d.value);
});
var svg_line = d3.select('#'+id).append('svg')
.attr("id", "graph_div")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append('g')
.attr('transform', "translate("+ margin.left +","+ margin.top +")");
var div = d3.select('body').append('div')
.attr('class', 'tooltip')
.style('opacity', 0);
//add div tooltip
d3.json(url)
.then(function(data){
data.forEach(function(d) {
d.date_label = d.date;
d.date = parseTime(d.date);
d.value = +d.value;
});
// fit the data
x.domain(d3.extent(data, function(d) { return d.date; }));
//x.domain(data.map(function (d) { return d.date; })); //E
y.domain([0, d3.max(data, function(d){ return d.value ; })]);
//line
svg_line.append("path")
.data([data])
.attr("class", "line_graph")
.attr("d", line);
// add X axis
svg_line.append("g")
.attr("transform", "translate(0," + height + ")")
.call(d3.axisBottom(x))
.selectAll("text")
.style("text-anchor", "end")
.attr("transform", "rotate(-45)" );
// Add the Y Axis
svg_line.append("g")
.call(d3.axisLeft(y));
//add a dot circle
svg_line.selectAll('dot')
.data(data).enter()
.append('circle')
.attr('r', 2)
.attr('cx', function(d) { return x(d.date); })
.attr('cy', function(d) { return y(d.value); })
.on('mouseover', function(d) {
div.transition().style('opacity', .9);
div.html('' + d.date_label+ '<br/>' + d.value).style('left', (d3.event.pageX) + 'px')
.style("left", (d3.event.pageX) + "px")
.style("top", (d3.event.pageY - 28) + "px");
})
.on('mouseout', function(d)
{
div.transition().style('opacity', 0);
});
});
}
*/
</script>
</body>
</html>

229
var/www/modules/hiddenServices/templates/hiddenServices.html
View file

@ -12,11 +12,17 @@
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet"> <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet"> <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet"> <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/daterangepicker.min.css') }}" rel="stylesheet" type="text/css" />
<!-- JS --> <!-- JS -->
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script> <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/moment.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.daterangepicker.min.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script> <script language="javascript" src="{{ url_for('static', filename='js/d3.min.js') }}"></script>
<style> <style>
.input-group .form-control {
position: unset;
}
div.tooltip { div.tooltip {
position: absolute; position: absolute;
text-align: center; text-align: center;
@ -60,7 +66,7 @@
<th>Status</th> <th>Status</th>
</tr> </tr>
</thead> </thead>
<tbody> <tbody id="tbody_last_crawled">
{% for metadata_onion in last_onions %} {% for metadata_onion in last_onions %}
<tr> <tr>
<td><a target="_blank" href="{{ url_for('hiddenServices.onion_domain') }}?onion_domain={{ metadata_onion['domain'] }}">{{ metadata_onion['domain'] }}</a></td> <td><a target="_blank" href="{{ url_for('hiddenServices.onion_domain') }}?onion_domain={{ metadata_onion['domain'] }}">{{ metadata_onion['domain'] }}</a></td>
@ -84,6 +90,58 @@
</div> </div>
<div class="col-md-6"> <div class="col-md-6">
<div class="panel panel-info" style="text-align:center;">
<div class="panel-heading">
Select domains by date range :
<br>
<br>
<form action="{{ url_for('hiddenServices.get_onions_by_daterange') }}" id="hash_selector_form" method='post'>
<div class="row">
<div class="col-md-6">
<div class="input-group">
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
<input class="form-control" id="date-range-from" placeholder="yyyy-mm-dd" value="{{ date_from }}" name="date_from">
</div>
<div class="input-group">
<span class="input-group-addon"><i class="fa fa-calendar fa" aria-hidden="true"></i></span>
<input class="form-control" id="date-range-to" placeholder="yyyy-mm-dd" value="{{ date_to }}" name="date_to">
</div>
<div class="checkbox">
<label>
<input type="checkbox" name="domains_tags" value="True" checked>
<div style="color:#286090; display:inline-block">
Domains Tags <i class="fa fa-tags"></i>
</div>
</label>
</div>
</div>
<div class="col-md-6">
<div class="checkbox">
<label>
<input type="checkbox" name="domains_up" value="True" checked>
<div style="color:Green; display:inline-block">
Domains UP <i class="fa fa-check-circle"></i>
</div>
</label>
</div>
<div class="checkbox">
<label>
<input type="checkbox" name="domains_down" value="True">
<div style="color:Red; display:inline-block">
Domains DOWN <i class="fa fa-times-circle"></i>
</div>
</label>
</div>
</div>
</div>
<button class="btn btn-primary" style="text-align:center;">
<i class="fa fa-eye-slash"></i> Show Onions
</button>
<form>
</div>
</div>
<div class="panel panel-info"> <div class="panel panel-info">
<div class="panel-heading"> <div class="panel-heading">
<i class="fa fa-eye-slash"></i> Domains Crawled Today <i class="fa fa-eye-slash"></i> Domains Crawled Today
@ -93,41 +151,81 @@
<tbody> <tbody>
<tr> <tr>
<td> <td>
<a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_up=True">
<div style="color:Green; display:inline-block"> <div style="color:Green; display:inline-block">
<i class="fa fa-check-circle fa-2x"></i> <i class="fa fa-check-circle fa-2x"></i>
Domains UP Domains UP
</div> </div>
</a>
</td> </td>
<td> <td>
<a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_up=True">
<div style="color:Green; display:inline-block"> <div style="color:Green; display:inline-block">
{{ statDomains['domains_up'] }} <div id="text_domain_up">{{ statDomains['domains_up'] }}</div>
</div> </div>
</a>
</td> </td>
</tr> </tr>
<tr> <tr>
<td> <td>
<a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_down=True">
<div style="color:Red; display:inline-block"> <div style="color:Red; display:inline-block">
<i class="fa fa-times-circle fa-2x"></i> <i class="fa fa-times-circle fa-2x"></i>
Domains DOWN Domains DOWN
</div> </div>
</a>
</td> </td>
<td> <td>
<a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_down=True">
<div style="color:Red; display:inline-block"> <div style="color:Red; display:inline-block">
{{ statDomains['domains_down'] }} <div id="text_domain_down">{{ statDomains['domains_down'] }}</div>
</div> </div>
</a>
</td> </td>
</tr> </tr>
<tr> <tr>
<td>Crawled Domains</td> <td>
<td>{{ statDomains['total'] }}</td> <a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_up=True&domains_down=True">
Crawled Domains
</a>
</td>
<td>
<a target="_blank" href="{{ url_for('hiddenServices.show_domains_by_daterange') }}?date_from={{ date_from }}&domains_up=True&domains_down=True">
<div id="text_total_domains">{{ statDomains['total'] }}</div>
</a>
</td>
</tr> </tr>
<tr> <tr>
<td>Domains in Queue</td> <td>Domains in Queue</td>
<td>{{ statDomains['domains_queue'] }}</td> <td><div id="text_domain_queue">{{ statDomains['domains_queue'] }}</div></td>
</tr> </tr>
</tbody> </tbody>
</table> </table>
</div> </div>
<div class="panel panel-info" {%if not crawler_metadata%}hidden{%endif%} id="panel_crawler">
<div class="panel-heading">
Crawlers Status
</div>
<table class="table table-hover table-striped">
<tbody id="tbody_crawler_info">
{% for crawler in crawler_metadata %}
<tr>
<td>
<i class="fa fa-{%if crawler['status']%}check{%else%}times{%endif%}-circle fa-2x" style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};"></i> {{crawler['crawler_info']}}
</td>
<td>
{{crawler['crawling_domain']}}
</td>
<td style="color:{%if crawler['status']%}Green{%else%}Red{%endif%};">
{{crawler['status_info']}}
</td>
</tr>
{% endfor %}
</tbody>
</table>
</div>
</div> </div>
</div> </div>
@ -137,14 +235,59 @@
<script> <script>
var all_graph = {}; var all_graph = {};
var to_refresh = false
$(document).ready(function(){ $(document).ready(function(){
activePage = "page-hiddenServices" activePage = "page-hiddenServices"
$("#"+activePage).addClass("active"); $("#"+activePage).addClass("active");
$('#date-range-from').dateRangePicker({
separator : ' to ',
getValue: function()
{
if ($('#date-range-from').val() && $('#date-range-to').val() )
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
else
return '';
},
setValue: function(s,s1,s2)
{
$('#date-range-from').val(s1);
$('#date-range-to').val(s2);
}
});
$('#date-range-to').dateRangePicker({
separator : ' to ',
getValue: function()
{
if ($('#date-range-from').val() && $('#date-range-to').val() )
return $('#date-range-from').val() + ' to ' + $('#date-range-to').val();
else
return '';
},
setValue: function(s,s1,s2)
{
$('#date-range-from').val(s1);
$('#date-range-to').val(s2);
}
});
all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hiddenServices.domain_crawled_7days_json') }}?type=onion"); all_graph.line_chart = create_line_chart('graph_line', "{{ url_for('hiddenServices.domain_crawled_7days_json') }}?type=onion");
$( window ).focus(function() {
to_refresh = true
refresh_list_crawled();
});
$( window ).blur(function() {
to_refresh = false
});
to_refresh = true
refresh_list_crawled();
}); });
$(window).on("resize", function() { $(window).on("resize", function() {
all_graph.onResize(); all_graph.onResize();
}); });
</script> </script>
<script> <script>
@ -228,6 +371,80 @@ d3.json(url)
}); });
}); });
} }
</script>
<script>
function refresh_list_crawled(){
$.getJSON("{{ url_for('hiddenServices.last_crawled_domains_with_stats_json') }}",
function(data) {
var tableRef = document.getElementById('tbody_last_crawled');
$("#tbody_last_crawled").empty()
for (var i = 0; i < data.last_onions.length; i++) {
var data_domain = data.last_onions[i]
var newRow = tableRef.insertRow(tableRef.rows.length);
var newCell = newRow.insertCell(0);
newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.onion_domain') }}?onion_domain="+data_domain['domain']+"\">"+data_domain['domain']+"</a></td>";
newCell = newRow.insertCell(1);
newCell.innerHTML = "<td>"+data_domain['first_seen'].substr(0, 4)+"/"+data_domain['first_seen'].substr(4, 2)+"/"+data_domain['first_seen'].substr(6, 2)+"</td>"
newCell = newRow.insertCell(2);
newCell.innerHTML = "<td>"+data_domain['last_check'].substr(0, 4)+"/"+data_domain['last_check'].substr(4, 2)+"/"+data_domain['last_check'].substr(6, 2)+"</td>"
newCell = newRow.insertCell(3);
newCell.innerHTML = "<td><div style=\"color:"+data_domain['status_color']+"; display:inline-block\"><i class=\"fa "+data_domain['status_icon']+" fa-2x\"></i>"+data_domain['status_text']+"</div></td>"
}
var statDomains = data.statDomains
document.getElementById('text_domain_up').innerHTML = statDomains['domains_up']
document.getElementById('text_domain_down').innerHTML = statDomains['domains_down']
document.getElementById('text_domain_queue').innerHTML = statDomains['domains_queue']
document.getElementById('text_total_domains').innerHTML = statDomains['total']
if(data.crawler_metadata.length!=0){
$("#tbody_crawler_info").empty();
var tableRef = document.getElementById('tbody_crawler_info');
for (var i = 0; i < data.crawler_metadata.length; i++) {
var crawler = data.crawler_metadata[i];
var newRow = tableRef.insertRow(tableRef.rows.length);
var text_color;
var icon;
if(crawler['status']){
text_color = 'Green';
icon = 'check';
} else {
text_color = 'Red';
icon = 'times';
}
var newCell = newRow.insertCell(0);
newCell.innerHTML = "<td><i class=\"fa fa-"+icon+"-circle fa-2x\" style=\"color:"+text_color+";\"></i>"+crawler['crawler_info']+"</td>";
newCell = newRow.insertCell(1);
newCell.innerHTML = "<td><a target=\"_blank\" href=\"{{ url_for('hiddenServices.onion_domain') }}?onion_domain="+crawler['crawling_domain']+"\">"+crawler['crawling_domain']+"</a></td>";
newCell = newRow.insertCell(2);
newCell.innerHTML = "<td><div style=\"color:"+text_color+";\">"+crawler['status_info']+"</div></td>";
$("#panel_crawler").show();
}
} else {
$("#panel_crawler").hide();
}
}
);
if (to_refresh) {
setTimeout("refresh_list_crawled()", 10000);
}
}
</script> </script>
</body> </body>

21
var/www/modules/search/Flask_search.py
View file

@ -16,6 +16,8 @@ from whoosh import index
from whoosh.fields import Schema, TEXT, ID from whoosh.fields import Schema, TEXT, ID
from whoosh.qparser import QueryParser from whoosh.qparser import QueryParser
import time
# ============ VARIABLES ============ # ============ VARIABLES ============
import Flask_config import Flask_config
@ -55,8 +57,8 @@ def get_index_list(selected_index=""):
if os.path.isdir(os.path.join(baseindexpath, dirs)): if os.path.isdir(os.path.join(baseindexpath, dirs)):
value = dirs value = dirs
name = to_iso_date(dirs) + " - " + \ name = to_iso_date(dirs) + " - " + \
str(get_dir_size(dirs) / (1000*1000)) + " Mb " + \ str(get_dir_size(dirs) / (1000*1000)) + " Mb " #+ \
"(" + str(get_item_count(dirs)) + " Items" + ")" #"(" + str(get_item_count(dirs))''' + " Items" + ")"
flag = dirs==selected_index.split('/')[-1] flag = dirs==selected_index.split('/')[-1]
if dirs == "old_index": if dirs == "old_index":
temp = [value, name, flag] temp = [value, name, flag]
@ -66,6 +68,7 @@ def get_index_list(selected_index=""):
index_list.sort(reverse=True, key=lambda x: x[0]) index_list.sort(reverse=True, key=lambda x: x[0])
if len(temp) != 0: if len(temp) != 0:
index_list.append(temp) index_list.append(temp)
return index_list return index_list
def get_dir_size(directory): def get_dir_size(directory):
@ -108,6 +111,7 @@ def search():
else: else:
selected_index = os.path.join(baseindexpath, index_name) selected_index = os.path.join(baseindexpath, index_name)
''' temporary disabled
# Search filename # Search filename
for path in r_serv_pasteName.smembers(q[0]): for path in r_serv_pasteName.smembers(q[0]):
r.append(path) r.append(path)
@ -119,13 +123,14 @@ def search():
curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
paste_date.append(curr_date) paste_date.append(curr_date)
paste_size.append(paste._get_p_size()) paste_size.append(paste._get_p_size())
'''
# Search full line # Search full line
schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
ix = index.open_dir(selected_index) ix = index.open_dir(selected_index)
with ix.searcher() as searcher: with ix.searcher() as searcher:
query = QueryParser("content", ix.schema).parse(" ".join(q)) query = QueryParser("content", ix.schema).parse("".join(q))
results = searcher.search_page(query, 1, pagelen=num_elem_to_get) results = searcher.search_page(query, 1, pagelen=num_elem_to_get)
for x in results: for x in results:
r.append(x.items()[0][1]) r.append(x.items()[0][1])
@ -159,15 +164,18 @@ def search():
results = searcher.search(query) results = searcher.search(query)
num_res = len(results) num_res = len(results)
index_list = get_index_list()
index_min = 1 index_min = 1
index_max = len(get_index_list()) index_max = len(index_list)
return render_template("search.html", r=r, c=c, return render_template("search.html", r=r, c=c,
query=request.form['query'], paste_date=paste_date, query=request.form['query'], paste_date=paste_date,
paste_size=paste_size, char_to_display=max_preview_modal, paste_size=paste_size, char_to_display=max_preview_modal,
num_res=num_res, index_min=index_min, index_max=index_max, num_res=num_res, index_min=index_min, index_max=index_max,
bootstrap_label=bootstrap_label, bootstrap_label=bootstrap_label,
paste_tags=paste_tags, paste_tags=paste_tags,
index_list=get_index_list(selected_index) index_list=index_list
) )
@ -212,6 +220,7 @@ def get_more_search_result():
p_tags = r_serv_metadata.smembers('tag:'+path) p_tags = r_serv_metadata.smembers('tag:'+path)
l_tags = [] l_tags = []
for tag in p_tags: for tag in p_tags:
complete_tag = tag
tag = tag.split('=') tag = tag.split('=')
if len(tag) > 1: if len(tag) > 1:
if tag[1] != '': if tag[1] != '':
@ -223,7 +232,7 @@ def get_more_search_result():
else: else:
tag = tag[0] tag = tag[0]
l_tags.append(tag) l_tags.append( (tag, complete_tag) )
list_tags.append(l_tags) list_tags.append(l_tags)
to_return = {} to_return = {}

4
var/www/modules/search/templates/search.html
View file

@ -201,7 +201,9 @@
var curr_preview = data.preview_array[i].replace(/\"/g, "\'"); var curr_preview = data.preview_array[i].replace(/\"/g, "\'");
var tag = "" var tag = ""
for(j=0; j<data.list_tags[i].length; j++) { for(j=0; j<data.list_tags[i].length; j++) {
tag = tag + "<span class=\"label label-" + data.bootstrap_label[j % 5] + " pull-left\">" + data.list_tags[j] + "</span>" tag = tag + "<a href=\"{{ url_for('Tags.get_tagged_paste') }}?ltags=" + data.list_tags[i][j][1] + ">"
+ "<span class=\"label label-" + data.bootstrap_label[j % 5] + " pull-left\">" + data.list_tags[i][j][0]
+ "</span>" + "</a>"
} }
search_table.row.add( [ search_table.row.add( [
init_num_of_elements_in_table+((offset))+i+1, init_num_of_elements_in_table+((offset))+i+1,

69
var/www/modules/terms/Flask_terms.py
View file

@ -10,7 +10,7 @@ import redis
import datetime import datetime
import calendar import calendar
import flask import flask
from flask import Flask, render_template, jsonify, request, Blueprint from flask import Flask, render_template, jsonify, request, Blueprint, url_for, redirect
import re import re
import Paste import Paste
from pprint import pprint from pprint import pprint
@ -24,6 +24,8 @@ cfg = Flask_config.cfg
baseUrl = Flask_config.baseUrl baseUrl = Flask_config.baseUrl
r_serv_term = Flask_config.r_serv_term r_serv_term = Flask_config.r_serv_term
r_serv_cred = Flask_config.r_serv_cred r_serv_cred = Flask_config.r_serv_cred
r_serv_db = Flask_config.r_serv_db
bootstrap_label = Flask_config.bootstrap_label
terms = Blueprint('terms', __name__, template_folder='templates') terms = Blueprint('terms', __name__, template_folder='templates')
@ -51,6 +53,7 @@ TrackedTermsNotificationEnabled_Name = "TrackedNotifications"
# same value as in `bin/NotificationHelper.py` # same value as in `bin/NotificationHelper.py`
# Keys will be e.g. TrackedNotificationEmails_<TERMNAME> # Keys will be e.g. TrackedNotificationEmails_<TERMNAME>
TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_" TrackedTermsNotificationEmailsPrefix_Name = "TrackedNotificationEmails_"
TrackedTermsNotificationTagsPrefix_Name = "TrackedNotificationTags_"
'''CRED''' '''CRED'''
REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+' REGEX_CRED = '[a-z]+|[A-Z]{3,}|[A-Z]{1,2}[a-z]+|[0-9]+'
@ -130,6 +133,12 @@ def mixUserName(supplied, extensive=False):
filtered_usernames.append(usr) filtered_usernames.append(usr)
return filtered_usernames return filtered_usernames
def save_tag_to_auto_push(list_tag):
for tag in set(list_tag):
#limit tag length
if len(tag) > 49:
tag = tag[0:48]
r_serv_db.sadd('list_export_tags', tag)
# ============ ROUTES ============ # ============ ROUTES ============
@ -152,6 +161,7 @@ def terms_management():
# Maps a specific term to the associated email addresses # Maps a specific term to the associated email addresses
notificationEMailTermMapping = {} notificationEMailTermMapping = {}
notificationTagsTermMapping = {}
#Regex #Regex
trackReg_list = [] trackReg_list = []
@ -159,7 +169,8 @@ def terms_management():
trackReg_list_num_of_paste = [] trackReg_list_num_of_paste = []
for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name): for tracked_regex in r_serv_term.smembers(TrackedRegexSet_Name):
notificationEMailTermMapping[tracked_regex] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)) ) notificationEMailTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_regex)
notificationTagsTermMapping[tracked_regex] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_regex)
if tracked_regex not in notificationEnabledDict: if tracked_regex not in notificationEnabledDict:
notificationEnabledDict[tracked_regex] = False notificationEnabledDict[tracked_regex] = False
@ -185,8 +196,8 @@ def terms_management():
for tracked_set in r_serv_term.smembers(TrackedSetSet_Name): for tracked_set in r_serv_term.smembers(TrackedSetSet_Name):
tracked_set = tracked_set tracked_set = tracked_set
notificationEMailTermMapping[tracked_set] = "\n".join( (r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)) ) notificationEMailTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_set)
notificationTagsTermMapping[tracked_set] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_set)
if tracked_set not in notificationEnabledDict: if tracked_set not in notificationEnabledDict:
notificationEnabledDict[tracked_set] = False notificationEnabledDict[tracked_set] = False
@ -211,7 +222,8 @@ def terms_management():
track_list_num_of_paste = [] track_list_num_of_paste = []
for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name): for tracked_term in r_serv_term.smembers(TrackedTermsSet_Name):
notificationEMailTermMapping[tracked_term] = "\n".join( r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term)) notificationEMailTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationEmailsPrefix_Name + tracked_term)
notificationTagsTermMapping[tracked_term] = r_serv_term.smembers(TrackedTermsNotificationTagsPrefix_Name + tracked_term)
if tracked_term not in notificationEnabledDict: if tracked_term not in notificationEnabledDict:
notificationEnabledDict[tracked_term] = False notificationEnabledDict[tracked_term] = False
@ -244,7 +256,8 @@ def terms_management():
track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste, track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste,
trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste, trackReg_list_values=trackReg_list_values, trackReg_list_num_of_paste=trackReg_list_num_of_paste,
trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste, trackSet_list_values=trackSet_list_values, trackSet_list_num_of_paste=trackSet_list_num_of_paste,
per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, notificationEMailTermMapping=notificationEMailTermMapping) per_paste=per_paste, notificationEnabledDict=notificationEnabledDict, bootstrap_label=bootstrap_label,
notificationEMailTermMapping=notificationEMailTermMapping, notificationTagsTermMapping=notificationTagsTermMapping)
@terms.route("/terms_management_query_paste/") @terms.route("/terms_management_query_paste/")
@ -313,6 +326,7 @@ def terms_management_action():
action = request.args.get('action') action = request.args.get('action')
term = request.args.get('term') term = request.args.get('term')
notificationEmailsParam = request.args.get('emailAddresses') notificationEmailsParam = request.args.get('emailAddresses')
input_tags = request.args.get('tags')
if action is None or term is None or notificationEmailsParam is None: if action is None or term is None or notificationEmailsParam is None:
return "None" return "None"
@ -320,11 +334,8 @@ def terms_management_action():
if section == "followTerm": if section == "followTerm":
if action == "add": if action == "add":
# Strip all whitespace
notificationEmailsParam = "".join(notificationEmailsParam.split())
# Make a list of all passed email addresses # Make a list of all passed email addresses
notificationEmails = notificationEmailsParam.split(",") notificationEmails = notificationEmailsParam.split()
validNotificationEmails = [] validNotificationEmails = []
# check for valid email addresses # check for valid email addresses
@ -334,6 +345,8 @@ def terms_management_action():
if re.match(r"[^@]+@[^@]+\.[^@]+", email): if re.match(r"[^@]+@[^@]+\.[^@]+", email):
validNotificationEmails.append(email) validNotificationEmails.append(email)
# create tags list
list_tags = input_tags.split()
# check if regex/set or simple term # check if regex/set or simple term
#regex #regex
@ -345,6 +358,10 @@ def terms_management_action():
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email) r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term, email)
# enable notifications by default # enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term) r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term)
# add tags list
for tag in list_tags:
r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term, tag)
save_tag_to_auto_push(list_tags)
#set #set
elif term.startswith('\\') and term.endswith('\\'): elif term.startswith('\\') and term.endswith('\\'):
@ -363,6 +380,10 @@ def terms_management_action():
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email) r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + set_to_add, email)
# enable notifications by default # enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add) r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, set_to_add)
# add tags list
for tag in list_tags:
r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + set_to_add, tag)
save_tag_to_auto_push(list_tags)
#simple term #simple term
else: else:
@ -373,6 +394,10 @@ def terms_management_action():
r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email) r_serv_term.sadd(TrackedTermsNotificationEmailsPrefix_Name + term.lower(), email)
# enable notifications by default # enable notifications by default
r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower()) r_serv_term.sadd(TrackedTermsNotificationEnabled_Name, term.lower())
# add tags list
for tag in list_tags:
r_serv_term.sadd(TrackedTermsNotificationTagsPrefix_Name + term.lower(), tag)
save_tag_to_auto_push(list_tags)
elif action == "toggleEMailNotification": elif action == "toggleEMailNotification":
# get the current state # get the current state
@ -397,6 +422,8 @@ def terms_management_action():
# delete the associated notification emails too # delete the associated notification emails too
r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term) r_serv_term.delete(TrackedTermsNotificationEmailsPrefix_Name + term)
# delete the associated tags set
r_serv_term.delete(TrackedTermsNotificationTagsPrefix_Name + term)
elif section == "blacklistTerm": elif section == "blacklistTerm":
if action == "add": if action == "add":
@ -413,6 +440,28 @@ def terms_management_action():
to_return["term"] = term to_return["term"] = term
return jsonify(to_return) return jsonify(to_return)
@terms.route("/terms_management/delete_terms_tags", methods=['POST'])
def delete_terms_tags():
term = request.form.get('term')
tags_to_delete = request.form.getlist('tags_to_delete')
if term is not None and tags_to_delete is not None:
for tag in tags_to_delete:
r_serv_term.srem(TrackedTermsNotificationTagsPrefix_Name + term, tag)
return redirect(url_for('terms.terms_management'))
else:
return 'None args', 400
@terms.route("/terms_management/delete_terms_email", methods=['GET'])
def delete_terms_email():
term = request.args.get('term')
email = request.args.get('email')
if term is not None and email is not None:
r_serv_term.srem(TrackedTermsNotificationEmailsPrefix_Name + term, email)
return redirect(url_for('terms.terms_management'))
else:
return 'None args', 400
@terms.route("/terms_plot_tool/") @terms.route("/terms_plot_tool/")

215
var/www/modules/terms/templates/terms_management.html
View file

@ -36,6 +36,12 @@
white-space:pre-wrap; white-space:pre-wrap;
word-wrap:break-word; word-wrap:break-word;
} }
.mouse_pointer{
cursor: pointer;
}
.lb-md {
font-size: 16px;
}
</style> </style>
</head> </head>
<body> <body>
@ -74,6 +80,7 @@
<div class="row"> <div class="row">
<div class="col-lg-12"> <div class="col-lg-12">
<div class="row"> <div class="row">
{% set uniq_id = namespace(modal_id=0)%}
<div class="col-lg-12"> <div class="col-lg-12">
<label class="switch"> <label class="switch">
<input id="per_paste" class="switch-input" value="per_paste" type="checkbox" onclick="reload_per_paste()"> <input id="per_paste" class="switch-input" value="per_paste" type="checkbox" onclick="reload_per_paste()">
@ -98,7 +105,8 @@
<div class="form-group input-group" style="margin-bottom: 30px;"> <div class="form-group input-group" style="margin-bottom: 30px;">
<span class="input-group-addon"><span class="fa fa-eye"></span></span> <span class="input-group-addon"><span class="fa fa-eye"></span></span>
<input id="followTermInput" class="form-control" placeholder="Term to track." type="text" style="max-width: 400px;"> <input id="followTermInput" class="form-control" placeholder="Term to track." type="text" style="max-width: 400px;">
<input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (comma separated)" type="text" style="max-width: 400px;"> <input id="followTermEMailNotificationReceiversInput" class="form-control" placeholder="Notification E-Mails (optional, space separated)" type="text" style="max-width: 400px;">
<input id="followTermTag" class="form-control" placeholder="Tags (optional, space separated)" type="text" style="max-width: 400px;">
<button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button> <button id="followTermBtn" class="btn btn-success btn-interaction" style="margin-left: 10px;" data-section="followTerm" data-action="add"> Add term</button>
</div> </div>
@ -119,7 +127,57 @@
<!-- SET --> <!-- SET -->
{% for set in trackSet_list %} {% for set in trackSet_list %}
<tr style="background-color: #cdffca;"> <tr style="background-color: #cdffca;">
<td>{{ set }}</td> <td>
<span class="term_name">{{ set }}</span>
<div>
{% for tag in notificationTagsTermMapping[set] %}
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
</a>
{% endfor %}
{% if notificationTagsTermMapping[set] %}
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
<div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
<div class="modal-dialog">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
<h2 class="text-center">Remove Custom Tag</h2>
</div>
<div class="modal-body">
<form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
{% for tag in notificationTagsTermMapping[set] %}
<div class="form-check">
<input type="hidden" class="form-control" name="term" value="{{ set }}">
<input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
<label class="form-check-label">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
</label>
<br>
</div>
{% endfor %}
</form>
</div>
<div class="modal-footer">
<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
<span class="glyphicon glyphicon-trash"></span>
<span class="label-icon">Remove Tags</span>
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
{% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
{% endif %}
</div>
</td>
<td>{{ trackSet_list_values[loop.index0][3] }}</td> <td>{{ trackSet_list_values[loop.index0][3] }}</td>
<td>{{ trackSet_list_values[loop.index0][0] }}</td> <td>{{ trackSet_list_values[loop.index0][0] }}</td>
<td>{{ trackSet_list_values[loop.index0][1] }}</td> <td>{{ trackSet_list_values[loop.index0][1] }}</td>
@ -130,13 +188,73 @@
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button> <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ set }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ set }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[set] %} checked {% endif %}> &nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ set }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[set] %} checked {% endif %}>
</p></td> </p></td>
<td style="white-space:pre">{{ notificationEMailTermMapping[set] }}</td> <td>
{% for email in notificationEMailTermMapping[set] %}
<a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{set}}">
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
<span class="glyphicon glyphicon-trash" style="color:Red;" ></span>
</div>
</a>
{{ email }}
<br>
{% endfor %}
</td>
</tr> </tr>
{% endfor %} {% endfor %}
<!-- REGEX --> <!-- REGEX -->
{% for regex in trackReg_list %} {% for regex in trackReg_list %}
<tr style="background-color: #fffdca;"> <tr style="background-color: #fffdca;">
<td>{{ regex }}</td> <td>
<span class="term_name">{{ regex }}</span>
<div>
{% for tag in notificationTagsTermMapping[regex] %}
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
</a>
{% endfor %}
{% if notificationTagsTermMapping[regex] %}
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
<div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
<div class="modal-dialog">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
<h2 class="text-center">Remove Custom Tag</h2>
</div>
<div class="modal-body">
<form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
{% for tag in notificationTagsTermMapping[regex] %}
<div class="form-check">
<input type="hidden" class="form-control" name="term" value="{{ regex }}">
<input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
<label class="form-check-label">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
</label>
<br>
</div>
{% endfor %}
</form>
</div>
<div class="modal-footer">
<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
<span class="glyphicon glyphicon-trash"></span>
<span class="label-icon">Remove Tags</span>
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
{% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
{% endif %}
</div>
</td>
<td>{{ trackReg_list_values[loop.index0][3] }}</td> <td>{{ trackReg_list_values[loop.index0][3] }}</td>
<td>{{ trackReg_list_values[loop.index0][0] }}</td> <td>{{ trackReg_list_values[loop.index0][0] }}</td>
<td>{{ trackReg_list_values[loop.index0][1] }}</td> <td>{{ trackReg_list_values[loop.index0][1] }}</td>
@ -147,13 +265,73 @@
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ regex }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button> <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ regex }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ regex }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[regex] %} checked {% endif %}> &nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ regex }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[regex] %} checked {% endif %}>
</p></td> </p></td>
<td style="white-space:pre">{{ notificationEMailTermMapping[regex] }}</td> <td>
{% for email in notificationEMailTermMapping[regex] %}
<a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{regex}}">
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
<span class="glyphicon glyphicon-trash" style="color:Red;"></span>
</div>
</a>
{{ email }}
<br>
{% endfor %}
</td>
</tr> </tr>
{% endfor %} {% endfor %}
<!-- Normal term --> <!-- Normal term -->
{% for term in track_list %} {% for term in track_list %}
<tr> <tr>
<td>{{ term }}</td> <td>
<span class="term_name">{{ term }}</span>
<div>
{% for tag in notificationTagsTermMapping[term] %}
<a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag }}">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} pull-left">{{ tag }}</span>
</a>
{% endfor %}
{% if notificationTagsTermMapping[term] %}
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="modal" data-target="#edit_custom_tag_modal_{{ uniq_id.modal_id }}" data-placement="right" title="Edit Tags List"><i class="fa fa-pencil" style="color:Red;"></i></div>
<div id="edit_custom_tag_modal_{{ uniq_id.modal_id }}" class="modal fade" role="dialog">
<div class="modal-dialog">
<!-- Modal content-->
<div id="mymodalcontent" class="modal-content">
<div class="modal-header" style="border-bottom: 4px solid #48c9b0; background-color: #48c9b0; color: #ffffff;">
<h2 class="text-center">Remove Custom Tag</h2>
</div>
<div class="modal-body">
<form action="{{ url_for('terms.delete_terms_tags') }}" id="checkboxForm" method='post'>
{% for tag in notificationTagsTermMapping[term] %}
<div class="form-check">
<input type="hidden" class="form-control" name="term" value="{{ term }}">
<input type="checkbox" class="form-check-input" name="tags_to_delete" value="{{ tag }}">
<label class="form-check-label">
<span class="label label-{{ bootstrap_label[loop.index0 % 5] }} lb-md">{{ tag }}</span>
</label>
<br>
</div>
{% endfor %}
</form>
</div>
<div class="modal-footer">
<button class="btn btn-danger" type="submit" form="checkboxForm" value="Submit">
<span class="glyphicon glyphicon-trash"></span>
<span class="label-icon">Remove Tags</span>
</button>
<button type="button" class="btn btn-default" data-dismiss="modal" >Close</button>
</div>
</div>
</div>
</div>
{% set uniq_id.modal_id = uniq_id.modal_id + 1 %}
{% endif %}
</div>
</td>
<td>{{ track_list_values[loop.index0][3] }}</td> <td>{{ track_list_values[loop.index0][3] }}</td>
<td>{{ track_list_values[loop.index0][0] }}</td> <td>{{ track_list_values[loop.index0][0] }}</td>
<td>{{ track_list_values[loop.index0][1] }}</td> <td>{{ track_list_values[loop.index0][1] }}</td>
@ -164,7 +342,17 @@
<button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ term }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button> <button class="btn-link btn-interaction" data-toggle="tooltip" data-placement="left" title="Remove this term" data-content="{{ term }}" data-section="followTerm" data-action="delete"><span class="glyphicon glyphicon-trash"></span></button>
&nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ term }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[term] %} checked {% endif %}> &nbsp; &nbsp;<input id="checkBoxEMailAlerts" type="checkbox" title="Toggle E-Mail notifications" class="btn-link btn-interaction" data-content="{{ term }}" data-section="followTerm" data-action="toggleEMailNotification" {% if notificationEnabledDict[term] %} checked {% endif %}>
</p></td> </p></td>
<td style="white-space:pre">{{ notificationEMailTermMapping[term] }}</td> <td>
{% for email in notificationEMailTermMapping[term] %}
<a href="{{ url_for('terms.delete_terms_email') }}?email={{email}}&term={{term}}">
<div class="btn-link btn-interaction pull-right mouse_pointer" data-toggle="tooltip" data-placement="left" data-original-title="Remove this email">
<span class="glyphicon glyphicon-trash" style="color:Red;"></span>
</div>
</a>
{{ email }}
<br>
{% endfor %}
</td>
</tr> </tr>
{% endfor %} {% endfor %}
</tbody> </tbody>
@ -351,17 +539,19 @@ function perform_binding() {
function perform_operation(){ function perform_operation(){
var curr_section = $(this).attr('data-section'); var curr_section = $(this).attr('data-section');
var curr_action = $(this).attr('data-action'); var curr_action = $(this).attr('data-action');
var row_tr = $(this).closest("tr");
if (curr_action == "add") { if (curr_action == "add") {
var curr_term = $('#'+curr_section+'Input').val(); var curr_term = $('#'+curr_section+'Input').val();
var email_addresses = $('#followTermEMailNotificationReceiversInput').val(); var email_addresses = $('#followTermEMailNotificationReceiversInput').val();
var tags = $('#followTermTag').val();
} else { } else {
var curr_term = $(this).attr('data-content'); var curr_term = $(this).attr('data-content');
var email_addresses = ""; var email_addresses = "";
} }
var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses}; var data_to_send = { section: curr_section, action: curr_action, term: curr_term, emailAddresses: email_addresses, tags: tags};
if (curr_term != "") { if (curr_term != "") {
console.log(data_to_send); //console.log(data_to_send);
$.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){ $.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){
if(status == "success") { if(status == "success") {
var json = data; var json = data;
@ -373,12 +563,7 @@ function perform_operation(){
reload_per_paste(); reload_per_paste();
}); });
} else if (json.action == "delete") { } else if (json.action == "delete") {
// Find indexes of row which have the term in the first column row_tr.remove()
var index = table_track.rows().eq( 0 ).filter( function (rowIdx) {
console.log(table_track.cell( rowIdx, 0 ).data())
return table_track.cell( rowIdx, 0 ).data() === json.term;
} );
table_track.rows(index).remove().draw( false );
} }
} else if(json.section == "blacklistTerm"){ } else if(json.section == "blacklistTerm"){
if(json.action == "add") { if(json.action == "add") {

8
var/www/update_thirdparty.sh
View file

@ -86,11 +86,11 @@ popd
#active virtualenv #active virtualenv
source ./../../AILENV/bin/activate source ./../../AILENV/bin/activate
#Update MISP Taxonomies and Galaxies #Update MISP Taxonomies and Galaxies
python3 -m pip install git+https://github.com/MISP/PyTaxonomies python3 -m pip install git+https://github.com/MISP/PyTaxonomies --upgrade
python3 -m pip install git+https://github.com/MISP/PyMISPGalaxies python3 -m pip install git+https://github.com/MISP/PyMISPGalaxies --upgrade
#Update PyMISP #Update PyMISP
python3 -m pip install git+https://github.com/MISP/PyMISP python3 -m pip install git+https://github.com/MISP/PyMISP --upgrade
#Update the Hive #Update the Hive
python3 -m pip install git+https://github.com/TheHive-Project/TheHive4py python3 -m pip install git+https://github.com/TheHive-Project/TheHive4py --upgrade