mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 00:28:22 +00:00
Merge pull request #56 from mokaddem/mokaddem-testing
New statistics modules and usage of Faup
This commit is contained in:
GitHub
commit
a4c2ba915e
19 changed files with 1006 additions and 16 deletions
|
|
@ -66,7 +66,7 @@ if __name__ == "__main__":
|
|||
# FUNCTIONS #
|
||||
publisher.info("Script Categ started")
|
||||
|
||||
categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential']
|
||||
categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve']
|
||||
tmp_dict = {}
|
||||
for filename in categories:
|
||||
bname = os.path.basename(filename)
|
||||
|
|
|
|||
57
bin/Cve.py
Executable file
57
bin/Cve.py
Executable file
|
|
@ -0,0 +1,57 @@
|
|||
#!/usr/bin/env python2
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Template for new modules
|
||||
"""
|
||||
|
||||
import time
|
||||
import re
|
||||
from pubsublogger import publisher
|
||||
from packages import Paste
|
||||
from Helper import Process
|
||||
|
||||
|
||||
def search_cve(message):
|
||||
filepath, count = message.split()
|
||||
paste = Paste.Paste(filepath)
|
||||
content = paste.get_p_content()
|
||||
# regex to find CVE
|
||||
reg_cve = re.compile(r'(CVE-)[1-2]\d{1,4}-\d{1,5}')
|
||||
# list of the regex results in the Paste, may be null
|
||||
results = set(reg_cve.findall(content))
|
||||
|
||||
# if the list is greater than 2, we consider the Paste may contain a list of cve
|
||||
if len(results) > 0:
|
||||
print('{} contains CVEs'.format(paste.p_name))
|
||||
publisher.warning('{} contains CVEs'.format(paste.p_name))
|
||||
|
||||
if __name__ == '__main__':
|
||||
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
|
||||
# Port of the redis instance used by pubsublogger
|
||||
publisher.port = 6380
|
||||
# Script is the default channel used for the modules.
|
||||
publisher.channel = 'Script'
|
||||
|
||||
# Section name in bin/packages/modules.cfg
|
||||
config_section = 'Cve'
|
||||
|
||||
# Setup the I/O queues
|
||||
p = Process(config_section)
|
||||
|
||||
# Sent to the logging a description of the module
|
||||
publisher.info("Run CVE module")
|
||||
|
||||
# Endless loop getting messages from the input queue
|
||||
while True:
|
||||
# Get one message from the input queue
|
||||
message = p.get_from_set()
|
||||
if message is None:
|
||||
publisher.debug("{} queue is empty, waiting".format(config_section))
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
# Do something with the message from the queue
|
||||
search_cve(message)
|
||||
|
||||
# (Optional) Send that thing to the next queue
|
||||
#p.populate_set_out(something_has_been_done)
|
||||
|
|
@ -39,7 +39,7 @@ def main():
|
|||
if message is not None:
|
||||
PST = Paste.Paste(message)
|
||||
else:
|
||||
publisher.debug("Script DomClassifier is idling 10s")
|
||||
publisher.debug("Script DomClassifier is idling 1s")
|
||||
time.sleep(1)
|
||||
continue
|
||||
paste = PST.get_p_content()
|
||||
|
|
|
|||
|
|
@ -33,11 +33,12 @@ if __name__ == "__main__":
|
|||
# DB OBJECT & HASHS ( DISK )
|
||||
# FIXME increase flexibility
|
||||
dico_redis = {}
|
||||
for year in xrange(2013, 2015):
|
||||
for year in xrange(2013, 2017):
|
||||
for month in xrange(0, 16):
|
||||
dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Level_DB", "host"), port=year,
|
||||
db=month)
|
||||
#print("dup: "+str(year)+str(month).zfill(2)+"\n")
|
||||
|
||||
# FUNCTIONS #
|
||||
publisher.info("Script duplicate started")
|
||||
|
|
@ -81,7 +82,7 @@ if __name__ == "__main__":
|
|||
bloop_path_set.add(filebloompath)
|
||||
|
||||
# UNIQUE INDEX HASHS TABLE
|
||||
r_serv0 = dico_redis["201300"]
|
||||
r_serv0 = dico_redis["201600"]
|
||||
r_serv0.incr("current_index")
|
||||
index = r_serv0.get("current_index")+str(PST.p_date)
|
||||
# HASHTABLES PER MONTH (because of r_serv1 changing db)
|
||||
|
|
|
|||
|
|
@ -66,6 +66,7 @@ function launching_lvldb {
|
|||
lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
|
||||
db1_y='2013'
|
||||
db2_y='2014'
|
||||
db3_y='2016'
|
||||
nb_db=13
|
||||
|
||||
screen -dmS "LevelDB"
|
||||
|
|
@ -75,6 +76,8 @@ function launching_lvldb {
|
|||
screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
|
||||
sleep 0.1
|
||||
screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
|
||||
sleep 0.1
|
||||
screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
|
||||
}
|
||||
|
||||
function launching_logs {
|
||||
|
|
@ -127,6 +130,16 @@ function launching_scripts {
|
|||
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
|
||||
sleep 0.1
|
||||
screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
|
||||
}
|
||||
|
||||
#If no params, display the help
|
||||
|
|
|
|||
32
bin/Url.py
32
bin/Url.py
|
|
@ -7,6 +7,8 @@ import dns.exception
|
|||
from packages import Paste
|
||||
from packages import lib_refine
|
||||
from pubsublogger import publisher
|
||||
from pyfaup.faup import Faup
|
||||
import re
|
||||
|
||||
# Country and ASN lookup
|
||||
from cymru.ip2asn.dns import DNSClient as ip2asn
|
||||
|
|
@ -16,6 +18,13 @@ import ipaddress
|
|||
|
||||
from Helper import Process
|
||||
|
||||
# Used to prevent concat with empty fields due to url parsing
|
||||
def avoidNone(str):
|
||||
if str is None:
|
||||
return ""
|
||||
else:
|
||||
return str
|
||||
|
||||
if __name__ == "__main__":
|
||||
publisher.port = 6380
|
||||
publisher.channel = "Script"
|
||||
|
|
@ -41,6 +50,7 @@ if __name__ == "__main__":
|
|||
|
||||
message = p.get_from_set()
|
||||
prec_filename = None
|
||||
faup = Faup()
|
||||
|
||||
url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
|
||||
|
||||
|
|
@ -53,19 +63,27 @@ if __name__ == "__main__":
|
|||
PST = Paste.Paste(filename)
|
||||
client = ip2asn()
|
||||
for x in PST.get_regex(url_regex):
|
||||
scheme, credential, subdomain, domain, host, tld, \
|
||||
port, resource_path, query_string, f1, f2, f3, \
|
||||
f4 = x
|
||||
matching_url = re.search(url_regex, PST.get_p_content())
|
||||
url = matching_url.group(0)
|
||||
|
||||
to_send = "{} {}".format(url, PST._get_p_date())
|
||||
p.populate_set_out(to_send, 'Url')
|
||||
|
||||
faup.decode(url)
|
||||
domain = faup.get_domain()
|
||||
subdomain = faup.get_subdomain()
|
||||
f1 = None
|
||||
|
||||
domains_list.append(domain)
|
||||
p.populate_set_out(x, 'Url')
|
||||
publisher.debug('{} Published'.format(x))
|
||||
|
||||
publisher.debug('{} Published'.format(url))
|
||||
|
||||
if f1 == "onion":
|
||||
print domain
|
||||
|
||||
hostl = unicode(subdomain+domain)
|
||||
hostl = unicode(avoidNone(subdomain)+avoidNone(domain))
|
||||
try:
|
||||
socket.setdefaulttimeout(2)
|
||||
socket.setdefaulttimeout(1)
|
||||
ip = socket.gethostbyname(unicode(hostl))
|
||||
except:
|
||||
# If the resolver is not giving any IPv4 address,
|
||||
|
|
|
|||
95
bin/WebStats.py
Executable file
95
bin/WebStats.py
Executable file
|
|
@ -0,0 +1,95 @@
|
|||
#!/usr/bin/env python2
|
||||
# -*-coding:UTF-8 -*
|
||||
"""
|
||||
Template for new modules
|
||||
"""
|
||||
|
||||
import time
|
||||
import datetime
|
||||
import re
|
||||
import redis
|
||||
import os
|
||||
from packages import lib_words
|
||||
from pubsublogger import publisher
|
||||
from packages import Paste
|
||||
from Helper import Process
|
||||
from pyfaup.faup import Faup
|
||||
|
||||
def analyse(field_name):
|
||||
field = url_parsed[field_name]
|
||||
if field is not None:
|
||||
prev_score = r_serv1.hget(field, date)
|
||||
if prev_score is not None:
|
||||
r_serv1.hset(field, date, int(prev_score) + 1)
|
||||
else:
|
||||
r_serv1.hset(field, date, 1)
|
||||
|
||||
if __name__ == '__main__':
|
||||
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
|
||||
# Port of the redis instance used by pubsublogger
|
||||
publisher.port = 6380
|
||||
# Script is the default channel used for the modules.
|
||||
publisher.channel = 'Script'
|
||||
|
||||
# Section name in bin/packages/modules.cfg
|
||||
config_section = 'WebStats'
|
||||
|
||||
# Setup the I/O queues
|
||||
p = Process(config_section)
|
||||
|
||||
# Sent to the logging a description of the module
|
||||
publisher.info("Makes statistics about valid URL")
|
||||
|
||||
# REDIS #
|
||||
r_serv1 = redis.StrictRedis(
|
||||
host=p.config.get("Redis_Level_DB", "host"),
|
||||
port=p.config.get("Redis_Level_DB", "port"),
|
||||
db=p.config.get("Redis_Level_DB", "db"))
|
||||
|
||||
# FILE CURVE SECTION #
|
||||
csv_path_proto = os.path.join(os.environ['AIL_HOME'],
|
||||
p.config.get("Directories", "protocolstrending_csv"))
|
||||
protocolsfile_path = os.path.join(os.environ['AIL_HOME'],
|
||||
p.config.get("Directories", "protocolsfile"))
|
||||
|
||||
csv_path_tld = os.path.join(os.environ['AIL_HOME'],
|
||||
p.config.get("Directories", "tldstrending_csv"))
|
||||
tldsfile_path = os.path.join(os.environ['AIL_HOME'],
|
||||
p.config.get("Directories", "tldsfile"))
|
||||
|
||||
faup = Faup()
|
||||
generate_new_graph = False
|
||||
# Endless loop getting messages from the input queue
|
||||
while True:
|
||||
# Get one message from the input queue
|
||||
message = p.get_from_set()
|
||||
|
||||
if message is None:
|
||||
if generate_new_graph:
|
||||
generate_new_graph = False
|
||||
print 'Building graph'
|
||||
today = datetime.date.today()
|
||||
year = today.year
|
||||
month = today.month
|
||||
|
||||
lib_words.create_curve_with_word_file(r_serv1, csv_path_proto,
|
||||
protocolsfile_path, year,
|
||||
month)
|
||||
|
||||
lib_words.create_curve_with_word_file(r_serv1, csv_path_tld,
|
||||
tldsfile_path, year,
|
||||
month)
|
||||
|
||||
publisher.debug("{} queue is empty, waiting".format(config_section))
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
else:
|
||||
generate_new_graph = True
|
||||
# Do something with the message from the queue
|
||||
url, date = message.split()
|
||||
faup.decode(url)
|
||||
url_parsed = faup.get()
|
||||
|
||||
analyse('scheme') #Scheme analysis
|
||||
analyse('tld') #Tld analysis
|
||||
35
bin/empty_queue.py
Executable file
35
bin/empty_queue.py
Executable file
|
|
@ -0,0 +1,35 @@
|
|||
#!/usr/bin/env python2
|
||||
# -*-coding:UTF-8 -*
|
||||
|
||||
"""
|
||||
The Empty queue module
|
||||
====================
|
||||
|
||||
This simple module can be used to clean all queues.
|
||||
|
||||
Requirements:
|
||||
-------------
|
||||
|
||||
|
||||
"""
|
||||
import redis
|
||||
import os
|
||||
import time
|
||||
from packages import Paste
|
||||
from pubsublogger import publisher
|
||||
from Helper import Process
|
||||
|
||||
if __name__ == "__main__":
|
||||
publisher.port = 6380
|
||||
publisher.channel = "Script"
|
||||
|
||||
config_section = ['Global', 'Duplicates', 'Indexer', 'Attributes', 'Lines', 'DomClassifier', 'Tokenize', 'Curve', 'Categ', 'CreditCards', 'Mail', 'Onion', 'DumpValidOnion', 'Web', 'WebStats', 'Release', 'Credential', 'Cve', 'Phone', 'SourceCode', 'Keys']
|
||||
|
||||
for queue in config_section:
|
||||
print 'dropping: ' + queue
|
||||
p = Process(queue)
|
||||
while True:
|
||||
message = p.get_from_set()
|
||||
if message is None:
|
||||
break
|
||||
|
||||
|
|
@ -27,7 +27,7 @@ subscribe = Redis_Words
|
|||
|
||||
[Categ]
|
||||
subscribe = Redis_Global
|
||||
publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode
|
||||
publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve
|
||||
|
||||
[CreditCards]
|
||||
subscribe = Redis_CreditCards
|
||||
|
|
@ -47,12 +47,18 @@ subscribe = Redis_ValidOnion
|
|||
subscribe = Redis_Web
|
||||
publish = Redis_Url,ZMQ_Url
|
||||
|
||||
[WebStats]
|
||||
subscribe = Redis_Url
|
||||
|
||||
[Release]
|
||||
subscribe = Redis_Global
|
||||
|
||||
[Credential]
|
||||
subscribe = Redis_Credential
|
||||
|
||||
[Cve]
|
||||
subscribe = Redis_Cve
|
||||
|
||||
[Phone]
|
||||
subscribe = Redis_Global
|
||||
|
||||
|
|
|
|||
1
files/Cve
Normal file
1
files/Cve
Normal file
|
|
@ -0,0 +1 @@
|
|||
CVE
|
||||
100
files/protocolsfile
Normal file
100
files/protocolsfile
Normal file
|
|
@ -0,0 +1,100 @@
|
|||
afs
|
||||
file
|
||||
ftp
|
||||
z39.50
|
||||
z39.50r
|
||||
z39.50s
|
||||
vemmi
|
||||
urn
|
||||
nfs
|
||||
dict
|
||||
acap
|
||||
rtspu
|
||||
rtsp
|
||||
rtsps
|
||||
tip
|
||||
pop
|
||||
cid
|
||||
mid
|
||||
data
|
||||
thismessage
|
||||
service
|
||||
shttp
|
||||
fax
|
||||
modem
|
||||
tv
|
||||
sip
|
||||
sips
|
||||
go
|
||||
icap
|
||||
h323
|
||||
ipp
|
||||
xmlrpc.beep
|
||||
xmlrpc.beeps
|
||||
tftp
|
||||
mupdate
|
||||
pres
|
||||
im
|
||||
mtqp
|
||||
tel
|
||||
iris
|
||||
iris.beep
|
||||
crid
|
||||
snmp
|
||||
tag
|
||||
wais
|
||||
prospero
|
||||
soap.beep
|
||||
soap.beeps
|
||||
telnet
|
||||
gopher
|
||||
cap
|
||||
info
|
||||
dns
|
||||
ldap
|
||||
dav
|
||||
opaquelocktoken
|
||||
msrp
|
||||
msrps
|
||||
dtn
|
||||
imap
|
||||
xmpp
|
||||
iax
|
||||
news
|
||||
nntp
|
||||
snews
|
||||
sms
|
||||
rsync
|
||||
sieve
|
||||
geo
|
||||
mailto
|
||||
jms
|
||||
mailserver
|
||||
ipn
|
||||
tn3270
|
||||
ws
|
||||
wss
|
||||
xcon
|
||||
xcon-userid
|
||||
about
|
||||
aaa
|
||||
aaas
|
||||
session
|
||||
ni
|
||||
nih
|
||||
reload
|
||||
ham
|
||||
stun
|
||||
stuns
|
||||
turn
|
||||
turns
|
||||
http
|
||||
https
|
||||
coap
|
||||
coaps
|
||||
rtmfp
|
||||
ipps
|
||||
pkcs11
|
||||
acct
|
||||
example
|
||||
vnc
|
||||
263
files/tldsfile
Normal file
263
files/tldsfile
Normal file
|
|
@ -0,0 +1,263 @@
|
|||
com
|
||||
org
|
||||
net
|
||||
int
|
||||
edu
|
||||
gov
|
||||
mil
|
||||
arpa
|
||||
ac
|
||||
ad
|
||||
ae
|
||||
af
|
||||
ag
|
||||
ai
|
||||
al
|
||||
am
|
||||
an
|
||||
ao
|
||||
aq
|
||||
ar
|
||||
as
|
||||
at
|
||||
au
|
||||
aw
|
||||
ax
|
||||
az
|
||||
ba
|
||||
bb
|
||||
bd
|
||||
be
|
||||
bf
|
||||
bg
|
||||
bh
|
||||
bi
|
||||
bj
|
||||
bl
|
||||
bm
|
||||
bn
|
||||
bo
|
||||
bq
|
||||
br
|
||||
bs
|
||||
bt
|
||||
bv
|
||||
bw
|
||||
by
|
||||
bz
|
||||
ca
|
||||
cc
|
||||
cd
|
||||
cf
|
||||
cg
|
||||
ch
|
||||
ci
|
||||
ck
|
||||
cl
|
||||
cm
|
||||
cn
|
||||
co
|
||||
cr
|
||||
cu
|
||||
cv
|
||||
cw
|
||||
cx
|
||||
cy
|
||||
cz
|
||||
de
|
||||
dj
|
||||
dk
|
||||
dm
|
||||
do
|
||||
dz
|
||||
ec
|
||||
ee
|
||||
eg
|
||||
eh
|
||||
er
|
||||
es
|
||||
et
|
||||
eu
|
||||
fi
|
||||
fj
|
||||
fk
|
||||
fm
|
||||
fo
|
||||
fr
|
||||
ga
|
||||
gb
|
||||
gd
|
||||
ge
|
||||
gf
|
||||
gg
|
||||
gh
|
||||
gi
|
||||
gl
|
||||
gm
|
||||
gn
|
||||
gp
|
||||
gq
|
||||
gr
|
||||
gs
|
||||
gt
|
||||
gu
|
||||
gw
|
||||
gy
|
||||
hk
|
||||
hm
|
||||
hn
|
||||
hr
|
||||
ht
|
||||
hu
|
||||
id
|
||||
ie
|
||||
il
|
||||
im
|
||||
in
|
||||
io
|
||||
iq
|
||||
ir
|
||||
is
|
||||
it
|
||||
je
|
||||
jm
|
||||
jo
|
||||
jp
|
||||
ke
|
||||
kg
|
||||
kh
|
||||
ki
|
||||
km
|
||||
kn
|
||||
kp
|
||||
kr
|
||||
kw
|
||||
ky
|
||||
kz
|
||||
la
|
||||
lb
|
||||
lc
|
||||
li
|
||||
lk
|
||||
lr
|
||||
ls
|
||||
lt
|
||||
lu
|
||||
lv
|
||||
ly
|
||||
ma
|
||||
mc
|
||||
md
|
||||
me
|
||||
mf
|
||||
mg
|
||||
mh
|
||||
mk
|
||||
ml
|
||||
mm
|
||||
mn
|
||||
mo
|
||||
mp
|
||||
mq
|
||||
mr
|
||||
ms
|
||||
mt
|
||||
mu
|
||||
mv
|
||||
mw
|
||||
mx
|
||||
my
|
||||
mz
|
||||
na
|
||||
nc
|
||||
ne
|
||||
nf
|
||||
ng
|
||||
ni
|
||||
nl
|
||||
no
|
||||
np
|
||||
nr
|
||||
nu
|
||||
nz
|
||||
om
|
||||
pa
|
||||
pe
|
||||
pf
|
||||
pg
|
||||
ph
|
||||
pk
|
||||
pl
|
||||
pm
|
||||
pn
|
||||
pr
|
||||
ps
|
||||
pt
|
||||
pw
|
||||
py
|
||||
qa
|
||||
re
|
||||
ro
|
||||
rs
|
||||
ru
|
||||
rw
|
||||
sa
|
||||
sb
|
||||
sc
|
||||
sd
|
||||
se
|
||||
sg
|
||||
sh
|
||||
si
|
||||
sj
|
||||
sk
|
||||
sl
|
||||
sm
|
||||
sn
|
||||
so
|
||||
sr
|
||||
ss
|
||||
st
|
||||
su
|
||||
sv
|
||||
sx
|
||||
sy
|
||||
sz
|
||||
tc
|
||||
td
|
||||
tf
|
||||
tg
|
||||
th
|
||||
tj
|
||||
tk
|
||||
tl
|
||||
tm
|
||||
tn
|
||||
to
|
||||
tp
|
||||
tr
|
||||
tt
|
||||
tv
|
||||
tw
|
||||
tz
|
||||
ua
|
||||
ug
|
||||
uk
|
||||
um
|
||||
us
|
||||
uy
|
||||
uz
|
||||
va
|
||||
vc
|
||||
ve
|
||||
vg
|
||||
vi
|
||||
vn
|
||||
vu
|
||||
wf
|
||||
ws
|
||||
ye
|
||||
yt
|
||||
za
|
||||
zm
|
||||
zw
|
||||
|
|
@ -96,5 +96,14 @@ def wordstrending():
|
|||
return render_template("Wordstrending.html")
|
||||
|
||||
|
||||
@app.route("/protocolstrending/")
|
||||
def protocolstrending():
|
||||
return render_template("Protocolstrending.html")
|
||||
|
||||
@app.route("/tldstrending/")
|
||||
def tldstrending():
|
||||
return render_template("Tldstrending.html")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
app.run(host='0.0.0.0', port=7000, threaded=True)
|
||||
|
|
|
|||
196
var/www/templates/Protocolstrending.html
Normal file
196
var/www/templates/Protocolstrending.html
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Analysis Information Leak framework Dashboard</title>
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
|
||||
<!-- JS -->
|
||||
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div id="wrapper">
|
||||
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
|
||||
<div class="navbar-header">
|
||||
<ul class="nav navbar-nav">
|
||||
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
|
||||
</div>
|
||||
<!-- /.navbar-top-links -->
|
||||
<div class="navbar-default sidebar" role="navigation">
|
||||
<div class="sidebar-collapse">
|
||||
<ul class="nav" id="side-menu">
|
||||
<li class="sidebar-search">
|
||||
<div class="input-group custom-search-form">
|
||||
<input type="text" class="form-control" placeholder="Search Paste">
|
||||
<span class="input-group-btn">
|
||||
<button class="btn btn-default" type="button">
|
||||
<i class="fa fa-search"></i>
|
||||
</button>
|
||||
</span>
|
||||
</div>
|
||||
<!-- /input-group -->
|
||||
</li>
|
||||
</ul>
|
||||
<!-- /#side-menu -->
|
||||
</div>
|
||||
<!-- /.sidebar-collapse -->
|
||||
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
|
||||
</div>
|
||||
<!-- /.navbar-static-side -->
|
||||
</nav>
|
||||
<div id="page-wrapper">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<h1 class="page-header">ProtocolsTrendings</h1>
|
||||
</div>
|
||||
<!-- /.col-lg-12 -->
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<i class="fa fa-bar-chart-o fa-fw"></i> Protocols Trend
|
||||
<div class="pull-right">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown">
|
||||
Actions
|
||||
<span class="caret"></span>
|
||||
</button>
|
||||
<ul class="dropdown-menu pull-right" role="menu">
|
||||
<li><a href="#" id="linear">Linear Scale</a>
|
||||
</li>
|
||||
<li><a href="#" id="log">Log Scale</a>
|
||||
</li>
|
||||
<li><a href="#" id="unzoom" onclick="unzoomGraph()">Unzoom</a>
|
||||
</li>
|
||||
<li class="divider"></li>
|
||||
<li><a href="#" id="edit_graph">Edit graph words</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.panel-heading -->
|
||||
<div class="panel-body">
|
||||
<!-- <div id="ProtocolsTrending" style="width:100%;"></div> -->
|
||||
<div id="ProtocolsTrending" style="width:100%; height:800px;"></div>
|
||||
</div>
|
||||
<!-- /.panel-body -->
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
</div>
|
||||
<!-- /#page-wrapper -->
|
||||
<script type="text/javascript">
|
||||
g2 = new Dygraph(
|
||||
document.getElementById("ProtocolsTrending"),
|
||||
// path to CSV file
|
||||
"{{ url_for('static', filename='csv/protocolstrendingdata.csv') }}",
|
||||
//"../csv/wordstrendingdata.csv",
|
||||
//window.csv,
|
||||
{
|
||||
rollPeriod: 1,
|
||||
showRoller: true,
|
||||
//drawPoints: true,
|
||||
//fillGraph: true,
|
||||
logscale: true,
|
||||
animatedZooms: true,
|
||||
labelsKMB: true,
|
||||
highlightCircleSize: 3,
|
||||
highlightSeriesOpts: {
|
||||
strokeWidth: 3,
|
||||
strokeBorderWidth: 1,
|
||||
highlightCircleSize: 5,
|
||||
},
|
||||
underlayCallback: function(canvas, area, g) {
|
||||
canvas.fillStyle = "rgba(255, 193, 37, 0.5)";
|
||||
|
||||
function highlight_period(x_start, x_end) {
|
||||
var canvas_left_x = g.toDomXCoord(x_start);
|
||||
var canvas_right_x = g.toDomXCoord(x_end);
|
||||
var canvas_width = canvas_right_x - canvas_left_x;
|
||||
canvas.fillRect(canvas_left_x, area.y, canvas_width, area.h);
|
||||
}
|
||||
|
||||
var min_data_x = g.getValue(0,0);
|
||||
var max_data_x = g.getValue(g.numRows()-1,0);
|
||||
|
||||
// get day of week
|
||||
var d = new Date(min_data_x);
|
||||
var dow = d.getUTCDay();
|
||||
var ds = d.toUTCString();
|
||||
|
||||
var w = min_data_x;
|
||||
// starting on Sunday is a special case
|
||||
if (dow == 0) {
|
||||
highlight_period(w,w+12*3600*1000);
|
||||
}
|
||||
// find first saturday
|
||||
while (dow != 5) {
|
||||
w += 24*3600*1000;
|
||||
d = new Date(w);
|
||||
dow = d.getUTCDay();
|
||||
}
|
||||
|
||||
// shift back 1/2 day to center highlight around the point for the day
|
||||
w -= 12*3600*1000;
|
||||
while (w < max_data_x) {
|
||||
var start_x_highlight = w;
|
||||
var end_x_highlight = w + 2*24*3600*1000;
|
||||
// make sure we don't try to plot outside the graph
|
||||
if (start_x_highlight < min_data_x) {
|
||||
start_x_highlight = min_data_x;
|
||||
}
|
||||
if (end_x_highlight > max_data_x) {
|
||||
end_x_highlight = max_data_x;
|
||||
}
|
||||
highlight_period(start_x_highlight,end_x_highlight);
|
||||
// calculate start of highlight for next Saturday
|
||||
w += 7*24*3600*1000;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
onclick = function(ev) {
|
||||
if (g2.isSeriesLocked()) {
|
||||
g2.clearSelection();
|
||||
}
|
||||
else {
|
||||
g2.setSelection(g2.getSelection(), g2.getHighlightSeries(), true);
|
||||
}
|
||||
};
|
||||
g2.updateOptions({clickCallback: onclick}, true);
|
||||
|
||||
var linear = document.getElementById("linear");
|
||||
var log = document.getElementById("log");
|
||||
linear.onclick = function() { setLog(false); }
|
||||
log.onclick = function() { setLog(true); }
|
||||
var setLog = function(val) {
|
||||
g2.updateOptions({ logscale: val });
|
||||
linear.disabled = !val;
|
||||
log.disabled = val;
|
||||
}
|
||||
function unzoomGraph() {
|
||||
g2.updateOptions({
|
||||
dateWindow:null,
|
||||
valueRange:null
|
||||
});
|
||||
}
|
||||
</script>
|
||||
</div>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
196
var/www/templates/Tldstrending.html
Normal file
196
var/www/templates/Tldstrending.html
Normal file
|
|
@ -0,0 +1,196 @@
|
|||
<!DOCTYPE html>
|
||||
<html>
|
||||
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||||
|
||||
<title>Analysis Information Leak framework Dashboard</title>
|
||||
|
||||
<!-- Core CSS -->
|
||||
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
|
||||
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
|
||||
<!-- JS -->
|
||||
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
|
||||
<script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
|
||||
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
|
||||
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<div id="wrapper">
|
||||
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
|
||||
<div class="navbar-header">
|
||||
<ul class="nav navbar-nav">
|
||||
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
|
||||
</div>
|
||||
<!-- /.navbar-top-links -->
|
||||
<div class="navbar-default sidebar" role="navigation">
|
||||
<div class="sidebar-collapse">
|
||||
<ul class="nav" id="side-menu">
|
||||
<li class="sidebar-search">
|
||||
<div class="input-group custom-search-form">
|
||||
<input type="text" class="form-control" placeholder="Search Paste">
|
||||
<span class="input-group-btn">
|
||||
<button class="btn btn-default" type="button">
|
||||
<i class="fa fa-search"></i>
|
||||
</button>
|
||||
</span>
|
||||
</div>
|
||||
<!-- /input-group -->
|
||||
</li>
|
||||
</ul>
|
||||
<!-- /#side-menu -->
|
||||
</div>
|
||||
<!-- /.sidebar-collapse -->
|
||||
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
|
||||
</div>
|
||||
<!-- /.navbar-static-side -->
|
||||
</nav>
|
||||
<div id="page-wrapper">
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<h1 class="page-header">Top Level Domain Trending</h1>
|
||||
</div>
|
||||
<!-- /.col-lg-12 -->
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
<div class="row">
|
||||
<div class="col-lg-12">
|
||||
<div class="panel panel-default">
|
||||
<div class="panel-heading">
|
||||
<i class="fa fa-bar-chart-o fa-fw"></i> Top Level Domain Trending
|
||||
<div class="pull-right">
|
||||
<div class="btn-group">
|
||||
<button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown">
|
||||
Actions
|
||||
<span class="caret"></span>
|
||||
</button>
|
||||
<ul class="dropdown-menu pull-right" role="menu">
|
||||
<li><a href="#" id="linear">Linear Scale</a>
|
||||
</li>
|
||||
<li><a href="#" id="log">Log Scale</a>
|
||||
</li>
|
||||
<li><a href="#" id="unzoom" onclick="unzoomGraph()">Unzoom</a>
|
||||
</li>
|
||||
<li class="divider"></li>
|
||||
<li><a href="#" id="edit_graph">Edit graph words</a>
|
||||
</li>
|
||||
</ul>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.panel-heading -->
|
||||
<div class="panel-body">
|
||||
<!-- <div id="WordTrending" style="width:100%;"></div> -->
|
||||
<div id="TldsTrending" style="width:100%; height:800px;"></div>
|
||||
</div>
|
||||
<!-- /.panel-body -->
|
||||
</div>
|
||||
</div>
|
||||
<!-- /.row -->
|
||||
</div>
|
||||
<!-- /#page-wrapper -->
|
||||
<script type="text/javascript">
|
||||
g2 = new Dygraph(
|
||||
document.getElementById("TldsTrending"),
|
||||
// path to CSV file
|
||||
"{{ url_for('static', filename='csv/tldstrendingdata.csv') }}",
|
||||
//"../csv/wordstrendingdata.csv",
|
||||
//window.csv,
|
||||
{
|
||||
rollPeriod: 1,
|
||||
showRoller: true,
|
||||
//drawPoints: true,
|
||||
//fillGraph: true,
|
||||
logscale: true,
|
||||
animatedZooms: true,
|
||||
labelsKMB: true,
|
||||
highlightCircleSize: 3,
|
||||
highlightSeriesOpts: {
|
||||
strokeWidth: 3,
|
||||
strokeBorderWidth: 1,
|
||||
highlightCircleSize: 5,
|
||||
},
|
||||
underlayCallback: function(canvas, area, g) {
|
||||
canvas.fillStyle = "rgba(255, 193, 37, 0.5)";
|
||||
|
||||
function highlight_period(x_start, x_end) {
|
||||
var canvas_left_x = g.toDomXCoord(x_start);
|
||||
var canvas_right_x = g.toDomXCoord(x_end);
|
||||
var canvas_width = canvas_right_x - canvas_left_x;
|
||||
canvas.fillRect(canvas_left_x, area.y, canvas_width, area.h);
|
||||
}
|
||||
|
||||
var min_data_x = g.getValue(0,0);
|
||||
var max_data_x = g.getValue(g.numRows()-1,0);
|
||||
|
||||
// get day of week
|
||||
var d = new Date(min_data_x);
|
||||
var dow = d.getUTCDay();
|
||||
var ds = d.toUTCString();
|
||||
|
||||
var w = min_data_x;
|
||||
// starting on Sunday is a special case
|
||||
if (dow == 0) {
|
||||
highlight_period(w,w+12*3600*1000);
|
||||
}
|
||||
// find first saturday
|
||||
while (dow != 5) {
|
||||
w += 24*3600*1000;
|
||||
d = new Date(w);
|
||||
dow = d.getUTCDay();
|
||||
}
|
||||
|
||||
// shift back 1/2 day to center highlight around the point for the day
|
||||
w -= 12*3600*1000;
|
||||
while (w < max_data_x) {
|
||||
var start_x_highlight = w;
|
||||
var end_x_highlight = w + 2*24*3600*1000;
|
||||
// make sure we don't try to plot outside the graph
|
||||
if (start_x_highlight < min_data_x) {
|
||||
start_x_highlight = min_data_x;
|
||||
}
|
||||
if (end_x_highlight > max_data_x) {
|
||||
end_x_highlight = max_data_x;
|
||||
}
|
||||
highlight_period(start_x_highlight,end_x_highlight);
|
||||
// calculate start of highlight for next Saturday
|
||||
w += 7*24*3600*1000;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
onclick = function(ev) {
|
||||
if (g2.isSeriesLocked()) {
|
||||
g2.clearSelection();
|
||||
}
|
||||
else {
|
||||
g2.setSelection(g2.getSelection(), g2.getHighlightSeries(), true);
|
||||
}
|
||||
};
|
||||
g2.updateOptions({clickCallback: onclick}, true);
|
||||
|
||||
var linear = document.getElementById("linear");
|
||||
var log = document.getElementById("log");
|
||||
linear.onclick = function() { setLog(false); }
|
||||
log.onclick = function() { setLog(true); }
|
||||
var setLog = function(val) {
|
||||
g2.updateOptions({ logscale: val });
|
||||
linear.disabled = !val;
|
||||
log.disabled = val;
|
||||
}
|
||||
function unzoomGraph() {
|
||||
g2.updateOptions({
|
||||
dateWindow:null,
|
||||
valueRange:null
|
||||
});
|
||||
}
|
||||
</script>
|
||||
</div>
|
||||
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
|
||||
</body>
|
||||
|
||||
</html>
|
||||
|
|
@ -24,7 +24,7 @@
|
|||
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
|
||||
<div class="navbar-header">
|
||||
<ul class="nav navbar-nav">
|
||||
<li><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li class="active"><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a><li></ul>
|
||||
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
|
||||
</div>
|
||||
<!-- /.navbar-top-links -->
|
||||
<div class="navbar-default sidebar" role="navigation">
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@
|
|||
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
|
||||
<div class="navbar-header">
|
||||
<ul class="nav navbar-nav">
|
||||
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a><li></ul>
|
||||
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
|
||||
</div>
|
||||
<!-- /.navbar-top-links -->
|
||||
<div class="navbar-default sidebar" role="navigation">
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@
|
|||
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
|
||||
<div class="navbar-header">
|
||||
<ul class="nav navbar-nav">
|
||||
<li><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a><li></ul>
|
||||
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
|
||||
</div>
|
||||
<!-- /.navbar-top-links -->
|
||||
<div class="navbar-default sidebar" role="navigation">
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ mv temp/startbootstrap-sb-admin-2-1.0.2 temp/sb-admin-2
|
|||
JQVERSION="1.11.1"
|
||||
wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js
|
||||
|
||||
wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js
|
||||
#wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js
|
||||
|
||||
#Here to fix an error about an hard dependency in a obscur script of bootstrap..
|
||||
wget http://code.jquery.com/jquery-1.4.2.js -O ./static/js/jquery-1.4.2.js
|
||||
|
|
|
|||
Loading…
Reference in a new issue