Merge pull request #56 from mokaddem/mokaddem-testing

New statistics modules and usage of Faup
This commit is contained in:
Alexandre Dulaunoy 2016-07-12 10:10:09 +02:00 committed by GitHub
commit a4c2ba915e
19 changed files with 1006 additions and 16 deletions

View file

@ -66,7 +66,7 @@ if __name__ == "__main__":
# FUNCTIONS #
publisher.info("Script Categ started")
categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential']
categories = ['CreditCards', 'Mail', 'Onion', 'Web', 'Credential', 'Cve']
tmp_dict = {}
for filename in categories:
bname = os.path.basename(filename)

57
bin/Cve.py Executable file
View file

@ -0,0 +1,57 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
"""
Template for new modules
"""
import time
import re
from pubsublogger import publisher
from packages import Paste
from Helper import Process
def search_cve(message):
filepath, count = message.split()
paste = Paste.Paste(filepath)
content = paste.get_p_content()
# regex to find CVE
reg_cve = re.compile(r'(CVE-)[1-2]\d{1,4}-\d{1,5}')
# list of the regex results in the Paste, may be null
results = set(reg_cve.findall(content))
# if the list is greater than 2, we consider the Paste may contain a list of cve
if len(results) > 0:
print('{} contains CVEs'.format(paste.p_name))
publisher.warning('{} contains CVEs'.format(paste.p_name))
if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
# Port of the redis instance used by pubsublogger
publisher.port = 6380
# Script is the default channel used for the modules.
publisher.channel = 'Script'
# Section name in bin/packages/modules.cfg
config_section = 'Cve'
# Setup the I/O queues
p = Process(config_section)
# Sent to the logging a description of the module
publisher.info("Run CVE module")
# Endless loop getting messages from the input queue
while True:
# Get one message from the input queue
message = p.get_from_set()
if message is None:
publisher.debug("{} queue is empty, waiting".format(config_section))
time.sleep(1)
continue
# Do something with the message from the queue
search_cve(message)
# (Optional) Send that thing to the next queue
#p.populate_set_out(something_has_been_done)

View file

@ -39,7 +39,7 @@ def main():
if message is not None:
PST = Paste.Paste(message)
else:
publisher.debug("Script DomClassifier is idling 10s")
publisher.debug("Script DomClassifier is idling 1s")
time.sleep(1)
continue
paste = PST.get_p_content()

View file

@ -33,11 +33,12 @@ if __name__ == "__main__":
# DB OBJECT & HASHS ( DISK )
# FIXME increase flexibility
dico_redis = {}
for year in xrange(2013, 2015):
for year in xrange(2013, 2017):
for month in xrange(0, 16):
dico_redis[str(year)+str(month).zfill(2)] = redis.StrictRedis(
host=p.config.get("Redis_Level_DB", "host"), port=year,
db=month)
#print("dup: "+str(year)+str(month).zfill(2)+"\n")
# FUNCTIONS #
publisher.info("Script duplicate started")
@ -81,7 +82,7 @@ if __name__ == "__main__":
bloop_path_set.add(filebloompath)
# UNIQUE INDEX HASHS TABLE
r_serv0 = dico_redis["201300"]
r_serv0 = dico_redis["201600"]
r_serv0.incr("current_index")
index = r_serv0.get("current_index")+str(PST.p_date)
# HASHTABLES PER MONTH (because of r_serv1 changing db)

View file

@ -66,6 +66,7 @@ function launching_lvldb {
lvdbdir="${AIL_HOME}/LEVEL_DB_DATA/"
db1_y='2013'
db2_y='2014'
db3_y='2016'
nb_db=13
screen -dmS "LevelDB"
@ -75,6 +76,8 @@ function launching_lvldb {
screen -S "LevelDB" -X screen -t "2013" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2013/ -P '$db1_y' -M '$nb_db'; read x'
sleep 0.1
screen -S "LevelDB" -X screen -t "2014" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2014/ -P '$db2_y' -M '$nb_db'; read x'
sleep 0.1
screen -S "LevelDB" -X screen -t "2016" bash -c 'redis-leveldb -H '$lvdbhost' -D '$lvdbdir'2016/ -P '$db3_y' -M '$nb_db'; read x'
}
function launching_logs {
@ -127,6 +130,16 @@ function launching_scripts {
screen -S "Script" -X screen -t "Curve" bash -c './Curve.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Indexer" bash -c './Indexer.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Keys" bash -c './Keys.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Phone" bash -c './Phone.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Release" bash -c './Release.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "Cve" bash -c './Cve.py; read x'
sleep 0.1
screen -S "Script" -X screen -t "WebStats" bash -c './WebStats.py; read x'
}
#If no params, display the help

View file

@ -7,6 +7,8 @@ import dns.exception
from packages import Paste
from packages import lib_refine
from pubsublogger import publisher
from pyfaup.faup import Faup
import re
# Country and ASN lookup
from cymru.ip2asn.dns import DNSClient as ip2asn
@ -16,6 +18,13 @@ import ipaddress
from Helper import Process
# Used to prevent concat with empty fields due to url parsing
def avoidNone(str):
if str is None:
return ""
else:
return str
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
@ -41,6 +50,7 @@ if __name__ == "__main__":
message = p.get_from_set()
prec_filename = None
faup = Faup()
url_regex = "(http|https|ftp)\://([a-zA-Z0-9\.\-]+(\:[a-zA-Z0-9\.&%\$\-]+)*@)*((25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9])\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[1-9]|0)\.(25[0-5]|2[0-4][0-9]|[0-1]{1}[0-9]{2}|[1-9]{1}[0-9]{1}|[0-9])|localhost|([a-zA-Z0-9\-]+\.)*[a-zA-Z0-9\-]+\.(com|edu|gov|int|mil|net|org|biz|arpa|info|name|pro|aero|coop|museum|[a-zA-Z]{2}))(\:[0-9]+)*(/($|[a-zA-Z0-9\.\,\?\'\\\+&%\$#\=~_\-]+))*"
@ -53,19 +63,27 @@ if __name__ == "__main__":
PST = Paste.Paste(filename)
client = ip2asn()
for x in PST.get_regex(url_regex):
scheme, credential, subdomain, domain, host, tld, \
port, resource_path, query_string, f1, f2, f3, \
f4 = x
matching_url = re.search(url_regex, PST.get_p_content())
url = matching_url.group(0)
to_send = "{} {}".format(url, PST._get_p_date())
p.populate_set_out(to_send, 'Url')
faup.decode(url)
domain = faup.get_domain()
subdomain = faup.get_subdomain()
f1 = None
domains_list.append(domain)
p.populate_set_out(x, 'Url')
publisher.debug('{} Published'.format(x))
publisher.debug('{} Published'.format(url))
if f1 == "onion":
print domain
hostl = unicode(subdomain+domain)
hostl = unicode(avoidNone(subdomain)+avoidNone(domain))
try:
socket.setdefaulttimeout(2)
socket.setdefaulttimeout(1)
ip = socket.gethostbyname(unicode(hostl))
except:
# If the resolver is not giving any IPv4 address,

95
bin/WebStats.py Executable file
View file

@ -0,0 +1,95 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
"""
Template for new modules
"""
import time
import datetime
import re
import redis
import os
from packages import lib_words
from pubsublogger import publisher
from packages import Paste
from Helper import Process
from pyfaup.faup import Faup
def analyse(field_name):
field = url_parsed[field_name]
if field is not None:
prev_score = r_serv1.hget(field, date)
if prev_score is not None:
r_serv1.hset(field, date, int(prev_score) + 1)
else:
r_serv1.hset(field, date, 1)
if __name__ == '__main__':
# If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh)
# Port of the redis instance used by pubsublogger
publisher.port = 6380
# Script is the default channel used for the modules.
publisher.channel = 'Script'
# Section name in bin/packages/modules.cfg
config_section = 'WebStats'
# Setup the I/O queues
p = Process(config_section)
# Sent to the logging a description of the module
publisher.info("Makes statistics about valid URL")
# REDIS #
r_serv1 = redis.StrictRedis(
host=p.config.get("Redis_Level_DB", "host"),
port=p.config.get("Redis_Level_DB", "port"),
db=p.config.get("Redis_Level_DB", "db"))
# FILE CURVE SECTION #
csv_path_proto = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "protocolstrending_csv"))
protocolsfile_path = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "protocolsfile"))
csv_path_tld = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "tldstrending_csv"))
tldsfile_path = os.path.join(os.environ['AIL_HOME'],
p.config.get("Directories", "tldsfile"))
faup = Faup()
generate_new_graph = False
# Endless loop getting messages from the input queue
while True:
# Get one message from the input queue
message = p.get_from_set()
if message is None:
if generate_new_graph:
generate_new_graph = False
print 'Building graph'
today = datetime.date.today()
year = today.year
month = today.month
lib_words.create_curve_with_word_file(r_serv1, csv_path_proto,
protocolsfile_path, year,
month)
lib_words.create_curve_with_word_file(r_serv1, csv_path_tld,
tldsfile_path, year,
month)
publisher.debug("{} queue is empty, waiting".format(config_section))
time.sleep(1)
continue
else:
generate_new_graph = True
# Do something with the message from the queue
url, date = message.split()
faup.decode(url)
url_parsed = faup.get()
analyse('scheme') #Scheme analysis
analyse('tld') #Tld analysis

35
bin/empty_queue.py Executable file
View file

@ -0,0 +1,35 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
"""
The Empty queue module
====================
This simple module can be used to clean all queues.
Requirements:
-------------
"""
import redis
import os
import time
from packages import Paste
from pubsublogger import publisher
from Helper import Process
if __name__ == "__main__":
publisher.port = 6380
publisher.channel = "Script"
config_section = ['Global', 'Duplicates', 'Indexer', 'Attributes', 'Lines', 'DomClassifier', 'Tokenize', 'Curve', 'Categ', 'CreditCards', 'Mail', 'Onion', 'DumpValidOnion', 'Web', 'WebStats', 'Release', 'Credential', 'Cve', 'Phone', 'SourceCode', 'Keys']
for queue in config_section:
print 'dropping: ' + queue
p = Process(queue)
while True:
message = p.get_from_set()
if message is None:
break

View file

@ -27,7 +27,7 @@ subscribe = Redis_Words
[Categ]
subscribe = Redis_Global
publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode
publish = Redis_CreditCards,Redis_Mail,Redis_Onion,Redis_Web,Redis_Credential,Redis_SourceCode,Redis_Cve
[CreditCards]
subscribe = Redis_CreditCards
@ -47,12 +47,18 @@ subscribe = Redis_ValidOnion
subscribe = Redis_Web
publish = Redis_Url,ZMQ_Url
[WebStats]
subscribe = Redis_Url
[Release]
subscribe = Redis_Global
[Credential]
subscribe = Redis_Credential
[Cve]
subscribe = Redis_Cve
[Phone]
subscribe = Redis_Global

1
files/Cve Normal file
View file

@ -0,0 +1 @@
CVE

100
files/protocolsfile Normal file
View file

@ -0,0 +1,100 @@
afs
file
ftp
z39.50
z39.50r
z39.50s
vemmi
urn
nfs
dict
acap
rtspu
rtsp
rtsps
tip
pop
cid
mid
data
thismessage
service
shttp
fax
modem
tv
sip
sips
go
icap
h323
ipp
xmlrpc.beep
xmlrpc.beeps
tftp
mupdate
pres
im
mtqp
tel
iris
iris.beep
crid
snmp
tag
wais
prospero
soap.beep
soap.beeps
telnet
gopher
cap
info
dns
ldap
dav
opaquelocktoken
msrp
msrps
dtn
imap
xmpp
iax
news
nntp
snews
sms
rsync
sieve
geo
mailto
jms
mailserver
ipn
tn3270
ws
wss
xcon
xcon-userid
about
aaa
aaas
session
ni
nih
reload
ham
stun
stuns
turn
turns
http
https
coap
coaps
rtmfp
ipps
pkcs11
acct
example
vnc

263
files/tldsfile Normal file
View file

@ -0,0 +1,263 @@
com
org
net
int
edu
gov
mil
arpa
ac
ad
ae
af
ag
ai
al
am
an
ao
aq
ar
as
at
au
aw
ax
az
ba
bb
bd
be
bf
bg
bh
bi
bj
bl
bm
bn
bo
bq
br
bs
bt
bv
bw
by
bz
ca
cc
cd
cf
cg
ch
ci
ck
cl
cm
cn
co
cr
cu
cv
cw
cx
cy
cz
de
dj
dk
dm
do
dz
ec
ee
eg
eh
er
es
et
eu
fi
fj
fk
fm
fo
fr
ga
gb
gd
ge
gf
gg
gh
gi
gl
gm
gn
gp
gq
gr
gs
gt
gu
gw
gy
hk
hm
hn
hr
ht
hu
id
ie
il
im
in
io
iq
ir
is
it
je
jm
jo
jp
ke
kg
kh
ki
km
kn
kp
kr
kw
ky
kz
la
lb
lc
li
lk
lr
ls
lt
lu
lv
ly
ma
mc
md
me
mf
mg
mh
mk
ml
mm
mn
mo
mp
mq
mr
ms
mt
mu
mv
mw
mx
my
mz
na
nc
ne
nf
ng
ni
nl
no
np
nr
nu
nz
om
pa
pe
pf
pg
ph
pk
pl
pm
pn
pr
ps
pt
pw
py
qa
re
ro
rs
ru
rw
sa
sb
sc
sd
se
sg
sh
si
sj
sk
sl
sm
sn
so
sr
ss
st
su
sv
sx
sy
sz
tc
td
tf
tg
th
tj
tk
tl
tm
tn
to
tp
tr
tt
tv
tw
tz
ua
ug
uk
um
us
uy
uz
va
vc
ve
vg
vi
vn
vu
wf
ws
ye
yt
za
zm
zw

View file

@ -96,5 +96,14 @@ def wordstrending():
return render_template("Wordstrending.html")
@app.route("/protocolstrending/")
def protocolstrending():
return render_template("Protocolstrending.html")
@app.route("/tldstrending/")
def tldstrending():
return render_template("Tldstrending.html")
if __name__ == "__main__":
app.run(host='0.0.0.0', port=7000, threaded=True)

View file

@ -0,0 +1,196 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
<div class="navbar-header">
<ul class="nav navbar-nav">
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
</div>
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
<div class="input-group custom-search-form">
<input type="text" class="form-control" placeholder="Search Paste">
<span class="input-group-btn">
<button class="btn btn-default" type="button">
<i class="fa fa-search"></i>
</button>
</span>
</div>
<!-- /input-group -->
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header">ProtocolsTrendings</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="row">
<div class="col-lg-12">
<div class="panel panel-default">
<div class="panel-heading">
<i class="fa fa-bar-chart-o fa-fw"></i> Protocols Trend
<div class="pull-right">
<div class="btn-group">
<button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown">
Actions
<span class="caret"></span>
</button>
<ul class="dropdown-menu pull-right" role="menu">
<li><a href="#" id="linear">Linear Scale</a>
</li>
<li><a href="#" id="log">Log Scale</a>
</li>
<li><a href="#" id="unzoom" onclick="unzoomGraph()">Unzoom</a>
</li>
<li class="divider"></li>
<li><a href="#" id="edit_graph">Edit graph words</a>
</li>
</ul>
</div>
</div>
</div>
<!-- /.panel-heading -->
<div class="panel-body">
<!-- <div id="ProtocolsTrending" style="width:100%;"></div> -->
<div id="ProtocolsTrending" style="width:100%; height:800px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
<script type="text/javascript">
g2 = new Dygraph(
document.getElementById("ProtocolsTrending"),
// path to CSV file
"{{ url_for('static', filename='csv/protocolstrendingdata.csv') }}",
//"../csv/wordstrendingdata.csv",
//window.csv,
{
rollPeriod: 1,
showRoller: true,
//drawPoints: true,
//fillGraph: true,
logscale: true,
animatedZooms: true,
labelsKMB: true,
highlightCircleSize: 3,
highlightSeriesOpts: {
strokeWidth: 3,
strokeBorderWidth: 1,
highlightCircleSize: 5,
},
underlayCallback: function(canvas, area, g) {
canvas.fillStyle = "rgba(255, 193, 37, 0.5)";
function highlight_period(x_start, x_end) {
var canvas_left_x = g.toDomXCoord(x_start);
var canvas_right_x = g.toDomXCoord(x_end);
var canvas_width = canvas_right_x - canvas_left_x;
canvas.fillRect(canvas_left_x, area.y, canvas_width, area.h);
}
var min_data_x = g.getValue(0,0);
var max_data_x = g.getValue(g.numRows()-1,0);
// get day of week
var d = new Date(min_data_x);
var dow = d.getUTCDay();
var ds = d.toUTCString();
var w = min_data_x;
// starting on Sunday is a special case
if (dow == 0) {
highlight_period(w,w+12*3600*1000);
}
// find first saturday
while (dow != 5) {
w += 24*3600*1000;
d = new Date(w);
dow = d.getUTCDay();
}
// shift back 1/2 day to center highlight around the point for the day
w -= 12*3600*1000;
while (w < max_data_x) {
var start_x_highlight = w;
var end_x_highlight = w + 2*24*3600*1000;
// make sure we don't try to plot outside the graph
if (start_x_highlight < min_data_x) {
start_x_highlight = min_data_x;
}
if (end_x_highlight > max_data_x) {
end_x_highlight = max_data_x;
}
highlight_period(start_x_highlight,end_x_highlight);
// calculate start of highlight for next Saturday
w += 7*24*3600*1000;
}
}
});
onclick = function(ev) {
if (g2.isSeriesLocked()) {
g2.clearSelection();
}
else {
g2.setSelection(g2.getSelection(), g2.getHighlightSeries(), true);
}
};
g2.updateOptions({clickCallback: onclick}, true);
var linear = document.getElementById("linear");
var log = document.getElementById("log");
linear.onclick = function() { setLog(false); }
log.onclick = function() { setLog(true); }
var setLog = function(val) {
g2.updateOptions({ logscale: val });
linear.disabled = !val;
log.disabled = val;
}
function unzoomGraph() {
g2.updateOptions({
dateWindow:null,
valueRange:null
});
}
</script>
</div>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
</body>
</html>

View file

@ -0,0 +1,196 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery-1.4.2.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
<div class="navbar-header">
<ul class="nav navbar-nav">
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
</div>
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
<div class="input-group custom-search-form">
<input type="text" class="form-control" placeholder="Search Paste">
<span class="input-group-btn">
<button class="btn btn-default" type="button">
<i class="fa fa-search"></i>
</button>
</span>
</div>
<!-- /input-group -->
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header">Top Level Domain Trending</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
<div class="row">
<div class="col-lg-12">
<div class="panel panel-default">
<div class="panel-heading">
<i class="fa fa-bar-chart-o fa-fw"></i> Top Level Domain Trending
<div class="pull-right">
<div class="btn-group">
<button type="button" class="btn btn-default btn-xs dropdown-toggle" data-toggle="dropdown">
Actions
<span class="caret"></span>
</button>
<ul class="dropdown-menu pull-right" role="menu">
<li><a href="#" id="linear">Linear Scale</a>
</li>
<li><a href="#" id="log">Log Scale</a>
</li>
<li><a href="#" id="unzoom" onclick="unzoomGraph()">Unzoom</a>
</li>
<li class="divider"></li>
<li><a href="#" id="edit_graph">Edit graph words</a>
</li>
</ul>
</div>
</div>
</div>
<!-- /.panel-heading -->
<div class="panel-body">
<!-- <div id="WordTrending" style="width:100%;"></div> -->
<div id="TldsTrending" style="width:100%; height:800px;"></div>
</div>
<!-- /.panel-body -->
</div>
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
<script type="text/javascript">
g2 = new Dygraph(
document.getElementById("TldsTrending"),
// path to CSV file
"{{ url_for('static', filename='csv/tldstrendingdata.csv') }}",
//"../csv/wordstrendingdata.csv",
//window.csv,
{
rollPeriod: 1,
showRoller: true,
//drawPoints: true,
//fillGraph: true,
logscale: true,
animatedZooms: true,
labelsKMB: true,
highlightCircleSize: 3,
highlightSeriesOpts: {
strokeWidth: 3,
strokeBorderWidth: 1,
highlightCircleSize: 5,
},
underlayCallback: function(canvas, area, g) {
canvas.fillStyle = "rgba(255, 193, 37, 0.5)";
function highlight_period(x_start, x_end) {
var canvas_left_x = g.toDomXCoord(x_start);
var canvas_right_x = g.toDomXCoord(x_end);
var canvas_width = canvas_right_x - canvas_left_x;
canvas.fillRect(canvas_left_x, area.y, canvas_width, area.h);
}
var min_data_x = g.getValue(0,0);
var max_data_x = g.getValue(g.numRows()-1,0);
// get day of week
var d = new Date(min_data_x);
var dow = d.getUTCDay();
var ds = d.toUTCString();
var w = min_data_x;
// starting on Sunday is a special case
if (dow == 0) {
highlight_period(w,w+12*3600*1000);
}
// find first saturday
while (dow != 5) {
w += 24*3600*1000;
d = new Date(w);
dow = d.getUTCDay();
}
// shift back 1/2 day to center highlight around the point for the day
w -= 12*3600*1000;
while (w < max_data_x) {
var start_x_highlight = w;
var end_x_highlight = w + 2*24*3600*1000;
// make sure we don't try to plot outside the graph
if (start_x_highlight < min_data_x) {
start_x_highlight = min_data_x;
}
if (end_x_highlight > max_data_x) {
end_x_highlight = max_data_x;
}
highlight_period(start_x_highlight,end_x_highlight);
// calculate start of highlight for next Saturday
w += 7*24*3600*1000;
}
}
});
onclick = function(ev) {
if (g2.isSeriesLocked()) {
g2.clearSelection();
}
else {
g2.setSelection(g2.getSelection(), g2.getHighlightSeries(), true);
}
};
g2.updateOptions({clickCallback: onclick}, true);
var linear = document.getElementById("linear");
var log = document.getElementById("log");
linear.onclick = function() { setLog(false); }
log.onclick = function() { setLog(true); }
var setLog = function(val) {
g2.updateOptions({ logscale: val });
linear.disabled = !val;
log.disabled = val;
}
function unzoomGraph() {
g2.updateOptions({
dateWindow:null,
valueRange:null
});
}
</script>
</div>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
</body>
</html>

View file

@ -24,7 +24,7 @@
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
<div class="navbar-header">
<ul class="nav navbar-nav">
<li><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li class="active"><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a><li></ul>
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
</div>
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">

View file

@ -35,7 +35,7 @@
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
<div class="navbar-header">
<ul class="nav navbar-nav">
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a><li></ul>
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
</div>
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">

View file

@ -24,7 +24,7 @@
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
<div class="navbar-header">
<ul class="nav navbar-nav">
<li><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a><li></ul>
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li><li><a href="{{ url_for('wordstrending') }}"><i class="glyphicon glyphicon-stats"></i> WordsTrendings</a></li><li><a href="{{ url_for('protocolstrending') }}"><i class="glyphicon glyphicon-stats"></i> ProtocolsTrendings</a></li><li><a href="{{ url_for('tldstrending') }}"><i class="glyphicon glyphicon-stats"></i> Top Level Domain Trending</a></li></ul>
</div>
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">

View file

@ -17,7 +17,7 @@ mv temp/startbootstrap-sb-admin-2-1.0.2 temp/sb-admin-2
JQVERSION="1.11.1"
wget http://code.jquery.com/jquery-${JQVERSION}.js -O ./static/js/jquery.js
wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js
#wget https://collabdev.googlecode.com/svn-history/r5/trunk/static/js/jquery.timers-1.0.0.js -O ./static/js/jquery.timers-1.0.0.js
#Here to fix an error about an hard dependency in a obscur script of bootstrap..
wget http://code.jquery.com/jquery-1.4.2.js -O ./static/js/jquery-1.4.2.js