AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-23 06:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #127 from mokaddem/auto-web-interface

Browse source 
Auto web interface
This commit is contained in:
Alexandre Dulaunoy 2017-07-17 12:17:34 +02:00 committed by GitHub
commit b18e0dad92
48 changed files with 478 additions and 497 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

30
var/www/Flask_base_template.py Normal file
View file

@ -0,0 +1,30 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
'''
Flask functions and routes for the trending modules page
'''
import redis
from flask import Flask, render_template, jsonify, request, Blueprint
# ============ VARIABLES ============
import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
MODULENAME = Blueprint('MODULENAME', __name__, template_folder='templates')
# ============ FUNCTIONS ============
def one():
return 1
# ============= ROUTES ==============
@MODULENAME.route("/MODULENAME/", methods=['GET'])
def MODULENAME_page():
return render_template("MODULENAME.html")
# ========= REGISTRATION =========
app.register_blueprint(MODULENAME)

82
var/www/Flask_server.py
View file

@ -9,10 +9,12 @@ import time
import calendar
from flask import Flask, render_template, jsonify, request
import flask
import importlib
import os
from os.path import join
import sys
sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/'))
sys.path.append('./Flasks/')
sys.path.append('./modules/')
import Paste
from Date import Date
@ -25,16 +27,70 @@ cfg = Flask_config.cfg
Flask_config.app = Flask(__name__, static_url_path='/static/')
app = Flask_config.app
# import routes and functions from modules
import Flask_dashboard
import Flask_trendingcharts
import Flask_trendingmodules
import Flask_browsepastes
import Flask_sentiment
import Flask_terms
import Flask_search
import Flask_showpaste
# ========= HEADER GENERATION ========
# Get headers items that should be ignored (not displayed)
toIgnoreModule = set()
try:
with open('templates/ignored_modules.txt', 'r') as f:
lines = f.read().splitlines()
for line in lines:
toIgnoreModule.add(line)
except IOError:
f = open('templates/ignored_modules.txt', 'w')
f.close()
# Dynamically import routes and functions from modules
# Also, prepare header.html
to_add_to_header_dico = {}
for root, dirs, files in os.walk('modules/'):
sys.path.append(join(root))
# Ignore the module
curr_dir = root.split('/')[1]
if curr_dir in toIgnoreModule:
continue
for name in files:
module_name = root.split('/')[-2]
if name.startswith('Flask_') and name.endswith('.py'):
if name == 'Flask_config.py':
continue
name = name.strip('.py')
#print('importing {}'.format(name))
importlib.import_module(name)
elif name == 'header_{}.html'.format(module_name):
with open(join(root, name), 'r') as f:
to_add_to_header_dico[module_name] = f.read()
#create header.html
complete_header = ""
with open('templates/header_base.html', 'r') as f:
complete_header = f.read()
modified_header = complete_header
#Add the header in the supplied order
for module_name, txt in to_add_to_header_dico.items():
to_replace = '<!--{}-->'.format(module_name)
if to_replace in complete_header:
modified_header = modified_header.replace(to_replace, txt)
del to_add_to_header_dico[module_name]
#Add the header for no-supplied order
to_add_to_header = []
for module_name, txt in to_add_to_header_dico.items():
to_add_to_header.append(txt)
modified_header = modified_header.replace('<!--insert here-->', '\n'.join(to_add_to_header))
#Write the header.html file
with open('templates/header.html', 'w') as f:
f.write(modified_header)
# ========= JINJA2 FUNCTIONS ========
def list_len(s):
return len(s)
app.jinja_env.filters['list_len'] = list_len
@ -51,6 +107,12 @@ def add_header(response):
response.headers['Cache-Control'] = 'public, max-age=0'
return response
# ========== ROUTES ============
@app.route('/searchbox/')
def searchbox():
return render_template("searchbox.html")
# ============ MAIN ============
if __name__ == "__main__":

219
var/www/Flasks/Flask_corpus.py
View file

@ -1,219 +0,0 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
'''
Flask functions and routes for the trending modules page
'''
import redis
import datetime
import calendar
import flask
from flask import Flask, render_template, jsonify, request
import Paste
# ============ VARIABLES ============
import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
r_serv_corpus = Flask_config.r_serv_corpus
# ============ FUNCTIONS ============
def Corpus_getValueOverRange(word, startDate, num_day):
passed_days = 0
oneDay = 60*60*24
to_return = []
curr_to_return = 0
for timestamp in range(startDate, startDate - max(num_day)*oneDay, -oneDay):
value = r_serv_corpus.hget(timestamp, word)
curr_to_return += int(value) if value is not None else 0
for i in num_day:
if passed_days == i-1:
to_return.append(curr_to_return)
passed_days += 1
return to_return
# ============ ROUTES ============
@app.route("/corpus_management/")
def corpus_management():
TrackedCorpusSet_Name = "TrackedSetCorpusSet"
TrackedCorpusDate_Name = "TrackedCorpusDate"
today = datetime.datetime.now()
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
today_timestamp = calendar.timegm(today.timetuple())
track_list = []
track_list_values = []
track_list_num_of_paste = []
for tracked_corpus in r_serv_corpus.smembers(TrackedCorpusSet_Name):
track_list.append(tracked_corpus)
value_range = Corpus_getValueOverRange(tracked_corpus, today_timestamp, [1, 7, 31])
corpus_date = r_serv_corpus.hget(TrackedCorpusDate_Name, tracked_corpus)
set_paste_name = "tracked_" + tracked_corpus
track_list_num_of_paste.append(r_serv_corpus.scard(set_paste_name))
corpus_date = datetime.datetime.utcfromtimestamp(int(corpus_date)) if corpus_date is not None else "No date recorded"
value_range.append(corpus_date)
track_list_values.append(value_range)
return render_template("corpus_management.html", black_list=black_list, track_list=track_list, track_list_values=track_list_values, track_list_num_of_paste=track_list_num_of_paste)
@app.route("/corpus_management_query_paste/")
def corpus_management_query_paste():
corpus = request.args.get('corpus')
TrackedCorpusSet_Name = "TrackedSetCorpusSet"
paste_info = []
set_paste_name = "tracked_" + corpus
track_list_path = r_serv_corpus.smembers(set_paste_name)
for path in track_list_path:
paste = Paste.Paste(path)
p_date = str(paste._get_p_date())
p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4]
p_source = paste.p_source
p_encoding = paste._get_p_encoding()
p_size = paste.p_size
p_mime = paste.p_mime
p_lineinfo = paste.get_lines_info()
p_content = paste.get_p_content().decode('utf-8', 'ignore')
if p_content != 0:
p_content = p_content[0:400]
paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content})
return jsonify(paste_info)
@app.route("/corpus_management_query/")
def corpus_management_query():
TrackedCorpusDate_Name = "TrackedCorpusDate"
corpus = request.args.get('corpus')
today = datetime.datetime.now()
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
today_timestamp = calendar.timegm(today.timetuple())
value_range = corpus_getValueOverRange(corpus, today_timestamp, [1, 7, 31])
corpus_date = r_serv_corpus.hget(TrackedCorpusDate_Name, corpus)
corpus_date = datetime.datetime.utcfromtimestamp(int(corpus_date)) if corpus_date is not None else "No date recorded"
value_range.append(str(corpus_date))
return jsonify(value_range)
@app.route("/corpus_management_action/", methods=['GET'])
def corpus_management_action():
TrackedCorpusSet_Name = "TrackedSetCorpusSet"
TrackedCorpusDate_Name = "TrackedCorpusDate"
today = datetime.datetime.now()
today = today.replace(microsecond=0)
today_timestamp = calendar.timegm(today.timetuple())
section = request.args.get('section')
action = request.args.get('action')
corpus = request.args.get('corpus')
if action is None or corpus is None:
return "None"
else:
if section == "followCorpus":
if action == "add":
r_serv_corpus.sadd(TrackedCorpusSet_Name, corpus.lower())
r_serv_corpus.hset(TrackedCorpusDate_Name, corpus, today_timestamp)
else:
r_serv_corpus.srem(TrackedCorpusSet_Name, corpus.lower())
else:
return "None"
to_return = {}
to_return["section"] = section
to_return["action"] = action
to_return["corpus"] = corpus
return jsonify(to_return)
@app.route("/corpus_plot_tool/")
def corpus_plot_tool():
corpus = request.args.get('corpus')
if corpus is not None:
return render_template("corpus_plot_tool.html", corpus=corpus)
else:
return render_template("corpus_plot_tool.html", corpus="")
@app.route("/corpus_plot_tool_data/")
def corpus_plot_tool_data():
oneDay = 60*60*24
range_start = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
range_start = range_start.replace(hour=0, minute=0, second=0, microsecond=0)
range_start = calendar.timegm(range_start.timetuple())
range_end = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_end')))) if request.args.get('range_end') is not None else 0;
range_end = range_end.replace(hour=0, minute=0, second=0, microsecond=0)
range_end = calendar.timegm(range_end.timetuple())
corpus = request.args.get('corpus')
if corpus is None:
return "None"
else:
value_range = []
for timestamp in range(range_start, range_end+oneDay, oneDay):
value = r_serv_corpus.hget(timestamp, corpus)
curr_value_range = int(value) if value is not None else 0
value_range.append([timestamp, curr_value_range])
value_range.insert(0,corpus)
return jsonify(value_range)
@app.route("/corpus_plot_top/")
def corpus_plot_top():
return render_template("corpus_plot_top.html")
@app.route("/corpus_plot_top_data/")
def corpus_plot_top_data():
oneDay = 60*60*24
today = datetime.datetime.now()
today = today.replace(hour=0, minute=0, second=0, microsecond=0)
today_timestamp = calendar.timegm(today.timetuple())
set_day = "TopCorpusFreq_set_day_" + str(today_timestamp)
set_week = "TopCorpusFreq_set_week";
set_month = "TopCorpusFreq_set_month";
the_set = request.args.get('set')
num_day = int(request.args.get('num_day'))
if the_set is None:
return "None"
else:
to_return = []
if the_set == "TopCorpusFreq_set_day":
the_set += "_" + str(today_timestamp)
for corpus, tot_value in r_serv_corpus.zrevrangebyscore(the_set, '+inf', '-inf', withscores=True, start=0, num=20):
position = {}
position['day'] = r_serv_corpus.zrevrank(set_day, corpus)
position['day'] = position['day']+1 if position['day'] is not None else "<20"
position['week'] = r_serv_corpus.zrevrank(set_week, corpus)
position['week'] = position['week']+1 if position['week'] is not None else "<20"
position['month'] = r_serv_corpus.zrevrank(set_month, corpus)
position['month'] = position['month']+1 if position['month'] is not None else "<20"
value_range = []
for timestamp in range(today_timestamp, today_timestamp - num_day*oneDay, -oneDay):
value = r_serv_corpus.hget(timestamp, corpus)
curr_value_range = int(value) if value is not None else 0
value_range.append([timestamp, curr_value_range])
to_return.append([corpus, value_range, tot_value, position])
return jsonify(to_return)

44
var/www/create_new_web_module.py Executable file
View file

@ -0,0 +1,44 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
"Hepler to create a new webpage associated with a module."
import os
def createModuleFolder(modulename):
path_module = os.path.join('modules', modulename)
os.mkdir(path_module)
# create html template
with open('templates/base_template.html', 'r') as templateFile:
template = templateFile.read()
template = template.replace('MODULENAME', modulename)
os.mkdir(os.path.join(path_module, 'templates'))
with open(os.path.join(os.path.join(path_module, 'templates'), modulename+'.html'), 'w') as toWriteTemplate:
toWriteTemplate.write(template)
# create html header template
with open('templates/header_base_template.html', 'r') as header_templateFile:
header = header_templateFile.read()
header = header.replace('MODULENAME', modulename)
with open(os.path.join(os.path.join(path_module, 'templates'), 'header_{}.html'.format(modulename) ), 'w') as toWriteHeader:
toWriteHeader.write(header)
#create flask template
with open('Flask_base_template.py', 'r') as flaskFile:
flask = flaskFile.read()
flask = flask.replace('MODULENAME', modulename)
with open(os.path.join(path_module, 'Flask_{}.py'.format(modulename)), 'w') as toWriteFlask:
toWriteFlask.write(flask)
def main():
rep1 = raw_input('New module name: ')
createModuleFolder(rep1)
if __name__ == '__main__':
main()

0
var/www/Flasks/Flask_config.py → var/www/modules/Flask_config.py
View file

13
var/www/Flasks/Flask_browsepastes.py → var/www/modules/browsepastes/Flask_browsepastes.py
View file

@ -7,7 +7,7 @@
import redis
import json
import flask
from flask import Flask, render_template, jsonify, request
from flask import Flask, render_template, jsonify, request, Blueprint
import Paste
@ -19,6 +19,9 @@ cfg = Flask_config.cfg
max_preview_char = Flask_config.max_preview_char
max_preview_modal = Flask_config.max_preview_modal
r_serv_db = Flask_config.r_serv_db
browsepastes = Blueprint('browsepastes', __name__, template_folder='templates')
# ============ FUNCTIONS ============
def getPastebyType(server, module_name):
@ -51,13 +54,13 @@ def event_stream_getImportantPasteByModule(module_name):
# ============ ROUTES ============
@app.route("/browseImportantPaste/", methods=['GET'])
@browsepastes.route("/browseImportantPaste/", methods=['GET'])
def browseImportantPaste():
module_name = request.args.get('moduleName')
return render_template("browse_important_paste.html")
@app.route("/importantPasteByModule/", methods=['GET'])
@browsepastes.route("/importantPasteByModule/", methods=['GET'])
def importantPasteByModule():
module_name = request.args.get('moduleName')
@ -92,9 +95,11 @@ def importantPasteByModule():
char_to_display=max_preview_modal,
finished=finished)
@app.route("/_getImportantPasteByModule")
@browsepastes.route("/_getImportantPasteByModule")
def getImportantPasteByModule():
module_name = request.args.get('moduleName')
return flask.Response(event_stream_getImportantPasteByModule(module_name), mimetype="text/event-stream")
# ========= REGISTRATION =========
app.register_blueprint(browsepastes)

23
var/www/templates/browse_important_paste.html → var/www/modules/browsepastes/templates/browse_important_paste.html
View file

@ -35,24 +35,7 @@
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
{% include 'searchbox.html' %}
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
{% include 'navbar.html' %}
<!-- Modal -->
<div id="mymodal" class="modal fade" role="dialog">
@ -130,7 +113,7 @@
$("#"+activePage).addClass("active");
var dataPath = 'credential';
$.get("{{ url_for('importantPasteByModule') }}"+"?moduleName="+dataPath, function(data, status){
$.get("{{ url_for('browsepastes.importantPasteByModule') }}"+"?moduleName="+dataPath, function(data, status){
$('#'+dataPath+'-tab').html(data);
});
});
@ -144,7 +127,7 @@ var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_fo
$('.nav-tabs a').on('shown.bs.tab', function(event){
var dataPath = $(event.target).attr('data-attribute-name');
$.get("{{ url_for('importantPasteByModule') }}"+"?moduleName="+dataPath, function(data, status){
$.get("{{ url_for('browsepastes.importantPasteByModule') }}"+"?moduleName="+dataPath, function(data, status){
var currentTab = $('[name].active').children();
$('#'+previous_tab.attr('data-attribute-name')+'-tab').html(loading_gif);
currentTab.removeClass( "active" );

1
var/www/modules/browsepastes/templates/header_browsepastes.html Normal file
View file

@ -0,0 +1 @@
<li id='page-browse'><a href="{{ url_for('browsepastes.browseImportantPaste') }}"><i class="fa fa-search-plus "></i> Browse important pastes</a></li>

22
var/www/templates/important_paste_by_module.html → var/www/modules/browsepastes/templates/important_paste_by_module.html
View file

@ -9,16 +9,14 @@
</tr>
</thead>
<tbody>
{% set i = 0 %}
{% for path in all_path %}
<tr>
<td> {{ i + 1 }}</td>
<td><a target="_blank" href="{{ url_for('showsavedpaste') }}?paste={{path}}&num={{i+1}}">{{ path }}</a></td>
<td>{{ paste_date[i] }}</td>
<td>{{ paste_linenum[i] }}</td>
<td><p><span class="glyphicon glyphicon-info-sign" data-toggle="tooltip" data-placement="left" title="{{ content[i] }} "></span> <button type="button" class="btn-link" data-num="{{ i + 1 }}" data-toggle="modal" data-target="#mymodal" data-url="{{ url_for('showsavedpaste') }}?paste={{ path }}&num={{ i+1 }}" data-path="{{ path }}"><span class="fa fa-search-plus"></span></button></p></td>
<td> {{ loop.index0 }}</td>
<td><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{path}}&num={{loop.index0+1}}">{{ path }}</a></td>
<td>{{ paste_date[loop.index0] }}</td>
<td>{{ paste_linenum[loop.index0] }}</td>
<td><p><span class="glyphicon glyphicon-info-sign" data-toggle="tooltip" data-placement="left" title="{{ content[loop.index0] }} "></span> <button type="button" class="btn-link" data-num="{{ loop.index0 + 1 }}" data-toggle="modal" data-target="#mymodal" data-url="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{ path }}&num={{ loop.index0+1 }}" data-path="{{ path }}"><span class="fa fa-search-plus"></span></button></p></td>
</tr>
{% set i = i + 1 %}
{% endfor %}
</tbody>
</table>
@ -47,7 +45,7 @@ function deploy_source() {
if(typeof(EventSource) !== "undefined" && typeof(source) !== "") {
$("#load_more_json_button1").show();
$("#load_more_json_button2").show();
var source = new EventSource("{{ url_for('getImportantPasteByModule') }}"+"?moduleName="+moduleName);
var source = new EventSource("{{ url_for('browsepastes.getImportantPasteByModule') }}"+"?moduleName="+moduleName);
source.onmessage = function(event) {
var feed = jQuery.parseJSON( event.data );
curr_numElem = parseInt($("#myTable_"+moduleName).attr('data-numElem'));
@ -93,10 +91,10 @@ function add_entries_X(to_add) {
elem_added++;
search_table.row.add( [
feed.index,
"<a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+feed.path+"&num="+feed.index+"\"> "+ feed.path +"</a>",
"<a target=\"_blank\" href=\"{{ url_for('showsavedpastes.showsavedpaste') }}?paste="+feed.path+"&num="+feed.index+"\"> "+ feed.path +"</a>",
feed.date,
feed.linenum,
"<p><span class=\"glyphicon glyphicon-info-sign\" data-toggle=\"tooltip\" data-placement=\"left\" title=\""+feed.content.replace(/\"/g, "\'").replace(/\r/g, "\'").replace(/\n/g, "\'")+"\"></span> <button type=\"button\" class=\"btn-link\" data-num=\""+feed.index+"\" data-toggle=\"modal\" data-target=\"#mymodal\" data-url=\"{{ url_for('showsavedpaste') }}?paste="+feed.path+"&num="+feed.index+"\" data-path=\""+feed.path+"\"><span class=\"fa fa-search-plus\"></span></button></p>"
"<p><span class=\"glyphicon glyphicon-info-sign\" data-toggle=\"tooltip\" data-placement=\"left\" title=\""+feed.content.replace(/\"/g, "\'").replace(/\r/g, "\'").replace(/\n/g, "\'")+"\"></span> <button type=\"button\" class=\"btn-link\" data-num=\""+feed.index+"\" data-toggle=\"modal\" data-target=\"#mymodal\" data-url=\"{{ url_for('showsavedpastes.showsavedpaste') }}?paste="+feed.path+"&num="+feed.index+"\" data-path=\""+feed.path+"\"><span class=\"fa fa-search-plus\"></span></button></p>"
] ).draw( false );
$("#myTable_"+moduleName).attr('data-numElem', curr_numElem+1);
}
@ -195,7 +193,7 @@ $(document).ready(function(){
$("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
event.preventDefault();
var modal=$(this);
var url = " {{ url_for('showpreviewpaste') }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
var url = " {{ url_for('showsavedpastes.showpreviewpaste') }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
last_clicked_paste = $(this).attr('data-num');
$.get(url, function (data) {
@ -221,7 +219,7 @@ $(document).ready(function(){
// On click, donwload all paste's content
$("#load-more-button").on("click", function (event) {
if (complete_paste == null) { //Donwload only once
$.get("{{ url_for('getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
$.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
complete_paste = data;
update_preview();
});

14
var/www/Flasks/Flask_dashboard.py → var/www/modules/dashboard/Flask_dashboard.py
View file

@ -8,7 +8,7 @@ import json
import datetime
import flask
from flask import Flask, render_template, jsonify, request
from flask import Flask, render_template, jsonify, request, Blueprint
# ============ VARIABLES ============
import Flask_config
@ -17,6 +17,9 @@ app = Flask_config.app
cfg = Flask_config.cfg
r_serv = Flask_config.r_serv
r_serv_log = Flask_config.r_serv_log
dashboard = Blueprint('dashboard', __name__, template_folder='templates')
# ============ FUNCTIONS ============
def event_stream():
@ -51,18 +54,21 @@ def get_queues(r):
# ============ ROUTES ============
@app.route("/_logs")
@dashboard.route("/_logs")
def logs():
return flask.Response(event_stream(), mimetype="text/event-stream")
@app.route("/_stuff", methods=['GET'])
@dashboard.route("/_stuff", methods=['GET'])
def stuff():
return jsonify(row1=get_queues(r_serv))
@app.route("/")
@dashboard.route("/")
def index():
default_minute = cfg.get("Flask", "minute_processed_paste")
threshold_stucked_module = cfg.getint("Module_ModuleInformation", "threshold_stucked_module")
return render_template("index.html", default_minute = default_minute, threshold_stucked_module=threshold_stucked_module)
# ========= REGISTRATION =========
app.register_blueprint(dashboard)

0
var/www/templates/Queue_live_Monitoring.html → var/www/modules/dashboard/templates/Queue_live_Monitoring.html
View file

1
var/www/modules/dashboard/templates/header_dashboard.html Normal file
View file

@ -0,0 +1 @@
<li id='page-index'><a href="{{ url_for('dashboard.index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li>

6
var/www/templates/index.html → var/www/modules/dashboard/templates/index.html
View file

@ -43,7 +43,7 @@
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
<a href="{{ url_for('dashboard.index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
{% include 'searchbox.html' %}
<!-- /input-group -->
</li>
@ -80,7 +80,7 @@
</table>
</div>
<div class="table-responsive", id="queueing" style="margin-top:10px; font-size: small;"></div>
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
<a href="{{ url_for('dashboard.index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
@ -176,7 +176,7 @@
<!-- /#page-wrapper -->
</div>
<script> var url_showSavedPath = "{{ url_for('showsavedpaste') }}"; </script>
<script> var url_showSavedPath = "{{ url_for('showsavedpastes.showsavedpaste') }}"; </script>
<script type="text/javascript" src="{{ url_for('static', filename='js/indexjavascript.js')}}"></script>
<script>

0
var/www/templates/searchbox.html → var/www/modules/dashboard/templates/searchbox.html
View file

30
var/www/modules/rawSkeleton/Flask_rawSkeleton.py Normal file
View file

@ -0,0 +1,30 @@
#!/usr/bin/env python2
# -*-coding:UTF-8 -*
'''
Flask functions and routes for the trending modules page
'''
import redis
from flask import Flask, render_template, jsonify, request, Blueprint
# ============ VARIABLES ============
import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
rawSkeleton = Blueprint('rawSkeleton', __name__, template_folder='templates')
# ============ FUNCTIONS ============
def one():
return 1
# ============= ROUTES ==============
@rawSkeleton.route("/rawSkeleton/", methods=['GET'])
def skeleton_page():
return render_template("rawSkeleton.html")
# ========= REGISTRATION =========
app.register_blueprint(rawSkeleton)

1
var/www/modules/rawSkeleton/templates/header_rawSkeleton.html Normal file
View file

@ -0,0 +1 @@
<li id='page-rawSkeleton'><a href="{{ url_for('rawSkeleton.skeleton_page') }}"><i class="glyphicon glyphicon-new-window"></i> rawSkeleton page</a></li>

47
var/www/modules/rawSkeleton/templates/rawSkeleton.html Normal file
View file

@ -0,0 +1,47 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.pie.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
</head>
<body>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-rawSkeleton" >rawSkeleton</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
<script>
$(document).ready(function(){
activePage = "page-rawSkeleton"
$("#"+activePage).addClass("active");
});
</script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
</body>
</html>

10
var/www/Flasks/Flask_search.py → var/www/modules/search/Flask_search.py
View file

@ -9,7 +9,7 @@ import json
import os
import datetime
import flask
from flask import Flask, render_template, jsonify, request
from flask import Flask, render_template, jsonify, request, Blueprint
import Paste
from whoosh import index
@ -30,6 +30,8 @@ baseindexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path"))
indexRegister_path = os.path.join(os.environ['AIL_HOME'],
cfg.get("Indexer", "register"))
searches = Blueprint('searches', __name__, template_folder='templates')
# ============ FUNCTIONS ============
def get_current_index():
with open(indexRegister_path, "r") as f:
@ -84,7 +86,7 @@ def to_iso_date(timestamp):
# ============ ROUTES ============
@app.route("/search", methods=['POST'])
@searches.route("/search", methods=['POST'])
def search():
query = request.form['query']
q = []
@ -144,7 +146,7 @@ def search():
)
@app.route("/get_more_search_result", methods=['POST'])
@searches.route("/get_more_search_result", methods=['POST'])
def get_more_search_result():
query = request.form['query']
q = []
@ -193,3 +195,5 @@ def get_more_search_result():
return jsonify(to_return)
# ========= REGISTRATION =========
app.register_blueprint(searches)

34
var/www/templates/search.html → var/www/modules/search/templates/search.html
View file

@ -37,23 +37,7 @@
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
{% include 'searchbox.html' %}
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
{% include 'navbar.html' %}
<!-- Modal -->
<div id="mymodal" class="modal fade" role="dialog">
@ -113,10 +97,10 @@
{% for path in r %}
<tr>
<td>{{ loop.index0 + 1 }}</td>
<td><a target="_blank" href="{{ url_for('showsavedpaste') }}?paste={{ path }}&num={{ loop.index0+1 }}"> {{ path }}</a></td>
<td><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{ path }}&num={{ loop.index0+1 }}"> {{ path }}</a></td>
<td>{{ paste_date[loop.index0] }}</td>
<td>{{ paste_size[loop.index0] }}</td>
<td><p><span class="glyphicon glyphicon-info-sign" data-toggle="tooltip" data-placement="left" title="{{ c[loop.index0] }}"></span> <button type="button" class="btn-link" data-num="{{ loop.index0 + 1 }}" data-toggle="modal" data-target="#mymodal" data-url="{{ url_for('showsavedpaste') }}?paste={{ path }}&num={{ loop.index0+1 }}" data-path="{{ path }}"><span class="fa fa-search-plus"></span></button></p></td>
<td><p><span class="glyphicon glyphicon-info-sign" data-toggle="tooltip" data-placement="left" title="{{ c[loop.index0] }}"></span> <button type="button" class="btn-link" data-num="{{ loop.index0 + 1 }}" data-toggle="modal" data-target="#mymodal" data-url="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{ path }}&num={{ loop.index0+1 }}" data-path="{{ path }}"><span class="fa fa-search-plus"></span></button></p></td>
</tr>
{% endfor %}
</tbody>
@ -169,7 +153,7 @@
$('#index_name').on('change', function() {
var form = document.createElement('form');
form.setAttribute("method", 'post');
form.setAttribute("action", "{{ url_for('search') }}");
form.setAttribute("action", "{{ url_for('searches.search') }}");
var input1 = document.createElement('input');
input1.setAttribute("type", "hidden");
@ -202,16 +186,16 @@
function load_search_50_data() {
var options = { query: query, page_offset: page_offset, index_name: $("#index_name").val() };
$.post( "{{ url_for('get_more_search_result') }}", options).done(function( data ) {
$.post( "{{ url_for('searches.get_more_search_result') }}", options).done(function( data ) {
for(i=0; i<data.path_array.length; i++) {
var curr_preview = data.preview_array[i].replace(/\"/g, "\'");
search_table.row.add( [
init_num_of_elements_in_table+((offset))+i+1,
"<a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+data.path_array[i]+"&num="+i+"\"> "+ data.path_array[i] +"</a>",
"<a target=\"_blank\" href=\"{{ url_for('showsavedpastes.showsavedpaste') }}?paste="+data.path_array[i]+"&num="+i+"\"> "+ data.path_array[i] +"</a>",
data.date_array[i],
data.size_array[i],
"<p><span class=\"glyphicon glyphicon-info-sign\" data-toggle=\"tooltip\" data-placement=\"left\" title=\""+curr_preview+"\"></span> <button type=\"button\" class=\"btn-link\" data-num=\""+i+"\" data-toggle=\"modal\" data-target=\"#mymodal\" data-url=\"{{ url_for('showsavedpaste') }}?paste="+data.path_array[i]+"&num="+i+"\" data-path=\""+data.path_array[i]+"\"><span class=\"fa fa-search-plus\"></span></button></p>"
"<p><span class=\"glyphicon glyphicon-info-sign\" data-toggle=\"tooltip\" data-placement=\"left\" title=\""+curr_preview+"\"></span> <button type=\"button\" class=\"btn-link\" data-num=\""+i+"\" data-toggle=\"modal\" data-target=\"#mymodal\" data-url=\"{{ url_for('showsavedpastes.showsavedpaste') }}?paste="+data.path_array[i]+"&num="+i+"\" data-path=\""+data.path_array[i]+"\"><span class=\"fa fa-search-plus\"></span></button></p>"
] ).draw( false );
}
offset = offset + data.path_array.length;
@ -284,7 +268,7 @@
// On click, get html content from url and update the corresponding modal
$("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
var modal=$(this);
var url = " {{ url_for('showpreviewpaste') }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
var url = " {{ url_for('showsavedpastes.showpreviewpaste') }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
last_clicked_paste = $(this).attr('data-num');
$.get(url, function (data) {
@ -311,7 +295,7 @@
// On click, donwload all paste's content
$("#load-more-button").off('click.download').on("click.download", function (event) {
if (complete_paste == null) { //Donwload only once
$.get("{{ url_for('getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
$.get("{{ url_for('showsavedpastes.getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
complete_paste = data;
update_preview();
});

16
var/www/Flasks/Flask_sentiment.py → var/www/modules/sentiment/Flask_sentiment.py
View file

@ -9,7 +9,7 @@ import datetime
import calendar
from Date import Date
import flask
from flask import Flask, render_template, jsonify, request
from flask import Flask, render_template, jsonify, request, Blueprint
import Paste
@ -20,6 +20,9 @@ app = Flask_config.app
cfg = Flask_config.cfg
r_serv_charts = Flask_config.r_serv_charts
r_serv_sentiment = Flask_config.r_serv_sentiment
sentiments = Blueprint('sentiments', __name__, template_folder='templates')
# ============ FUNCTIONS ============
def get_date_range(num_day):
@ -34,12 +37,12 @@ def get_date_range(num_day):
# ============ ROUTES ============
@app.route("/sentiment_analysis_trending/")
@sentiments.route("/sentiment_analysis_trending/")
def sentiment_analysis_trending():
return render_template("sentiment_analysis_trending.html")
@app.route("/sentiment_analysis_getplotdata/", methods=['GET'])
@sentiments.route("/sentiment_analysis_getplotdata/", methods=['GET'])
def sentiment_analysis_getplotdata():
# Get the top providers based on number of pastes
oneHour = 60*60
@ -85,13 +88,13 @@ def sentiment_analysis_getplotdata():
@app.route("/sentiment_analysis_plot_tool/")
@sentiments.route("/sentiment_analysis_plot_tool/")
def sentiment_analysis_plot_tool():
return render_template("sentiment_analysis_plot_tool.html")
@app.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET'])
@sentiments.route("/sentiment_analysis_plot_tool_getdata/", methods=['GET'])
def sentiment_analysis_plot_tool_getdata():
getProviders = request.args.get('getProviders')
@ -134,4 +137,5 @@ def sentiment_analysis_plot_tool_getdata():
return jsonify(to_return)
# ========= REGISTRATION =========
app.register_blueprint(sentiments)

7
var/www/modules/sentiment/templates/header_sentiment.html Normal file
View file

@ -0,0 +1,7 @@
<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-heart"></i> Sentiment Analysis
<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-bar-chart-o"> </i> Sentiment trending</a></li>
<li><a href="{{ url_for('sentiments.sentiment_analysis_plot_tool') }}"><i class="fa fa-wrench"> </i> Sentiment plot Tool</a></li>
</ul>
</li>

20
var/www/templates/sentiment_analysis_plot_tool.html → var/www/modules/sentiment/templates/sentiment_analysis_plot_tool.html
View file

@ -31,25 +31,7 @@
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
{% include 'searchbox.html' %}
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
</div>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">

20
var/www/templates/sentiment_analysis_trending.html → var/www/modules/sentiment/templates/sentiment_analysis_trending.html
View file

@ -84,25 +84,7 @@
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
{% include 'searchbox.html' %}
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
</div>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">

15
var/www/Flasks/Flask_showpaste.py → var/www/modules/showpaste/Flask_showpaste.py
View file

@ -7,7 +7,7 @@
import redis
import json
import flask
from flask import Flask, render_template, jsonify, request
from flask import Flask, render_template, jsonify, request, Blueprint
import Paste
@ -20,6 +20,9 @@ r_serv_pasteName = Flask_config.r_serv_pasteName
max_preview_char = Flask_config.max_preview_char
max_preview_modal = Flask_config.max_preview_modal
tlsh_to_percent = Flask_config.tlsh_to_percent
showsavedpastes = Blueprint('showsavedpastes', __name__, template_folder='templates')
# ============ FUNCTIONS ============
def showpaste(content_range):
@ -89,22 +92,20 @@ def showpaste(content_range):
return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list)
# ============ ROUTES ============
@app.route("/showsavedpaste/") #completely shows the paste in a new tab
@showsavedpastes.route("/showsavedpaste/") #completely shows the paste in a new tab
def showsavedpaste():
return showpaste(0)
@app.route("/showpreviewpaste/")
@showsavedpastes.route("/showpreviewpaste/")
def showpreviewpaste():
num = request.args.get('num', '')
return "|num|"+num+"|num|"+showpaste(max_preview_modal)
@app.route("/getmoredata/")
@showsavedpastes.route("/getmoredata/")
def getmoredata():
requested_path = request.args.get('paste', '')
paste = Paste.Paste(requested_path)
@ -112,3 +113,5 @@ def getmoredata():
to_return = p_content[max_preview_modal-1:]
return to_return
# ========= REGISTRATION =========
app.register_blueprint(showsavedpastes)

2
var/www/templates/show_saved_paste.html → var/www/modules/showpaste/templates/show_saved_paste.html
View file

@ -71,7 +71,7 @@
<td>{{ hashtype_list[i] }}</td>
<td>Similarity: {{ simil_list[i] }}%</td>
<td>{{ date_list[i] }}</td>
<td><a target="_blank" href="{{ url_for('showsavedpaste') }}?paste={{ dup_path }}" id='dup_path'>{{ dup_path }}</a></td>
<td><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{ dup_path }}" id='dup_path'>{{ dup_path }}</a></td>
</tr>
{% set i = i + 1 %}
{% endfor %}

22
var/www/Flasks/Flask_terms.py → var/www/modules/terms/Flask_terms.py
View file

@ -8,7 +8,7 @@ import redis
import datetime
import calendar
import flask
from flask import Flask, render_template, jsonify, request
from flask import Flask, render_template, jsonify, request, Blueprint
import re
import Paste
@ -19,6 +19,8 @@ app = Flask_config.app
cfg = Flask_config.cfg
r_serv_term = Flask_config.r_serv_term
terms = Blueprint('terms', __name__, template_folder='templates')
DEFAULT_MATCH_PERCENT = 50
#tracked
@ -53,7 +55,7 @@ def Term_getValueOverRange(word, startDate, num_day, per_paste=""):
# ============ ROUTES ============
@app.route("/terms_management/")
@terms.route("/terms_management/")
def terms_management():
per_paste = request.args.get('per_paste')
if per_paste == "1" or per_paste is None:
@ -131,7 +133,7 @@ def terms_management():
per_paste=per_paste)
@app.route("/terms_management_query_paste/")
@terms.route("/terms_management_query_paste/")
def terms_management_query_paste():
term = request.args.get('term')
paste_info = []
@ -164,7 +166,7 @@ def terms_management_query_paste():
return jsonify(paste_info)
@app.route("/terms_management_query/")
@terms.route("/terms_management_query/")
def terms_management_query():
TrackedTermsDate_Name = "TrackedTermDate"
BlackListTermsDate_Name = "BlackListTermDate"
@ -186,7 +188,7 @@ def terms_management_query():
return jsonify(value_range)
@app.route("/terms_management_action/", methods=['GET'])
@terms.route("/terms_management_action/", methods=['GET'])
def terms_management_action():
today = datetime.datetime.now()
today = today.replace(microsecond=0)
@ -254,7 +256,7 @@ def terms_management_action():
@app.route("/terms_plot_tool/")
@terms.route("/terms_plot_tool/")
def terms_plot_tool():
term = request.args.get('term')
if term is not None:
@ -263,7 +265,7 @@ def terms_plot_tool():
return render_template("terms_plot_tool.html", term="")
@app.route("/terms_plot_tool_data/")
@terms.route("/terms_plot_tool_data/")
def terms_plot_tool_data():
oneDay = 60*60*24
range_start = datetime.datetime.utcfromtimestamp(int(float(request.args.get('range_start')))) if request.args.get('range_start') is not None else 0;
@ -293,14 +295,14 @@ def terms_plot_tool_data():
return jsonify(value_range)
@app.route("/terms_plot_top/")
@terms.route("/terms_plot_top/")
def terms_plot_top():
per_paste = request.args.get('per_paste')
per_paste = per_paste if per_paste is not None else 1
return render_template("terms_plot_top.html", per_paste=per_paste)
@app.route("/terms_plot_top_data/")
@terms.route("/terms_plot_top_data/")
def terms_plot_top_data():
oneDay = 60*60*24
today = datetime.datetime.now()
@ -346,3 +348,5 @@ def terms_plot_top_data():
return jsonify(to_return)
# ========= REGISTRATION =========
app.register_blueprint(terms)

8
var/www/modules/terms/templates/header_terms.html Normal file
View file

@ -0,0 +1,8 @@
<li id='page-termsfrequency'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-eye"></i> Terms frequency
<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="{{ url_for('terms.terms_management') }}"><i class="fa fa-gear "> </i> Terms managements</a></li>
<li><a href="{{ url_for('terms.terms_plot_top') }}"><i class="glyphicon glyphicon-fire"> </i> Terms plot top</a></li>
<li><a href="{{ url_for('terms.terms_plot_tool') }}"><i class="fa fa-wrench"> </i> Terms plot tool</a></li>
</ul>
</li>

36
var/www/templates/terms_management.html → var/www/modules/terms/templates/terms_management.html
View file

@ -57,25 +57,7 @@
</div>
</div>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
{% include 'searchbox.html' %}
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
</div>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
@ -242,7 +224,7 @@
<script>
function reload_per_paste() {
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
window.location.href = {{ url_for('terms_management') }}+"?per_paste="+checked;
window.location.href = {{ url_for('terms.terms_management') }}+"?per_paste="+checked;
}
@ -289,7 +271,7 @@
//console.log(data);
event.preventDefault();
var the_modal=$(this);
var url = "{{ url_for('terms_management_query_paste') }}?term=" + encodeURIComponent($(this).attr('data-term'));
var url = "{{ url_for('terms.terms_management_query_paste') }}?term=" + encodeURIComponent($(this).attr('data-term'));
$.getJSON(url, function (data) {
if (data.length != 0) {
var html_to_add = "";
@ -315,7 +297,7 @@
html_to_add += "<td>"+curr_data.size+"</td>";
html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";
html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content.replace(/\"/g, "\'")+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+curr_data.path+"&num=0\"> <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content.replace(/\"/g, "\'")+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('showsavedpastes.showsavedpaste') }}?paste="+curr_data.path+"&num=0\"> <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
html_to_add += "</tr>";
}
@ -323,11 +305,11 @@
html_to_add += "</table>";
$("#mymodalbody").html(html_to_add);
$("[data-toggle=popover]").popover();
$("#button_show_plot").attr("href", "{{ url_for('terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
$("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
$('#modal-table').DataTable();
} else {
$("#mymodalbody").html("No paste containing this term has been received yet.");
$("#button_show_plot").attr("href", "{{ url_for('terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
$("#button_show_plot").attr("href", "{{ url_for('terms.terms_plot_tool')}}"+"?term="+the_modal.attr('data-term') );
}
});
@ -364,14 +346,14 @@ function perform_operation(){
if (curr_term != "") {
console.log(data_to_send);
$.get("{{ url_for('terms_management_action') }}", data_to_send, function(data, status){
$.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){
if(status == "success") {
var json = data;
if(json.section == "followTerm") {
if(json.action == "add") {
// query data
$.get("{{ url_for('terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
$.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"left\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"followTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
table_track.row.add( [ json.term, data2[3], data2[0], data2[1], data2[2], 0, action_button ] ).draw( false );
perform_binding();
@ -386,7 +368,7 @@ function perform_operation(){
}
} else if(json.section == "blacklistTerm"){
if(json.action == "add") {
$.get("{{ url_for('terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
$.get("{{ url_for('terms.terms_management_query') }}", { term: json.term, section: json.section }, function(data2, status){
console.log(data2);
var action_button = "<button class=\"btn-link btn-interaction\" data-toggle=\"tooltip\" data-placement=\"right\" title=\"Remove this term\" data-content=\"" + json.term + "\" data-section=\"blacklistTerm\" data-action=\"delete\"><span class=\"glyphicon glyphicon-trash\"></span></button>"
table_black.row.add( [ json.term, data2[3], action_button ] ).draw( false );

26
var/www/templates/terms_plot_tool.html → var/www/modules/terms/templates/terms_plot_tool.html
View file

@ -32,25 +32,7 @@
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
{% include 'searchbox.html' %}
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
</div>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
@ -214,7 +196,7 @@ function plotData() {
var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
$.getJSON("{{ url_for('terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
$.getJSON("{{ url_for('terms.terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
graph_data = [];
var to_plot = [];
var curr_data = [];
@ -252,7 +234,7 @@ function addData() {
var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
$.getJSON("{{ url_for('terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
$.getJSON("{{ url_for('terms.terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
var curr_data = [];
for(i=1; i<data.length; i++) {
curr_data.push([data[i][0]*1000, data[i][1]]);
@ -273,7 +255,7 @@ function replot() {
var range_end = new Date($( ".sliderRange" ).slider( "values", 1 )).getTime() / 1000;
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
promises.push($.getJSON("{{ url_for('terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
promises.push($.getJSON("{{ url_for('terms.terms_plot_tool_data') }}", { range_start: range_start, range_end: range_end, term: term, per_paste: checked }, function(data, status){
var curr_data = [];
for(i=1; i<data.length; i++) {
curr_data.push([data[i][0]*1000, data[i][1]]);

30
var/www/templates/terms_plot_top.html → var/www/modules/terms/templates/terms_plot_top.html
View file

@ -33,25 +33,7 @@
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
{% include 'searchbox.html' %}
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
</div>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
@ -297,7 +279,7 @@
function reload_per_paste() {
var checked = $("#per_paste").prop( "checked" ) ? 1 : 0;
window.location.href = {{ url_for('terms_plot_top') }}+"?per_paste="+checked;
window.location.href = {{ url_for('terms.terms_plot_top') }}+"?per_paste="+checked;
}
</script>
@ -357,7 +339,7 @@ var plot_month;
var promises = []; // Used to know when everything has been received
promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_today, num_day: 5, per_paste: per_paste }, function(data, status){
promises.push($.getJSON("{{ url_for('terms.terms_plot_top_data') }}", { set: set_today, num_day: 5, per_paste: per_paste }, function(data, status){
data.sort(function(a, b){return b[2]-a[2];});
// Sort data
var table_today = $("#table-today")
@ -398,7 +380,7 @@ promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_today
}));
promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_week, num_day: 7, per_paste: per_paste }, function(data, status){
promises.push($.getJSON("{{ url_for('terms.terms_plot_top_data') }}", { set: set_week, num_day: 7, per_paste: per_paste }, function(data, status){
data.sort(function(a, b){return b[2]-a[2];});
// Sort data
var table = $("#table-week")
@ -438,7 +420,7 @@ promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_week,
});
}));
promises.push($.getJSON("{{ url_for('terms_plot_top_data') }}", { set: set_month, num_day: 31, per_paste: per_paste }, function(data, status){
promises.push($.getJSON("{{ url_for('terms.terms_plot_top_data') }}", { set: set_month, num_day: 31, per_paste: per_paste }, function(data, status){
data.sort(function(a, b){return b[2]-a[2];});
// Sort data
var table = $("#table-month")
@ -507,7 +489,7 @@ function perform_operation(){
var data_to_send = { section: curr_section, action:"add", term: curr_term};
$.get("{{ url_for('terms_management_action') }}", data_to_send, function(data, status){
$.get("{{ url_for('terms.terms_management_action') }}", data_to_send, function(data, status){
if(status == "success") {
location.reload();
}

15
var/www/Flasks/Flask_trendingcharts.py → var/www/modules/trendingcharts/Flask_trendingcharts.py
View file

@ -8,7 +8,7 @@ import redis
import datetime
from Date import Date
import flask
from flask import Flask, render_template, jsonify, request
from flask import Flask, render_template, jsonify, request, Blueprint
# ============ VARIABLES ============
import Flask_config
@ -16,6 +16,9 @@ import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
r_serv_charts = Flask_config.r_serv_charts
trendings = Blueprint('trendings', __name__, template_folder='templates')
# ============ FUNCTIONS ============
def get_date_range(num_day):
@ -30,7 +33,7 @@ def get_date_range(num_day):
# ============ ROUTES ============
@app.route("/_progressionCharts", methods=['GET'])
@trendings.route("/_progressionCharts", methods=['GET'])
def progressionCharts():
attribute_name = request.args.get('attributeName')
trending_name = request.args.get('trendingName')
@ -53,21 +56,23 @@ def progressionCharts():
keyw_value = r_serv_charts.zrevrangebyscore(redis_progression_name, '+inf', '-inf', withscores=True, start=0, num=10)
return jsonify(keyw_value)
@app.route("/wordstrending/")
@trendings.route("/wordstrending/")
def wordstrending():
default_display = cfg.get("Flask", "default_display")
return render_template("Wordstrending.html", default_display = default_display)
@app.route("/protocolstrending/")
@trendings.route("/protocolstrending/")
def protocolstrending():
default_display = cfg.get("Flask", "default_display")
return render_template("Protocolstrending.html", default_display = default_display)
@app.route("/trending/")
@trendings.route("/trending/")
def trending():
default_display = cfg.get("Flask", "default_display")
return render_template("Trending.html", default_display = default_display)
# ========= REGISTRATION =========
app.register_blueprint(trendings)

20
var/www/templates/Trending.html → var/www/modules/trendingcharts/templates/Trending.html
View file

@ -30,24 +30,8 @@
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
{% include 'searchbox.html' %}
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">

1
var/www/modules/trendingcharts/templates/header_trendingcharts.html Normal file
View file

@ -0,0 +1 @@
<li id='page-trendingchart'><a href="{{ url_for('trendings.trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>

0
var/www/templates/trending_graphs/Domainstrending.html → var/www/modules/trendingcharts/templates/trending_graphs/Domainstrending.html
View file

0
var/www/templates/trending_graphs/Moduletrending.html → var/www/modules/trendingcharts/templates/trending_graphs/Moduletrending.html
View file

0
var/www/templates/trending_graphs/Protocolstrending.html → var/www/modules/trendingcharts/templates/trending_graphs/Protocolstrending.html
View file

0
var/www/templates/trending_graphs/Tldstrending.html → var/www/modules/trendingcharts/templates/trending_graphs/Tldstrending.html
View file

0
var/www/templates/trending_graphs/Wordstrending.html → var/www/modules/trendingcharts/templates/trending_graphs/Wordstrending.html
View file

13
var/www/Flasks/Flask_trendingmodules.py → var/www/modules/trendingmodules/Flask_trendingmodules.py
View file

@ -8,7 +8,7 @@ import redis
import datetime
from Date import Date
import flask
from flask import Flask, render_template, jsonify, request
from flask import Flask, render_template, jsonify, request, Blueprint
# ============ VARIABLES ============
import Flask_config
@ -16,6 +16,9 @@ import Flask_config
app = Flask_config.app
cfg = Flask_config.cfg
r_serv_charts = Flask_config.r_serv_charts
trendingmodules = Blueprint('trendingmodules', __name__, template_folder='templates')
# ============ FUNCTIONS ============
# Iterate over elements in the module provided and return the today data or the last data
@ -43,7 +46,7 @@ def get_date_range(num_day):
# ============ ROUTES ============
@app.route("/_moduleCharts", methods=['GET'])
@trendingmodules.route("/_moduleCharts", methods=['GET'])
def modulesCharts():
keyword_name = request.args.get('keywordName')
module_name = request.args.get('moduleName')
@ -69,7 +72,7 @@ def modulesCharts():
return jsonify(member_set)
@app.route("/_providersChart", methods=['GET'])
@trendingmodules.route("/_providersChart", methods=['GET'])
def providersChart():
keyword_name = request.args.get('keywordName')
module_name = request.args.get('moduleName')
@ -106,8 +109,10 @@ def providersChart():
return jsonify(member_set)
@app.route("/moduletrending/")
@trendingmodules.route("/moduletrending/")
def moduletrending():
return render_template("Moduletrending.html")
# ========= REGISTRATION =========
app.register_blueprint(trendingmodules)

20
var/www/templates/Moduletrending.html → var/www/modules/trendingmodules/templates/Moduletrending.html
View file

@ -22,24 +22,8 @@
</head>
<body>
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
{% include 'searchbox.html' %}
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
{% include 'navbar.html' %}
<div id="page-wrapper">
</br>
{% include 'trending_graphs/Moduletrending.html' %}

1
var/www/modules/trendingmodules/templates/header_trendingmodules.html Normal file
View file

@ -0,0 +1 @@
<li id='page-modulestats'><a href="{{ url_for('trendingmodules.moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>

2
var/www/static/js/indexjavascript.js
View file

@ -173,7 +173,7 @@ function create_log_table(obj_json) {
options_processed_pastes.legend.show = false;
var total_proc = $.plot("#global", [ getData("global", null) ], options_processed_pastes);
options_processed_pastes.legend.show = true;
options_processed_pastes.series.lines = { show: true };
options_processed_pastes.series.lines = { show: true, fill: true };
data_for_processed_paste["global"] = Array(totalPoints+1).join(0).split('');
var feederProc = $.plot("#Proc_feeder", [ getData(feeder, "Proc") ], options_processed_pastes);

48
var/www/templates/base_template.html Normal file
View file

@ -0,0 +1,48 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Analysis Information Leak framework Dashboard</title>
<!-- Core CSS -->
<link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
<link href="{{ url_for('static', filename='css/dygraph_gallery.css') }}" rel="stylesheet" type="text/css" />
<!-- JS -->
<script type="text/javascript" src="{{ url_for('static', filename='js/dygraph-combined.js') }}"></script>
<script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.pie.js') }}"></script>
<script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
</head>
<body>
{% include 'navbar.html' %}
<div id="page-wrapper">
<div class="row">
<div class="col-lg-12">
<h1 class="page-header" data-page="page-termsfrequency" >MODULENAME</h1>
</div>
<!-- /.col-lg-12 -->
</div>
<!-- /.row -->
</div>
<!-- /#page-wrapper -->
<script>
$(document).ready(function(){
activePage = "page-MODULENAME"
$("#"+activePage).addClass("active");
});
</script>
<script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
</body>
</html>

30
var/www/templates/header.html
View file

@ -1,23 +1,31 @@
<div class="navbar-header">
<ul class="nav navbar-nav">
<li id='page-index'><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li>
<li id='page-trendingchart'><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>
<li id='page-modulestats'><a href="{{ url_for('moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>
<li id='page-browse'><a href="{{ url_for('browseImportantPaste') }}"><i class="fa fa-search-plus "></i> Browse important pastes</a></li>
<li id='page-index'><a href="{{ url_for('dashboard.index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li>
<li id='page-trendingchart'><a href="{{ url_for('trendings.trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>
<li id='page-modulestats'><a href="{{ url_for('trendingmodules.moduletrending') }}"><i class="glyphicon glyphicon-stats"></i> Modules statistics</a></li>
<li id='page-browse'><a href="{{ url_for('browsepastes.browseImportantPaste') }}"><i class="fa fa-search-plus "></i> Browse important pastes</a></li>
<li id='page-sentiment'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-heart"></i> Sentiment Analysis
<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="{{ url_for('sentiment_analysis_trending') }}"><i class="fa fa-bar-chart-o"> </i> Sentiment trending</a></li>
<li><a href="{{ url_for('sentiment_analysis_plot_tool') }}"><i class="fa fa-wrench"> </i> Sentiment plot Tool</a></li>
<li><a href="{{ url_for('sentiments.sentiment_analysis_trending') }}"><i class="fa fa-bar-chart-o"> </i> Sentiment trending</a></li>
<li><a href="{{ url_for('sentiments.sentiment_analysis_plot_tool') }}"><i class="fa fa-wrench"> </i> Sentiment plot Tool</a></li>
</ul>
</li>
</li>
<li id='page-termsfrequency'><a class="dropdown-toggle" data-toggle="dropdown" href="#"><i class="fa fa-eye"></i> Terms frequency
<span class="caret"></span></a>
<ul class="dropdown-menu">
<li><a href="{{ url_for('terms_management') }}"><i class="fa fa-gear "> </i> Terms managements</a></li>
<li><a href="{{ url_for('terms_plot_top') }}"><i class="glyphicon glyphicon-fire"> </i> Terms plot top</a></li>
<li><a href="{{ url_for('terms_plot_tool') }}"><i class="fa fa-wrench"> </i> Terms plot tool</a></li>
<li><a href="{{ url_for('terms.terms_management') }}"><i class="fa fa-gear "> </i> Terms managements</a></li>
<li><a href="{{ url_for('terms.terms_plot_top') }}"><i class="glyphicon glyphicon-fire"> </i> Terms plot top</a></li>
<li><a href="{{ url_for('terms.terms_plot_tool') }}"><i class="fa fa-wrench"> </i> Terms plot tool</a></li>
</ul>
</li>
</li>
<li id='page-myNewModule'><a href="{{ url_for('myNewModule.myNewModule_page') }}"><i class="glyphicon glyphicon-new-window"></i> myNewModule page</a></li>
</ul>
</div>

11
var/www/templates/header_base.html Normal file
View file

@ -0,0 +1,11 @@
<div class="navbar-header">
<ul class="nav navbar-nav">
<!--dashboard-->
<!--trendingcharts-->
<!--trendingmodules-->
<!--browsepastes-->
<!--sentiment-->
<!--terms-->
<!--insert here-->
</ul>
</div>

1
var/www/templates/header_base_template.html Normal file
View file

@ -0,0 +1 @@
<li id='page-MODULENAME'><a href="{{ url_for('MODULENAME.MODULENAME_page') }}"><i class="glyphicon glyphicon-new-window"></i> MODULENAME </a></li>

1
var/www/templates/ignored_modules.txt Normal file
View file

@ -0,0 +1 @@
rawSkeleton

19
var/www/templates/navbar.html Normal file
View file

@ -0,0 +1,19 @@
<div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
{% include 'header.html' %}
<!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse">
<ul class="nav" id="side-menu">
<li class="sidebar-search">
{% include 'searchbox.html' %}
</li>
</ul>
<!-- /#side-menu -->
</div>
<!-- /.sidebar-collapse -->
<a href="{{ url_for('dashboard.index') }}"><img src="{{ url_for('static', filename='image/AIL.png') }}" /></a>
</div>
<!-- /.navbar-static-side -->
</nav>
</div>