AIL/ail-framework
7
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2024-11-23 06:37:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Added support of pasteName in search + added header in search.html + supressed mixed indent and trailing in Flask.py

Browse source
This commit is contained in:
Mokaddem 2016-08-17 14:21:31 +02:00
parent c10f28d9df
commit 24072ece36
2 changed files with 44 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

72
var/www/Flask_server.py
View file

@ -55,7 +55,7 @@ r_serv_sentiment = redis.StrictRedis(
port=cfg.getint("Redis_Level_DB_Sentiment", "port"), port=cfg.getint("Redis_Level_DB_Sentiment", "port"),
db=cfg.getint("Redis_Level_DB_Sentiment", "db")) db=cfg.getint("Redis_Level_DB_Sentiment", "db"))
app = Flask(__name__, static_url_path='/static/') app = Flask(__name__, static_url_path='/static/')
@ -113,7 +113,7 @@ def parseStringToList2(the_string):
return res return res
def showpaste(content_range): def showpaste(content_range):
requested_path = request.args.get('paste', '') requested_path = request.args.get('paste', '')
paste = Paste.Paste(requested_path) paste = Paste.Paste(requested_path)
p_date = str(paste._get_p_date()) p_date = str(paste._get_p_date())
@ -136,7 +136,7 @@ def showpaste(content_range):
dup_list[2] = int(((tlsh_to_percent - float(dup_list[2])) / tlsh_to_percent)*100) dup_list[2] = int(((tlsh_to_percent - float(dup_list[2])) / tlsh_to_percent)*100)
else: else:
dup_list[2] = int(dup_list[2]) dup_list[2] = int(dup_list[2])
p_duplicate_full_list.sort(lambda x,y: cmp(x[2], y[2]), reverse=True) p_duplicate_full_list.sort(lambda x,y: cmp(x[2], y[2]), reverse=True)
# Combine multiple duplicate paste name and format for display # Combine multiple duplicate paste name and format for display
@ -165,7 +165,7 @@ def showpaste(content_range):
p_hashtype_list.append(hash_type) p_hashtype_list.append(hash_type)
if content_range != 0: if content_range != 0:
p_content = p_content[0:content_range] p_content = p_content[0:content_range]
return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list) return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list)
@ -189,7 +189,7 @@ def get_date_range(num_day):
# return format: [('passed_days', num_of_passed_days), ('elem_name1', elem_value1), ('elem_name2', elem_value2)]] # return format: [('passed_days', num_of_passed_days), ('elem_name1', elem_value1), ('elem_name2', elem_value2)]]
def get_top_relevant_data(server, module_name): def get_top_relevant_data(server, module_name):
redis_progression_name_set = 'top_'+ module_name +'_set' redis_progression_name_set = 'top_'+ module_name +'_set'
days = 0 days = 0
for date in get_date_range(15): for date in get_date_range(15):
member_set = [] member_set = []
for keyw in server.smembers(redis_progression_name_set): for keyw in server.smembers(redis_progression_name_set):
@ -232,19 +232,19 @@ def progressionCharts():
attribute_name = request.args.get('attributeName') attribute_name = request.args.get('attributeName')
trending_name = request.args.get('trendingName') trending_name = request.args.get('trendingName')
bar_requested = True if request.args.get('bar') == "true" else False bar_requested = True if request.args.get('bar') == "true" else False
if (bar_requested): if (bar_requested):
num_day = int(request.args.get('days')) num_day = int(request.args.get('days'))
bar_values = [] bar_values = []
date_range = get_date_range(num_day) date_range = get_date_range(num_day)
# Retreive all data from the last num_day # Retreive all data from the last num_day
for date in date_range: for date in date_range:
curr_value = r_serv_charts.hget(attribute_name, date) curr_value = r_serv_charts.hget(attribute_name, date)
bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)]) bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
bar_values.insert(0, attribute_name) bar_values.insert(0, attribute_name)
return jsonify(bar_values) return jsonify(bar_values)
else: else:
redis_progression_name = 'top_progression_'+trending_name redis_progression_name = 'top_progression_'+trending_name
redis_progression_name_set = 'top_progression_'+trending_name+'_set' redis_progression_name_set = 'top_progression_'+trending_name+'_set'
@ -259,25 +259,25 @@ def progressionCharts():
if len(member_set) == 0: if len(member_set) == 0:
member_set.append(("No relevant data", int(100))) member_set.append(("No relevant data", int(100)))
return jsonify(member_set) return jsonify(member_set)
@app.route("/_moduleCharts", methods=['GET']) @app.route("/_moduleCharts", methods=['GET'])
def modulesCharts(): def modulesCharts():
keyword_name = request.args.get('keywordName') keyword_name = request.args.get('keywordName')
module_name = request.args.get('moduleName') module_name = request.args.get('moduleName')
bar_requested = True if request.args.get('bar') == "true" else False bar_requested = True if request.args.get('bar') == "true" else False
if (bar_requested): if (bar_requested):
num_day = int(request.args.get('days')) num_day = int(request.args.get('days'))
bar_values = [] bar_values = []
date_range = get_date_range(num_day) date_range = get_date_range(num_day)
# Retreive all data from the last num_day # Retreive all data from the last num_day
for date in date_range: for date in date_range:
curr_value = r_serv_charts.hget(date, module_name+'-'+keyword_name) curr_value = r_serv_charts.hget(date, module_name+'-'+keyword_name)
bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)]) bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], int(curr_value if curr_value is not None else 0)])
bar_values.insert(0, keyword_name) bar_values.insert(0, keyword_name)
return jsonify(bar_values) return jsonify(bar_values)
else: else:
member_set = get_top_relevant_data(r_serv_charts, module_name) member_set = get_top_relevant_data(r_serv_charts, module_name)
if len(member_set) == 0: if len(member_set) == 0:
@ -290,12 +290,12 @@ def providersChart():
keyword_name = request.args.get('keywordName') keyword_name = request.args.get('keywordName')
module_name = request.args.get('moduleName') module_name = request.args.get('moduleName')
bar_requested = True if request.args.get('bar') == "true" else False bar_requested = True if request.args.get('bar') == "true" else False
if (bar_requested): if (bar_requested):
num_day = int(request.args.get('days')) num_day = int(request.args.get('days'))
bar_values = [] bar_values = []
date_range = get_date_range(num_day) date_range = get_date_range(num_day)
# Retreive all data from the last num_day # Retreive all data from the last num_day
for date in date_range: for date in date_range:
curr_value_size = r_serv_charts.hget(keyword_name+'_'+'size', date) curr_value_size = r_serv_charts.hget(keyword_name+'_'+'size', date)
@ -310,7 +310,7 @@ def providersChart():
bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], curr_value]) bar_values.append([date[0:4]+'/'+date[4:6]+'/'+date[6:8], curr_value])
bar_values.insert(0, keyword_name) bar_values.insert(0, keyword_name)
return jsonify(bar_values) return jsonify(bar_values)
else: else:
redis_provider_name_set = 'top_size_set' if module_name == "size" else 'providers_set' redis_provider_name_set = 'top_size_set' if module_name == "size" else 'providers_set'
@ -322,7 +322,7 @@ def providersChart():
keyw_value_size = r_serv_charts.hget(redis_provider_name_size, get_date_range(0)[0]) keyw_value_size = r_serv_charts.hget(redis_provider_name_size, get_date_range(0)[0])
keyw_value_size = keyw_value_size if keyw_value_size is not None else 0.0 keyw_value_size = keyw_value_size if keyw_value_size is not None else 0.0
keyw_value_num = r_serv_charts.hget(redis_provider_name_num, get_date_range(0)[0]) keyw_value_num = r_serv_charts.hget(redis_provider_name_num, get_date_range(0)[0])
if keyw_value_num is not None: if keyw_value_num is not None:
keyw_value_num = int(keyw_value_num) keyw_value_num = int(keyw_value_num)
else: else:
@ -351,7 +351,21 @@ def search():
c = [] #preview of the paste content c = [] #preview of the paste content
paste_date = [] paste_date = []
paste_size = [] paste_size = []
# Search # Search filename
from os import walk
for (dirpath, dirnames, filenames) in walk(os.path.join(os.environ['AIL_HOME'], 'PASTES/')):
if q[0] in filenames:
r.append(dirpath+'/'+q[0])
paste = Paste.Paste(dirpath+'/'+q[0])
content = paste.get_p_content().decode('utf8', 'ignore')
content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
c.append(content[0:content_range])
curr_date = str(paste._get_p_date())
curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
paste_date.append(curr_date)
paste_size.append(paste._get_p_size())
# Search full line
from whoosh import index from whoosh import index
from whoosh.fields import Schema, TEXT, ID from whoosh.fields import Schema, TEXT, ID
schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT) schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
@ -367,11 +381,11 @@ def search():
paste = Paste.Paste(x.items()[0][1]) paste = Paste.Paste(x.items()[0][1])
content = paste.get_p_content().decode('utf8', 'ignore') content = paste.get_p_content().decode('utf8', 'ignore')
content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
c.append(content[0:content_range]) c.append(content[0:content_range])
curr_date = str(paste._get_p_date()) curr_date = str(paste._get_p_date())
curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
paste_date.append(curr_date) paste_date.append(curr_date)
paste_size.append(paste._get_p_size()) paste_size.append(paste._get_p_size())
return render_template("search.html", r=r, c=c, query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal) return render_template("search.html", r=r, c=c, query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal)
@ -424,11 +438,11 @@ def importantPasteByModule():
paste = Paste.Paste(path) paste = Paste.Paste(path)
content = paste.get_p_content().decode('utf8', 'ignore') content = paste.get_p_content().decode('utf8', 'ignore')
content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
all_content.append(content[0:content_range]) all_content.append(content[0:content_range])
curr_date = str(paste._get_p_date()) curr_date = str(paste._get_p_date())
curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:] curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
paste_date.append(curr_date) paste_date.append(curr_date)
paste_linenum.append(paste.get_lines_info()[0]) paste_linenum.append(paste.get_lines_info()[0])
return render_template("important_paste_by_module.html", all_path=all_path, content=all_content, paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal) return render_template("important_paste_by_module.html", all_path=all_path, content=all_content, paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal)
@ -452,11 +466,11 @@ def sentiment_analysis_getplotdata():
to_return = {} to_return = {}
for cur_provider in r_serv_charts.smembers('providers_set'): for cur_provider in r_serv_charts.smembers('providers_set'):
cur_provider_name = cur_provider + '_' cur_provider_name = cur_provider + '_'
list_date = {} list_date = {}
for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour): for cur_timestamp in range(int(dateStart_timestamp), int(dateStart_timestamp)-sevenDays-oneHour, -oneHour):
cur_set_name = cur_provider_name + str(cur_timestamp) cur_set_name = cur_provider_name + str(cur_timestamp)
list_value = [] list_value = []
for cur_id in r_serv_sentiment.smembers(cur_set_name): for cur_id in r_serv_sentiment.smembers(cur_set_name):
cur_value = r_serv_sentiment.get(cur_id) cur_value = r_serv_sentiment.get(cur_id)
@ -494,7 +508,7 @@ def sentiment_analysis_plot_tool_getdata():
date2 = (Qdate.split('-')[1]).split('.') date2 = (Qdate.split('-')[1]).split('.')
date2 = datetime.date(int(date2[2]), int(date2[1]), int(date2[0])) date2 = datetime.date(int(date2[2]), int(date2[1]), int(date2[0]))
timestamp1 = calendar.timegm(date1.timetuple()) timestamp1 = calendar.timegm(date1.timetuple())
timestamp2 = calendar.timegm(date2.timetuple()) timestamp2 = calendar.timegm(date2.timetuple())
@ -504,10 +518,10 @@ def sentiment_analysis_plot_tool_getdata():
to_return = {} to_return = {}
for cur_provider in query: for cur_provider in query:
list_date = {} list_date = {}
cur_provider_name = cur_provider + '_' cur_provider_name = cur_provider + '_'
for cur_timestamp in range(int(timestamp1), int(timestamp2)+oneDay, oneHour): for cur_timestamp in range(int(timestamp1), int(timestamp2)+oneDay, oneHour):
cur_set_name = cur_provider_name + str(cur_timestamp) cur_set_name = cur_provider_name + str(cur_timestamp)
list_value = [] list_value = []
for cur_id in r_serv_sentiment.smembers(cur_set_name): for cur_id in r_serv_sentiment.smembers(cur_set_name):
cur_value = r_serv_sentiment.get(cur_id) cur_value = r_serv_sentiment.get(cur_id)
@ -535,7 +549,7 @@ def getmoredata():
paste = Paste.Paste(requested_path) paste = Paste.Paste(requested_path)
p_content = paste.get_p_content().decode('utf-8', 'ignore') p_content = paste.get_p_content().decode('utf-8', 'ignore')
to_return = p_content[max_preview_modal-1:] to_return = p_content[max_preview_modal-1:]
return to_return return to_return
if __name__ == "__main__": if __name__ == "__main__":

7
var/www/templates/search.html
View file

@ -42,12 +42,7 @@
<div id="wrapper"> <div id="wrapper">
<nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0"> <nav class="navbar navbar-default navbar-static-top" role="navigation" style="margin-bottom: 0">
<div class="navbar-header"> {% include 'header.html' %}
<ul class="nav navbar-nav">
<li class="active"><a href="{{ url_for('index') }}"><i class="fa fa-dashboard fa-fw"></i> Dashboard</a></li>
<li><a href="{{ url_for('trending') }}"><i class="glyphicon glyphicon-stats"></i> Trending charts</a></li>
</ul>
</div>
<!-- /.navbar-top-links --> <!-- /.navbar-top-links -->
<div class="navbar-default sidebar" role="navigation"> <div class="navbar-default sidebar" role="navigation">
<div class="sidebar-collapse"> <div class="sidebar-collapse">