From 8c1eeea6e67af61cb2035fe3c08cdf7b6b213ddd Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Mon, 4 Jul 2016 11:43:38 +0200 Subject: [PATCH 01/19] created new graph pannel for further url stats and dissociate graph script from html --- var/www/static/js/plot-graph.js | 100 ++++++++++++++++++++ var/www/templates/Tldstrending.html | 137 ++++++++-------------------- 2 files changed, 140 insertions(+), 97 deletions(-) create mode 100644 var/www/static/js/plot-graph.js diff --git a/var/www/static/js/plot-graph.js b/var/www/static/js/plot-graph.js new file mode 100644 index 00000000..22f2c178 --- /dev/null +++ b/var/www/static/js/plot-graph.js @@ -0,0 +1,100 @@ +function Graph(id_pannel, path){ + this.path = path; + this.id_pannel = id_pannel; + + g2 = new Dygraph( + document.getElementById(this.id_pannel), + // path to CSV file + //"{{ url_for('static', filename='csv/tldstrendingdata.csv') }}", + //"../static//csv/tldstrendingdata.csv", + this.path, + //window.csv, + { + rollPeriod: 1, + showRoller: true, + //drawPoints: true, + //fillGraph: true, + logscale: true, + animatedZooms: true, + labelsKMB: true, + highlightCircleSize: 3, + highlightSeriesOpts: { + strokeWidth: 3, + strokeBorderWidth: 1, + highlightCircleSize: 5, + }, + underlayCallback: function(canvas, area, g) { + canvas.fillStyle = "rgba(255, 193, 37, 0.5)"; + + function highlight_period(x_start, x_end) { + var canvas_left_x = g.toDomXCoord(x_start); + var canvas_right_x = g.toDomXCoord(x_end); + var canvas_width = canvas_right_x - canvas_left_x; + canvas.fillRect(canvas_left_x, area.y, canvas_width, area.h); + } + + var min_data_x = g.getValue(0,0); + var max_data_x = g.getValue(g.numRows()-1,0); + + // get day of week + var d = new Date(min_data_x); + var dow = d.getUTCDay(); + var ds = d.toUTCString(); + + var w = min_data_x; + // starting on Sunday is a special case + if (dow == 0) { + highlight_period(w,w+12*3600*1000); + } + // find first saturday + while (dow != 5) { + w += 24*3600*1000; + d = new Date(w); + dow = d.getUTCDay(); + } + + // shift back 1/2 day to center highlight around the point for the day + w -= 12*3600*1000; + while (w < max_data_x) { + var start_x_highlight = w; + var end_x_highlight = w + 2*24*3600*1000; + // make sure we don't try to plot outside the graph + if (start_x_highlight < min_data_x) { + start_x_highlight = min_data_x; + } + if (end_x_highlight > max_data_x) { + end_x_highlight = max_data_x; + } + highlight_period(start_x_highlight,end_x_highlight); + // calculate start of highlight for next Saturday + w += 7*24*3600*1000; + } + } + }); + + onclick = function(ev) { + if (g2.isSeriesLocked()) { + g2.clearSelection(); + } + else { + g2.setSelection(g2.getSelection(), g2.getHighlightSeries(), true); + } + }; + g2.updateOptions({clickCallback: onclick}, true); + + var linear = document.getElementById("linear"); + var log = document.getElementById("log"); + linear.onclick = function() { setLog(false); } + log.onclick = function() { setLog(true); } + var setLog = function(val) { + g2.updateOptions({ logscale: val }); + linear.disabled = !val; + log.disabled = val; + } + function unzoomGraph() { + g2.updateOptions({ + dateWindow:null, + valueRange:null + }); + } +} diff --git a/var/www/templates/Tldstrending.html b/var/www/templates/Tldstrending.html index e2706f21..6bc96cba 100644 --- a/var/www/templates/Tldstrending.html +++ b/var/www/templates/Tldstrending.html @@ -84,111 +84,54 @@
- +
+
+
+
+ Top Domain Trending +
+
+ + +
+
+
+ +
+ +
+
+ +
+
- + + + + From 7ff9b9a583261ab0a1c470120545e4c0643efc69 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Tue, 5 Jul 2016 16:53:03 +0200 Subject: [PATCH 02/19] Added DomainTrending seems working. Started search features with related html pages, not finish yet. --- bin/WebStats.py | 90 ++++++++++++++++++++++--- bin/packages/Date.py | 9 +++ bin/packages/Paste.py | 4 +- bin/packages/lib_words.py | 44 +++++++++++- var/www/Flask_server.py | 19 +++++- var/www/templates/Tldstrending.html | 2 +- var/www/templates/search.html | 71 ++++++++++++++++++- var/www/templates/show_saved_paste.html | 48 +++++++++++++ 8 files changed, 269 insertions(+), 18 deletions(-) create mode 100644 var/www/templates/show_saved_paste.html diff --git a/bin/WebStats.py b/bin/WebStats.py index 5da443a8..cac352af 100755 --- a/bin/WebStats.py +++ b/bin/WebStats.py @@ -10,19 +10,72 @@ import re import redis import os from packages import lib_words +from packages.Date import Date from pubsublogger import publisher from packages import Paste from Helper import Process from pyfaup.faup import Faup -def analyse(field_name): +# Config Var +threshold_need_to_look = 50 +range_to_look = 10 +threshold_to_plot = 1 #500% +to_plot = set() +clean_frequency = 10 #minutes + +def analyse(server, field_name): field = url_parsed[field_name] if field is not None: - prev_score = r_serv1.hget(field, date) + prev_score = server.hget(field, date) if prev_score is not None: - r_serv1.hset(field, date, int(prev_score) + 1) + server.hset(field, date, int(prev_score) + 1) else: - r_serv1.hset(field, date, 1) + server.hset(field, date, 1) + +def analyse_and_progression(server, field_name): + field = url_parsed[field_name] + if field is not None: + prev_score = server.hget(field, date) + if prev_score is not None: + print field + ' prev_score:' + prev_score + server.hset(field, date, int(prev_score) + 1) + if int(prev_score) + 1 > threshold_need_to_look: #threshold for false possitive + if(check_for_progression(server, field, date)): + to_plot.add(field) + else: + server.hset(field, date, 1) + +def check_for_progression(server, field, date): + previous_data = set() + tot_sum = 0 + for i in range(0, range_to_look): + curr_value = server.hget(field, Date(date).substract_day(i)) + if curr_value is None: #no further data + break + else: + curr_value = int(curr_value) + previous_data.add(curr_value) + tot_sum += curr_value + if i == 0: + today_val = curr_value + + + print 'totsum='+str(tot_sum) + print 'div='+str(tot_sum/today_val) + if tot_sum/today_val >= threshold_to_plot: + return True + else: + return False + +def clean_to_plot(): + temp_to_plot = set() + curr_date = datetime.date.today() + date = Date(str(curr_date.year)+str(curr_date.month)+str(curr_date.day)) + + for elem in to_plot: + if(check_for_progression(field, date)): + temp_to_plot.add(elem) + to_plot = temp_to_plot if __name__ == '__main__': # If you wish to use an other port of channel, do not forget to run a subscriber accordingly (see launch_logs.sh) @@ -45,6 +98,11 @@ if __name__ == '__main__': host=p.config.get("Redis_Level_DB", "host"), port=p.config.get("Redis_Level_DB", "port"), db=p.config.get("Redis_Level_DB", "db")) + + r_serv2 = redis.StrictRedis( + host=p.config.get("Redis_Level_DB_Domain", "host"), + port=p.config.get("Redis_Level_DB_Domain", "port"), + db=p.config.get("Redis_Level_DB_Domain", "db")) # FILE CURVE SECTION # csv_path_proto = os.path.join(os.environ['AIL_HOME'], @@ -57,6 +115,10 @@ if __name__ == '__main__': tldsfile_path = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "tldsfile")) + csv_path_domain = os.path.join(os.environ['AIL_HOME'], + p.config.get("Directories", "domainstrending_csv")) + + faup = Faup() generate_new_graph = False # Endless loop getting messages from the input queue @@ -71,17 +133,22 @@ if __name__ == '__main__': today = datetime.date.today() year = today.year month = today.month - + print 'b1' lib_words.create_curve_with_word_file(r_serv1, csv_path_proto, protocolsfile_path, year, month) - + print 'b2' lib_words.create_curve_with_word_file(r_serv1, csv_path_tld, tldsfile_path, year, month) - + print 'b3' + lib_words.create_curve_with_list(r_serv2, csv_path_domain, + to_plot, year, + month) + print 'end building' publisher.debug("{} queue is empty, waiting".format(config_section)) - time.sleep(1) + print 'sleeping' + time.sleep(5) continue else: @@ -91,5 +158,8 @@ if __name__ == '__main__': faup.decode(url) url_parsed = faup.get() - analyse('scheme') #Scheme analysis - analyse('tld') #Tld analysis + analyse(r_serv1, 'scheme') #Scheme analysis + analyse(r_serv1, 'tld') #Tld analysis + analyse_and_progression(r_serv2, 'domain') #Domain analysis + print "to_plot:" + print to_plot diff --git a/bin/packages/Date.py b/bin/packages/Date.py index 4abb0910..ce02636a 100644 --- a/bin/packages/Date.py +++ b/bin/packages/Date.py @@ -30,3 +30,12 @@ class Date(object): def _set_day(self, day): self.day = day + + def substract_day(self, numDay): + import datetime + computed_date = datetime.date(int(self.year), int(self.month), int(self.day)) - datetime.timedelta(numDay) + comp_year = str(computed_date.year) + comp_month = str(computed_date.month).zfill(2) + comp_day = str(computed_date.day).zfill(2) + return comp_year + comp_month + comp_day + diff --git a/bin/packages/Paste.py b/bin/packages/Paste.py index 3b81c7fe..cc5c41ad 100755 --- a/bin/packages/Paste.py +++ b/bin/packages/Paste.py @@ -186,7 +186,9 @@ class Paste(object): if the paste doesn't contain any human dictionnary words ..seealso: git@github.com:saffsd/langid.py.git - """ + FIXME: This procedure is using more than 20% of CPU + + """ identifier = LanguageIdentifier.from_modelstring(model, norm_probs=True) return identifier.classify(self.get_p_content()) diff --git a/bin/packages/lib_words.py b/bin/packages/lib_words.py index 3c065ed0..b2cf418b 100644 --- a/bin/packages/lib_words.py +++ b/bin/packages/lib_words.py @@ -81,13 +81,14 @@ def create_curve_with_word_file(r_serv, csvfilename, feederfilename, year, month to keep the timeline of the curve correct. """ + threshold = 50 first_day = date(year, month, 01) last_day = date(year, month, calendar.monthrange(year, month)[1]) words = [] with open(feederfilename, 'rb') as f: # words of the files - words = sorted([word.strip() for word in f]) + words = sorted([word.strip() for word in f if word.strip()[0:2]!='//' ]) headers = ['Date'] + words with open(csvfilename+'.csv', 'wb') as f: @@ -102,6 +103,47 @@ def create_curve_with_word_file(r_serv, csvfilename, feederfilename, year, month # from the 1srt day to the last of the list for word in words: value = r_serv.hget(word, curdate) + if value is None: + row.append(0) + else: + # if the word have a value for the day + # FIXME Due to performance issues (too many tlds, leads to more than 7s to perform this procedure), I added a threshold + if value >= threshold: + row.append(value) + writer.writerow(row) + +def create_curve_with_list(server, csvfilename, to_plot, year, month): + """Create a csv file used with dygraph. + + :param r_serv: -- connexion to redis database + :param csvfilename: -- the path to the .csv file created + :param to_plot: -- the list which contain a words to plot. + :param year: -- (integer) The year to process + :param month: -- (integer) The month to process + + This function create a .csv file using datas in redis. + It's checking if the words contained in to_plot and + their respectives values by days exists. + + """ + + first_day = date(year, month, 01) + last_day = date(year, month, calendar.monthrange(year, month)[1]) + words = sorted(to_plot) + + headers = ['Date'] + words + with open(csvfilename+'.csv', 'wb') as f: + writer = csv.writer(f) + writer.writerow(headers) + + # for each days + for dt in rrule(DAILY, dtstart=first_day, until=last_day): + row = [] + curdate = dt.strftime("%Y%m%d") + row.append(curdate) + # from the 1srt day to the last of the list + for word in words: + value = server.hget(word, curdate) if value is None: row.append(0) else: diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 9c152cab..a11b1920 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -7,7 +7,9 @@ import json from flask import Flask, render_template, jsonify, request import flask import os - +import sys +sys.path.append(os.path.join(os.environ['AIL_BIN'], 'packages/')) +import Paste # CONFIG # configfile = os.path.join(os.environ['AIL_BIN'], 'packages/config.cfg') @@ -18,6 +20,7 @@ if not os.path.exists(configfile): cfg = ConfigParser.ConfigParser() cfg.read(configfile) +max_preview_char = 500 # REDIS # r_serv = redis.StrictRedis( @@ -49,6 +52,10 @@ def get_queues(r): r.hgetall("queues").iteritems()] +def list_len(s): + return len(s) +app.jinja_env.filters['list_len'] = list_len + @app.route("/_logs") def logs(): return flask.Response(event_stream(), mimetype="text/event-stream") @@ -65,6 +72,7 @@ def search(): q = [] q.append(query) r = [] + c = [] # Search from whoosh import index from whoosh.fields import Schema, TEXT, ID @@ -78,7 +86,10 @@ def search(): results = searcher.search(query, limit=None) for x in results: r.append(x.items()[0][1]) - return render_template("search.html", r=r) + content = Paste.Paste(x.items()[0][1]).get_p_content() + content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 + c.append(content[0:content_range]) + return render_template("search.html", r=r, c=c) @app.route("/") def index(): @@ -104,6 +115,10 @@ def protocolstrending(): def tldstrending(): return render_template("Tldstrending.html") +@app.route("/showsavedpaste/") +def showsavedpaste(): + return render_template("show_saved_paste.html") + if __name__ == "__main__": app.run(host='0.0.0.0', port=7000, threaded=True) diff --git a/var/www/templates/Tldstrending.html b/var/www/templates/Tldstrending.html index 6bc96cba..fe86001c 100644 --- a/var/www/templates/Tldstrending.html +++ b/var/www/templates/Tldstrending.html @@ -130,7 +130,7 @@ diff --git a/var/www/templates/search.html b/var/www/templates/search.html index feb323dd..b9b7caeb 100644 --- a/var/www/templates/search.html +++ b/var/www/templates/search.html @@ -16,6 +16,16 @@ + @@ -39,6 +49,26 @@ + + + + + +
@@ -53,10 +83,26 @@
- - {% for result in r %} - +
{{ result }}
+ + + + + + + + + {% set i = 0 %} + {% for path in r %} + {% set prev_content = c[i] %} + + + + + + {% set i = i + 1 %} {% endfor %} +
#PathAction
{{ i + 1 }} {{ path }}
@@ -69,4 +115,23 @@ + + + + + diff --git a/var/www/templates/show_saved_paste.html b/var/www/templates/show_saved_paste.html new file mode 100644 index 00000000..0468ca2a --- /dev/null +++ b/var/www/templates/show_saved_paste.html @@ -0,0 +1,48 @@ + + + + Paste information + + + + + +

Paste:

+

{{ request.args.get('paste') }}

+ +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
DateSourceEncodingLanguageSizeMimeNumber of line
JohnDoejohn@example.com
MaryMoemary@example.com
JulyDooleyjuly@example.com
+ + + + From 9209d48e308696b0e90da3c1d37968c5608ce67b Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Wed, 6 Jul 2016 14:30:32 +0200 Subject: [PATCH 03/19] searching now working with preview, details and redirection to the paste display page. --- var/www/Flask_server.py | 35 ++++++++++++++++++++++--- var/www/templates/search.html | 26 +++++++++++++----- var/www/templates/show_saved_paste.html | 32 +++++++++++----------- 3 files changed, 67 insertions(+), 26 deletions(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index a11b1920..6bf31038 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -20,7 +20,10 @@ if not os.path.exists(configfile): cfg = ConfigParser.ConfigParser() cfg.read(configfile) -max_preview_char = 500 + +max_preview_char = cfg.get("Flask", "max_preview_char") +max_preview_modal = cfg.get("Flask", "max_preview_modal") + # REDIS # r_serv = redis.StrictRedis( @@ -86,7 +89,7 @@ def search(): results = searcher.search(query, limit=None) for x in results: r.append(x.items()[0][1]) - content = Paste.Paste(x.items()[0][1]).get_p_content() + content = Paste.Paste(x.items()[0][1]).get_p_content().decode('utf8', 'ignore') content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 c.append(content[0:content_range]) return render_template("search.html", r=r, c=c) @@ -117,7 +120,33 @@ def tldstrending(): @app.route("/showsavedpaste/") def showsavedpaste(): - return render_template("show_saved_paste.html") + requested_path = request.args.get('paste', '') + paste = Paste.Paste(requested_path) + p_date = str(paste._get_p_date()) + p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4] + p_source = paste.p_source + p_encoding = paste._get_p_encoding() + p_language = paste._get_p_language() + p_size = paste.p_size + p_mime = paste.p_mime + p_lineinfo = paste.get_lines_info() + p_content = paste.get_p_content().decode('utf-8', 'ignore') + return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content) + +@app.route("/showpreviewpaste/") +def showpreviewpaste(): + requested_path = request.args.get('paste', '') + paste = Paste.Paste(requested_path) + p_date = str(paste._get_p_date()) + p_date = p_date[6:]+'/'+p_date[4:6]+'/'+p_date[0:4] + p_source = paste.p_source + p_encoding = paste._get_p_encoding() + p_language = paste._get_p_language() + p_size = paste.p_size + p_mime = paste.p_mime + p_lineinfo = paste.get_lines_info() + p_content = paste.get_p_content()[0:max_preview_modal].decode('utf-8', 'ignore') + return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content) if __name__ == "__main__": diff --git a/var/www/templates/search.html b/var/www/templates/search.html index b9b7caeb..1feef403 100644 --- a/var/www/templates/search.html +++ b/var/www/templates/search.html @@ -25,6 +25,13 @@ max-height: 500px; font-size: 13px; } + xmp { + white-space:pre-wrap; + word-wrap:break-word; + } + .modal-backdrop.fade { + opacity: 0; + } @@ -57,14 +64,14 @@
-
-

Some text in the modal.

+
+

Loading paste information...

+
-
@@ -98,7 +105,7 @@ {{ i + 1 }} {{ path }} -

+

{% set i = i + 1 %} {% endfor %} @@ -119,6 +126,7 @@ @@ -127,11 +135,17 @@ $(document).ready(function(){ // On click, get html content from url and update the corresponding modal $("[data-toggle='modal']").on("click", function (event) { event.preventDefault(); - var url = $(this).attr('data-url'); - var modal_id = $(this).attr('data-target'); + var modal=$(this) + var url = " {{ url_for('showpreviewpaste') }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num'); $.get(url, function (data) { $("#mymodalbody").html(data); + $("#button_show_path").attr('href', $(modal).attr('data-url')); + $("#button_show_path").show('fast'); }); }); + $("#mymodal").on('hidden.bs.modal', function () { + $("#mymodalbody").html("

Loading paste information...

"); + $("#button_show_path").attr('href', ''); + }); diff --git a/var/www/templates/show_saved_paste.html b/var/www/templates/show_saved_paste.html index 0468ca2a..0accffb1 100644 --- a/var/www/templates/show_saved_paste.html +++ b/var/www/templates/show_saved_paste.html @@ -7,7 +7,7 @@ -

Paste:

+

Paste: {{ request.args.get('num') }}

{{ request.args.get('paste') }}


@@ -19,30 +19,28 @@ Source Encoding Language - Size + Size (Kb) Mime - Number of line + Number of lines + Max line length - John - Doe - john@example.com - - - Mary - Moe - mary@example.com - - - July - Dooley - july@example.com + {{ date }} + {{ source }} + {{ encoding }} + {{ language }} + {{ size }} + {{ mime }} + {{ lineinfo.0 }} + {{ lineinfo.1 }} +

Content:

+

{{ content }}

+ - From 3345124219147064c9d3e6137f43f58a393c4d8a Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Wed, 6 Jul 2016 14:48:27 +0200 Subject: [PATCH 04/19] Added pannel in search modal + small fix related to config --- .gitignore | 1 + var/www/Flask_server.py | 4 +- var/www/templates/show_saved_paste.html | 60 ++++++++++++++----------- 3 files changed, 36 insertions(+), 29 deletions(-) diff --git a/.gitignore b/.gitignore index 264aaa5f..88b623fa 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ # Temp files *.swp *.pyc +*.swo # Install Dirs AILENV diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 6bf31038..a9a03fa8 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -21,8 +21,8 @@ if not os.path.exists(configfile): cfg = ConfigParser.ConfigParser() cfg.read(configfile) -max_preview_char = cfg.get("Flask", "max_preview_char") -max_preview_modal = cfg.get("Flask", "max_preview_modal") +max_preview_char = int(cfg.get("Flask", "max_preview_char")) +max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # REDIS # diff --git a/var/www/templates/show_saved_paste.html b/var/www/templates/show_saved_paste.html index 0accffb1..d8872523 100644 --- a/var/www/templates/show_saved_paste.html +++ b/var/www/templates/show_saved_paste.html @@ -12,35 +12,41 @@

- - - - - - - - - - - - - - - - - - - - - - - - - -
DateSourceEncodingLanguageSize (Kb)MimeNumber of linesMax line length
{{ date }}{{ source }}{{ encoding }}{{ language }}{{ size }}{{ mime }}{{ lineinfo.0 }}{{ lineinfo.1 }}
- +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + +
DateSourceEncodingLanguageSize (Kb)MimeNumber of linesMax line length
{{ date }}{{ source }}{{ encoding }}{{ language }}{{ size }}{{ mime }}{{ lineinfo.0 }}{{ lineinfo.1 }}
+
+

Content:

{{ content }}

+
+
+ From f29620cf969cfdb8b6779ae19ecf61dc42be4503 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Wed, 6 Jul 2016 16:54:27 +0200 Subject: [PATCH 05/19] Added load more content in search modal preview. Works fine but may duplicate content (need to fix). --- var/www/Flask_server.py | 15 ++++++++++++++- var/www/templates/search.html | 24 +++++++++++++++++++++++- var/www/templates/show_saved_paste.html | 4 ++-- 3 files changed, 39 insertions(+), 4 deletions(-) diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index a9a03fa8..e87864a9 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -23,7 +23,7 @@ cfg.read(configfile) max_preview_char = int(cfg.get("Flask", "max_preview_char")) max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) - +index_prev = 0 # used if the user want to load more paste content # REDIS # r_serv = redis.StrictRedis( @@ -148,6 +148,19 @@ def showpreviewpaste(): p_content = paste.get_p_content()[0:max_preview_modal].decode('utf-8', 'ignore') return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content) +@app.route("/getmoredata/") +def getmoredata(): + requested_path = request.args.get('paste', '') + index_prev = int(request.args.get('index', '')) + paste = Paste.Paste(requested_path) + + p_content = paste.get_p_content().decode('utf-8', 'ignore') + final_index = (index_prev+1)*max_preview_modal + if final_index > len(p_content)-1: # prevent out of bound + final_index = len(p_content)-1 + + to_return = p_content[index_prev*max_preview_modal:final_index] + return to_return if __name__ == "__main__": app.run(host='0.0.0.0', port=7000, threaded=True) diff --git a/var/www/templates/search.html b/var/www/templates/search.html index 1feef403..218bf606 100644 --- a/var/www/templates/search.html +++ b/var/www/templates/search.html @@ -135,17 +135,39 @@ $(document).ready(function(){ // On click, get html content from url and update the corresponding modal $("[data-toggle='modal']").on("click", function (event) { event.preventDefault(); - var modal=$(this) + var modal=$(this); var url = " {{ url_for('showpreviewpaste') }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num'); $.get(url, function (data) { $("#mymodalbody").html(data); + + var button = ''; + $("#mymodalbody").children(".panel-default").append(button); + $("#button_show_path").attr('href', $(modal).attr('data-url')); $("#button_show_path").show('fast'); + + var index_preview = 0 // used for load more preview + + // On load more content click, replace paste content + $("#load-more-button").on("click", function (event) { + $.get("{{ url_for('getmoredata') }}"+"?paste="+$(modal).attr('data-path')+"&index="+index_preview, function(data, status){ + index_preview++; + console.log($("#mymodalbody").find("#panel-body")); + var new_content = $('

'+data+'

').hide(); + $("#mymodalbody").find("#panel-body").append(new_content); + new_content.show('fast'); + }); + }); + }); }); + + // When the modal goes out, refresh it to normal content $("#mymodal").on('hidden.bs.modal', function () { $("#mymodalbody").html("

Loading paste information...

"); $("#button_show_path").attr('href', ''); }); + + diff --git a/var/www/templates/show_saved_paste.html b/var/www/templates/show_saved_paste.html index d8872523..427b6a9a 100644 --- a/var/www/templates/show_saved_paste.html +++ b/var/www/templates/show_saved_paste.html @@ -41,9 +41,9 @@ -
+

Content:

-

{{ content }}

+

{{ content }}

From 74b337f1a105950e261d804a015fa248f9b94891 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Thu, 7 Jul 2016 09:31:04 +0200 Subject: [PATCH 06/19] search preview modal do not duplicate anymore but still needs improvements --- var/www/templates/search.html | 25 +++++++++++++++++-------- 1 file changed, 17 insertions(+), 8 deletions(-) diff --git a/var/www/templates/search.html b/var/www/templates/search.html index 218bf606..deb53537 100644 --- a/var/www/templates/search.html +++ b/var/www/templates/search.html @@ -132,6 +132,10 @@ $(document).ready(function(){ diff --git a/var/www/templates/show_saved_paste.html b/var/www/templates/show_saved_paste.html index 427b6a9a..924fd2da 100644 --- a/var/www/templates/show_saved_paste.html +++ b/var/www/templates/show_saved_paste.html @@ -43,7 +43,7 @@

Content:

-

{{ content }}

+

{{ content }}

From 465244e1cec1b04a03636f72c3f46ed0891e2087 Mon Sep 17 00:00:00 2001 From: Mokaddem Date: Thu, 7 Jul 2016 16:38:00 +0200 Subject: [PATCH 08/19] Added dynamic table sorting in search page. (Still need to add dependencies) --- bin/packages/Paste.py | 3 +++ var/www/Flask_server.py | 17 +++++++---------- var/www/templates/search.html | 14 ++++++++++++-- 3 files changed, 22 insertions(+), 12 deletions(-) diff --git a/bin/packages/Paste.py b/bin/packages/Paste.py index cc5c41ad..bedf36b0 100755 --- a/bin/packages/Paste.py +++ b/bin/packages/Paste.py @@ -198,6 +198,9 @@ class Paste(object): def _get_p_date(self): return self.p_date + def _get_p_size(self): + return self.p_size + def _get_hash_lines(self, min=1, start=1, jump=10): """ Returning all the lines of the paste hashed. diff --git a/var/www/Flask_server.py b/var/www/Flask_server.py index 546d92c6..924c0501 100755 --- a/var/www/Flask_server.py +++ b/var/www/Flask_server.py @@ -76,6 +76,8 @@ def search(): q.append(query) r = [] c = [] + paste_date = [] + paste_size = [] # Search from whoosh import index from whoosh.fields import Schema, TEXT, ID @@ -89,10 +91,13 @@ def search(): results = searcher.search(query, limit=None) for x in results: r.append(x.items()[0][1]) - content = Paste.Paste(x.items()[0][1]).get_p_content().decode('utf8', 'ignore') + paste = Paste.Paste(x.items()[0][1]) + content = paste.get_p_content().decode('utf8', 'ignore') content_range = max_preview_char if len(content)>max_preview_char else len(content)-1 c.append(content[0:content_range]) - return render_template("search.html", r=r, c=c, char_to_display=max_preview_modal) + paste_date.append(paste._get_p_date()) + paste_size.append(paste._get_p_size()) + return render_template("search.html", r=r, c=c, paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal) @app.route("/") def index(): @@ -156,14 +161,6 @@ def getmoredata(): requested_path = request.args.get('paste', '') paste = Paste.Paste(requested_path) p_content = paste.get_p_content().decode('utf-8', 'ignore') - '''final_index = (index_prev+1)*max_preview_modal - if final_index > len(p_content)-1: # prevent out of bound - final_index = len(p_content)-1 - ''' - #to_return = p_content[index_prev*max_preview_modal:final_index] - - #correct_index = len(p_content) if max_preview_modal > len(p_content) else max_preview_modal - #to_return = str(p_content[correct_index:]) to_return = p_content[max_preview_modal:] return to_return diff --git a/var/www/templates/search.html b/var/www/templates/search.html index 9f0c3b4f..451f46df 100644 --- a/var/www/templates/search.html +++ b/var/www/templates/search.html @@ -12,10 +12,14 @@ + + + + @@ -45,7 +34,7 @@