Merge branch 'website-improvement' into production

bin/Credential.py

@@ -48,7 +48,7 @@ if __name__ == "__main__":
         if sites_set:
             message += ' Related websites: {}'.format(', '.join(sites_set))
 
-        to_print = 'Credential;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, message)
+        to_print = 'Credential;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, message, paste.p_path)
 
         print('\n '.join(creds))
 

bin/CreditCards.py

@@ -63,14 +63,14 @@ if __name__ == "__main__":
                 to_print = 'CreditCard;{};{};{};'.format(
                     paste.p_source, paste.p_date, paste.p_name)
                 if (len(creditcard_set) > 0):
-                    publisher.warning('{}Checked {} valid number(s)'.format(
+                    publisher.warning('{}Checked {} valid number(s);{}'.format(
-                        to_print, len(creditcard_set)))
+                        to_print, len(creditcard_set), paste.p_path))
                     #Send to duplicate
                     p.populate_set_out(filename, 'Duplicate')
                     #send to Browse_warning_paste
                     p.populate_set_out('creditcard;{}'.format(filename), 'BrowseWarningPaste')
                 else:
-                    publisher.info('{}CreditCard related'.format(to_print))
+                    publisher.info('{}CreditCard related;{}'.format(to_print, paste.p_path))
         else:
             publisher.debug("Script creditcard is idling 1m")
             time.sleep(10)

bin/DomClassifier.py

@@ -51,13 +51,13 @@ def main():
                 localizeddomains = c.include(expression=cc_tld)
                 if localizeddomains:
                     print(localizeddomains)
-                    publisher.warning('DomainC;{};{};{};Checked {} located in {}'.format(
+                    publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
-                        PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld))
+                        PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc_tld, PST.p_path))
                 localizeddomains = c.localizedomain(cc=cc)
                 if localizeddomains:
                     print(localizeddomains)
-                    publisher.warning('DomainC;{};{};{};Checked {} located in {}'.format(
+                    publisher.warning('DomainC;{};{};{};Checked {} located in {};{}'.format(
-                        PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc))
+                        PST.p_source, PST.p_date, PST.p_name, localizeddomains, cc, PST.p_path))
         except IOError:
             print "CRC Checksum Failed on :", PST.p_path
             publisher.error('Duplicate;{};{};{};CRC Checksum Failed'.format(

bin/Duplicates.py

@@ -131,8 +131,10 @@ if __name__ == "__main__":
                                 # index of paste
                                 index_current = r_serv_dico.get(dico_hash)
                                 paste_path = r_serv_dico.get(index_current)
+                                paste_date = r_serv_dico.get(index_current+'_date')
+                                paste_date = paste_date if paste_date != None else "No date available"
                                 if paste_path != None:
-                                    hash_dico[dico_hash] = (hash_type, paste_path, percent)
+                                    hash_dico[dico_hash] = (hash_type, paste_path, percent, paste_date)
 
                                 print '['+hash_type+'] '+'comparing: ' + str(PST.p_path[44:]) + '  and  ' + str(paste_path[44:]) + ' percentage: ' + str(percent)
                         except Exception,e:
@@ -142,6 +144,7 @@ if __name__ == "__main__":
             # Add paste in DB after checking to prevent its analysis twice
             # hash_type_i -> index_i  AND  index_i -> PST.PATH
             r_serv1.set(index, PST.p_path)
+            r_serv1.set(index+'_date', PST._get_p_date())
             r_serv1.sadd("INDEX", index)
             # Adding hashes in Redis
             for hash_type, paste_hash in paste_hashes.iteritems():
@@ -152,7 +155,7 @@ if __name__ == "__main__":
 
             # if there is data in this dictionnary
             if len(hash_dico) != 0:
-                # paste_tuple = (paste_path, percent)
+                # paste_tuple = (hash_type, date, paste_path, percent)
                 for dico_hash, paste_tuple in hash_dico.items():
                     dupl.append(paste_tuple)
 
@@ -162,7 +165,7 @@ if __name__ == "__main__":
                 if dupl != []:
                     PST.__setattr__("p_duplicate", dupl)
                     PST.save_attribute_redis("p_duplicate", dupl)
-                    publisher.info('{}Detected {}'.format(to_print, len(dupl)))
+                    publisher.info('{}Detected {};{}'.format(to_print, len(dupl), PST.p_path))
                     print '{}Detected {}'.format(to_print, len(dupl))
 
                 y = time.time()

bin/Mail.py

@@ -55,9 +55,9 @@ if __name__ == "__main__":
                                              list(MX_values[1])))
 
                     pprint.pprint(MX_values)
-                    to_print = 'Mails;{};{};{};Checked {} e-mail(s)'.\
+                    to_print = 'Mails;{};{};{};Checked {} e-mail(s);{}'.\
                         format(PST.p_source, PST.p_date, PST.p_name,
-                               MX_values[0])
+                               MX_values[0], PST.p_path)
                     if MX_values[0] > is_critical:
                         publisher.warning(to_print)
                         #Send to duplicate

bin/Onion.py

@@ -133,8 +133,8 @@ if __name__ == "__main__":
                                                     PST.p_name)
                 if len(domains_list) > 0:
 
-                    publisher.warning('{}Detected {} .onion(s)'.format(
+                    publisher.warning('{}Detected {} .onion(s);{}'.format(
-                        to_print, len(domains_list)))
+                        to_print, len(domains_list),PST.p_path))
                     now = datetime.datetime.now()
                     path = os.path.join('onions', str(now.year).zfill(4),
                                         str(now.month).zfill(2),
@@ -144,9 +144,9 @@ if __name__ == "__main__":
                                                         PST.p_date,
                                                         PST.p_name)
                     for url in fetch(p, r_cache, urls, domains_list, path):
-                        publisher.warning('{}Checked {}'.format(to_print, url))
+                        publisher.warning('{}Checked {};{}'.format(to_print, url, PST.p_path))
                 else:
-                    publisher.info('{}Onion related'.format(to_print))
+                    publisher.info('{}Onion related;{}'.format(to_print, PST.p_path))
 
             prec_filename = filename
         else:

bin/Release.py

@@ -34,7 +34,7 @@ if __name__ == "__main__":
         if len(releases) == 0:
             continue
 
-        to_print = 'Release;{};{};{};{} releases'.format(paste.p_source, paste.p_date, paste.p_name, len(releases))
+        to_print = 'Release;{};{};{};{} releases;{}'.format(paste.p_source, paste.p_date, paste.p_name, len(releases), paste.p_path)
         if len(releases) > 30:
             publisher.warning(to_print)
         else:

bin/SQLInjectionDetection.py

@@ -69,7 +69,7 @@ def analyse(url, path):
         if (result_path > 1) or (result_query > 1):
             print "Detected SQL in URL: "
             print urllib2.unquote(url)
-            to_print = 'SQLInjection;{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL")
+            to_print = 'SQLInjection;{};{};{};{};{}'.format(paste.p_source, paste.p_date, paste.p_name, "Detected SQL in URL", paste.p_path)
             publisher.warning(to_print)
             #Send to duplicate
             p.populate_set_out(path, 'Duplicate')

bin/Web.py

@@ -131,8 +131,8 @@ if __name__ == "__main__":
                                              list(A_values[1])))
 
                     pprint.pprint(A_values)
-                    publisher.info('Url;{};{};{};Checked {} URL'.format(
+                    publisher.info('Url;{};{};{};Checked {} URL;{}'.format(
-                        PST.p_source, PST.p_date, PST.p_name, A_values[0]))
+                        PST.p_source, PST.p_date, PST.p_name, A_values[0], PST.p_path))
             prec_filename = filename
 
         else:

var/www/Flask_server.py

@@ -78,6 +78,27 @@ def event_stream():
         if msg['type'] == 'pmessage' and level != "DEBUG":
             yield 'data: %s\n\n' % json.dumps(msg)
 
+def event_stream_getImportantPasteByModule(module_name):
+    index = 0
+    all_pastes_list = getPastebyType(r_serv_db, module_name)
+    for path in all_pastes_list:
+        index += 1
+        paste = Paste.Paste(path)
+        content = paste.get_p_content().decode('utf8', 'ignore')
+        content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
+        curr_date = str(paste._get_p_date())
+        curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
+        data = {}
+        data["module"] = module_name
+        data["index"] = index
+        data["path"] = path
+        data["content"] = content[0:content_range]
+        data["linenum"] = paste.get_lines_info()[0]
+        data["date"] = curr_date
+        data["char_to_display"] = max_preview_modal
+        data["finished"] = True if index == len(all_pastes_list) else False
+        yield 'retry: 100000\ndata: %s\n\n' % json.dumps(data) #retry to avoid reconnection of the browser
+
 
 def get_queues(r):
     # We may want to put the llen in a pipeline to do only one query.
@@ -122,6 +143,7 @@ def showpaste(content_range):
     p_duplicate_full_list = json.loads(paste._get_p_duplicate())
     p_duplicate_list = []
     p_simil_list = []
+    p_date_list = []
     p_hashtype_list = []
 
 
@@ -149,20 +171,29 @@ def showpaste(content_range):
 
         hash_types = str(hash_types).replace("[","").replace("]","") if len(hash_types)==1 else str(hash_types)
         comp_vals = str(comp_vals).replace("[","").replace("]","") if len(comp_vals)==1 else str(comp_vals)
-        new_dup_list.append([hash_types.replace("'", ""), p_duplicate_full_list[dup_list_index][1], comp_vals])
+        if len(p_duplicate_full_list[dup_list_index]) > 3:
+            try:
+                date_paste = str(int(p_duplicate_full_list[dup_list_index][3]))
+                date_paste = date_paste[0:4]+"-"+date_paste[4:6]+"-"+date_paste[6:8]
+            except ValueError:
+                date_paste = str(p_duplicate_full_list[dup_list_index][3])
+        else:
+            date_paste = "No date available"
+        new_dup_list.append([hash_types.replace("'", ""), p_duplicate_full_list[dup_list_index][1], comp_vals, date_paste])
 
     # Create the list to pass to the webpage
     for dup_list in new_dup_list:
-        hash_type, path, simil_percent = dup_list
+        hash_type, path, simil_percent, date_paste = dup_list
         p_duplicate_list.append(path)
         p_simil_list.append(simil_percent)
         p_hashtype_list.append(hash_type)
+        p_date_list.append(date_paste)
 
     if content_range != 0:
        p_content = p_content[0:content_range]
 
 
-    return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list)
+    return render_template("show_saved_paste.html", date=p_date, source=p_source, encoding=p_encoding, language=p_language, size=p_size, mime=p_mime, lineinfo=p_lineinfo, content=p_content, initsize=len(p_content), duplicate_list = p_duplicate_list, simil_list = p_simil_list, hashtype_list = p_hashtype_list, date_list=p_date_list)
 
 def getPastebyType(server, module_name):
     all_path = []
@@ -326,11 +357,10 @@ def search():
     c = [] #preview of the paste content
     paste_date = []
     paste_size = []
+    num_elem_to_get = 50
 
     # Search filename
-    print r_serv_pasteName.smembers(q[0])
     for path in r_serv_pasteName.smembers(q[0]):
-        print path
         r.append(path)
         paste = Paste.Paste(path)
         content = paste.get_p_content().decode('utf8', 'ignore')
@@ -351,7 +381,7 @@ def search():
     from whoosh.qparser import QueryParser
     with ix.searcher() as searcher:
         query = QueryParser("content", ix.schema).parse(" ".join(q))
-        results = searcher.search(query, limit=None)
+        results = searcher.search_page(query, 1, pagelen=num_elem_to_get)
         for x in results:
             r.append(x.items()[0][1])
             paste = Paste.Paste(x.items()[0][1])
@@ -362,7 +392,57 @@ def search():
             curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
             paste_date.append(curr_date)
             paste_size.append(paste._get_p_size())
-    return render_template("search.html", r=r, c=c, query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal)
+        results = searcher.search(query)
+        num_res = len(results)
+
+    return render_template("search.html", r=r, c=c, query=request.form['query'], paste_date=paste_date, paste_size=paste_size, char_to_display=max_preview_modal, num_res=num_res)
+
+
+@app.route("/get_more_search_result", methods=['POST'])
+def get_more_search_result():
+    query = request.form['query']
+    q = []
+    q.append(query)
+    page_offset = int(request.form['page_offset'])
+    num_elem_to_get = 50
+
+    path_array = []
+    preview_array = []
+    date_array = []
+    size_array = []
+
+    from whoosh import index
+    from whoosh.fields import Schema, TEXT, ID
+    schema = Schema(title=TEXT(stored=True), path=ID(stored=True), content=TEXT)
+
+    indexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path"))
+    ix = index.open_dir(indexpath)
+    from whoosh.qparser import QueryParser
+    with ix.searcher() as searcher:
+        query = QueryParser("content", ix.schema).parse(" ".join(q))
+        results = searcher.search_page(query, page_offset, num_elem_to_get)   
+        for x in results:
+            path_array.append(x.items()[0][1])
+            paste = Paste.Paste(x.items()[0][1])
+            content = paste.get_p_content().decode('utf8', 'ignore')
+            content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
+            preview_array.append(content[0:content_range])
+            curr_date = str(paste._get_p_date())
+            curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
+            date_array.append(curr_date)
+            size_array.append(paste._get_p_size())
+        to_return = {}
+        to_return["path_array"] = path_array
+        to_return["preview_array"] = preview_array
+        to_return["date_array"] = date_array
+        to_return["size_array"] = size_array
+        print "len(path_array)="+str(len(path_array))
+        if len(path_array) < num_elem_to_get: #pagelength
+            to_return["moreData"] = False
+        else:
+            to_return["moreData"] = True
+
+    return jsonify(to_return)
 
 
 @app.route("/")
@@ -408,19 +488,30 @@ def importantPasteByModule():
     paste_date = []
     paste_linenum = []
     all_path = []
+    allPastes = getPastebyType(r_serv_db, module_name)
 
-    for path in getPastebyType(r_serv_db, module_name):
+    for path in allPastes[0:10]:
         all_path.append(path)
         paste = Paste.Paste(path)
         content = paste.get_p_content().decode('utf8', 'ignore')
         content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
-        all_content.append(content[0:content_range])
+        all_content.append(content[0:content_range].replace("\"", "\'").replace("\r", " ").replace("\n", " "))
         curr_date = str(paste._get_p_date())
         curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
         paste_date.append(curr_date)
         paste_linenum.append(paste.get_lines_info()[0])
 
-    return render_template("important_paste_by_module.html", all_path=all_path, content=all_content, paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal)
+    if len(allPastes) > 10:
+        finished = "" 
+    else:
+        finished = "display: none;"
+
+    return render_template("important_paste_by_module.html", moduleName=module_name, all_path=all_path, content=all_content, paste_date=paste_date, paste_linenum=paste_linenum, char_to_display=max_preview_modal, finished=finished)
+
+@app.route("/_getImportantPasteByModule")
+def getImportantPasteByModule():
+    module_name = request.args.get('moduleName')
+    return flask.Response(event_stream_getImportantPasteByModule(module_name), mimetype="text/event-stream")
 
 @app.route("/moduletrending/")
 def moduletrending():
@@ -735,7 +826,8 @@ def showsavedpaste():
 
 @app.route("/showpreviewpaste/")
 def showpreviewpaste():
-    return showpaste(max_preview_modal)
+    num = request.args.get('num', '')
+    return "|num|"+num+"|num|"+showpaste(max_preview_modal)
 
 
 @app.route("/getmoredata/")

var/www/static/js/indexjavascript.js

@@ -109,11 +109,11 @@ function create_log_table(obj_json) {
     var pdate = document.createElement('TD')
     var nam = document.createElement('TD')
     var msage = document.createElement('TD')
+    var inspect = document.createElement('TD')
 
     var chansplit = obj_json.channel.split('.');
     var parsedmess = obj_json.data.split(';');
 
-
     if (parsedmess[0] == "Global"){
         var paste_processed = parsedmess[4].split(" ")[2];
         window.paste_num_tabvar = paste_processed;
@@ -139,7 +139,7 @@ function create_log_table(obj_json) {
         source_url = "http://"+parsedmess[1]+"/"+parsedmess[3].split(".")[0];
     }
     source_link.setAttribute("HREF",source_url);
-    source_link.setAttribute("TARGET", "_blank")
+    source_link.setAttribute("TARGET", "_blank");
     source_link.appendChild(document.createTextNode(parsedmess[1]));
 
     src.appendChild(source_link);
@@ -169,6 +169,18 @@ function create_log_table(obj_json) {
 
     msage.appendChild(document.createTextNode(message.join(" ")));
 
+    var paste_path = parsedmess[5];
+    var url_to_saved_paste = url_showSavedPath+"?paste="+paste_path+"&num="+parsedmess[0];
+
+    var action_icon_a = document.createElement("A");
+    action_icon_a.setAttribute("TARGET", "_blank");
+    action_icon_a.setAttribute("HREF", url_to_saved_paste);
+    var action_icon_span = document.createElement('SPAN');
+    action_icon_span.className = "fa fa-search-plus";
+    action_icon_a.appendChild(action_icon_span);
+ 
+    inspect.appendChild(action_icon_a);
+
     tr.appendChild(time)
     tr.appendChild(chan);
     tr.appendChild(level);
@@ -177,6 +189,7 @@ function create_log_table(obj_json) {
     tr.appendChild(pdate);
     tr.appendChild(nam);
     tr.appendChild(msage);
+    tr.appendChild(inspect);
 
     if (tr.className == document.getElementById("checkbox_log_info").value && document.getElementById("checkbox_log_info").checked  == true) {
            tableBody.appendChild(tr);
@@ -219,6 +232,19 @@ function create_queue_table() {
         tr.appendChild(th);
     }
 
+    if ((glob_tabvar.row1).length == 0) {
+        var tr = document.createElement('TR');
+        var td = document.createElement('TD');
+        var td2 = document.createElement('TD');
+        td.appendChild(document.createTextNode("No running queues"));
+        td2.appendChild(document.createTextNode("Or no feed"));
+        td.className += " danger";
+        td2.className += " danger";
+        tr.appendChild(td);
+        tr.appendChild(td2);
+        tableBody.appendChild(tr);
+    }
+    else {
         for(i = 0; i < (glob_tabvar.row1).length;i++){
             var tr = document.createElement('TR')
             for(j = 0; j < 2; j++){
@@ -241,6 +267,7 @@ function create_queue_table() {
                 tr.className += " success";
             tableBody.appendChild(tr);
         }
+    }
     Tablediv.appendChild(table);
 }
 

var/www/templates/browse_important_paste.html

@@ -30,9 +30,6 @@
          white-space:pre-wrap;
          word-wrap:break-word;
       }
-      .modal-backdrop.fade {
-         opacity: 0;
-      }
   </style>
 
 </head>

var/www/templates/important_paste_by_module.html

@@ -1,4 +1,4 @@
-<table class="table table-striped table-bordered table-hover" id="myTable">
+<table class="table table-striped table-bordered table-hover" id="myTable_{{ moduleName }}">
     <thead>
          <tr>
             <th>#</th>
@@ -23,12 +23,110 @@
     </tbody>
 </table>
 </br>
+<div id="nbr_entry" class="alert alert-info">
+ 
+</div>
+<div id="div_stil_data">
+    <button id="load_more_json_button1" type="button" class="btn btn-default" onclick="add_entries(100)" style="display: none">Load 100 entries</button>
+    <button id="load_more_json_button2" type="button" class="btn btn-warning" onclick="add_entries(300)" style="display: none">Load 300 entries</button>
+    <img id="loading_gif_browse" src="{{url_for('static', filename='image/loading.gif') }}" heigt="20" width="20" style="margin: 2px; {{ finished }}"></div>
 </br>
 
+
+
 <script>
+var json_array = [];
+var all_data_received = false;
+var curr_numElem;
+var elem_added = 0;
+var tot_num_entry = 0;
+
+function deploy_source() {
+    var button_load_more_displayed = false;
+
+    if(typeof(EventSource) !== "undefined" && typeof(source) !== "") {
+        var source = new EventSource("{{ url_for('getImportantPasteByModule') }}"+"?moduleName="+moduleName);
+        source.onmessage = function(event) {
+            var feed = jQuery.parseJSON( event.data );
+            curr_numElem = parseInt($("#myTable_"+moduleName).attr('data-numElem'));
+            if (feed.index > curr_numElem & feed.module == moduleName) { // Avoid doubling the pastes
+                json_array.push(feed);
+                tot_num_entry++;
+                $("#load_more_json_button1").show();
+                $("#load_more_json_button2").show();
+                $("#nbr_entry").text(tot_num_entry + " entries available, " + (tot_num_entry - elem_added) + " not displayed");
+
+                $("#myTable_"+moduleName).attr('data-numElem', curr_numElem+1);
+
+                if(feed.index > 100 && !button_load_more_displayed) {
+                    button_load_more_displayed = true;
+                    add_entries_X(20);
+                }
+                if(feed.finished) {
+                    $("#loading_gif_browse").hide();
+                    source.close();
+                    all_data_received = true;
+                    add_entries_X(10);
+                }
+            }
+        };
+    } else {
+        console.log("Sorry, your browser does not support server-sent events...");
+    }
+}
+
+function add_entries(iter) { //Used to disable the button before going to the big loop
+    $("#load_more_json_button1").attr('disabled','disabled');
+    $("#load_more_json_button2").attr('disabled','disabled');
+    setTimeout(function() { add_entries_X(iter);}, 50);
+}
+
+function add_entries_X(to_add) {
+    for(i=0; i<to_add; i++) {
+        if(json_array.length == 0 && all_data_received) {
+            $("#load_more_json_button1").hide();
+            $("#load_more_json_button2").hide();
+            $("#nbr_entry").hide();
+            return false;
+        } else {
+            var feed = json_array.shift();
+            elem_added++;
+             search_table.row.add( [
+                 feed.index,
+                "<a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+feed.path+"&num="+feed.index+"\"> "+ feed.path +"</a>",
+                 feed.date,
+                 feed.linenum,
+                 "<p><span class=\"glyphicon glyphicon-info-sign\" data-toggle=\"tooltip\" data-placement=\"left\" title=\""+feed.content.replace(/\"/g, "\'").replace(/\r/g, "\'").replace(/\n/g, "\'")+"\"></span> <button type=\"button\" class=\"btn-link\" data-num=\""+feed.index+"\" data-toggle=\"modal\" data-target=\"#mymodal\" data-url=\"{{ url_for('showsavedpaste') }}?paste="+feed.path+"&num="+feed.index+"\" data-path=\""+feed.path+"\"><span class=\"fa fa-search-plus\"></span></button></p>"
+             ] ).draw( false );
+             $("#myTable_"+moduleName).attr('data-numElem', curr_numElem+1);
+        }
+    }
+    $("#load_more_json_button1").removeAttr('disabled');
+    $("#load_more_json_button2").removeAttr('disabled');
+    $("#nbr_entry").text(tot_num_entry + " entries available, " + (tot_num_entry - elem_added) + " not displayed");
+    return true;
+}
+
+
+
+
+</script>
+
+
+<script>
+var moduleName = "{{ moduleName }}";
+var search_table;
+var last_clicked_paste;
+var can_change_modal_content = true;
+$("#myTable_"+moduleName).attr('data-numElem', "{{ all_path|length }}");
+
 $(document).ready(function(){
     $('[data-toggle="tooltip"]').tooltip();
-    $('#myTable').dataTable({ "order": [[ 2, "desc" ]] });
+
+    search_table = $('#myTable_'+moduleName).DataTable({ "order": [[ 2, "desc" ]] });
+
+    deploy_source();
+
 });
 </script>
 
@@ -40,39 +138,9 @@ $(document).ready(function(){
     var char_to_display = {{ char_to_display }};
     var start_index = 0;
 
-    // On click, get html content from url and update the corresponding modal
-    $("[data-toggle='modal']").on("click.openmodal", function (event) {
-        event.preventDefault();
-        var modal=$(this);
-        var url = " {{ url_for('showpreviewpaste')  }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
-        $.get(url, function (data) {
-            $("#mymodalbody").html(data);
-            var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
-            button.tooltip();
-            $("#mymodalbody").children(".panel-default").append(button);
-
-            $("#button_show_path").attr('href', $(modal).attr('data-url'));
-            $("#button_show_path").show('fast');
-            $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
-            if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
-                nothing_to_display();
-            }
-            // On click, donwload all paste's content
-            $("#load-more-button").on("click", function (event) {
-                if (complete_paste == null) { //Donwload only once
-                    $.get("{{ url_for('getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
-                        complete_paste = data;
-                        update_preview();
-                    });
-                } else {
-                    update_preview();
-                }
-            });
-        });
-    });
-
     // When the modal goes out, refresh it to normal content
     $("#mymodal").on('hidden.bs.modal', function () {
+        can_change_modal_content = true;
         $("#mymodalbody").html("<p>Loading paste information...</p>");
         var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
         $("#mymodalbody").append(loading_gif); // Show the loading GIF
@@ -110,17 +178,29 @@ $(document).ready(function(){
    }
 
 
-
    // Use to bind the button with the new displayed data
    // (The bind do not happens if the dataTable is in tabs and the clicked data is in another page)
-   $('#myTable').on( 'draw.dt', function () {
+
+   search_table.on( 'draw.dt', function () {
+       // Bind tooltip each time we draw a new page
+       $('[data-toggle="tooltip"]').tooltip();
        // On click, get html content from url and update the corresponding modal
-       $("[data-toggle='modal']").unbind('click.openmodal').on("click.openmodal", function (event) {
+       $("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
            event.preventDefault();
            var modal=$(this);
            var url = " {{ url_for('showpreviewpaste')  }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
+           last_clicked_paste =  $(this).attr('data-num');
            $.get(url, function (data) {
-               $("#mymodalbody").html(data);
+
+               // verify that the reveived data is really the current clicked paste. Otherwise, ignore it. 
+               var received_num = parseInt(data.split("|num|")[1]);
+               if (received_num == last_clicked_paste && can_change_modal_content) {
+                   can_change_modal_content = false;
+
+                   // clear data by removing html, body, head tags. prevent dark modal background stack bug.
+                   var cleared_data = data.split("<body>")[1].split("</body>")[0];
+                   $("#mymodalbody").html(cleared_data);
+
                    var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
                    button.tooltip();
                    $("#mymodalbody").children(".panel-default").append(button);
@@ -142,6 +222,9 @@ $(document).ready(function(){
                            update_preview();
                        }
                    });
+               } else if (can_change_modal_content) {
+                   $("#mymodalbody").html("Ignoring previous not finished query of paste #" + received_num);
+               }
            });
        });
    } );

var/www/templates/index.html

@@ -140,6 +140,7 @@
 						<th>Date</th>
 						<th>Paste name</th>
 						<th>Message</th>
+						<th>Actions</th>
 						</tr>
 					</thead>
 					<tbody id="tab_body">
@@ -153,6 +154,7 @@
         <!-- /#page-wrapper -->
 
     </div>
+    <script> var url_showSavedPath = "{{ url_for('showsavedpaste') }}"; </script>
     <script type="text/javascript" src="{{ url_for('static', filename='js/indexjavascript.js')}}"></script>
 
     <script>

var/www/templates/search.html

@@ -31,9 +31,7 @@
          white-space:pre-wrap;
          word-wrap:break-word;
       }
-      .modal-backdrop.fade {
+
-         opacity: 0;
-      }
   </style>
 
 </head>
@@ -84,7 +82,7 @@
                     </br>
                     <div class="panel panel-default">
                         <div class="panel-heading">
-                            <i class="glyphicon glyphicon-search"></i> {{ r|length }} Results for "<strong>{{ query }}</strong>"
+                            <i class="glyphicon glyphicon-search"></i> <b id="numberOfRes">{{ r|length }}</b> Results for "<strong>{{ query }}</strong>
                             <div class="pull-right">
 
                             </div>
@@ -101,7 +99,7 @@
                                       <th>Action</th>
                                    </tr>
                               </thead>
-                              <tbody>
+                              <tbody id="table_body">
                           {% set i = 0 %}
                           {% for path in r %}
                                 <tr>
@@ -115,6 +113,10 @@
                           {% endfor %}
                               </tbody>
                           </table>
+                          <div id="div_stil_data">
+                              <button id="load_more_json_button1" type="button" class="btn btn-default" onclick="add_entries();" style="display: none">Load 50 entries</button>
+                              <strong> Totalling {{ num_res }} results related to paste content </strong>
+                          </div>
                         </div>
                         <!-- /.panel-body -->
                     </div>
@@ -124,15 +126,78 @@
         <!-- /#page-wrapper -->
     </div>
     <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
+</br>
 </body>
 
 <!-- enable tooltip and dataTable -->
 <script>
+    var search_table;
+    var last_clicked_paste;
+    var can_change_modal_content = true;
+    var page_offset;
+    var offset;
+    var all_loaded;
+    var init_num_of_elements_in_table;
+    var query;
+    var pagelen = 50;
+
     $(document).ready(function(){
         $('[data-toggle="tooltip"]').tooltip();
+
         $("#button_show_path").hide();
-	$('#myTable').dataTable();
+        search_table = $('#myTable').DataTable();
+
+        query = "{{ query }}";
+        offset = 0;
+        page_offset = 2; //page 1 already loaded
+        all_loaded = false;
+        init_num_of_elements_in_table = parseInt("{{ r|length }}"); // Comes from the file search
+
+
+        if (init_num_of_elements_in_table == pagelen) {
+            $("#load_more_json_button1").show();
+        }
     });
+
+</script>
+
+<script>
+
+// Loop to recover all the data from  get_more_search_results
+// And add it dynamically top the dataTable
+    function add_entries() { //Used to disable the button before going to the big loop
+        $("#load_more_json_button1").attr('disabled','disabled');
+        setTimeout(function() { load_search_50_data();}, 50);
+    }
+
+    function load_search_50_data() {
+        var options = { query: query, page_offset: page_offset };
+        $.post( "{{ url_for('get_more_search_result') }}", options).done(function( data ) {
+
+            for(i=0; i<data.path_array.length; i++) {
+                var curr_preview = data.preview_array[i].replace(/\"/g, "\'");
+                search_table.row.add( [
+                    init_num_of_elements_in_table+((offset))+i+1,
+                   "<a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+data.path_array[i]+"&num="+i+"\"> "+ data.path_array[i] +"</a>",
+                    data.date_array[i],
+                    data.size_array[i],
+                    "<p><span class=\"glyphicon glyphicon-info-sign\" data-toggle=\"tooltip\" data-placement=\"left\" title=\""+curr_preview+"\"></span> <button type=\"button\" class=\"btn-link\" data-num=\""+i+"\" data-toggle=\"modal\" data-target=\"#mymodal\" data-url=\"{{ url_for('showsavedpaste') }}?paste="+data.path_array[i]+"&num="+i+"\" data-path=\""+data.path_array[i]+"\"><span class=\"fa fa-search-plus\"></span></button></p>"
+                ] ).draw( false );
+            }
+            offset = offset + data.path_array.length;
+            page_offset = page_offset+1;
+            $("#numberOfRes").text(parseInt($("#numberOfRes").text()) + data.path_array.length);
+            if (data.moreData == true) {
+                //continue
+            } else {
+                all_loaded = true;
+                $("#load_more_json_button1").hide();
+            }
+            $("#load_more_json_button1").removeAttr('disabled');
+            return data.path_array.length;
+        });
+    }
+
 </script>
 
 <!-- Dynamically update the modal -->
@@ -143,39 +208,9 @@
     var char_to_display = {{ char_to_display }};
     var start_index = 0;
 
-    // On click, get html content from url and update the corresponding modal
-    $("[data-toggle='modal']").on("click", function (event) {
-        event.preventDefault();
-        var modal=$(this);
-        var url = " {{ url_for('showpreviewpaste')  }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
-        $.get(url, function (data) {
-            $("#mymodalbody").html(data);
-            var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
-            button.tooltip();
-            $("#mymodalbody").children(".panel-default").append(button);
-
-            $("#button_show_path").attr('href', $(modal).attr('data-url'));
-            $("#button_show_path").show('fast');
-            $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
-            if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
-                nothing_to_display();
-            }
-            // On click, donwload all paste's content
-            $("#load-more-button").on("click", function (event) {
-                if (complete_paste == null) { //Donwload only once
-                    $.get("{{ url_for('getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
-                        complete_paste = data;
-                        update_preview();
-                    });
-                } else {
-                    update_preview();
-                }
-            });
-        });
-    });
-
     // When the modal goes out, refresh it to normal content
     $("#mymodal").on('hidden.bs.modal', function () {
+        can_change_modal_content = true;
         $("#mymodalbody").html("<p>Loading paste information...</p>");
         var loading_gif = "<img id='loading-gif-modal' class='img-center' src=\"{{url_for('static', filename='image/loading.gif') }}\" height='26' width='26' style='margin: 4px;'>";
         $("#mymodalbody").append(loading_gif); // Show the loading GIF
@@ -211,5 +246,55 @@
        new_content.show('fast');
        $("#load-more-button").hide();
    }
+
+
+   $('#myTable').on( 'draw.dt', function () {
+       // Bind tooltip each time we draw a new page
+       $('[data-toggle="tooltip"]').tooltip();
+       // On click, get html content from url and update the corresponding modal
+       $("[data-toggle='modal']").off('click.openmodal').on("click.openmodal", function (event) {
+           var modal=$(this);
+           var url = " {{ url_for('showpreviewpaste')  }}?paste=" + $(this).attr('data-path') + "&num=" + $(this).attr('data-num');
+           last_clicked_paste =  $(this).attr('data-num');
+           $.get(url, function (data) {
+
+               // verify that the reveived data is really the current clicked paste. Otherwise, ignore it. 
+               var received_num = parseInt(data.split("|num|")[1]);
+               if (received_num == last_clicked_paste && can_change_modal_content) {
+                   can_change_modal_content = false;
+
+                   // clear data by removing html, body, head tags. prevent dark modal background stack bug.
+                   var cleared_data = data.split("<body>")[1].split("</body>")[0];
+                   $("#mymodalbody").html(cleared_data);
+                   setTimeout(function() { $('#tableDup').DataTable(); }, 150);
+
+                   var button = $('<button type="button" id="load-more-button" class="btn btn-info btn-xs center-block" data-url="' + $(modal).attr('data-path') +'" data-toggle="tooltip" data-placement="bottom" title="Load more content"><span class="glyphicon glyphicon-download"></span></button>');
+                   button.tooltip();
+                   $("#mymodalbody").children(".panel-default").append(button);
+   
+                   $("#button_show_path").attr('href', $(modal).attr('data-url'));
+                   $("#button_show_path").show('fast');
+                   $("#loading-gif-modal").css("visibility", "hidden"); // Hide the loading GIF
+                   if ($("[data-initsize]").attr('data-initsize') < char_to_display) { // All the content is displayed
+                       nothing_to_display();
+                   }
+                   // On click, donwload all paste's content
+                   $("#load-more-button").off('click.download').on("click.download", function (event) {
+                       if (complete_paste == null) { //Donwload only once
+                           $.get("{{ url_for('getmoredata') }}"+"?paste="+$(modal).attr('data-path'), function(data, status){
+                               complete_paste = data;
+                               update_preview();
+                           });
+                       } else {
+                           update_preview();
+                       }
+                   });
+               } else if (can_change_modal_content) {
+                   $("#mymodalbody").html("Ignoring previous not finished query of paste #" + received_num);
+               }
+           });
+       });
+   } );
+
 </script>
 </html>

var/www/templates/sentiment_analysis_trending.html

@@ -205,24 +205,24 @@
                                         </thead>
                                         <tbody>
                                           <tr>
-                                              <td class="worst1 moodtable_worst">worst1</td>
+                                              <td class="worst1 moodtable_worst">no data</td>
-                                              <td class="best1 moodtable_best">best1</td>
+                                              <td class="best1 moodtable_best">no data</td>
                                           </tr>
                                           <tr>
-                                              <td class="worst2 moodtable_worst">worst2</td>
+                                              <td class="worst2 moodtable_worst">no data</td>
-                                              <td class="best2 moodtable_best">best2</td>
+                                              <td class="best2 moodtable_best">no data</td>
                                           </tr>
                                           <tr>
-                                              <td class="worst3 moodtable_worst">worst3</td>
+                                              <td class="worst3 moodtable_worst">no data</td>
-                                              <td class="best3 moodtable_best">best3</td>
+                                              <td class="best3 moodtable_best">no data</td>
                                           </tr>
                                           <tr>
-                                              <td class="worst4 moodtable_worst">worst4</td>
+                                              <td class="worst4 moodtable_worst">no data</td>
-                                              <td class="best4 moodtable_best">best4</td>
+                                              <td class="best4 moodtable_best">no data</td>
                                           </tr>
                                           <tr>
-                                              <td class="worst5 moodtable_worst">worst5</td>
+                                              <td class="worst5 moodtable_worst">no data</td>
-                                              <td class="best5 moodtable_best">best5</td>
+                                              <td class="best5 moodtable_best">no data</td>
                                           </tr>
                                         </tbody>
                                     </table>

var/www/templates/show_saved_paste.html

@@ -4,16 +4,25 @@
   <title>Paste information</title>
   <meta charset="utf-8">
   <meta name="viewport" content="width=device-width, initial-scale=1">
+
+   <link href="{{ url_for('static', filename='css/bootstrap.min.css') }}" rel="stylesheet">
+   <link href="{{ url_for('static', filename='font-awesome/css/font-awesome.css') }}" rel="stylesheet">
+   <link href="{{ url_for('static', filename='css/sb-admin-2.css') }}" rel="stylesheet">
+   <link href="{{ url_for('static', filename='css/dataTables.bootstrap.css') }}" rel="stylesheet" type="text/css" />
+   <script language="javascript" src="{{ url_for('static', filename='js/jquery.js')}}"></script>
+   <script src="{{ url_for('static', filename='js/bootstrap.min.js') }}"></script>
+   <script src="{{ url_for('static', filename='js/jquery.dataTables.min.js') }}"></script>
+   <script src="{{ url_for('static', filename='js/dataTables.bootstrap.js') }}"></script>
+   <script src="{{ url_for('static', filename='js/jquery.flot.js') }}"></script>
+   <script src="{{ url_for('static', filename='js/jquery.flot.time.js') }}"></script>
+   <script src="{{ url_for('static', filename='js/jquery.flot.stack.js') }}"></script>     
 </head>
 <body>
-
-  <h2> Paste: {{ request.args.get('num') }}</h2>
-  <h3> {{ request.args.get('paste') }} </h3>
-
-  <hr></br>
-
     <div class="panel panel-default">
       <div class="panel-heading">
+        <h1 class="page-header" >Paste: {{ request.args.get('paste') }}</h1>
+        <h2 class="page-header" >({{ request.args.get('num') }})</h2>
+  
         <table class="table table-condensed">
           <thead>
             <tr>
@@ -46,19 +55,27 @@
           <h3> No Duplicate </h3>
       {% else %}
           <h3> Duplicate list: </h3>
-      <table style="width:100%">
+          <table id="tableDup" class="table table-striped table-bordered">
           {% set i = 0 %}
+          <thead>
           <tr>
-          <th style="text-align:left;">Hash type</th><th style="text-align:left;">Paste info</th>
+              <th>Hash type</th>
+              <th>Paste info</th>
+              <th>Date</th>
+              <th>Path</th> 
           </tr>
+          </thead>
+          <tbody>
           {% for dup_path in duplicate_list %}
               <tr>
                   <td>{{ hashtype_list[i] }}</td>
                   <td>Similarity: {{ simil_list[i] }}%</td>
+                  <td>{{ date_list[i] }}</td>
                   <td><a target="_blank" href="{{ url_for('showsavedpaste') }}?paste={{ dup_path }}" id='dup_path'>{{ dup_path }}</a></td>
               </tr>
               {% set i = i + 1 %}
           {% endfor %}
+          </tbody>
           </table>
       {% endif %}
         <h3> Content: </h3>
@@ -66,6 +83,8 @@
       </div>
     </div>
   
-
 </body>
+<script>
+    $('#tableDup').DataTable();
+</script>
 </html>

var/www/templates/terms_management.html

@@ -111,7 +111,7 @@
                                         <th>Day occurence</th>
                                         <th>Week occurence</th>
                                         <th>Month occurence</th>
-                                        <th># Concerned pastes</th>
+                                        <th># tracked paste</th>
                                         <th>Action</th>
                                      </tr>
                                 </thead>
@@ -198,6 +198,9 @@
 
 <!-- import graph function -->
 <script>
+    var table_track;
+    var table_black;
+
     $(document).ready(function(){
         activePage = $('h1.page-header').attr('data-page');
         $("#"+activePage).addClass("active");
@@ -207,6 +210,14 @@
 	table_track = $('#myTable').DataTable();
 	table_black = $('#myTable2').DataTable();
 
+        table_track.on( 'draw.dt', function () {
+            perform_binding();
+        });
+        table_black.on( 'draw.dt', function () {
+            perform_binding();
+        });
+
+
         $("#followTermInput").keyup(function(event){
             if(event.keyCode == 13){
                 $("#followTermBtn").click();
@@ -254,7 +265,7 @@
                                 html_to_add += "<td>"+curr_data.size+"</td>";
                                 html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
                                 html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";
-                                html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+curr_data.path+"&num=0\">  <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
+                                html_to_add += "<td><div class=\"row\"><button class=\"btn btn-xs btn-default\" data-toggle=\"popover\" data-placement=\"left\" data-content=\""+curr_data.content.replace(/\"/g, "\'")+"\">Preview content</button><a target=\"_blank\" href=\"{{ url_for('showsavedpaste') }}?paste="+curr_data.path+"&num=0\">  <button type=\"button\" class=\"btn btn-xs btn-info\">Show Paste</button></a></div></td>";
 
                             html_to_add += "</tr>";
                     }