Merge pull request #164 from mokaddem/showDiff

Show differences between duplicates

bin/Mixer.py

@@ -20,6 +20,7 @@ Depending on the configuration, this module will process the feed as follow:
             - Else, do not process it but keep track for statistics on duplicate
 
     operation_mode 3: "Don't look if duplicated content"
+        - SImply do not bother to check if it is a duplicate
         - Simply do not bother to check if it is a duplicate
 
 Note that the hash of the content is defined as the sha1(gzip64encoded).

bin/packages/config.cfg.sample

@@ -28,6 +28,8 @@ max_preview_modal = 800
 default_display = 10
 #Number of minutes displayed for the number of processed pastes.
 minute_processed_paste = 10
+#Maximum line length authorized to make a diff between duplicates 
+DiffMaxLineLength = 10000
 
 #### Modules ####
 [Categ]

var/www/modules/Flask_config.py

@@ -63,3 +63,4 @@ max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of
 max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
 
 tlsh_to_percent = 1000.0 #Use to display the estimated percentage instead of a raw value
+DiffMaxLineLength =  int(cfg.get("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value

var/www/modules/showpaste/Flask_showpaste.py

@@ -7,7 +7,8 @@
 import redis
 import json
 import flask
-from flask import Flask, render_template, jsonify, request, Blueprint
+from flask import Flask, render_template, jsonify, request, Blueprint, make_response
+import difflib
 
 import Paste
 
@@ -20,6 +21,7 @@ r_serv_pasteName = Flask_config.r_serv_pasteName
 max_preview_char = Flask_config.max_preview_char
 max_preview_modal = Flask_config.max_preview_modal
 tlsh_to_percent = Flask_config.tlsh_to_percent
+DiffMaxLineLength = Flask_config.DiffMaxLineLength
 
 showsavedpastes = Blueprint('showsavedpastes', __name__, template_folder='templates')
 
@@ -113,5 +115,21 @@ def getmoredata():
     to_return = p_content[max_preview_modal-1:]
     return to_return
 
+@showsavedpastes.route("/showDiff/")
+def showDiff():
+    s1 = request.args.get('s1', '')
+    s2 = request.args.get('s2', '')
+    p1 = Paste.Paste(s1)
+    p2 = Paste.Paste(s2)
+    maxLengthLine1 = p1.get_lines_info()[1]
+    maxLengthLine2 = p2.get_lines_info()[1]
+    if maxLengthLine1 > DiffMaxLineLength or maxLengthLine2 > DiffMaxLineLength:
+        return "Can't make the difference as the lines are too long."
+    htmlD = difflib.HtmlDiff()
+    lines1 = p1.get_p_content().decode('utf8', 'ignore').splitlines()
+    lines2 = p2.get_p_content().decode('utf8', 'ignore').splitlines()
+    the_html = htmlD.make_file(lines1, lines2)
+    return the_html
+
 # ========= REGISTRATION =========
 app.register_blueprint(showsavedpastes)

var/www/modules/showpaste/templates/show_saved_paste.html

@@ -63,6 +63,7 @@
               <th>Paste info</th>
               <th>Date</th>
               <th>Path</th>
+              <th>Action</th>
           </tr>
           </thead>
           <tbody>
@@ -72,6 +73,7 @@
                   <td>Similarity: {{ simil_list[i] }}%</td>
                   <td>{{ date_list[i] }}</td>
                   <td><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{ dup_path }}" id='dup_path'>{{ dup_path }}</a></td>
+                  <td><a target="_blank" href="{{ url_for('showsavedpastes.showDiff') }}?s1={{ request.args.get('paste') }}&s2={{ dup_path }}" class="fa fa-columns" title="Show differences"></a></td>
               </tr>
               {% set i = i + 1 %}
           {% endfor %}