chg: [Tags] add backend, daterange selector

bin/Tags.py

@@ -17,8 +17,8 @@ from pubsublogger import publisher
 from Helper import Process
 from packages import Paste
 
-def get_paste_date(paste_filename):
+def get_item_date(item_filename):
-    l_directory = paste_filename.split('/')
+    l_directory = item_filename.split('/')
     return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
 
 def set_tag_metadata(tag, date):
@@ -75,14 +75,15 @@ if __name__ == '__main__':
             if res == 1:
                 print("new tags added : {}".format(tag))
             # add the path to the tag set
-            curr_date = datetime.date.today().strftime("%Y%m%d")
+            #curr_date = datetime.date.today().strftime("%Y%m%d")
-            res = server.sadd('{}:{}'.format(tag, curr_date), path)
+            item_date = get_item_date(path)
+            res = server.sadd('{}:{}'.format(tag, item_date), path)
             if res == 1:
                 print("new paste: {}".format(path))
                 print("   tagged: {}".format(tag))
-                set_tag_metadata(tag, curr_date)
+                set_tag_metadata(tag, item_date)
             server_metadata.sadd('tag:{}'.format(path), tag)
 
             curr_date = datetime.date.today().strftime("%Y%m%d")
-            server.hincrby('daily_tags:{}'.format(curr_date), tag, 1)
+            server.hincrby('daily_tags:{}'.format(item_date), tag, 1)
             p.populate_set_out(message, 'MISP_The_Hive_feeder')

var/www/modules/Flask_config.py

@@ -154,6 +154,8 @@ if baseUrl != '':
 max_preview_char = int(cfg.get("Flask", "max_preview_char")) # Maximum number of character to display in the tooltip
 max_preview_modal = int(cfg.get("Flask", "max_preview_modal")) # Maximum number of character to display in the modal
 
+max_tags_result = 50
+
 DiffMaxLineLength =  int(cfg.get("Flask", "DiffMaxLineLength"))#Use to display the estimated percentage instead of a raw value
 
 bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']

var/www/modules/Tags/Flask_Tags.py

@@ -28,6 +28,7 @@ r_serv_statistics = Flask_config.r_serv_statistics
 max_preview_char = Flask_config.max_preview_char
 max_preview_modal = Flask_config.max_preview_modal
 bootstrap_label = Flask_config.bootstrap_label
+max_tags_result = Flask_config.max_tags_result
 PASTES_FOLDER = Flask_config.PASTES_FOLDER
 
 Tags = Blueprint('Tags', __name__, template_folder='templates')
@@ -68,20 +69,61 @@ def get_tags_with_synonyms(tag):
     else:
         return {'name':tag,'id':tag}
 
+def get_item_date(item_filename):
+    l_directory = item_filename.split('/')
+    return '{}{}{}'.format(l_directory[-4], l_directory[-3], l_directory[-2])
+
+def substract_date(date_from, date_to):
+    date_from = datetime.date(int(date_from[0:4]), int(date_from[4:6]), int(date_from[6:8]))
+    date_to = datetime.date(int(date_to[0:4]), int(date_to[4:6]), int(date_to[6:8]))
+    delta = date_to - date_from # timedelta
+    l_date = []
+    for i in range(delta.days + 1):
+        date = date_from + datetime.timedelta(i)
+        l_date.append( date.strftime('%Y%m%d') )
+    return l_date
+
+def get_all_dates_range(date_from, date_to):
+    all_dates = {}
+    date_range = []
+    if date_from is not None and date_to is not None:
+        #change format
+        try:
+            if len(date_from) != 8:
+                date_from = date_from[0:4] + date_from[5:7] + date_from[8:10]
+                date_to = date_to[0:4] + date_to[5:7] + date_to[8:10]
+            date_range = substract_date(date_from, date_to)
+        except:
+            pass
+
+    if not date_range:
+        date_range.append(datetime.date.today().strftime("%Y%m%d"))
+        date_from = date_range[0][0:4] + '-' + date_range[0][4:6] + '-' + date_range[0][6:8]
+        date_to = date_from
+
+    else:
+        date_from = date_from[0:4] + '-' + date_from[4:6] + '-' + date_from[6:8]
+        date_to = date_to[0:4] + '-' + date_to[4:6] + '-' + date_to[6:8]
+    all_dates['date_from'] = date_from
+    all_dates['date_to'] = date_to
+    all_dates['date_range'] = date_range
+    return all_dates
+
 
 # ============= ROUTES ==============
 
 @Tags.route("/tags/", methods=['GET'])
 def Tags_page():
-    current_date = datetime.date.today().strftime("%Y-%m-%d")
+    date_from = request.args.get('date_from')
+    date_to = request.args.get('date_to')
+
+    dates = get_all_dates_range(date_from, date_to)
 
     tags = request.args.get('ltags')
     if tags is None:
         return render_template("Tags.html", date_from=current_date, date_to=current_date)
 
     else:
-
-
         tags = request.args.get('ltags')
 
         list_tags = tags.split(',')
@@ -97,11 +139,23 @@ def Tags_page():
                 print('empty')
             # 1 tag
             elif len(list_tags) < 2:
-                tagged_pastes = r_serv_tags.smembers(list_tags[0])
+                tagged_pastes = []
+                for date in dates['date_range']:
+                    tagged_pastes.extend(r_serv_tags.smembers('{}:{}'.format(list_tags[0], date)))
 
             # 2 tags or more
             else:
-                tagged_pastes = r_serv_tags.sinter(list_tags[0], *list_tags[1:])
+                tagged_pastes = []
+                for date in dates['date_range']:
+                    tag_keys = []
+                    for tag in list_tags:
+                        tag_keys.append('{}:{}'.format(tag, date))
+
+                    if len(tag_keys) > 1:
+                        daily_items = r_serv_tags.sinter(tag_keys[0], *tag_keys[1:])
+                    else:
+                        daily_items = r_serv_tags.sinter(tag_keys[0])
+                    tagged_pastes.extend(daily_items)
 
         else :
             return 'INCORRECT INPUT'
@@ -113,7 +167,21 @@ def Tags_page():
         allPastes = list(tagged_pastes)
         paste_tags = []
 
-        for path in allPastes[0:50]: ######################moduleName
+        try:
+            page = int(request.args.get('page'))
+        except:
+            page = 1
+        if page <= 0:
+            page = 1
+        nb_page_max = len(tagged_pastes)/(max_tags_result)
+        if not nb_page_max.is_integer():
+            nb_page_max = int(nb_page_max)+1
+        if page > nb_page_max:
+            page = nb_page_max
+        start = max_tags_result*(page -1)
+        stop = max_tags_result*page
+
+        for path in allPastes[start:stop]: ######################moduleName
             all_path.append(path)
             paste = Paste.Paste(path)
             content = paste.get_p_content()
@@ -153,8 +221,9 @@ def Tags_page():
                 all_path=all_path,
                 tags=tags,
                 list_tag = list_tag,
-                date_from=current_date,
+                date_from=dates['date_from'],
-                date_to=current_date,
+                date_to=dates['date_to'],
+                page=page, nb_page_max=nb_page_max,
                 paste_tags=paste_tags,
                 bootstrap_label=bootstrap_label,
                 content=all_content,

var/www/modules/Tags/templates/Tags.html

@@ -142,6 +142,42 @@
 
 				      </tbody>
 				    </table>
+
+						<div class="d-flex justify-content-center border-top mt-2">
+							<nav class="mt-4" aria-label="...">
+								<ul class="pagination">
+									<li class="page-item {%if page==1%}disabled{%endif%}">
+										<a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">Previous</a>
+									</li>
+
+									{%if page>3%}
+										<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page=1&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">1</a></li>
+										<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
+										<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page-1}}</a></li>
+										<li class="page-item active"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page}}</a></li>
+									{%else%}
+									{%if page>2%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-2}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page-2}}</a></li>{%endif%}
+									{%if page>1%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page-1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page-1}}</a></li>{%endif%}
+									<li class="page-item active"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page}}</a></li>
+									{%endif%}
+
+									{%if nb_page_max-page>3%}
+										<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page+1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{page+1}}</a></li>
+										<li class="page-item disabled"><a class="page-link" aria-disabled="true" href="#">...</a></li>
+										<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{nb_page_max}}</a></li>
+									{%else%}
+										{%if nb_page_max-page>2%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max-2}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{nb_page_max-2}}</a></li>{%endif%}
+										{%if nb_page_max-page>1%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max-1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{nb_page_max-1}}</a></li>{%endif%}
+										{%if nb_page_max-page>0%}<li class="page-item"><a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{nb_page_max}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}">{{nb_page_max}}</a></li>{%endif%}
+									{%endif%}
+
+									<li class="page-item {%if page==nb_page_max%}disabled{%endif%}">
+										<a class="page-link" href="{{ url_for('Tags.Tags_page') }}?page={{page+1}}&date_from={{date_from}}&date_to={{date_to}}&ltags={{tags}}" aria-disabled="true">Next</a>
+									</li>
+								</ul>
+							</nav>
+						</div>
+
 					{%endif%}
 				</div>
 
@@ -247,7 +283,9 @@ $(document).ready(function(){
 <script>
 function searchTags() {
 	var data = ltags.getValue();
-	window.location.replace("{{ url_for('Tags.Tags_page') }}?ltags=" + data);
+	var date_from = $('#date-range-from-input').val();
+	var date_to =$('#date-range-to-input').val();
+	window.location.replace("{{ url_for('Tags.Tags_page') }}?date_from="+date_from+"&date_to="+date_to+"&ltags=" + data);
 }
 function emptyTags() {
 	ltags.clear();

var/www/modules/showpaste/templates/show_saved_item_min.html

@@ -241,6 +241,7 @@
       });
   </script>
 
+{% if crawler_metadata['get_metadata'] %}
   <script>
   var ctx = canvas.getContext('2d'), img = new Image();
 
@@ -287,6 +288,7 @@
 
   blocks.addEventListener('change', pixelate, false);
   </script>
+{% endif %}
 
   <div id="container-show-more" class="text-center">