chg: [Update v1.5] update tracked terms path

OVERVIEW.md

@@ -30,7 +30,7 @@ Redis and ARDB overview
 
 # Database Map:
 
-## DB2 - TermFreq: (TODO: update/refractor DB)
+## DB2 - TermFreq:
 
 ##### Set:
 | Key | Value |

update/v1.4/Update-ARDB_Metadata.py

@@ -7,6 +7,14 @@ import time
 import redis
 import configparser
 
+def update_tracked_terms(main_key, tracked_container_key):
+    for tracked_item in r_serv_term.smembers(main_key):
+        all_items = r_serv_term.smembers(tracked_container_key.format(tracked_item))
+        for item_path in all_items:
+            if PASTES_FOLDER in item_path:
+                new_item_path = item_path.replace(PASTES_FOLDER, '', 1)
+                r_serv_term.sadd(tracked_container_key.format(tracked_item), new_item_path)
+                r_serv_term.srem(tracked_container_key.format(tracked_item), item_path)
 
 def update_hash_item(has_type):
     #get all hash items:
@@ -70,6 +78,12 @@ if __name__ == '__main__':
         db=cfg.getint("ARDB_Tags", "db"),
         decode_responses=True)
 
+    r_serv_term = redis.StrictRedis(
+        host=cfg.get("ARDB_TermFreq", "host"),
+        port=cfg.getint("ARDB_TermFreq", "port"),
+        db=cfg.getint("ARDB_TermFreq", "db"),
+        decode_responses=True)
+
     r_serv_onion = redis.StrictRedis(
         host=cfg.get("ARDB_Onion", "host"),
         port=cfg.getint("ARDB_Onion", "port"),
@@ -117,6 +131,15 @@ if __name__ == '__main__':
 
     end = time.time()
 
+    ## update tracked term/set/regex
+    # update tracked term
+    update_tracked_terms('TrackedSetTermSet', 'tracked_{}')
+    # update tracked set
+    update_tracked_terms('TrackedSetSet', 'set_{}')
+    # update tracked regex
+    update_tracked_terms('TrackedRegexSet', 'regex_{}')
+    ##
+
     print('Updating ARDB_Metadata Done => {} paths: {} s'.format(index, end - start))
     print()
 

var/www/modules/terms/Flask_terms.py

@@ -281,14 +281,13 @@ def terms_management_query_paste():
         p_date = str(paste._get_p_date())
         p_date = p_date[0:4]+'/'+p_date[4:6]+'/'+p_date[6:8]
         p_source = paste.p_source
-        p_encoding = paste._get_p_encoding()
         p_size = paste.p_size
         p_mime = paste.p_mime
         p_lineinfo = paste.get_lines_info()
         p_content = paste.get_p_content()
         if p_content != 0:
             p_content = p_content[0:400]
-        paste_info.append({"path": path, "date": p_date, "source": p_source, "encoding": p_encoding, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content})
+        paste_info.append({"path": path, "date": p_date, "source": p_source, "size": p_size, "mime": p_mime, "lineinfo": p_lineinfo, "content": p_content})
 
     return jsonify(paste_info)
 

var/www/modules/terms/templates/terms_management.html

@@ -443,7 +443,6 @@ function bindEventsForCurrentPage() {
                 html_to_add += "<tr>";
                 html_to_add += "<th>Source</th>";
                 html_to_add += "<th>Date</th>";
-                html_to_add += "<th>Encoding</th>";
                 html_to_add += "<th>Size (Kb)</th>";
                 html_to_add += "<th># lines</th>";
                 html_to_add += "<th>Max length</th>";
@@ -456,7 +455,6 @@ function bindEventsForCurrentPage() {
                     html_to_add += "<tr>";
                     html_to_add += "<td>"+curr_data.source+"</td>";
                     html_to_add += "<td>"+curr_data.date+"</td>";
-                    html_to_add += "<td>"+curr_data.encoding+"</td>";
                     html_to_add += "<td>"+curr_data.size+"</td>";
                     html_to_add += "<td>"+curr_data.lineinfo[0]+"</td>";
                     html_to_add += "<td>"+curr_data.lineinfo[1]+"</td>";