fix: [redis cache] remove old paste_name db

.gitignore

@@ -44,6 +44,9 @@ configs/update.cfg
 update/current_version
 files
 
+# Trackers
+bin/trackers/yara/custom-rules/*
+
 # Helper
 bin/helper/gen_cert/rootCA.*
 bin/helper/gen_cert/server.*

bin/ModuleStats.py

@@ -57,7 +57,7 @@ def compute_most_posted(server, message):
             print(redis_progression_name_set)
 
 
-def compute_provider_info(server_trend, server_pasteName, path):
+def compute_provider_info(server_trend, path):
     redis_all_provider = 'all_provider_set'
 
     paste = Paste.Paste(path)
@@ -71,7 +71,6 @@ def compute_provider_info(server_trend, server_pasteName, path):
     redis_providers_name_set = 'providers_set_' + paste_date
 
     # Add/Update in Redis
-    server_pasteName.sadd(paste_baseName, path)
     server_trend.sadd(redis_all_provider, paste_provider)
 
     num_paste = int(server_trend.hincrby(paste_provider+'_num', paste_date, 1))
@@ -137,12 +136,6 @@ if __name__ == '__main__':
         db=p.config.get("ARDB_Trending", "db"),
         decode_responses=True)
 
-    r_serv_pasteName = redis.StrictRedis(
-        host=p.config.get("Redis_Paste_Name", "host"),
-        port=p.config.get("Redis_Paste_Name", "port"),
-        db=p.config.get("Redis_Paste_Name", "db"),
-        decode_responses=True)
-
     # Endless loop getting messages from the input queue
     while True:
         # Get one message from the input queue
@@ -159,4 +152,4 @@ if __name__ == '__main__':
             if len(message.split(';')) > 1:
                 compute_most_posted(r_serv_trend, message)
             else:
-                compute_provider_info(r_serv_trend, r_serv_pasteName, message)
+                compute_provider_info(r_serv_trend, message)

configs/core.cfg.sample

@@ -147,11 +147,6 @@ host = localhost
 port = 6379
 db = 1
 
-[Redis_Paste_Name]
-host = localhost
-port = 6379
-db = 2
-
 [Redis_Mixer_Cache]
 host = localhost
 port = 6381

var/www/modules/Flask_config.py

@@ -26,7 +26,6 @@ r_serv_charts = config_loader.get_redis_conn("ARDB_Trending")
 r_serv_sentiment = config_loader.get_redis_conn("ARDB_Sentiment")
 r_serv_term = config_loader.get_redis_conn("ARDB_Tracker")
 r_serv_cred = config_loader.get_redis_conn("ARDB_TermCred")
-r_serv_pasteName = config_loader.get_redis_conn("Redis_Paste_Name")
 r_serv_tags = config_loader.get_redis_conn("ARDB_Tags")
 r_serv_metadata = config_loader.get_redis_conn("ARDB_Metadata")
 r_serv_db = config_loader.get_redis_conn("ARDB_DB")

var/www/modules/search/Flask_search.py

@@ -27,7 +27,6 @@ import Flask_config
 app = Flask_config.app
 config_loader = Flask_config.config_loader
 baseUrl = Flask_config.baseUrl
-r_serv_pasteName = Flask_config.r_serv_pasteName
 r_serv_metadata = Flask_config.r_serv_metadata
 max_preview_char = Flask_config.max_preview_char
 max_preview_modal = Flask_config.max_preview_modal
@@ -116,17 +115,7 @@ def search():
         selected_index = os.path.join(baseindexpath, index_name)
 
     ''' temporary disabled
-    # Search filename
+    # # TODO: search by filename/item id
-    for path in r_serv_pasteName.smembers(q[0]):
-        r.append(path)
-        paste = Paste.Paste(path)
-        content = paste.get_p_content()
-        content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
-        c.append(content[0:content_range])
-        curr_date = str(paste._get_p_date())
-        curr_date = curr_date[0:4]+'/'+curr_date[4:6]+'/'+curr_date[6:]
-        paste_date.append(curr_date)
-        paste_size.append(paste._get_p_size())
     '''
 
     # Search full line

var/www/modules/showpaste/Flask_showpaste.py

@@ -32,11 +32,9 @@ import Flask_config
 
 app = Flask_config.app
 baseUrl = Flask_config.baseUrl
-r_serv_pasteName = Flask_config.r_serv_pasteName
 r_serv_metadata = Flask_config.r_serv_metadata
 r_serv_tags = Flask_config.r_serv_tags
 r_serv_statistics = Flask_config.r_serv_statistics
-r_serv_onion = Flask_config.r_serv_onion
 max_preview_char = Flask_config.max_preview_char
 max_preview_modal = Flask_config.max_preview_modal
 DiffMaxLineLength = Flask_config.DiffMaxLineLength

var/www/templates/domains/card_img_domain.html

@@ -11,7 +11,11 @@
         <div class="card-body pb-0">
           <h5 class="card-title">
             <a target="_blank" href="{{ url_for('crawler_splash.showDomain') }}?domain={{dict_domain["id"]}}">
-              {{dict_domain["id"]}}
+              {% if 'hl-start' in  dict_domain %}
+                {{dict_domain["id"][:dict_domain['hl-start']]}}<mark>{{dict_domain["id"][dict_domain['hl-start']:dict_domain['hl-end']]}}</mark>{{dict_domain["id"][dict_domain['hl-end']:]}}
+              {% else %}
+                {{dict_domain["id"]}}
+              {% endif %}
             </a>
 
             {% if dict_domain["status"] %}