fix: [AIL] use only relative paths

pastes duplicates are fixed on the fly

bin/Global.py

@@ -52,7 +52,6 @@ if __name__ == '__main__':
     p = Process(config_section)
 
     PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes"))
-    print(PASTES_FOLDER)
 
     # LOGGING #
     publisher.info("Feed Script started to receive & publish.")
@@ -82,8 +81,6 @@ if __name__ == '__main__':
             continue
         # Creating the full filepath
         filename = os.path.join(PASTES_FOLDER, paste)
-        print(filename)
-        print(paste)
 
         dirname = os.path.dirname(filename)
         if not os.path.exists(dirname):

bin/Mixer.py

@@ -82,7 +82,7 @@ if __name__ == '__main__':
     ttl_key = cfg.getint("Module_Mixer", "ttl_duplicate")
     default_unnamed_feed_name = cfg.get("Module_Mixer", "default_unnamed_feed_name")
 
-    PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes"))
+    PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], p.config.get("Directories", "pastes")) + '/'
 
     # STATS #
     processed_paste = 0

bin/packages/HiddenServices.py

@@ -99,11 +99,7 @@ class HiddenServices(object):
         if father is None:
             return []
         l_crawled_pastes = []
-        paste_parent = father.replace(self.paste_directory+'/', '')
+        paste_childrens = self.r_serv_metadata.smembers('paste_children:{}'.format(father))
-        paste_childrens = self.r_serv_metadata.smembers('paste_children:{}'.format(paste_parent))
-        ## TODO: # FIXME: remove me
-        paste_children = self.r_serv_metadata.smembers('paste_children:{}'.format(father))
-        paste_childrens = paste_childrens | paste_children
         for children in paste_childrens:
             if self.domain in children:
                 l_crawled_pastes.append(children)
@@ -117,14 +113,9 @@ class HiddenServices(object):
 
         set_domain = set()
         for paste in l_paste:
-            paste_full = paste.replace(self.paste_directory+'/', '')
+            paste_childrens = self.r_serv_metadata.smembers('paste_children:{}'.format(paste))
-            paste_childrens = self.r_serv_metadata.smembers('paste_children:{}'.format(paste_full))
-            ## TODO: # FIXME: remove me
-            paste_children = self.r_serv_metadata.smembers('paste_children:{}'.format(paste))
-            paste_childrens = paste_childrens | paste_children
             for children in paste_childrens:
                 if not self.domain in children:
-                    print(children)
                     set_domain.add((children.split('.onion')[0]+'.onion').split('/')[-1])
 
         return set_domain
@@ -133,11 +124,7 @@ class HiddenServices(object):
         if father is None:
             return []
         l_crawled_pastes = []
-        paste_parent = father.replace(self.paste_directory+'/', '')
+        paste_childrens = self.r_serv_metadata.smembers('paste_children:{}'.format(father))
-        paste_childrens = self.r_serv_metadata.smembers('paste_children:{}'.format(paste_parent))
-        ## TODO: # FIXME: remove me
-        paste_children = self.r_serv_metadata.smembers('paste_children:{}'.format(father))
-        paste_childrens = paste_childrens | paste_children
         for children in paste_childrens:
             if not self.domain in children:
                 l_crawled_pastes.append(children)

bin/packages/Paste.py

@@ -82,14 +82,14 @@ class Paste(object):
             db=cfg.getint("ARDB_Metadata", "db"),
             decode_responses=True)
 
-        PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes"))
+        self.PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes"))
-        if PASTES_FOLDER not in p_path:
+        if self.PASTES_FOLDER not in p_path:
             self.p_rel_path = p_path
-            p_path = os.path.join(PASTES_FOLDER, p_path)
+            self.p_path = os.path.join(self.PASTES_FOLDER, p_path)
         else:
-            self.p_rel_path = None
+            self.p_path = p_path
+            self.p_rel_path = p_path.replace(self.PASTES_FOLDER+'/', '', 1)
 
-        self.p_path = p_path
         self.p_name = os.path.basename(self.p_path)
         self.p_size = round(os.path.getsize(self.p_path)/1024.0, 2)
         self.p_mime = magic.from_buffer("test", mime=True)
@@ -286,9 +286,13 @@ class Paste(object):
             return False, var
 
     def _get_p_duplicate(self):
-        self.p_duplicate = self.store_metadata.smembers('dup:'+self.p_path)
+        p_duplicate = self.store_metadata.smembers('dup:'+self.p_path)
-        if self.p_rel_path is not None:
+        # remove absolute path #fix-db
-            self.p_duplicate.union( self.store_metadata.smembers('dup:'+self.p_rel_path) )
+        if p_duplicate:
+            for duplicate_string in p_duplicate:
+                self.store_metadata.srem('dup:'+self.p_path, duplicate_string)
+                self.store_metadata.sadd('dup:'+self.p_rel_path, duplicate_string.replace(self.PASTES_FOLDER+'/', '', 1))
+        self.p_duplicate = self.store_metadata.smembers('dup:'+self.p_rel_path)
         if self.p_duplicate is not None:
             return list(self.p_duplicate)
         else:

var/www/modules/Flask_config.py

@@ -154,7 +154,7 @@ bootstrap_label = ['primary', 'success', 'danger', 'warning', 'info']
 
 UPLOAD_FOLDER = os.path.join(os.environ['AIL_FLASK'], 'submitted')
 
-PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes"))
+PASTES_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "pastes")) + '/'
 SCREENSHOT_FOLDER = os.path.join(os.environ['AIL_HOME'], cfg.get("Directories", "crawled_screenshot"))
 
 max_dashboard_logs = int(cfg.get("Flask", "max_dashboard_logs"))

var/www/modules/Tags/Flask_Tags.py

@@ -28,7 +28,6 @@ r_serv_statistics = Flask_config.r_serv_statistics
 max_preview_char = Flask_config.max_preview_char
 max_preview_modal = Flask_config.max_preview_modal
 bootstrap_label = Flask_config.bootstrap_label
-PASTES_FOLDER = Flask_config.PASTES_FOLDER
 
 Tags = Blueprint('Tags', __name__, template_folder='templates')
 

var/www/modules/browsepastes/Flask_browsepastes.py

@@ -23,6 +23,7 @@ max_preview_char = Flask_config.max_preview_char
 max_preview_modal = Flask_config.max_preview_modal
 r_serv_metadata = Flask_config.r_serv_metadata
 bootstrap_label = Flask_config.bootstrap_label
+PASTES_FOLDER = Flask_config.PASTES_FOLDER
 
 #init all lvlDB servers
 curYear = datetime.now().year
@@ -62,6 +63,7 @@ def event_stream_getImportantPasteByModule(module_name, year):
     paste_tags = []
 
     for path in all_pastes_list:
+        path = path.replace(PASTES_FOLDER, '', 1)
         index += 1
         paste = Paste.Paste(path)
         content = paste.get_p_content()
@@ -125,6 +127,7 @@ def importantPasteByModule():
     allPastes = getPastebyType(r_serv_db[currentSelectYear], module_name)
 
     for path in allPastes[0:10]:
+        path = path.replace(PASTES_FOLDER, '', 1)
         all_path.append(path)
         paste = Paste.Paste(path)
         content = paste.get_p_content()

var/www/modules/hiddenServices/Flask_hiddenServices.py

@@ -22,7 +22,6 @@ baseUrl = Flask_config.baseUrl
 r_serv_onion = Flask_config.r_serv_onion
 r_serv_metadata = Flask_config.r_serv_metadata
 bootstrap_label = Flask_config.bootstrap_label
-PASTES_FOLDER = Flask_config.PASTES_FOLDER
 
 hiddenServices = Blueprint('hiddenServices', __name__, template_folder='templates')
 
@@ -124,15 +123,13 @@ def onion_domain():
     origin_paste_name = h.get_origin_paste_name()
     origin_paste_tags = unpack_paste_tags(r_serv_metadata.smembers('tag:{}'.format(origin_paste)))
     paste_tags = []
-    path_name = []
     for path in l_pastes:
-        path_name.append(path.replace(PASTES_FOLDER+'/', ''))
         p_tags = r_serv_metadata.smembers('tag:'+path)
         paste_tags.append(unpack_paste_tags(p_tags))
 
     return render_template("showDomain.html", domain=onion_domain, last_check=last_check, first_seen=first_seen,
                             l_pastes=l_pastes, paste_tags=paste_tags, bootstrap_label=bootstrap_label,
-                            path_name=path_name, origin_paste_tags=origin_paste_tags, status=status,
+                            origin_paste_tags=origin_paste_tags, status=status,
                             origin_paste=origin_paste, origin_paste_name=origin_paste_name,
                             domain_tags=domain_tags, screenshot=screenshot)
 
@@ -143,7 +140,6 @@ def onion_son():
     h = HiddenServices(onion_domain, 'onion')
     l_pastes = h.get_last_crawled_pastes()
     l_son = h.get_domain_son(l_pastes)
-    print(l_son)
     return 'l_son'
 
 # ============= JSON ==============

var/www/modules/hiddenServices/templates/showDomain.html

@@ -105,7 +105,7 @@
 
 						        {% for path in l_pastes %}
 						          <tr>
-						              <td><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{path}}">{{ path_name[loop.index0] }}</a>
+						              <td><a target="_blank" href="{{ url_for('showsavedpastes.showsavedpaste') }}?paste={{path}}">{{ path }}</a>
 						                <div>
 						                  {% for tag in paste_tags[loop.index0] %}
 						                  <a href="{{ url_for('Tags.get_tagged_paste') }}?ltags={{ tag[1] }}">

var/www/modules/search/Flask_search.py

@@ -29,7 +29,7 @@ r_serv_metadata = Flask_config.r_serv_metadata
 max_preview_char = Flask_config.max_preview_char
 max_preview_modal = Flask_config.max_preview_modal
 bootstrap_label = Flask_config.bootstrap_label
-
+PASTES_FOLDER = Flask_config.PASTES_FOLDER
 
 baseindexpath = os.path.join(os.environ['AIL_HOME'], cfg.get("Indexer", "path"))
 indexRegister_path = os.path.join(os.environ['AIL_HOME'],
@@ -133,8 +133,8 @@ def search():
         query = QueryParser("content", ix.schema).parse("".join(q))
         results = searcher.search_page(query, 1, pagelen=num_elem_to_get)
         for x in results:
-            r.append(x.items()[0][1])
+            r.append(x.items()[0][1].replace(PASTES_FOLDER, '', 1))
-            path = x.items()[0][1]
+            path = x.items()[0][1].replace(PASTES_FOLDER, '', 1)
             paste = Paste.Paste(path)
             content = paste.get_p_content()
             content_range = max_preview_char if len(content)>max_preview_char else len(content)-1
@@ -208,6 +208,7 @@ def get_more_search_result():
         results = searcher.search_page(query, page_offset, num_elem_to_get)
         for x in results:
             path = x.items()[0][1]
+            path = path.replace(PASTES_FOLDER, '', 1)
             path_array.append(path)
             paste = Paste.Paste(path)
             content = paste.get_p_content()

var/www/modules/showpaste/Flask_showpaste.py

@@ -41,12 +41,15 @@ showsavedpastes = Blueprint('showsavedpastes', __name__, template_folder='templa
 # ============ FUNCTIONS ============
 
 def showpaste(content_range, requested_path):
-    if PASTES_FOLDER in requested_path:
+    if PASTES_FOLDER not in requested_path:
         # remove full path
+        requested_path_full = os.path.join(requested_path, PASTES_FOLDER)
+    else:
+        requested_path_full = requested_path
         requested_path = requested_path.replace(PASTES_FOLDER, '', 1)
-        #requested_path = os.path.join(PASTES_FOLDER, requested_path)
+
     # escape directory transversal
-    if os.path.commonprefix((os.path.realpath(requested_path),PASTES_FOLDER)) != PASTES_FOLDER:
+    if os.path.commonprefix((requested_path_full,PASTES_FOLDER)) != PASTES_FOLDER:
         return 'path transversal detected'
 
     vt_enabled = Flask_config.vt_enabled
@@ -122,12 +125,6 @@ def showpaste(content_range, requested_path):
     active_taxonomies = r_serv_tags.smembers('active_taxonomies')
 
     l_tags = r_serv_metadata.smembers('tag:'+requested_path)
-    print(l_tags)
-    if relative_path is not None:
-        print('union')
-        print(relative_path)
-        print(r_serv_metadata.smembers('tag:'+relative_path))
-        l_tags = l_tags.union( r_serv_metadata.smembers('tag:'+relative_path) )
 
     #active galaxies
     active_galaxies = r_serv_tags.smembers('active_galaxies')
@@ -280,6 +277,7 @@ def send_file_to_vt():
     paste = request.form['paste']
     hash = request.form['hash']
 
+    ## TODO:  # FIXME:  path transversal
     b64_full_path = os.path.join(os.environ['AIL_HOME'], b64_path)
     b64_content = ''
     with open(b64_full_path, 'rb') as f: