mirror of
https://github.com/ail-project/ail-framework.git
synced 2024-11-10 08:38:28 +00:00
fix: [Global Crawler] max filename size
This commit is contained in:
parent
423c7b1455
commit
7a4989ce10
3 changed files with 21 additions and 13 deletions
|
@ -23,23 +23,17 @@ Requirements
|
|||
import base64
|
||||
import os
|
||||
import time
|
||||
import uuid
|
||||
from pubsublogger import publisher
|
||||
|
||||
from Helper import Process
|
||||
|
||||
import magic
|
||||
import io
|
||||
#import gzip
|
||||
|
||||
'''
|
||||
def gunzip_bytes_obj(bytes_obj):
|
||||
in_ = io.BytesIO()
|
||||
in_.write(bytes_obj)
|
||||
in_.seek(0)
|
||||
with gzip.GzipFile(fileobj=in_, mode='rb') as fo:
|
||||
gunzipped_bytes_obj = fo.read()
|
||||
def rreplace(s, old, new, occurrence):
|
||||
li = s.rsplit(old, occurrence)
|
||||
return new.join(li)
|
||||
|
||||
return gunzipped_bytes_obj.decode()'''
|
||||
|
||||
if __name__ == '__main__':
|
||||
publisher.port = 6380
|
||||
|
@ -77,6 +71,12 @@ if __name__ == '__main__':
|
|||
processed_paste = 0
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
file_name_paste = paste.split('/')[-1]
|
||||
if len(file_name_paste)>255:
|
||||
new_file_name_paste = '{}{}.gz'.format(file_name_paste[:215], str(uuid.uuid4()))
|
||||
paste = rreplace(paste, file_name_paste, new_file_name_paste, 1)
|
||||
|
||||
# Creating the full filepath
|
||||
filename = os.path.join(os.environ['AIL_HOME'],
|
||||
p.config.get("Directories", "pastes"), paste)
|
||||
|
|
|
@ -198,8 +198,12 @@ if __name__ == "__main__":
|
|||
print(len(domains_list))
|
||||
if len(domains_list) > 0:
|
||||
|
||||
publisher.warning('{}Detected {} .onion(s);{}'.format(
|
||||
to_print, len(domains_list),PST.p_path))
|
||||
if not activate_crawler:
|
||||
publisher.warning('{}Detected {} .onion(s);{}'.format(
|
||||
to_print, len(domains_list),PST.p_path))
|
||||
else:
|
||||
publisher.info('{}Detected {} .onion(s);{}'.format(
|
||||
to_print, len(domains_list),PST.p_path))
|
||||
now = datetime.datetime.now()
|
||||
path = os.path.join('onions', str(now.year).zfill(4),
|
||||
str(now.month).zfill(2),
|
||||
|
|
|
@ -126,7 +126,11 @@ class TorSplashCrawler():
|
|||
print('Connection to proxy refused')
|
||||
else:
|
||||
|
||||
UUID = self.domains[0]+str(uuid.uuid4())
|
||||
#avoid filename too big
|
||||
if self.domains[0] > 225:
|
||||
UUID = self.domains[0][-215:]+str(uuid.uuid4())
|
||||
else
|
||||
UUID = self.domains[0]+str(uuid.uuid4())
|
||||
filename_paste = os.path.join(self.crawled_paste_filemame, UUID)
|
||||
relative_filename_paste = os.path.join(self.crawler_path, UUID)
|
||||
filename_screenshot = os.path.join(self.crawled_screenshot, UUID +'.png')
|
||||
|
|
Loading…
Reference in a new issue