AIL/ail-framework
8
0
Fork 0
mirror of https://github.com/ail-project/ail-framework.git synced 2025-01-18 08:26:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

fix: [title beautifullsoup] add signal, BeautifulSoup html.parser is stuck

Browse source
This commit is contained in:
terrtia 2025-01-07 17:21:58 +01:00
parent 8692d9b45b
commit 868da3c6bc
No known key found for this signature in database
GPG key ID: 1E1B1F50D84613D0
1 changed files with 24 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

29
bin/lib/crawlers.py
View file

@ -27,6 +27,19 @@ from pylacus import PyLacus
from pyfaup.faup import Faup
import signal
class TimeoutException(Exception):
pass
def timeout_handler(signum, frame):
raise TimeoutException
signal.signal(signal.SIGALRM, timeout_handler)
# interact with splash_crawler API
import requests
requests.packages.urllib3.disable_warnings(requests.packages.urllib3.exceptions.InsecureRequestWarning)
@ -310,12 +323,18 @@ def extract_favicon_from_html(html, url):
# # # # # # # #
def extract_title_from_html(html):
soup = BeautifulSoup(html, 'html.parser')
title = soup.title
if title:
title = title.string
signal.alarm(60)
try:
soup = BeautifulSoup(html, 'html.parser')
title = soup.title
if title:
return str(title)
title = title.string
if title:
return str(title)
except TimeoutException:
pass
else:
signal.alarm(0)
return ''
def extract_description_from_html(html):