Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 00:29:53 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #223 from vbanos/fix-AddressValueError

Browse Source 
Skip invalid outlink
This commit is contained in:
Barbara Miller 2021-05-25 17:12:35 -07:00 committed by GitHub
commit b59c4395ed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
brozzler/browser.py
View File

@ -28,6 +28,7 @@ import brozzler
from requests.structures import CaseInsensitiveDict from requests.structures import CaseInsensitiveDict
import datetime import datetime
import base64 import base64
from ipaddress import AddressValueError
from brozzler.chrome import Chrome from brozzler.chrome import Chrome
import socket import socket
import urlcanon import urlcanon
@ -619,8 +620,13 @@ class Browser:
if ('result' in message and 'result' in message['result'] if ('result' in message and 'result' in message['result']
and 'value' in message['result']['result']): and 'value' in message['result']['result']):
if message['result']['result']['value']: if message['result']['result']['value']:
return frozenset([str(urlcanon.whatwg(link)) for link in out = []
message['result']['result']['value'].split('\n')]) for link in message['result']['result']['value'].split('\n'):
try:
out.append(str(urlcanon.whatwg(link)))
except AddressValueError:
self.logger.warning('skip invalid outlink: %s', link)
return frozenset(out)
else: else:
# no links found # no links found
return frozenset() return frozenset()