Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 00:29:53 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #223 from vbanos/fix-AddressValueError

Browse Source 
Skip invalid outlink
This commit is contained in:
Barbara Miller 2021-05-25 17:12:35 -07:00 committed by GitHub
commit b59c4395ed
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 8 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
brozzler/browser.py
View File

@ -28,6 +28,7 @@ import brozzler
from requests.structures import CaseInsensitiveDict
import datetime
import base64
from ipaddress import AddressValueError
from brozzler.chrome import Chrome
import socket
import urlcanon
@ -619,8 +620,13 @@ class Browser:
if ('result' in message and 'result' in message['result']
and 'value' in message['result']['result']):
if message['result']['result']['value']:
return frozenset([str(urlcanon.whatwg(link)) for link in
message['result']['result']['value'].split('\n')])
out = []
for link in message['result']['result']['value'].split('\n'):
try:
out.append(str(urlcanon.whatwg(link)))
except AddressValueError:
self.logger.warning('skip invalid outlink: %s', link)
return frozenset(out)
else:
# no links found
return frozenset()