Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 16:49:56 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

restore scoping out of urls with unsupported schemes

Browse Source
This commit is contained in:
Noah Levitt 2016-04-21 11:40:08 -07:00
parent fee008266f
commit 1e52d1cf98
1 changed files with 5 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
brozzler/site.py
View File

@ -65,7 +65,11 @@ class Site(brozzler.BaseDictable):
self.scope["surt"] = new_scope_surt self.scope["surt"] = new_scope_surt
def is_in_scope(self, surt_, parent_page=None): def is_in_scope(self, surt_, parent_page=None):
if (parent_page and "max_hops" in self.scope if not surt_.startswith("http://") and not surt_.startswith("https://"):
# XXX doesn't belong here maybe (where? worker ignores unknown
# schemes?)
return False
elif (parent_page and "max_hops" in self.scope
and parent_page.hops_from_seed >= self.scope["max_hops"]): and parent_page.hops_from_seed >= self.scope["max_hops"]):
return False return False
elif surt_.startswith(self.scope["surt"]): elif surt_.startswith(self.scope["surt"]):