Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-07-26 00:05:42 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

WIP add an accept rule instead of modifying surt

Browse source 
in place for seed redirects
This commit is contained in:
Noah Levitt 2018-03-20 17:32:01 -07:00
parent 98ce67ef36
commit f26712ce93
1 changed files with 5 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

8
brozzler/model.py
View file

@ -203,10 +203,12 @@ class Site(doublethink.Document, ElapsedMixIn):
def note_seed_redirect(self, url): def note_seed_redirect(self, url):
new_scope_surt = brozzler.site_surt_canon(url).surt().decode("ascii") new_scope_surt = brozzler.site_surt_canon(url).surt().decode("ascii")
if not "accepts" in self.scope:
self.scope["accepts"] = []
if not new_scope_surt.startswith(self.scope["surt"]): if not new_scope_surt.startswith(self.scope["surt"]):
self.logger.info("changing site scope surt from {} to {}".format( self.logger.info(
self.scope["surt"], new_scope_surt)) "adding surt %s to scope accept rules", new_scope_surt)
self.scope["surt"] = new_scope_surt self.scope.accepts.append({"surt": new_scope_surt})
def extra_headers(self): def extra_headers(self):
hdrs = {} hdrs = {}