Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-02-24 00:29:53 -05:00
Code Issues Actions Packages Projects Releases Wiki Activity

WIP add an accept rule instead of modifying surt

Browse Source 
in place for seed redirects
This commit is contained in:
Noah Levitt 2018-03-20 17:32:01 -07:00
parent 98ce67ef36
commit f26712ce93
1 changed files with 5 additions and 3 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
brozzler/model.py
View File

@ -203,10 +203,12 @@ class Site(doublethink.Document, ElapsedMixIn):
def note_seed_redirect(self, url):
new_scope_surt = brozzler.site_surt_canon(url).surt().decode("ascii")
if not "accepts" in self.scope:
self.scope["accepts"] = []
if not new_scope_surt.startswith(self.scope["surt"]):
self.logger.info("changing site scope surt from {} to {}".format(
self.scope["surt"], new_scope_surt))
self.scope["surt"] = new_scope_surt
self.logger.info(
"adding surt %s to scope accept rules", new_scope_surt)
self.scope.accepts.append({"surt": new_scope_surt})
def extra_headers(self):
hdrs = {}