Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-04-20 23:56:34 -04:00
Code Issues Actions 1 Packages Projects Releases Wiki Activity

chore: formatting

Browse Source
This commit is contained in:
Adam Miller 2024-08-30 21:15:59 +00:00
parent 18a976f82c
commit 0dca9f3861
2 changed files with 13 additions and 4 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
brozzler/frontier.py
View File

@ -224,7 +224,11 @@ class RethinkDbFrontier:
index="priority_by_site",
)
.order_by(index=r.desc("priority_by_site"))
.filter(lambda page: r.or_(page.has_fields("retry_after").not_(), r.now() > page["retry_after"]))
.filter(
lambda page: r.or_(
page.has_fields("retry_after").not_(), r.now() > page["retry_after"]
)
)
.limit(1)
.update(
{"claimed": True, "last_claimed_by": worker_id}, return_changes="always"

11
brozzler/worker.py
View File

@ -505,15 +505,20 @@ class BrozzlerWorker:
except (brozzler.PageConnectionError, Exception) as e:
if isinstance(e, brozzler.PageConnectionError):
self.logger.error(
"Page status code possibly indicates connection failure between host and warcprox: site=%r page=%r", site, page, exc_info=True
"Page status code possibly indicates connection failure between host and warcprox: site=%r page=%r",
site,
page,
exc_info=True,
)
else:
self.logger.error(
"unexpected exception site=%r page=%r", site, page, exc_info=True
)
if page:
retry_delay = min(60, 60 * (1.5 ** page.failed_attempts))
page.retry_after = doublethink.utcnow() + datetime.timedelta(seconds=retry_delay)
retry_delay = min(60, 60 * (1.5**page.failed_attempts))
page.retry_after = doublethink.utcnow() + datetime.timedelta(
seconds=retry_delay
)
page.failed_attempts = (page.failed_attempts or 0) + 1
if page.failed_attempts >= brozzler.MAX_PAGE_FAILURES:
self.logger.info(