Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-07-20 21:48:52 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 1

chore: formatting

Browse source
This commit is contained in:
Adam Miller 2024-08-30 21:15:59 +00:00
parent 18a976f82c
commit 0dca9f3861
2 changed files with 13 additions and 4 deletions
Download patch file Download diff file Expand all files Collapse all files

6
brozzler/frontier.py
View file

@ -224,7 +224,11 @@ class RethinkDbFrontier:
index="priority_by_site", index="priority_by_site",
) )
.order_by(index=r.desc("priority_by_site")) .order_by(index=r.desc("priority_by_site"))
.filter(lambda page: r.or_(page.has_fields("retry_after").not_(), r.now() > page["retry_after"])) .filter(
lambda page: r.or_(
page.has_fields("retry_after").not_(), r.now() > page["retry_after"]
)
)
.limit(1) .limit(1)
.update( .update(
{"claimed": True, "last_claimed_by": worker_id}, return_changes="always" {"claimed": True, "last_claimed_by": worker_id}, return_changes="always"

11
brozzler/worker.py
View file

@ -505,15 +505,20 @@ class BrozzlerWorker:
except (brozzler.PageConnectionError, Exception) as e: except (brozzler.PageConnectionError, Exception) as e:
if isinstance(e, brozzler.PageConnectionError): if isinstance(e, brozzler.PageConnectionError):
self.logger.error( self.logger.error(
"Page status code possibly indicates connection failure between host and warcprox: site=%r page=%r", site, page, exc_info=True "Page status code possibly indicates connection failure between host and warcprox: site=%r page=%r",
site,
page,
exc_info=True,
) )
else: else:
self.logger.error( self.logger.error(
"unexpected exception site=%r page=%r", site, page, exc_info=True "unexpected exception site=%r page=%r", site, page, exc_info=True
) )
if page: if page:
retry_delay = min(60, 60 * (1.5 ** page.failed_attempts)) retry_delay = min(60, 60 * (1.5**page.failed_attempts))
page.retry_after = doublethink.utcnow() + datetime.timedelta(seconds=retry_delay) page.retry_after = doublethink.utcnow() + datetime.timedelta(
seconds=retry_delay
)
page.failed_attempts = (page.failed_attempts or 0) + 1 page.failed_attempts = (page.failed_attempts or 0) + 1
if page.failed_attempts >= brozzler.MAX_PAGE_FAILURES: if page.failed_attempts >= brozzler.MAX_PAGE_FAILURES:
self.logger.info( self.logger.info(