Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-09-26 03:00:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge branch 'master' of github.com:internetarchive/brozzler

Browse source
This commit is contained in:
Barbara Miller 2017-10-04 20:07:08 -07:00
commit 6dc6e71ddc
2 changed files with 2 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

2
brozzler/cli.py
View file

@ -196,7 +196,7 @@ def brozzle_page(argv=None):
enable_youtube_dl = False if args.skip_youtube_dl else True enable_youtube_dl = False if args.skip_youtube_dl else True
outlinks = worker.brozzle_page( outlinks = worker.brozzle_page(
browser, site, page, on_screenshot=on_screenshot, browser, site, page, on_screenshot=on_screenshot,
enable_youtube_dl=enable_youtube_dl) enable_youtube_dl=not args.skip_youtube_dl)
logging.info('outlinks: \n\t%s', '\n\t'.join(sorted(outlinks))) logging.info('outlinks: \n\t%s', '\n\t'.join(sorted(outlinks)))
except brozzler.ReachedLimit as e: except brozzler.ReachedLimit as e:
logging.error('reached limit %s', e) logging.error('reached limit %s', e)

3
brozzler/worker.py
View file

@ -487,9 +487,8 @@ class BrozzlerWorker:
page.blocked_by_robots = True page.blocked_by_robots = True
self._frontier.completed_page(site, page) self._frontier.completed_page(site, page)
else: else:
enable_youtube_dl = False if self._skip_youtube_dl else True
outlinks = self.brozzle_page(browser, site, page, outlinks = self.brozzle_page(browser, site, page,
enable_youtube_dl=enable_youtube_dl) enable_youtube_dl=not self._skip_youtube_dl)
self._frontier.completed_page(site, page) self._frontier.completed_page(site, page)
self._frontier.scope_and_schedule_outlinks( self._frontier.scope_and_schedule_outlinks(
site, page, outlinks) site, page, outlinks)