Merge branch 'internetarchive:master' into master

brozzler/browser.py

@@ -1,7 +1,7 @@
 '''
 brozzler/browser.py - manages the browsers for brozzler
 
-Copyright (C) 2014-2020 Internet Archive
+Copyright (C) 2014-2023 Internet Archive
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.

brozzler/chrome.py

@@ -1,7 +1,7 @@
 '''
 brozzler/chrome.py - manages the chrome/chromium browser for brozzler
 
-Copyright (C) 2014-2020 Internet Archive
+Copyright (C) 2014-2023 Internet Archive
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -135,7 +135,8 @@ class Chrome:
         return cookie_db
 
     def start(self, proxy=None, cookie_db=None, disk_cache_dir=None,
-              disk_cache_size=None, websocket_timeout=60):
+              disk_cache_size=None, websocket_timeout=60,
+              window_height=900, window_width=1400):
         '''
         Starts chrome/chromium process.
 
@@ -150,6 +151,7 @@ class Chrome:
             disk_cache_size: Forces the maximum disk space to be used by the disk
                 cache, in bytes. (default None)
             websocket_timeout: websocket timeout, in seconds
+            window_height, window_width: window height and width, in pixels
         Returns:
             websocket url to chrome window with about:blank loaded
         '''
@@ -172,7 +174,8 @@ class Chrome:
                 '--disable-renderer-backgrounding', '--disable-hang-monitor',
                 '--disable-background-timer-throttling', '--mute-audio',
                 '--disable-web-sockets',
-                '--window-size=1100,900', '--no-default-browser-check',
+                f'--window-size={window_width},{window_height}',
+                '--no-default-browser-check',
                 '--disable-first-run-ui', '--no-first-run',
                 '--homepage=about:blank', '--disable-direct-npapi-requests',
                 '--disable-web-security', '--disable-notifications',

brozzler/cli.py

@@ -2,7 +2,7 @@
 '''
 brozzler/cli.py - brozzler command line executables
 
-Copyright (C) 2014-2019 Internet Archive
+Copyright (C) 2014-2023 Internet Archive
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -156,6 +156,12 @@ def brozzle_page(argv=None):
     arg_parser.add_argument(
             '--browser_throughput', type=int, dest='download_throughput', default=-1,
             help='Chrome DevTools downloadThroughput for Network.emulateNetworkConditions')
+    arg_parser.add_argument(
+            '--browser_window_height', type=int, dest='window_height', default=900,
+            help='browser window height in pixels')
+    arg_parser.add_argument(
+            '--browser_window_width', type=int, dest='window_width', default=1400,
+            help='browser window width in pixels')
     arg_parser.add_argument(
             '--stealth', dest='stealth', action='store_true',
             help='Try to avoid web bot detection')
@@ -193,6 +199,8 @@ def brozzle_page(argv=None):
             simpler404=args.simpler404,
             screenshot_full_page=args.screenshot_full_page,
             download_throughput=args.download_throughput,
+            window_height=args.window_height,
+            window_width=args.window_width,
             stealth=args.stealth)
 
     def on_screenshot(screenshot_jpeg):
@@ -206,7 +214,7 @@ def brozzle_page(argv=None):
 
     browser = brozzler.Browser(chrome_exe=args.chrome_exe)
     try:
-        browser.start(proxy=args.proxy)
+        browser.start(proxy=args.proxy, window_height=args.window_height, window_width=args.window_width)
         outlinks = worker.brozzle_page(
                 browser, site, page, on_screenshot=on_screenshot,
                 enable_youtube_dl=not args.skip_youtube_dl)
@@ -326,6 +334,12 @@ def brozzler_worker(argv=None):
     arg_parser.add_argument(
             '--browser_throughput', type=int, dest='download_throughput', default=-1,
             help='Chrome DevTools downloadThroughput for Network.emulateNetworkConditions')
+    arg_parser.add_argument(
+            '--browser_window_height', type=int, dest='window_height', default=900,
+            help='browser window height in pixels')
+    arg_parser.add_argument(
+            '--browser_window_width', type=int, dest='window_width', default=1400,
+            help='browser window width in pixels')
     arg_parser.add_argument(
             '--warcprox-auto', dest='warcprox_auto', action='store_true',
             help=(

brozzler/worker.py

@@ -3,7 +3,7 @@ brozzler/worker.py - BrozzlerWorker brozzles pages from the frontier, meaning
 it runs youtube-dl on them, browses them and runs behaviors if appropriate,
 scopes and adds outlinks to the frontier
 
-Copyright (C) 2014-2018 Internet Archive
+Copyright (C) 2014-2023 Internet Archive
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -53,7 +53,8 @@ class BrozzlerWorker:
             skip_extract_outlinks=False, skip_visit_hashtags=False,
             skip_youtube_dl=False, simpler404=False, screenshot_full_page=False,
             page_timeout=300, behavior_timeout=900, extract_outlinks_timeout=60,
-            download_throughput=-1, stealth=False):
+            download_throughput=-1, stealth=False,
+            window_height=900, window_width=1400):
         self._frontier = frontier
         self._service_registry = service_registry
         self._max_browsers = max_browsers
@@ -71,6 +72,8 @@ class BrozzlerWorker:
         self._behavior_timeout = behavior_timeout
         self._extract_outlinks_timeout = extract_outlinks_timeout
         self._download_throughput = download_throughput
+        self._window_height = window_height
+        self._window_width = window_width
         self._stealth = stealth
 
         self._browser_pool = brozzler.browser.BrowserPool(
@@ -294,7 +297,9 @@ class BrozzlerWorker:
         if not browser.is_running():
             browser.start(
                     proxy=self._proxy_for(site),
-                    cookie_db=site.get('cookie_db'))
+                    cookie_db=site.get('cookie_db'),
+                    window_height=self._window_height,
+                    window_width=self._window_width)
         final_page_url, outlinks = browser.browse_page(
                 page.url, extra_headers=site.extra_headers(page),
                 behavior_parameters=site.get('behavior_parameters'),

brozzler/ydl.py

@@ -229,6 +229,7 @@ def _build_youtube_dl(worker, destdir, site, page):
             if d['postprocessor'] == 'FixupM3u8' and worker._using_warcprox(site):
                 _YoutubeDL._push_stitched_up_vid_to_warcprox(_YoutubeDL, site, d['info_dict'])
 
+    # default socket_timeout is 20 -- we hit it often when cluster is busy
     ydl_opts = {
         "outtmpl": "{}/ydl%(autonumber)s.out".format(destdir),
         "retries": 1,
@@ -237,6 +238,7 @@ def _build_youtube_dl(worker, destdir, site, page):
         "noprogress": True,
         "nopart": True,
         "no_color": True,
+        "socket_timeout": 40,
         "progress_hooks": [maybe_heartbeat_site_last_claimed],
         "postprocessor_hooks": [ydl_postprocess_hook],
 

setup.py

@@ -2,7 +2,7 @@
 '''
 setup.py - brozzler setup script
 
-Copyright (C) 2014-2022 Internet Archive
+Copyright (C) 2014-2023 Internet Archive
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -32,7 +32,7 @@ def find_package_data(package):
 
 setuptools.setup(
         name='brozzler',
-        version='1.5.32',
+        version='1.5.34',
         description='Distributed web crawling with browsers',
         url='https://github.com/internetarchive/brozzler',
         author='Noah Levitt',