From dff1fbb08bdc8f3ddbb50614e8cd61d0ac7a19fc Mon Sep 17 00:00:00 2001 From: Vangelis Banos Date: Tue, 17 Oct 2023 19:36:04 +0000 Subject: [PATCH 1/4] Update rethinkdb dependency The latest `warcprox` 2.5.1 requirement https://github.com/internetarchive/doublethink/blob/Py311/setup.py requires `rethinkdb>=2.4.9,<2.5` but Brozzler has `rethinkdb>=2.3,<2.4` and this creates a conflict if they are in the same virtualenv. We update Brozzler to use the same dependency. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b398b03..3614860 100644 --- a/setup.py +++ b/setup.py @@ -72,7 +72,7 @@ setuptools.setup( 'pillow>=5.2.0', 'urlcanon>=0.1.dev23', 'doublethink>=0.2.0', - 'rethinkdb>=2.3,<2.4', + 'rethinkdb>=2.4.9,<2.5', 'cerberus>=1.0.1', 'jinja2>=2.10', 'cryptography>=2.3', From 8c32c98431a4226f47b7480d8583a5270ca1ae55 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Tue, 17 Oct 2023 14:24:46 -0700 Subject: [PATCH 2/4] update doublethink dependency, too --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 3614860..ed4e5b1 100644 --- a/setup.py +++ b/setup.py @@ -71,7 +71,7 @@ setuptools.setup( 'websocket-client>=0.39.0,<=0.48.0', 'pillow>=5.2.0', 'urlcanon>=0.1.dev23', - 'doublethink>=0.2.0', + 'doublethink @ git+https://github.com/internetarchive/doublethink.git@Py311', 'rethinkdb>=2.4.9,<2.5', 'cerberus>=1.0.1', 'jinja2>=2.10', From d07cee8cf09b676a4cbe3d290a4251150ec099e2 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Tue, 17 Oct 2023 14:58:34 -0700 Subject: [PATCH 3/4] update rethinkdb imports --- brozzler/cli.py | 4 +++- brozzler/dashboard/__init__.py | 4 +++- brozzler/frontier.py | 4 +++- brozzler/pywb.py | 4 +++- brozzler/worker.py | 4 +++- 5 files changed, 15 insertions(+), 5 deletions(-) diff --git a/brozzler/cli.py b/brozzler/cli.py index a6d0aba..534dbe1 100755 --- a/brozzler/cli.py +++ b/brozzler/cli.py @@ -37,7 +37,9 @@ import warnings import yaml import shutil import base64 -import rethinkdb as r +import rethinkdb as rdb + +r = rdb.RethinkDB() def add_common_options(arg_parser, argv=None): argv = argv or sys.argv diff --git a/brozzler/dashboard/__init__.py b/brozzler/dashboard/__init__.py index fdef316..6e85b3c 100644 --- a/brozzler/dashboard/__init__.py +++ b/brozzler/dashboard/__init__.py @@ -31,10 +31,12 @@ import doublethink import json import os import importlib -import rethinkdb as r +import rethinkdb as rdb import yaml import base64 +r = rdb.RethinkDB() + app = flask.Flask(__name__) # configure with environment variables diff --git a/brozzler/frontier.py b/brozzler/frontier.py index 6e64e51..ac27047 100644 --- a/brozzler/frontier.py +++ b/brozzler/frontier.py @@ -21,10 +21,12 @@ import brozzler import random import time import datetime -import rethinkdb as r +import rethinkdb as rdb import doublethink import urlcanon +r = rdb.RethinkDB() + class UnexpectedDbResult(Exception): pass diff --git a/brozzler/pywb.py b/brozzler/pywb.py index 5932f0b..03d64b8 100644 --- a/brozzler/pywb.py +++ b/brozzler/pywb.py @@ -35,12 +35,14 @@ except ImportError as e: type(e).__name__, e) sys.exit(1) import doublethink -import rethinkdb as r +import rethinkdb as rdb import urlcanon import json import brozzler import argparse +r = rdb.RethinkDB() + class RethinkCDXSource(pywb.cdx.cdxsource.CDXSource): def __init__(self, servers, db, table): self.servers = servers diff --git a/brozzler/worker.py b/brozzler/worker.py index e8f7619..42c7c20 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -34,9 +34,11 @@ import doublethink import tempfile import urlcanon from requests.structures import CaseInsensitiveDict -import rethinkdb as r +import rethinkdb as rdb from . import ydl +r = rdb.RethinkDB() + class BrozzlerWorker: logger = logging.getLogger(__module__ + "." + __qualname__) From dc4097a9dfbdabf7910876e12eb4479a78e97003 Mon Sep 17 00:00:00 2001 From: Barbara Miller Date: Tue, 17 Oct 2023 15:01:51 -0700 Subject: [PATCH 4/4] better version number --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ed4e5b1..cc352a5 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_package_data(package): setuptools.setup( name='brozzler', - version='1.5.37', + version='1.5.38a1', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt',