diff --git a/brozzler/cli.py b/brozzler/cli.py index a6d0aba..534dbe1 100755 --- a/brozzler/cli.py +++ b/brozzler/cli.py @@ -37,7 +37,9 @@ import warnings import yaml import shutil import base64 -import rethinkdb as r +import rethinkdb as rdb + +r = rdb.RethinkDB() def add_common_options(arg_parser, argv=None): argv = argv or sys.argv diff --git a/brozzler/dashboard/__init__.py b/brozzler/dashboard/__init__.py index fdef316..6e85b3c 100644 --- a/brozzler/dashboard/__init__.py +++ b/brozzler/dashboard/__init__.py @@ -31,10 +31,12 @@ import doublethink import json import os import importlib -import rethinkdb as r +import rethinkdb as rdb import yaml import base64 +r = rdb.RethinkDB() + app = flask.Flask(__name__) # configure with environment variables diff --git a/brozzler/frontier.py b/brozzler/frontier.py index 6e64e51..ac27047 100644 --- a/brozzler/frontier.py +++ b/brozzler/frontier.py @@ -21,10 +21,12 @@ import brozzler import random import time import datetime -import rethinkdb as r +import rethinkdb as rdb import doublethink import urlcanon +r = rdb.RethinkDB() + class UnexpectedDbResult(Exception): pass diff --git a/brozzler/pywb.py b/brozzler/pywb.py index 5932f0b..03d64b8 100644 --- a/brozzler/pywb.py +++ b/brozzler/pywb.py @@ -35,12 +35,14 @@ except ImportError as e: type(e).__name__, e) sys.exit(1) import doublethink -import rethinkdb as r +import rethinkdb as rdb import urlcanon import json import brozzler import argparse +r = rdb.RethinkDB() + class RethinkCDXSource(pywb.cdx.cdxsource.CDXSource): def __init__(self, servers, db, table): self.servers = servers diff --git a/brozzler/worker.py b/brozzler/worker.py index e8f7619..42c7c20 100644 --- a/brozzler/worker.py +++ b/brozzler/worker.py @@ -34,9 +34,11 @@ import doublethink import tempfile import urlcanon from requests.structures import CaseInsensitiveDict -import rethinkdb as r +import rethinkdb as rdb from . import ydl +r = rdb.RethinkDB() + class BrozzlerWorker: logger = logging.getLogger(__module__ + "." + __qualname__) diff --git a/setup.py b/setup.py index b398b03..cc352a5 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_package_data(package): setuptools.setup( name='brozzler', - version='1.5.37', + version='1.5.38a1', description='Distributed web crawling with browsers', url='https://github.com/internetarchive/brozzler', author='Noah Levitt', @@ -71,8 +71,8 @@ setuptools.setup( 'websocket-client>=0.39.0,<=0.48.0', 'pillow>=5.2.0', 'urlcanon>=0.1.dev23', - 'doublethink>=0.2.0', - 'rethinkdb>=2.3,<2.4', + 'doublethink @ git+https://github.com/internetarchive/doublethink.git@Py311', + 'rethinkdb>=2.4.9,<2.5', 'cerberus>=1.0.1', 'jinja2>=2.10', 'cryptography>=2.3',