Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-08 06:22:23 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #262 from galgeek/pr261

Browse source 
@avdempsey approved this merge.
This commit is contained in:
Barbara Miller 2023-10-18 11:01:15 -07:00 • committed by GitHub
commit c1345e2c9f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 18 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

4
brozzler/cli.py
View file

@ -37,7 +37,9 @@ import warnings
import yaml
import shutil
import base64
import rethinkdb as r
import rethinkdb as rdb
r = rdb.RethinkDB()
def add_common_options(arg_parser, argv=None):
argv = argv or sys.argv

4
brozzler/dashboard/__init__.py
View file

@ -31,10 +31,12 @@ import doublethink
import json
import os
import importlib
import rethinkdb as r
import rethinkdb as rdb
import yaml
import base64
r = rdb.RethinkDB()
app = flask.Flask(__name__)
# configure with environment variables

4
brozzler/frontier.py
View file

@ -21,10 +21,12 @@ import brozzler
import random
import time
import datetime
import rethinkdb as r
import rethinkdb as rdb
import doublethink
import urlcanon
r = rdb.RethinkDB()
class UnexpectedDbResult(Exception):
pass

4
brozzler/pywb.py
View file

@ -35,12 +35,14 @@ except ImportError as e:
type(e).__name__, e)
sys.exit(1)
import doublethink
import rethinkdb as r
import rethinkdb as rdb
import urlcanon
import json
import brozzler
import argparse
r = rdb.RethinkDB()
class RethinkCDXSource(pywb.cdx.cdxsource.CDXSource):
def __init__(self, servers, db, table):
self.servers = servers

4
brozzler/worker.py
View file

@ -34,9 +34,11 @@ import doublethink
import tempfile
import urlcanon
from requests.structures import CaseInsensitiveDict
import rethinkdb as r
import rethinkdb as rdb
from . import ydl
r = rdb.RethinkDB()
class BrozzlerWorker:
logger = logging.getLogger(__module__ + "." + __qualname__)

6
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup(
name='brozzler',
version='1.5.37',
version='1.5.38a1',
description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler',
author='Noah Levitt',
@ -71,8 +71,8 @@ setuptools.setup(
'websocket-client>=0.39.0,<=0.48.0',
'pillow>=5.2.0',
'urlcanon>=0.1.dev23',
'doublethink>=0.2.0',
'rethinkdb>=2.3,<2.4',
'doublethink @ git+https://github.com/internetarchive/doublethink.git@Py311',
'rethinkdb>=2.4.9,<2.5',
'cerberus>=1.0.1',
'jinja2>=2.10',
'cryptography>=2.3',