Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-08-08 06:22:23 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merge pull request #262 from galgeek/pr261

Browse source 
@avdempsey approved this merge.
This commit is contained in:
Barbara Miller 2023-10-18 11:01:15 -07:00 • committed by GitHub
commit c1345e2c9f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
6 changed files with 18 additions and 8 deletions
Download patch file Download diff file Expand all files Collapse all files

4
brozzler/cli.py
View file

@ -37,7 +37,9 @@ import warnings
import yaml import yaml
import shutil import shutil
import base64 import base64
import rethinkdb as r import rethinkdb as rdb
r = rdb.RethinkDB()
def add_common_options(arg_parser, argv=None): def add_common_options(arg_parser, argv=None):
argv = argv or sys.argv argv = argv or sys.argv

4
brozzler/dashboard/__init__.py
View file

@ -31,10 +31,12 @@ import doublethink
import json import json
import os import os
import importlib import importlib
import rethinkdb as r import rethinkdb as rdb
import yaml import yaml
import base64 import base64
r = rdb.RethinkDB()
app = flask.Flask(__name__) app = flask.Flask(__name__)
# configure with environment variables # configure with environment variables

4
brozzler/frontier.py
View file

@ -21,10 +21,12 @@ import brozzler
import random import random
import time import time
import datetime import datetime
import rethinkdb as r import rethinkdb as rdb
import doublethink import doublethink
import urlcanon import urlcanon
r = rdb.RethinkDB()
class UnexpectedDbResult(Exception): class UnexpectedDbResult(Exception):
pass pass

4
brozzler/pywb.py
View file

@ -35,12 +35,14 @@ except ImportError as e:
type(e).__name__, e) type(e).__name__, e)
sys.exit(1) sys.exit(1)
import doublethink import doublethink
import rethinkdb as r import rethinkdb as rdb
import urlcanon import urlcanon
import json import json
import brozzler import brozzler
import argparse import argparse
r = rdb.RethinkDB()
class RethinkCDXSource(pywb.cdx.cdxsource.CDXSource): class RethinkCDXSource(pywb.cdx.cdxsource.CDXSource):
def __init__(self, servers, db, table): def __init__(self, servers, db, table):
self.servers = servers self.servers = servers

4
brozzler/worker.py
View file

@ -34,9 +34,11 @@ import doublethink
import tempfile import tempfile
import urlcanon import urlcanon
from requests.structures import CaseInsensitiveDict from requests.structures import CaseInsensitiveDict
import rethinkdb as r import rethinkdb as rdb
from . import ydl from . import ydl
r = rdb.RethinkDB()
class BrozzlerWorker: class BrozzlerWorker:
logger = logging.getLogger(__module__ + "." + __qualname__) logger = logging.getLogger(__module__ + "." + __qualname__)

6
setup.py
View file

@ -32,7 +32,7 @@ def find_package_data(package):
setuptools.setup( setuptools.setup(
name='brozzler', name='brozzler',
version='1.5.37', version='1.5.38a1',
description='Distributed web crawling with browsers', description='Distributed web crawling with browsers',
url='https://github.com/internetarchive/brozzler', url='https://github.com/internetarchive/brozzler',
author='Noah Levitt', author='Noah Levitt',
@ -71,8 +71,8 @@ setuptools.setup(
'websocket-client>=0.39.0,<=0.48.0', 'websocket-client>=0.39.0,<=0.48.0',
'pillow>=5.2.0', 'pillow>=5.2.0',
'urlcanon>=0.1.dev23', 'urlcanon>=0.1.dev23',
'doublethink>=0.2.0', 'doublethink @ git+https://github.com/internetarchive/doublethink.git@Py311',
'rethinkdb>=2.3,<2.4', 'rethinkdb>=2.4.9,<2.5',
'cerberus>=1.0.1', 'cerberus>=1.0.1',
'jinja2>=2.10', 'jinja2>=2.10',
'cryptography>=2.3', 'cryptography>=2.3',