import brozzler_ia as brozzler

This commit is contained in:
Barbara Miller 2024-11-03 14:32:09 -08:00
parent 07a6b9845a
commit afb73ceba8
17 changed files with 21 additions and 18 deletions

View File

@ -2,7 +2,7 @@
brozzler/__init__.py - __init__.py for brozzler package, contains some common
code
Copyright (C) 2014-2017 Internet Archive
Copyright (C) 2014-2024 Internet Archive
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -20,7 +20,7 @@ limitations under the License.
import logging
from pkg_resources import get_distribution as _get_distribution
__version__ = _get_distribution("brozzler").version
__version__ = _get_distribution("brozzler_ia").version
class ShutdownRequested(Exception):
@ -344,6 +344,7 @@ EPOCH_UTC = datetime.datetime.utcfromtimestamp(0.0).replace(tzinfo=doublethink.U
# we could make this configurable if there's a good reason
MAX_PAGE_FAILURES = 3
import brozzler_ia as brozzler
from brozzler.worker import BrozzlerWorker
from brozzler.robots import is_permitted_by_robots
from brozzler.frontier import RethinkDbFrontier

View File

@ -18,13 +18,12 @@ limitations under the License.
import logging
import time
import brozzler
import brozzler_ia as brozzler
import itertools
import json
import websocket
import time
import threading
import brozzler
from requests.structures import CaseInsensitiveDict
import datetime
import base64

View File

@ -22,7 +22,7 @@ import time
import threading
import subprocess
import os
import brozzler
import brozzler_ia as brozzler
import select
import re
import signal

View File

@ -18,7 +18,7 @@ limitations under the License.
"""
import argparse
import brozzler
import brozzler_ia as brozzler
import brozzler.worker
import datetime
import json

View File

@ -325,6 +325,7 @@ except ImportError:
def main(argv=None):
import argparse
import brozzler_ia as brozzler
import brozzler.cli
argv = argv or sys.argv

View File

@ -25,6 +25,7 @@ try:
import warcprox
import warcprox.main
import pywb
import brozzler_ia as brozzler
import brozzler.pywb
import wsgiref.simple_server
import wsgiref.handlers

View File

@ -17,7 +17,7 @@ limitations under the License.
"""
import logging
import brozzler
import brozzler_ia as brozzler
import random
import time
import datetime

View File

@ -17,7 +17,7 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
import brozzler
import brozzler_ia as brozzler
import base64
import cerberus
import copy

View File

@ -41,7 +41,7 @@ import doublethink
import rethinkdb as rdb
import urlcanon
import json
import brozzler
import brozzler_ia as brozzler
import argparse
r = rdb.RethinkDB()

View File

@ -24,7 +24,7 @@ limitations under the License.
import json
import logging
import brozzler
import brozzler_ia as brozzler
import reppy
import reppy.cache
import reppy.parser

View File

@ -19,7 +19,7 @@ limitations under the License.
"""
import logging
import brozzler
import brozzler_ia as brozzler
import brozzler.browser
import threading
import time

View File

@ -19,7 +19,7 @@ limitations under the License.
import logging
import yt_dlp
from yt_dlp.utils import match_filter_func
import brozzler
import brozzler_ia as brozzler
import urllib.request
import tempfile
import urlcanon

View File

@ -33,7 +33,7 @@ def find_package_data(package):
setuptools.setup(
name="brozzler",
name="brozzler_ia",
version="1.6.1",
description="Distributed web crawling with browsers",
url="https://github.com/internetarchive/brozzler",
@ -41,9 +41,9 @@ setuptools.setup(
author_email="nlevitt@archive.org",
long_description=open("README.rst", mode="rb").read().decode("UTF-8"),
license="Apache License 2.0",
packages=["brozzler", "brozzler.dashboard"],
packages=["brozzler_ia", "brozzler.dashboard"],
package_data={
"brozzler": ["js-templates/*.js*", "behaviors.yaml", "job_schema.yaml"],
"brozzler_ia": ["js-templates/*.js*", "behaviors.yaml", "job_schema.yaml"],
"brozzler.dashboard": find_package_data("brozzler.dashboard"),
},
entry_points={

View File

@ -17,6 +17,7 @@ See the License for the specific language governing permissions and
limitations under the License.
"""
import brozzler_ia as brozzler
import brozzler.cli
import pkg_resources
import pytest

View File

@ -26,7 +26,7 @@ import os
import socket
import doublethink
import time
import brozzler
import brozzler_ia as brozzler
import datetime
import requests
import subprocess

View File

@ -26,7 +26,7 @@ import time
import doublethink
import pytest
import brozzler
import brozzler_ia as brozzler
args = argparse.Namespace()
args.log_level = logging.INFO

View File

@ -21,7 +21,7 @@ import pytest
import http.server
import threading
import os
import brozzler
import brozzler_ia as brozzler
import brozzler.chrome
import brozzler.ydl
import logging