mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-02-23 08:09:48 -05:00
import brozzler_ia as brozzler
This commit is contained in:
parent
07a6b9845a
commit
afb73ceba8
@ -2,7 +2,7 @@
|
||||
brozzler/__init__.py - __init__.py for brozzler package, contains some common
|
||||
code
|
||||
|
||||
Copyright (C) 2014-2017 Internet Archive
|
||||
Copyright (C) 2014-2024 Internet Archive
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
@ -20,7 +20,7 @@ limitations under the License.
|
||||
import logging
|
||||
from pkg_resources import get_distribution as _get_distribution
|
||||
|
||||
__version__ = _get_distribution("brozzler").version
|
||||
__version__ = _get_distribution("brozzler_ia").version
|
||||
|
||||
|
||||
class ShutdownRequested(Exception):
|
||||
@ -344,6 +344,7 @@ EPOCH_UTC = datetime.datetime.utcfromtimestamp(0.0).replace(tzinfo=doublethink.U
|
||||
# we could make this configurable if there's a good reason
|
||||
MAX_PAGE_FAILURES = 3
|
||||
|
||||
import brozzler_ia as brozzler
|
||||
from brozzler.worker import BrozzlerWorker
|
||||
from brozzler.robots import is_permitted_by_robots
|
||||
from brozzler.frontier import RethinkDbFrontier
|
||||
|
@ -18,13 +18,12 @@ limitations under the License.
|
||||
|
||||
import logging
|
||||
import time
|
||||
import brozzler
|
||||
import brozzler_ia as brozzler
|
||||
import itertools
|
||||
import json
|
||||
import websocket
|
||||
import time
|
||||
import threading
|
||||
import brozzler
|
||||
from requests.structures import CaseInsensitiveDict
|
||||
import datetime
|
||||
import base64
|
||||
|
@ -22,7 +22,7 @@ import time
|
||||
import threading
|
||||
import subprocess
|
||||
import os
|
||||
import brozzler
|
||||
import brozzler_ia as brozzler
|
||||
import select
|
||||
import re
|
||||
import signal
|
||||
|
@ -18,7 +18,7 @@ limitations under the License.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import brozzler
|
||||
import brozzler_ia as brozzler
|
||||
import brozzler.worker
|
||||
import datetime
|
||||
import json
|
||||
|
@ -325,6 +325,7 @@ except ImportError:
|
||||
|
||||
def main(argv=None):
|
||||
import argparse
|
||||
import brozzler_ia as brozzler
|
||||
import brozzler.cli
|
||||
|
||||
argv = argv or sys.argv
|
||||
|
@ -25,6 +25,7 @@ try:
|
||||
import warcprox
|
||||
import warcprox.main
|
||||
import pywb
|
||||
import brozzler_ia as brozzler
|
||||
import brozzler.pywb
|
||||
import wsgiref.simple_server
|
||||
import wsgiref.handlers
|
||||
|
@ -17,7 +17,7 @@ limitations under the License.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import brozzler
|
||||
import brozzler_ia as brozzler
|
||||
import random
|
||||
import time
|
||||
import datetime
|
||||
|
@ -17,7 +17,7 @@ See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
|
||||
import brozzler
|
||||
import brozzler_ia as brozzler
|
||||
import base64
|
||||
import cerberus
|
||||
import copy
|
||||
|
@ -41,7 +41,7 @@ import doublethink
|
||||
import rethinkdb as rdb
|
||||
import urlcanon
|
||||
import json
|
||||
import brozzler
|
||||
import brozzler_ia as brozzler
|
||||
import argparse
|
||||
|
||||
r = rdb.RethinkDB()
|
||||
|
@ -24,7 +24,7 @@ limitations under the License.
|
||||
|
||||
import json
|
||||
import logging
|
||||
import brozzler
|
||||
import brozzler_ia as brozzler
|
||||
import reppy
|
||||
import reppy.cache
|
||||
import reppy.parser
|
||||
|
@ -19,7 +19,7 @@ limitations under the License.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import brozzler
|
||||
import brozzler_ia as brozzler
|
||||
import brozzler.browser
|
||||
import threading
|
||||
import time
|
||||
|
@ -19,7 +19,7 @@ limitations under the License.
|
||||
import logging
|
||||
import yt_dlp
|
||||
from yt_dlp.utils import match_filter_func
|
||||
import brozzler
|
||||
import brozzler_ia as brozzler
|
||||
import urllib.request
|
||||
import tempfile
|
||||
import urlcanon
|
||||
|
6
setup.py
6
setup.py
@ -33,7 +33,7 @@ def find_package_data(package):
|
||||
|
||||
|
||||
setuptools.setup(
|
||||
name="brozzler",
|
||||
name="brozzler_ia",
|
||||
version="1.6.1",
|
||||
description="Distributed web crawling with browsers",
|
||||
url="https://github.com/internetarchive/brozzler",
|
||||
@ -41,9 +41,9 @@ setuptools.setup(
|
||||
author_email="nlevitt@archive.org",
|
||||
long_description=open("README.rst", mode="rb").read().decode("UTF-8"),
|
||||
license="Apache License 2.0",
|
||||
packages=["brozzler", "brozzler.dashboard"],
|
||||
packages=["brozzler_ia", "brozzler.dashboard"],
|
||||
package_data={
|
||||
"brozzler": ["js-templates/*.js*", "behaviors.yaml", "job_schema.yaml"],
|
||||
"brozzler_ia": ["js-templates/*.js*", "behaviors.yaml", "job_schema.yaml"],
|
||||
"brozzler.dashboard": find_package_data("brozzler.dashboard"),
|
||||
},
|
||||
entry_points={
|
||||
|
@ -17,6 +17,7 @@ See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
"""
|
||||
|
||||
import brozzler_ia as brozzler
|
||||
import brozzler.cli
|
||||
import pkg_resources
|
||||
import pytest
|
||||
|
@ -26,7 +26,7 @@ import os
|
||||
import socket
|
||||
import doublethink
|
||||
import time
|
||||
import brozzler
|
||||
import brozzler_ia as brozzler
|
||||
import datetime
|
||||
import requests
|
||||
import subprocess
|
||||
|
@ -26,7 +26,7 @@ import time
|
||||
import doublethink
|
||||
import pytest
|
||||
|
||||
import brozzler
|
||||
import brozzler_ia as brozzler
|
||||
|
||||
args = argparse.Namespace()
|
||||
args.log_level = logging.INFO
|
||||
|
@ -21,7 +21,7 @@ import pytest
|
||||
import http.server
|
||||
import threading
|
||||
import os
|
||||
import brozzler
|
||||
import brozzler_ia as brozzler
|
||||
import brozzler.chrome
|
||||
import brozzler.ydl
|
||||
import logging
|
||||
|
Loading…
x
Reference in New Issue
Block a user