Git-Mirrors/brozzler
1
0
Fork 0
mirror of https://github.com/internetarchive/brozzler.git synced 2025-07-31 02:28:42 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

remove unnecessary imports, use find_spec

Browse source 
This was flagged by ruff check - if we just want to find out if a
package is available, and don't need to actually import it, we can
use importlib.util.find_spec() to resolve it. This can lead to a
moderate speedup too, since the import might be slow.
This commit is contained in:
Misty De Méo 2025-07-18 14:14:32 -07:00 • committed by Misty De Méo
parent 85ae741b5d
commit 0f0ae4fbc3
3 changed files with 9 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

10
brozzler/__init__.py
View file

@ -18,6 +18,7 @@ limitations under the License.
""" """
import datetime import datetime
import importlib.util
import logging import logging
import threading import threading
from importlib.metadata import version as _version from importlib.metadata import version as _version
@ -414,12 +415,7 @@ __all__ = [
"suggest_default_chrome_exe", "suggest_default_chrome_exe",
] ]
# TODO try using importlib.util.find_spec to test for dependency presence if importlib.util.find_spec("doublethink"):
# rather than try/except on import.
# See https://docs.astral.sh/ruff/rules/unused-import/#example
try:
import doublethink # noqa: F401
# All of these imports use doublethink for real and are unsafe # All of these imports use doublethink for real and are unsafe
# to do if doublethink is unavailable. # to do if doublethink is unavailable.
from brozzler.frontier import RethinkDbFrontier # noqa: F401 from brozzler.frontier import RethinkDbFrontier # noqa: F401
@ -447,8 +443,6 @@ try:
"InvalidJobConf", "InvalidJobConf",
] ]
) )
except ImportError:
pass
# we could make this configurable if there's a good reason # we could make this configurable if there's a good reason
MAX_PAGE_FAILURES = 3 MAX_PAGE_FAILURES = 3

9
brozzler/worker.py
View file

@ -19,6 +19,7 @@ limitations under the License.
""" """
import datetime import datetime
import importlib.util
import io import io
import json import json
import socket import socket
@ -100,14 +101,8 @@ class BrozzlerWorker:
if worker_id is not None: if worker_id is not None:
self.logger = self.logger.bind(worker_id=worker_id) self.logger = self.logger.bind(worker_id=worker_id)
# TODO try using importlib.util.find_spec to test for dependency
# presence rather than try/except on import.
# See https://docs.astral.sh/ruff/rules/unused-import/#example
# We definitely shouldn't ytdlp if the optional extra is missing # We definitely shouldn't ytdlp if the optional extra is missing
try: if not importlib.util.find_spec("yt_dlp"):
import yt_dlp # noqa: F401
except ImportError:
self.logger.info( self.logger.info(
"optional yt-dlp extra not installed; setting skip_youtube_dl to True" "optional yt-dlp extra not installed; setting skip_youtube_dl to True"
) )

11
tests/test_cli.py
View file

@ -18,6 +18,7 @@ limitations under the License.
""" """
import importlib.metadata import importlib.metadata
import importlib.util
import os import os
import subprocess import subprocess
@ -47,14 +48,12 @@ def console_scripts():
def cli_commands(): def cli_commands():
commands = set(console_scripts().keys()) commands = set(console_scripts().keys())
commands.remove("brozzler-wayback") commands.remove("brozzler-wayback")
try: if not importlib.util.find_spec("gunicorn"):
import gunicorn # noqa: F401
except ImportError:
commands.remove("brozzler-dashboard") commands.remove("brozzler-dashboard")
try:
import pywb # noqa: F401 if not importlib.util.find_spec("pywb"):
except ImportError:
commands.remove("brozzler-easy") commands.remove("brozzler-easy")
return commands return commands