mirror of
https://github.com/internetarchive/brozzler.git
synced 2025-07-22 22:40:47 -04:00
test: add CI (#329)
This adds two CI runs: a quick one that happens for every pull request and merge to master, and a longer one that happens daily. This also adds a new installation group to setup.py because the `easy` group isn't currently installable, and some of the dependencies specified there need to be present for the tests to run.
This commit is contained in:
parent
968cac0e10
commit
ab8970ff3e
8 changed files with 106 additions and 7 deletions
22
.github/workflows/daily.yaml
vendored
Normal file
22
.github/workflows/daily.yaml
vendored
Normal file
|
@ -0,0 +1,22 @@
|
||||||
|
name: Full test suite
|
||||||
|
|
||||||
|
on:
|
||||||
|
schedule:
|
||||||
|
- cron: "0 6 * * *" # 10PM Pacific daily
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
name: Run tests
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: '3.12'
|
||||||
|
|
||||||
|
- uses: ./.github/workflows/setup
|
||||||
|
|
||||||
|
- name: Run tests
|
||||||
|
run: |
|
||||||
|
py.test --tb=native --verbose tests
|
27
.github/workflows/setup/action.yml
vendored
Normal file
27
.github/workflows/setup/action.yml
vendored
Normal file
|
@ -0,0 +1,27 @@
|
||||||
|
name: Test setup
|
||||||
|
|
||||||
|
runs:
|
||||||
|
using: composite
|
||||||
|
steps:
|
||||||
|
- name: Install apt dependencies
|
||||||
|
run: |
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install libjpeg-dev chromium-browser
|
||||||
|
shell: bash
|
||||||
|
|
||||||
|
- name: Set up rethinkdb
|
||||||
|
run: |
|
||||||
|
wget -qO- https://download.rethinkdb.com/repository/raw/pubkey.gpg | sudo gpg --dearmor -o /usr/share/keyrings/rethinkdb-archive-keyrings.gpg
|
||||||
|
echo "deb [signed-by=/usr/share/keyrings/rethinkdb-archive-keyrings.gpg] https://download.rethinkdb.com/repository/ubuntu-$(lsb_release -cs) $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/rethinkdb.list
|
||||||
|
sudo apt-get update
|
||||||
|
sudo apt-get install rethinkdb
|
||||||
|
sudo cp /etc/rethinkdb/default.conf.sample /etc/rethinkdb/instances.d/instance1.conf
|
||||||
|
sudo /etc/init.d/rethinkdb restart
|
||||||
|
shell: bash
|
||||||
|
|
||||||
|
- name: Install pip dependencies
|
||||||
|
run: |
|
||||||
|
pip install .[rethinkdb,warcprox,yt-dlp]
|
||||||
|
# setuptools required by rethinkdb==2.4.9
|
||||||
|
pip install pytest setuptools
|
||||||
|
shell: bash
|
31
.github/workflows/tests.yml
vendored
Normal file
31
.github/workflows/tests.yml
vendored
Normal file
|
@ -0,0 +1,31 @@
|
||||||
|
name: Tests
|
||||||
|
|
||||||
|
on:
|
||||||
|
push:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
- master
|
||||||
|
pull_request:
|
||||||
|
branches:
|
||||||
|
- main
|
||||||
|
- master
|
||||||
|
|
||||||
|
jobs:
|
||||||
|
test:
|
||||||
|
name: Run tests
|
||||||
|
runs-on: ubuntu-latest
|
||||||
|
strategy:
|
||||||
|
matrix:
|
||||||
|
version: ['3.8', '3.12']
|
||||||
|
steps:
|
||||||
|
- uses: actions/checkout@v4
|
||||||
|
|
||||||
|
- uses: actions/setup-python@v5
|
||||||
|
with:
|
||||||
|
python-version: ${{ matrix.version }}
|
||||||
|
|
||||||
|
- uses: ./.github/workflows/setup
|
||||||
|
|
||||||
|
- name: Run tests
|
||||||
|
run: |
|
||||||
|
py.test --tb=native --verbose tests/test_cli.py tests/test_units.py
|
|
@ -19,9 +19,9 @@ limitations under the License.
|
||||||
|
|
||||||
import logging
|
import logging
|
||||||
import structlog
|
import structlog
|
||||||
from pkg_resources import get_distribution as _get_distribution
|
from importlib.metadata import version as _version
|
||||||
|
|
||||||
__version__ = _get_distribution("brozzler").version
|
__version__ = _version("brozzler")
|
||||||
|
|
||||||
|
|
||||||
class ShutdownRequested(Exception):
|
class ShutdownRequested(Exception):
|
||||||
|
|
|
@ -605,7 +605,7 @@ class BrozzlerWorker:
|
||||||
if page:
|
if page:
|
||||||
# Calculate backoff in seconds based on number of failed attempts.
|
# Calculate backoff in seconds based on number of failed attempts.
|
||||||
# Minimum of 60, max of 135 giving delays of 60, 90, 135, 135...
|
# Minimum of 60, max of 135 giving delays of 60, 90, 135, 135...
|
||||||
retry_delay = min(135, 60 * (1.5**page.failed_attempts))
|
retry_delay = min(135, 60 * (1.5 ** (page.failed_attempts or 0)))
|
||||||
page.retry_after = doublethink.utcnow() + datetime.timedelta(
|
page.retry_after = doublethink.utcnow() + datetime.timedelta(
|
||||||
seconds=retry_delay
|
seconds=retry_delay
|
||||||
)
|
)
|
||||||
|
|
3
setup.py
3
setup.py
|
@ -81,6 +81,9 @@ setuptools.setup(
|
||||||
extras_require={
|
extras_require={
|
||||||
"yt-dlp": ["yt-dlp>=2024.7.25"],
|
"yt-dlp": ["yt-dlp>=2024.7.25"],
|
||||||
"dashboard": ["flask>=1.0", "gunicorn>=19.8.1"],
|
"dashboard": ["flask>=1.0", "gunicorn>=19.8.1"],
|
||||||
|
"warcprox": [
|
||||||
|
"warcprox>=2.4.31",
|
||||||
|
],
|
||||||
"rethinkdb": [
|
"rethinkdb": [
|
||||||
"rethinkdb==2.4.9",
|
"rethinkdb==2.4.9",
|
||||||
"doublethink==0.4.9",
|
"doublethink==0.4.9",
|
||||||
|
|
|
@ -18,14 +18,24 @@ limitations under the License.
|
||||||
"""
|
"""
|
||||||
|
|
||||||
import brozzler.cli
|
import brozzler.cli
|
||||||
import pkg_resources
|
import importlib.metadata
|
||||||
import pytest
|
import pytest
|
||||||
import subprocess
|
import subprocess
|
||||||
import doublethink
|
import doublethink
|
||||||
|
|
||||||
|
|
||||||
|
def console_scripts():
|
||||||
|
# We do a dict comprehension here because the select filters aren't
|
||||||
|
# available until Python 3.10's importlib.
|
||||||
|
return {
|
||||||
|
ep.name: ep
|
||||||
|
for ep in importlib.metadata.distribution("brozzler").entry_points
|
||||||
|
if ep.group == "console_scripts"
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
def cli_commands():
|
def cli_commands():
|
||||||
commands = set(pkg_resources.get_entry_map("brozzler")["console_scripts"].keys())
|
commands = set(console_scripts().keys())
|
||||||
commands.remove("brozzler-wayback")
|
commands.remove("brozzler-wayback")
|
||||||
try:
|
try:
|
||||||
import gunicorn
|
import gunicorn
|
||||||
|
@ -40,8 +50,8 @@ def cli_commands():
|
||||||
|
|
||||||
@pytest.mark.parametrize("cmd", cli_commands())
|
@pytest.mark.parametrize("cmd", cli_commands())
|
||||||
def test_call_entrypoint(capsys, cmd):
|
def test_call_entrypoint(capsys, cmd):
|
||||||
entrypoint = pkg_resources.get_entry_map("brozzler")["console_scripts"][cmd]
|
entrypoint = console_scripts()[cmd]
|
||||||
callable = entrypoint.resolve()
|
callable = entrypoint.load()
|
||||||
with pytest.raises(SystemExit):
|
with pytest.raises(SystemExit):
|
||||||
callable(["/whatever/bin/%s" % cmd, "--version"])
|
callable(["/whatever/bin/%s" % cmd, "--version"])
|
||||||
out, err = capsys.readouterr()
|
out, err = capsys.readouterr()
|
||||||
|
|
|
@ -260,6 +260,9 @@ blocks:
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
||||||
|
# Some changes to the brozzler ydl interface not represented in this test
|
||||||
|
# https://github.com/internetarchive/brozzler/issues/330
|
||||||
|
@pytest.mark.xfail
|
||||||
def test_proxy_down():
|
def test_proxy_down():
|
||||||
"""
|
"""
|
||||||
Test all fetching scenarios raise `brozzler.ProxyError` when proxy is down.
|
Test all fetching scenarios raise `brozzler.ProxyError` when proxy is down.
|
||||||
|
@ -471,6 +474,9 @@ def test_thread_raise_second_with_block():
|
||||||
assert isinstance(thread_caught_exception, Exception2)
|
assert isinstance(thread_caught_exception, Exception2)
|
||||||
|
|
||||||
|
|
||||||
|
# brozzler.ydl.YoutubeDLSpy is missing
|
||||||
|
# https://github.com/internetarchive/brozzler/issues/330
|
||||||
|
@pytest.mark.xfail
|
||||||
def test_needs_browsing():
|
def test_needs_browsing():
|
||||||
# only one test case here right now, which exposed a bug
|
# only one test case here right now, which exposed a bug
|
||||||
|
|
||||||
|
|
Loading…
Add table
Add a link
Reference in a new issue