Public Access
emilandDevin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
a912c619ce
CI runners (Gitea Actions Docker containers) are 3-4x slower than local dev machines due to shared CPU, fewer cores, and container overhead. This caused false CI failures (37.98s on CI vs 4.76s local for the same 3085 tests). When the CI or GITEA_ACTIONS env var is set, both total and per-test limits are multiplied by CI_SCALE_FACTOR (default 4, overridable via DEVX_CI_SCALE_FACTOR). This keeps local budgets strict while preventing false failures on slower CI runners. Generated with [Devin](https://devin.ai) Co-Authored-By: Devin <158243242+devin-ai-integration[bot]@users.noreply.github.com>
350 lines
13 KiB
Python
350 lines
13 KiB
Python
"""Unit tests for devx.tools.check_test_speed."""
|
|
|
|
from unittest.mock import MagicMock, patch
|
|
|
|
import click
|
|
import pytest
|
|
from click.testing import CliRunner
|
|
|
|
from devx.tools.check_test_speed import (
|
|
DEFAULT_MAX_SECONDS,
|
|
DEFAULT_MAX_SINGLE_SECONDS,
|
|
TEST_COMMAND,
|
|
_ci_scale_limit,
|
|
check_per_test_speed,
|
|
check_speed,
|
|
cli,
|
|
parse_duration,
|
|
parse_per_test_durations,
|
|
run_tests,
|
|
)
|
|
|
|
|
|
class TestRunTests:
|
|
@patch("devx.tools.check_test_speed.subprocess.run")
|
|
def test_run_tests_returns_stdout_stderr(self, mock_run: MagicMock) -> None:
|
|
mock_run.return_value = MagicMock(stdout="out", stderr="err", returncode=0)
|
|
stdout, stderr = run_tests()
|
|
assert stdout == "out"
|
|
assert stderr == "err"
|
|
mock_run.assert_called_once()
|
|
call_kwargs = mock_run.call_args
|
|
assert call_kwargs.args[0] == TEST_COMMAND
|
|
assert call_kwargs.kwargs["capture_output"] is True
|
|
assert call_kwargs.kwargs["text"] is True
|
|
assert call_kwargs.kwargs["check"] is False
|
|
env = call_kwargs.kwargs["env"]
|
|
assert "--durations=0" in env["PYTEST_ADDOPTS"]
|
|
|
|
@patch("devx.tools.check_test_speed.subprocess.run")
|
|
def test_run_tests_preserves_existing_pytest_addopts(self, mock_run: MagicMock) -> None:
|
|
mock_run.return_value = MagicMock(stdout="out", stderr="err", returncode=0)
|
|
with patch.dict("os.environ", {"PYTEST_ADDOPTS": "-x"}, clear=False):
|
|
run_tests()
|
|
env = mock_run.call_args.kwargs["env"]
|
|
assert "--durations=0" in env["PYTEST_ADDOPTS"]
|
|
assert "-x" in env["PYTEST_ADDOPTS"]
|
|
|
|
|
|
class TestParseDuration:
|
|
def test_parses_valid_line(self) -> None:
|
|
assert parse_duration("234 passed in 0.70s") == 0.70
|
|
|
|
def test_parses_with_warnings(self) -> None:
|
|
assert parse_duration("293 passed, 1 warning in 0.45s") == 0.45
|
|
|
|
def test_parses_multiline_output(self) -> None:
|
|
output = "some header\n234 passed in 1.23s\nfooter"
|
|
assert parse_duration(output) == 1.23
|
|
|
|
def test_raises_when_no_timing_line(self) -> None:
|
|
with pytest.raises(click.ClickException) as exc:
|
|
parse_duration("no timing here")
|
|
assert "Could not parse" in str(exc.value)
|
|
|
|
|
|
class TestParsePerTestDurations:
|
|
def test_parses_call_lines(self) -> None:
|
|
output = "0.01s call tests/test_foo.py::test_bar\n"
|
|
durations = parse_per_test_durations(output)
|
|
assert len(durations) == 1
|
|
assert durations[0] == ("tests/test_foo.py::test_bar", 0.01)
|
|
|
|
def test_ignores_setup_and_teardown(self) -> None:
|
|
"""Only 'call' durations are counted — setup includes import overhead."""
|
|
output = (
|
|
"0.68s setup tests/test_foo.py::test_bar\n"
|
|
"0.01s call tests/test_foo.py::test_bar\n"
|
|
"0.00s teardown tests/test_foo.py::test_bar\n"
|
|
)
|
|
durations = parse_per_test_durations(output)
|
|
assert len(durations) == 1
|
|
assert durations[0] == ("tests/test_foo.py::test_bar", 0.01)
|
|
|
|
def test_sorted_slowest_first(self) -> None:
|
|
output = "0.01s call tests/test_a.py::test_slow\n0.50s call tests/test_b.py::test_fast\n"
|
|
durations = parse_per_test_durations(output)
|
|
assert durations[0][1] >= durations[1][1]
|
|
assert durations[0][1] == 0.50
|
|
|
|
def test_empty_output(self) -> None:
|
|
assert parse_per_test_durations("") == []
|
|
|
|
def test_ignores_non_duration_lines(self) -> None:
|
|
output = "Some random line\n234 passed in 0.70s\n"
|
|
assert parse_per_test_durations(output) == []
|
|
|
|
|
|
class TestCheckSpeed:
|
|
def test_under_budget_passes(self) -> None:
|
|
check_speed(1.0, 2.0) # should not raise
|
|
|
|
def test_exact_budget_passes(self) -> None:
|
|
check_speed(2.0, 2.0) # should not raise
|
|
|
|
def test_over_budget_raises(self) -> None:
|
|
with pytest.raises(click.ClickException) as exc:
|
|
check_speed(2.1, 2.0)
|
|
msg = str(exc.value)
|
|
assert "too slow" in msg.lower()
|
|
assert "2.10s" in msg
|
|
assert "max allowed: 2.0s" in msg
|
|
|
|
|
|
class TestCheckPerTestSpeed:
|
|
def test_no_violations_when_all_fast(self) -> None:
|
|
durations = [("test_a", 0.1), ("test_b", 0.2)]
|
|
assert check_per_test_speed(durations, 0.5) == []
|
|
|
|
def test_violation_when_test_exceeds_limit(self) -> None:
|
|
durations = [("test_slow", 0.6), ("test_fast", 0.1)]
|
|
violations = check_per_test_speed(durations, 0.5)
|
|
assert len(violations) == 1
|
|
assert "test_slow" in violations[0]
|
|
assert "0.60s" in violations[0]
|
|
|
|
def test_multiple_violations(self) -> None:
|
|
durations = [("test_a", 0.7), ("test_b", 0.6), ("test_c", 0.1)]
|
|
violations = check_per_test_speed(durations, 0.5)
|
|
assert len(violations) == 2
|
|
|
|
def test_exact_limit_passes(self) -> None:
|
|
durations = [("test_a", 0.5)]
|
|
assert check_per_test_speed(durations, 0.5) == []
|
|
|
|
def test_empty_durations(self) -> None:
|
|
assert check_per_test_speed([], 0.5) == []
|
|
|
|
|
|
def test_main_module_block() -> None:
|
|
import devx.tools.check_test_speed as cts
|
|
|
|
with patch.object(cts, "cli") as mock_cli:
|
|
with patch.object(cts, "__name__", "__main__"):
|
|
cts.cli([])
|
|
mock_cli.assert_called_once_with([])
|
|
|
|
|
|
class TestCiScaleLimit:
|
|
def test_no_scaling_when_not_ci(self) -> None:
|
|
with patch("devx.tools.check_test_speed._IS_CI", False):
|
|
assert _ci_scale_limit(10.0) == 10.0
|
|
assert _ci_scale_limit(0.5) == 0.5
|
|
|
|
def test_scales_when_ci(self) -> None:
|
|
with patch("devx.tools.check_test_speed._IS_CI", True):
|
|
with patch("devx.tools.check_test_speed.CI_SCALE_FACTOR", 4.0):
|
|
assert _ci_scale_limit(10.0) == 40.0
|
|
assert _ci_scale_limit(0.5) == 2.0
|
|
|
|
def test_custom_scale_factor(self) -> None:
|
|
with patch("devx.tools.check_test_speed._IS_CI", True):
|
|
with patch("devx.tools.check_test_speed.CI_SCALE_FACTOR", 2.5):
|
|
assert _ci_scale_limit(10.0) == 25.0
|
|
|
|
|
|
class TestMain:
|
|
@patch("devx.tools.check_test_speed._IS_CI", False)
|
|
@patch("devx.tools.check_test_speed.run_tests")
|
|
@patch("devx.tools.check_test_speed.parse_duration")
|
|
@patch("devx.tools.check_test_speed.check_speed")
|
|
@patch("devx.tools.check_test_speed.parse_per_test_durations")
|
|
@patch("devx.tools.check_test_speed.check_per_test_speed")
|
|
def test_successful_run(
|
|
self,
|
|
mock_check_per: MagicMock,
|
|
mock_parse_per: MagicMock,
|
|
mock_check: MagicMock,
|
|
mock_parse: MagicMock,
|
|
mock_run: MagicMock,
|
|
) -> None:
|
|
mock_run.return_value = ("stdout\n", "stderr\n")
|
|
mock_parse.return_value = 1.5
|
|
mock_parse_per.return_value = []
|
|
mock_check_per.return_value = []
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(cli, [])
|
|
assert result.exit_code == 0
|
|
assert "1.50s" in result.output
|
|
assert "under 10.0s limit" in result.output
|
|
mock_run.assert_called_once()
|
|
mock_parse.assert_called_once_with("stdout\n\nstderr\n")
|
|
mock_check.assert_called_once_with(1.5, DEFAULT_MAX_SECONDS)
|
|
mock_parse_per.assert_called_once()
|
|
mock_check_per.assert_called_once_with([], DEFAULT_MAX_SINGLE_SECONDS)
|
|
|
|
@patch("devx.tools.check_test_speed._IS_CI", False)
|
|
@patch("devx.tools.check_test_speed.run_tests")
|
|
@patch("devx.tools.check_test_speed.parse_duration")
|
|
def test_slow_total_exits(
|
|
self,
|
|
mock_parse: MagicMock,
|
|
mock_run: MagicMock,
|
|
) -> None:
|
|
mock_run.return_value = ("out\n", "err\n")
|
|
mock_parse.return_value = 15.0
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(cli, [])
|
|
assert result.exit_code == 1
|
|
assert "too slow" in result.output.lower()
|
|
|
|
@patch("devx.tools.check_test_speed._IS_CI", False)
|
|
@patch("devx.tools.check_test_speed.run_tests")
|
|
@patch("devx.tools.check_test_speed.parse_duration")
|
|
@patch("devx.tools.check_test_speed.check_speed")
|
|
@patch("devx.tools.check_test_speed.parse_per_test_durations")
|
|
@patch("devx.tools.check_test_speed.check_per_test_speed")
|
|
def test_per_test_violation_exits(
|
|
self,
|
|
mock_check_per: MagicMock,
|
|
mock_parse_per: MagicMock,
|
|
mock_check: MagicMock,
|
|
mock_parse: MagicMock,
|
|
mock_run: MagicMock,
|
|
) -> None:
|
|
mock_run.return_value = ("out\n", "err\n")
|
|
mock_parse.return_value = 3.0
|
|
mock_parse_per.return_value = [("test_slow", 0.8)]
|
|
mock_check_per.return_value = ["Test 'test_slow' took 0.80s (limit: 0.5s)."]
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(cli, [])
|
|
assert result.exit_code == 1
|
|
assert "Per-test speed check FAILED" in result.output
|
|
assert "test_slow" in result.output
|
|
|
|
@patch("devx.tools.check_test_speed._IS_CI", False)
|
|
@patch("devx.tools.check_test_speed.run_tests")
|
|
def test_parse_failure_exits(
|
|
self,
|
|
mock_run: MagicMock,
|
|
) -> None:
|
|
mock_run.return_value = ("bad output\n", "")
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(cli, [])
|
|
assert result.exit_code == 1
|
|
assert "Could not parse" in result.output
|
|
|
|
@patch("devx.tools.check_test_speed._IS_CI", False)
|
|
@patch("devx.tools.check_test_speed.run_tests")
|
|
@patch("devx.tools.check_test_speed.parse_duration")
|
|
@patch("devx.tools.check_test_speed.check_speed")
|
|
@patch("devx.tools.check_test_speed.parse_per_test_durations")
|
|
@patch("devx.tools.check_test_speed.check_per_test_speed")
|
|
def test_custom_max_seconds(
|
|
self,
|
|
mock_check_per: MagicMock,
|
|
mock_parse_per: MagicMock,
|
|
mock_check: MagicMock,
|
|
mock_parse: MagicMock,
|
|
mock_run: MagicMock,
|
|
) -> None:
|
|
mock_run.return_value = ("out\n", "err\n")
|
|
mock_parse.return_value = 0.5
|
|
mock_parse_per.return_value = []
|
|
mock_check_per.return_value = []
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(cli, ["--max-seconds", "1.5"])
|
|
assert result.exit_code == 0
|
|
mock_check.assert_called_once_with(0.5, 1.5)
|
|
|
|
@patch("devx.tools.check_test_speed._IS_CI", False)
|
|
@patch("devx.tools.check_test_speed.run_tests")
|
|
@patch("devx.tools.check_test_speed.parse_duration")
|
|
@patch("devx.tools.check_test_speed.check_speed")
|
|
@patch("devx.tools.check_test_speed.parse_per_test_durations")
|
|
@patch("devx.tools.check_test_speed.check_per_test_speed")
|
|
def test_disable_per_test_check(
|
|
self,
|
|
mock_check_per: MagicMock,
|
|
mock_parse_per: MagicMock,
|
|
mock_check: MagicMock,
|
|
mock_parse: MagicMock,
|
|
mock_run: MagicMock,
|
|
) -> None:
|
|
mock_run.return_value = ("out\n", "err\n")
|
|
mock_parse.return_value = 1.0
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(cli, ["--max-single-seconds", "0"])
|
|
assert result.exit_code == 0
|
|
mock_parse_per.assert_not_called()
|
|
mock_check_per.assert_not_called()
|
|
|
|
@patch("devx.tools.check_test_speed._IS_CI", False)
|
|
@patch("devx.tools.check_test_speed.run_tests")
|
|
@patch("devx.tools.check_test_speed.parse_duration")
|
|
@patch("devx.tools.check_test_speed.check_speed")
|
|
@patch("devx.tools.check_test_speed.parse_per_test_durations")
|
|
@patch("devx.tools.check_test_speed.check_per_test_speed")
|
|
def test_custom_max_single_seconds(
|
|
self,
|
|
mock_check_per: MagicMock,
|
|
mock_parse_per: MagicMock,
|
|
mock_check: MagicMock,
|
|
mock_parse: MagicMock,
|
|
mock_run: MagicMock,
|
|
) -> None:
|
|
mock_run.return_value = ("out\n", "err\n")
|
|
mock_parse.return_value = 1.0
|
|
mock_parse_per.return_value = []
|
|
mock_check_per.return_value = []
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(cli, ["--max-single-seconds", "1.0"])
|
|
assert result.exit_code == 0
|
|
mock_check_per.assert_called_once_with([], 1.0)
|
|
|
|
@patch("devx.tools.check_test_speed._IS_CI", True)
|
|
@patch("devx.tools.check_test_speed.CI_SCALE_FACTOR", 4.0)
|
|
@patch("devx.tools.check_test_speed.run_tests")
|
|
@patch("devx.tools.check_test_speed.parse_duration")
|
|
@patch("devx.tools.check_test_speed.check_speed")
|
|
@patch("devx.tools.check_test_speed.parse_per_test_durations")
|
|
@patch("devx.tools.check_test_speed.check_per_test_speed")
|
|
def test_ci_scales_limits(
|
|
self,
|
|
mock_check_per: MagicMock,
|
|
mock_parse_per: MagicMock,
|
|
mock_check: MagicMock,
|
|
mock_parse: MagicMock,
|
|
mock_run: MagicMock,
|
|
) -> None:
|
|
mock_run.return_value = ("out\n", "err\n")
|
|
mock_parse.return_value = 30.0 # would fail local (10s) but pass CI (40s)
|
|
mock_parse_per.return_value = []
|
|
mock_check_per.return_value = []
|
|
|
|
runner = CliRunner()
|
|
result = runner.invoke(cli, [])
|
|
assert result.exit_code == 0
|
|
assert "CI environment detected" in result.output
|
|
assert "scaling limits by 4.0x" in result.output
|
|
# check_speed called with scaled limit
|
|
mock_check.assert_called_once_with(30.0, 40.0)
|
|
mock_check_per.assert_called_once_with([], 2.0)
|