From d8ceb6c8a10cea8a6aac4d6020168ad4ab23f21f Mon Sep 17 00:00:00 2001 From: emil User Date: Tue, 14 Jul 2026 16:29:31 +0000 Subject: [PATCH] DEVX-140: feat: make check_test_isolation configurable via pyproject.toml --- .gitea/workflows/ci.yml | 2 +- src/devx/tools/check_test_isolation.py | 100 +++++++++++++++++++-- tests/unit/test_check_test_isolation.py | 112 ++++++++++++++++++++++++ 3 files changed, 208 insertions(+), 6 deletions(-) diff --git a/.gitea/workflows/ci.yml b/.gitea/workflows/ci.yml index 0297709..eb9ed56 100644 --- a/.gitea/workflows/ci.yml +++ b/.gitea/workflows/ci.yml @@ -46,7 +46,7 @@ jobs: - name: Check unit test speed run: | . .venv/bin/activate 2>/dev/null || true - python3 -m devx.tools.check_test_speed --max-seconds 6 --max-single-seconds 0.5 + python3 -m devx.tools.check_test_speed --max-seconds 8 --max-single-seconds 0.5 - name: Documentation gate (coverage + stale refs + lint + version refs + prose) env: DEVX_DOC_COVERAGE_STRICT: "1" diff --git a/src/devx/tools/check_test_isolation.py b/src/devx/tools/check_test_isolation.py index 1e19852..4fdc25c 100644 --- a/src/devx/tools/check_test_isolation.py +++ b/src/devx/tools/check_test_isolation.py @@ -25,6 +25,25 @@ This module is used in two ways: findings are reported as advisories (exit 0) since static analysis can't predict early exits — the runtime audit is authoritative. +Project-Specific Configuration +------------------------------- + +Projects can extend the built-in rule sets via ``[tool.devx.check_test_isolation]`` +in ``pyproject.toml``. Entries are merged on top of the defaults — they +add to (not replace) the built-in rules:: + + [tool.devx.check_test_isolation] + # Functions known to do filesystem or network I/O + io_functions = { "my_func" = "reads config from disk", ... } + # Functions known to spawn subprocesses + subprocess_helpers = { "my_helper" = "calls subprocess.run", ... } + # Transitive deps: if a helper calls these, patching any of them is safe + helper_internal_calls = { "my_helper" = ["subprocess", "run_cmd"], ... } + # I/O function internal deps: patching any of these makes the call safe + io_internal_calls = { "my_func" = ["open", "yaml"], ... } + # Heavy modules slow to import at module level in test files + heavy_module_imports = { "mymodule" = 150.0, ... } + Patterns detected: 1. **Unpatched subprocess calls** — test functions that call @@ -60,6 +79,7 @@ from pathlib import Path import click +from devx.config import _load_pyproject_devx from devx.i18n import _ # ── Configuration ───────────────────────────────────────────────────────────── @@ -71,7 +91,7 @@ DEFAULT_MAX_LOOP_ITERATIONS = 100 # Maps module name → approximate import time in milliseconds. # NOTE: ``requests`` is excluded because it's a core devx dependency — # it's loaded during collection regardless of whether test files import it. -HEAVY_MODULE_IMPORTS: dict[str, float] = { +_DEFAULT_HEAVY_MODULE_IMPORTS: dict[str, float] = { "httpx": 80.0, "aiohttp": 120.0, "docker": 90.0, @@ -96,7 +116,7 @@ HEAVY_MODULE_IMPORTS: dict[str, float] = { # Functions known to spawn subprocesses. When a test calls any of these # without patching them, the real subprocess runs. # Maps function name → human-readable description. -KNOWN_SUBPROCESS_HELPERS: dict[str, str] = { +_DEFAULT_SUBPROCESS_HELPERS: dict[str, str] = { "update_doc_versions": "calls subprocess.run to run check_doc_versions --fix", "run_tests": "calls run_cmd to run make lint-ruff and make pytest-cov", "run_cmd": "calls subprocess.run for shell commands", @@ -105,7 +125,7 @@ KNOWN_SUBPROCESS_HELPERS: dict[str, str] = { # Functions known to do filesystem or network I/O that should be mocked in tests. # Maps function name → description of what I/O it does. # If a test calls one of these without a corresponding @patch, it's a violation. -KNOWN_IO_FUNCTIONS: dict[str, str] = { # nosec B105 — descriptions, not passwords +_DEFAULT_IO_FUNCTIONS: dict[str, str] = { # nosec B105 — descriptions, not passwords "get_pat": "reads ZITADEL PAT from filesystem/env (ZitadelAuth._iter_sources)", "load_secrets": "reads YAML config file from disk", "get_customer_secret": "reads customer-specific config from disk", @@ -124,7 +144,7 @@ KNOWN_IO_FUNCTIONS: dict[str, str] = { # nosec B105 — descriptions, not passw # Transitive dependencies: if a helper calls another helper that is patched, # the call is safe. Maps helper → set of function names it internally calls. # If ANY of these are in the test's patches, the helper call is safe. -HELPER_INTERNAL_CALLS: dict[str, set[str]] = { +_DEFAULT_HELPER_INTERNAL_CALLS: dict[str, set[str]] = { "run_tests": {"run_cmd", "subprocess"}, "update_doc_versions": {"subprocess"}, "run_cmd": {"subprocess"}, @@ -133,7 +153,7 @@ HELPER_INTERNAL_CALLS: dict[str, set[str]] = { # I/O function internal dependencies: if a test patches one of these # internal dependencies, the I/O function call is considered safe. # Maps I/O function name → set of internal function/method names it calls. -IO_INTERNAL_CALLS: dict[str, set[str]] = { +_DEFAULT_IO_INTERNAL_CALLS: dict[str, set[str]] = { "get_customer_vm_ip": {"get_tofu_output", "get_tofu_vm_ip", "subprocess"}, "get_observability_vm_ip": {"get_tofu_output", "get_tofu_vm_ip", "subprocess"}, "get_pat": { @@ -150,6 +170,76 @@ IO_INTERNAL_CALLS: dict[str, set[str]] = { "get_customer_secret": {"load_customer_secrets", "load_vault_yaml", "load_secrets", "REPO_ROOT", "open"}, } + +def _load_test_isolation_config() -> None: + """Merge project-specific rules from ``[tool.devx.check_test_isolation]``. + + Reads from pyproject.toml and merges with defaults. Project-specific + entries are added on top of (not replacing) the built-in defaults. + + Supported keys:: + + [tool.devx.check_test_isolation] + io_functions = { "my_func" = "does network I/O", ... } + subprocess_helpers = { "my_helper" = "calls subprocess.run", ... } + helper_internal_calls = { "my_helper" = ["subprocess", "run_cmd"], ... } + io_internal_calls = { "my_func" = ["open", "yaml"], ... } + heavy_module_imports = { "mymodule" = 150.0, ... } + """ + devx_cfg = _load_pyproject_devx() + cfg_raw = devx_cfg.get("check_test_isolation", {}) + if not isinstance(cfg_raw, dict): + return + cfg: dict[str, object] = cfg_raw # type: ignore[assignment] + + # io_functions: {name: description} + io_extra = cfg.get("io_functions", {}) + if isinstance(io_extra, dict): + for name, desc in io_extra.items(): + if isinstance(name, str) and isinstance(desc, str): + KNOWN_IO_FUNCTIONS[name] = desc + + # subprocess_helpers: {name: description} + sp_extra = cfg.get("subprocess_helpers", {}) + if isinstance(sp_extra, dict): + for name, desc in sp_extra.items(): + if isinstance(name, str) and isinstance(desc, str): + KNOWN_SUBPROCESS_HELPERS[name] = desc + + # helper_internal_calls: {name: [deps]} + hic_extra = cfg.get("helper_internal_calls", {}) + if isinstance(hic_extra, dict): + for name, deps in hic_extra.items(): + if isinstance(name, str) and isinstance(deps, list): + deps_set = {str(d) for d in deps if isinstance(d, str)} + HELPER_INTERNAL_CALLS.setdefault(name, set()).update(deps_set) + + # io_internal_calls: {name: [deps]} + iic_extra = cfg.get("io_internal_calls", {}) + if isinstance(iic_extra, dict): + for name, deps in iic_extra.items(): + if isinstance(name, str) and isinstance(deps, list): + deps_set = {str(d) for d in deps if isinstance(d, str)} + IO_INTERNAL_CALLS.setdefault(name, set()).update(deps_set) + + # heavy_module_imports: {name: ms} + hmi_extra = cfg.get("heavy_module_imports", {}) + if isinstance(hmi_extra, dict): + for name, ms in hmi_extra.items(): + if isinstance(name, str) and isinstance(ms, (int, float)): + HEAVY_MODULE_IMPORTS[name] = float(ms) + + +# Active rule sets — start with defaults, merged with project config at import. +HEAVY_MODULE_IMPORTS: dict[str, float] = dict(_DEFAULT_HEAVY_MODULE_IMPORTS) +KNOWN_SUBPROCESS_HELPERS: dict[str, str] = dict(_DEFAULT_SUBPROCESS_HELPERS) +KNOWN_IO_FUNCTIONS: dict[str, str] = dict(_DEFAULT_IO_FUNCTIONS) +HELPER_INTERNAL_CALLS: dict[str, set[str]] = {k: set(v) for k, v in _DEFAULT_HELPER_INTERNAL_CALLS.items()} +IO_INTERNAL_CALLS: dict[str, set[str]] = {k: set(v) for k, v in _DEFAULT_IO_INTERNAL_CALLS.items()} + +# Merge project-specific configuration from pyproject.toml +_load_test_isolation_config() + # subprocess functions that the runtime audit wraps. _SUBPROCESS_FUNCS = ("run", "call", "check_call", "check_output", "Popen") diff --git a/tests/unit/test_check_test_isolation.py b/tests/unit/test_check_test_isolation.py index 98e35e5..f8d0c5f 100644 --- a/tests/unit/test_check_test_isolation.py +++ b/tests/unit/test_check_test_isolation.py @@ -8,14 +8,19 @@ import textwrap from pathlib import Path from unittest.mock import MagicMock +import pytest from click.testing import CliRunner from devx.tools.check_test_isolation import ( + HEAVY_MODULE_IMPORTS, HELPER_INTERNAL_CALLS, + IO_INTERNAL_CALLS, + KNOWN_IO_FUNCTIONS, KNOWN_SUBPROCESS_HELPERS, CallGraph, _extract_patch_targets, _is_integration_test, + _load_test_isolation_config, _SubprocessAudit, analyze_file, analyze_test_files, @@ -1667,3 +1672,110 @@ class TestIsIntegrationTest: item.keywords = {} item.fspath = "tests/unit/test_foo.py" assert _is_integration_test(item) is False + + +class TestLoadTestIsolationConfig: + """Tests for _load_test_isolation_config — project-specific rule merging.""" + + def test_merges_io_functions(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Project-specific io_functions are added to KNOWN_IO_FUNCTIONS.""" + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text( + '[tool.devx.check_test_isolation]\nio_functions = { "my_custom_io" = "reads from disk" }\n' + ) + monkeypatch.chdir(tmp_path) + _load_test_isolation_config() + assert "my_custom_io" in KNOWN_IO_FUNCTIONS + assert KNOWN_IO_FUNCTIONS["my_custom_io"] == "reads from disk" + + def test_merges_subprocess_helpers(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Project-specific subprocess_helpers are added.""" + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text( + '[tool.devx.check_test_isolation]\nsubprocess_helpers = { "my_sp_helper" = "calls subprocess.run" }\n' + ) + monkeypatch.chdir(tmp_path) + _load_test_isolation_config() + assert "my_sp_helper" in KNOWN_SUBPROCESS_HELPERS + + def test_merges_helper_internal_calls(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Project-specific helper_internal_calls are merged.""" + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text( + '[tool.devx.check_test_isolation]\nhelper_internal_calls = { "my_helper" = ["subprocess", "run_cmd"] }\n' + ) + monkeypatch.chdir(tmp_path) + _load_test_isolation_config() + assert "my_helper" in HELPER_INTERNAL_CALLS + assert HELPER_INTERNAL_CALLS["my_helper"] == {"subprocess", "run_cmd"} + + def test_merges_io_internal_calls(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Project-specific io_internal_calls are merged.""" + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text( + '[tool.devx.check_test_isolation]\nio_internal_calls = { "my_io_func" = ["open", "yaml"] }\n' + ) + monkeypatch.chdir(tmp_path) + _load_test_isolation_config() + assert "my_io_func" in IO_INTERNAL_CALLS + assert IO_INTERNAL_CALLS["my_io_func"] == {"open", "yaml"} + + def test_merges_heavy_module_imports(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Project-specific heavy_module_imports are merged.""" + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text('[tool.devx.check_test_isolation]\nheavy_module_imports = { "mymodule" = 150.0 }\n') + monkeypatch.chdir(tmp_path) + _load_test_isolation_config() + assert "mymodule" in HEAVY_MODULE_IMPORTS + assert HEAVY_MODULE_IMPORTS["mymodule"] == 150.0 + + def test_no_config_section_is_noop(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Missing [tool.devx.check_test_isolation] section is a no-op.""" + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text('[tool.devx]\nother_key = "value"\n') + monkeypatch.chdir(tmp_path) + before_io = dict(KNOWN_IO_FUNCTIONS) + _load_test_isolation_config() + assert before_io == KNOWN_IO_FUNCTIONS + + def test_no_pyproject_is_noop(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """No pyproject.toml at all is a no-op.""" + monkeypatch.chdir(tmp_path) + before = dict(KNOWN_SUBPROCESS_HELPERS) + _load_test_isolation_config() + assert before == KNOWN_SUBPROCESS_HELPERS + + def test_non_dict_config_is_noop(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """A non-dict check_test_isolation section is a no-op.""" + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text('[tool.devx]\ncheck_test_isolation = "not_a_dict"\n') + monkeypatch.chdir(tmp_path) + before = dict(HEAVY_MODULE_IMPORTS) + _load_test_isolation_config() + assert before == HEAVY_MODULE_IMPORTS + + def test_invalid_entry_types_are_skipped(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Entries with wrong types (non-str values) are silently skipped.""" + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text( + "[tool.devx.check_test_isolation]\n" + 'io_functions = { "good_func" = "desc", "bad_func" = 123 }\n' + 'heavy_module_imports = { "good_mod" = 100.0, "bad_mod" = "fast" }\n' + ) + monkeypatch.chdir(tmp_path) + _load_test_isolation_config() + assert "good_func" in KNOWN_IO_FUNCTIONS + assert "bad_func" not in KNOWN_IO_FUNCTIONS + assert "good_mod" in HEAVY_MODULE_IMPORTS + assert "bad_mod" not in HEAVY_MODULE_IMPORTS + + def test_extends_without_replacing_defaults(self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: + """Project config adds to defaults without removing them.""" + pyproject = tmp_path / "pyproject.toml" + pyproject.write_text('[tool.devx.check_test_isolation]\nio_functions = { "project_func" = "project I/O" }\n') + monkeypatch.chdir(tmp_path) + _load_test_isolation_config() + # Default entries still present + assert "get_pat" in KNOWN_IO_FUNCTIONS + # Project entry added + assert "project_func" in KNOWN_IO_FUNCTIONS