Public Access
919 lines
30 KiB
Python
919 lines
30 KiB
Python
"""Unit tests for devx.tools.check_test_isolation."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import textwrap
|
|
from pathlib import Path
|
|
|
|
from click.testing import CliRunner
|
|
|
|
from devx.tools.check_test_isolation import (
|
|
HELPER_INTERNAL_CALLS,
|
|
KNOWN_SUBPROCESS_HELPERS,
|
|
analyze_file,
|
|
analyze_test_files,
|
|
cli,
|
|
find_test_files,
|
|
)
|
|
|
|
|
|
def _write_test_file(tmp_path: Path, content: str) -> Path:
|
|
"""Write content to a test file and return the path."""
|
|
file = tmp_path / "test_example.py"
|
|
file.write_text(textwrap.dedent(content))
|
|
return file
|
|
|
|
|
|
class TestFindTestFiles:
|
|
def test_finds_test_files_in_directory(self, tmp_path: Path) -> None:
|
|
(tmp_path / "test_foo.py").touch()
|
|
(tmp_path / "test_bar.py").touch()
|
|
(tmp_path / "helper.py").touch()
|
|
result = find_test_files(tmp_path)
|
|
assert len(result) == 2
|
|
assert all(f.name.startswith("test_") for f in result)
|
|
|
|
def test_single_file(self, tmp_path: Path) -> None:
|
|
file = tmp_path / "test_single.py"
|
|
file.touch()
|
|
result = find_test_files(file)
|
|
assert result == [file]
|
|
|
|
def test_non_python_file(self, tmp_path: Path) -> None:
|
|
file = tmp_path / "test_readme.md"
|
|
file.touch()
|
|
result = find_test_files(file)
|
|
assert result == []
|
|
|
|
|
|
class TestAnalyzeFile:
|
|
def test_clean_file_no_violations(self, tmp_path: Path) -> None:
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
from unittest.mock import patch, MagicMock
|
|
|
|
class TestExample:
|
|
@patch("mymodule.subprocess.run")
|
|
def test_with_patch(self, mock_run: MagicMock) -> None:
|
|
mymodule.do_thing()
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_unpatched_subprocess_run(self, tmp_path: Path) -> None:
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import subprocess
|
|
|
|
class TestExample:
|
|
def test_direct_subprocess(self) -> None:
|
|
subprocess.run(["echo", "hi"])
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert len(violations) == 1
|
|
assert violations[0].category == "unpatched-subprocess"
|
|
assert "subprocess.run" in violations[0].message
|
|
|
|
def test_patched_subprocess_no_violation(self, tmp_path: Path) -> None:
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
from unittest.mock import patch, MagicMock
|
|
import subprocess
|
|
|
|
class TestExample:
|
|
@patch("subprocess.run")
|
|
def test_patched(self, mock_run: MagicMock) -> None:
|
|
subprocess.run(["echo", "hi"])
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_unpatched_time_sleep(self, tmp_path: Path) -> None:
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import time
|
|
|
|
class TestExample:
|
|
def test_with_sleep(self) -> None:
|
|
time.sleep(5)
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert len(violations) == 1
|
|
assert violations[0].category == "unpatched-sleep"
|
|
|
|
def test_patched_time_sleep_no_violation(self, tmp_path: Path) -> None:
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
from unittest.mock import patch, MagicMock
|
|
import time
|
|
|
|
class TestExample:
|
|
@patch("time.sleep")
|
|
def test_patched_sleep(self, mock_sleep: MagicMock) -> None:
|
|
time.sleep(5)
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_unpatched_known_helper(self, tmp_path: Path) -> None:
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
from mymodule import update_doc_versions
|
|
|
|
class TestExample:
|
|
def test_calls_helper(self) -> None:
|
|
update_doc_versions("1.0.0")
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert len(violations) == 1
|
|
assert violations[0].category == "unpatched-helper"
|
|
assert "update_doc_versions" in violations[0].message
|
|
|
|
def test_patched_helper_no_violation(self, tmp_path: Path) -> None:
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
from unittest.mock import patch, MagicMock
|
|
from mymodule import update_doc_versions
|
|
|
|
class TestExample:
|
|
@patch("mymodule.update_doc_versions")
|
|
def test_patched_helper(self, mock: MagicMock) -> None:
|
|
update_doc_versions("1.0.0")
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_helper_safe_when_subprocess_patched(self, tmp_path: Path) -> None:
|
|
"""update_doc_versions is safe if subprocess.run is patched."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
from unittest.mock import patch, MagicMock
|
|
from mymodule import update_doc_versions
|
|
|
|
class TestExample:
|
|
@patch("subprocess.run")
|
|
def test_subprocess_patched(self, mock: MagicMock) -> None:
|
|
update_doc_versions("1.0.0")
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_helper_safe_when_internal_dep_patched(self, tmp_path: Path) -> None:
|
|
"""run_tests is safe if run_cmd is patched (run_tests calls run_cmd)."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
from unittest.mock import patch, MagicMock
|
|
from mymodule import run_tests
|
|
|
|
class TestExample:
|
|
@patch("mymodule.run_cmd")
|
|
def test_run_cmd_patched(self, mock: MagicMock) -> None:
|
|
run_tests()
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_excessive_iterations(self, tmp_path: Path) -> None:
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
class TestExample:
|
|
def test_many_iterations(self) -> None:
|
|
for _ in range(500):
|
|
assert True
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert len(violations) == 1
|
|
assert violations[0].category == "excessive-iterations"
|
|
assert "500" in violations[0].message
|
|
|
|
def test_acceptable_iterations(self, tmp_path: Path) -> None:
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
class TestExample:
|
|
def test_few_iterations(self) -> None:
|
|
for _ in range(50):
|
|
assert True
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_range_with_start_stop(self, tmp_path: Path) -> None:
|
|
"""range(0, 500) should also be flagged."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
class TestExample:
|
|
def test_range_start_stop(self) -> None:
|
|
for _ in range(0, 500):
|
|
assert True
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert len(violations) == 1
|
|
assert violations[0].category == "excessive-iterations"
|
|
|
|
def test_subprocess_check_output(self, tmp_path: Path) -> None:
|
|
"""subprocess.check_output should also be flagged."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import subprocess
|
|
class TestExample:
|
|
def test_check_output(self) -> None:
|
|
result = subprocess.check_output(["echo", "hi"])
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert len(violations) == 1
|
|
assert violations[0].category == "unpatched-subprocess"
|
|
|
|
def test_subprocess_popen(self, tmp_path: Path) -> None:
|
|
"""subprocess.Popen should also be flagged."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import subprocess
|
|
class TestExample:
|
|
def test_popen(self) -> None:
|
|
p = subprocess.Popen(["echo", "hi"])
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert len(violations) == 1
|
|
assert violations[0].category == "unpatched-subprocess"
|
|
|
|
def test_attribute_style_patch(self, tmp_path: Path) -> None:
|
|
"""mock.patch.object style should be recognized."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
from unittest.mock import mock
|
|
import subprocess
|
|
class TestExample:
|
|
@mock.patch("subprocess.run")
|
|
def test_attr_patch(self, mock_run) -> None:
|
|
subprocess.run(["echo"])
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_subprocess_check_call(self, tmp_path: Path) -> None:
|
|
"""subprocess.check_call should also be flagged."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import subprocess
|
|
class TestExample:
|
|
def test_check_call(self) -> None:
|
|
subprocess.check_call(["echo", "hi"])
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert len(violations) == 1
|
|
assert violations[0].category == "unpatched-subprocess"
|
|
|
|
def test_subprocess_call(self, tmp_path: Path) -> None:
|
|
"""subprocess.call should also be flagged."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import subprocess
|
|
class TestExample:
|
|
def test_call(self) -> None:
|
|
subprocess.call(["echo", "hi"])
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert len(violations) == 1
|
|
assert violations[0].category == "unpatched-subprocess"
|
|
|
|
def test_non_subprocess_attribute_not_flagged(self, tmp_path: Path) -> None:
|
|
"""subprocess.something_else should not be flagged."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import subprocess
|
|
class TestExample:
|
|
def test_other(self) -> None:
|
|
x = subprocess.PIPE
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_async_test_function(self, tmp_path: Path) -> None:
|
|
"""Async test functions should be analyzed too."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import subprocess
|
|
class TestExample:
|
|
async def test_async(self) -> None:
|
|
subprocess.run(["echo"])
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert len(violations) == 1
|
|
assert violations[0].category == "unpatched-subprocess"
|
|
|
|
def test_call_with_no_name(self, tmp_path: Path) -> None:
|
|
"""Calls with complex expressions (e.g. lambda) should not crash."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
class TestExample:
|
|
def test_lambda_call(self) -> None:
|
|
(lambda: None)()
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_range_with_no_args(self, tmp_path: Path) -> None:
|
|
"""range() with no args should not crash."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
class TestExample:
|
|
def test_empty_range(self) -> None:
|
|
for _ in range():
|
|
pass
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_range_with_non_constant_stop(self, tmp_path: Path) -> None:
|
|
"""range(0, variable) should not be flagged (can't determine count)."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
class TestExample:
|
|
def test_variable_range(self) -> None:
|
|
n = 100
|
|
for _ in range(0, n):
|
|
pass
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_range_with_non_constant_start(self, tmp_path: Path) -> None:
|
|
"""range(variable, 500) should be flagged with stop value."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
class TestExample:
|
|
def test_variable_start(self) -> None:
|
|
s = 0
|
|
for _ in range(s, 500):
|
|
pass
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert len(violations) == 1
|
|
assert violations[0].category == "excessive-iterations"
|
|
|
|
def test_for_loop_with_non_range_call(self, tmp_path: Path) -> None:
|
|
"""for loop with a non-range call should not crash."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
class TestExample:
|
|
def test_iter_func(self) -> None:
|
|
for _ in list([1, 2, 3]):
|
|
pass
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_for_loop_with_list(self, tmp_path: Path) -> None:
|
|
"""for loop with a list literal should not crash."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
class TestExample:
|
|
def test_iter_list(self) -> None:
|
|
for _ in [1, 2, 3]:
|
|
pass
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_range_with_single_non_int_arg(self, tmp_path: Path) -> None:
|
|
"""range(variable) should not crash or flag."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
class TestExample:
|
|
def test_range_var(self) -> None:
|
|
n = 50
|
|
for _ in range(n):
|
|
pass
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_range_with_three_args(self, tmp_path: Path) -> None:
|
|
"""range(0, 500, 1) should be flagged (3 args, stop=500)."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
class TestExample:
|
|
def test_range_step(self) -> None:
|
|
for _ in range(0, 500, 1):
|
|
pass
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert len(violations) == 1
|
|
assert violations[0].category == "excessive-iterations"
|
|
|
|
def test_non_test_function_not_analyzed(self, tmp_path: Path) -> None:
|
|
"""Non-test functions should not be analyzed."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import subprocess
|
|
|
|
def helper_function() -> None:
|
|
subprocess.run(["echo", "hi"])
|
|
|
|
class TestExample:
|
|
def test_uses_helper(self) -> None:
|
|
helper_function()
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
# helper_function is not a test, so no violation for its subprocess call
|
|
# test_uses_helper calls helper_function, not subprocess directly
|
|
assert violations == []
|
|
|
|
def test_class_level_patch_satisfies_check(self, tmp_path: Path) -> None:
|
|
"""@patch on the class should satisfy the check for all methods."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
from unittest.mock import patch, MagicMock
|
|
import subprocess
|
|
|
|
@patch("subprocess.run")
|
|
class TestExample:
|
|
def test_method_a(self, mock: MagicMock) -> None:
|
|
subprocess.run(["echo", "a"])
|
|
|
|
def test_method_b(self, mock: MagicMock) -> None:
|
|
subprocess.run(["echo", "b"])
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_syntax_error_returns_violation(self, tmp_path: Path) -> None:
|
|
file = tmp_path / "test_broken.py"
|
|
file.write_text("def test(:\n pass\n")
|
|
violations = analyze_file(file)
|
|
assert len(violations) == 1
|
|
assert violations[0].category == "syntax-error"
|
|
|
|
|
|
class TestAnalyzeTestFiles:
|
|
def test_multiple_files(self, tmp_path: Path) -> None:
|
|
_write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import subprocess
|
|
class TestA:
|
|
def test_a(self) -> None:
|
|
subprocess.run(["echo"])
|
|
""",
|
|
)
|
|
file2 = tmp_path / "test_other.py"
|
|
file2.write_text(
|
|
textwrap.dedent("""
|
|
import time
|
|
class TestB:
|
|
def test_b(self) -> None:
|
|
time.sleep(1)
|
|
""")
|
|
)
|
|
violations = analyze_test_files(tmp_path)
|
|
assert len(violations) == 2
|
|
categories = {v.category for v in violations}
|
|
assert "unpatched-subprocess" in categories
|
|
assert "unpatched-sleep" in categories
|
|
|
|
def test_category_filter(self, tmp_path: Path) -> None:
|
|
_write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import subprocess
|
|
class TestA:
|
|
def test_a(self) -> None:
|
|
subprocess.run(["echo"])
|
|
""",
|
|
)
|
|
file2 = tmp_path / "test_other.py"
|
|
file2.write_text(
|
|
textwrap.dedent("""
|
|
import time
|
|
class TestB:
|
|
def test_b(self) -> None:
|
|
time.sleep(1)
|
|
""")
|
|
)
|
|
violations = analyze_test_files(tmp_path, categories={"unpatched-sleep"})
|
|
assert len(violations) == 1
|
|
assert violations[0].category == "unpatched-sleep"
|
|
|
|
|
|
class TestKnownHelpers:
|
|
def test_all_helpers_have_internal_calls(self) -> None:
|
|
"""Every known helper should have its internal calls documented."""
|
|
for helper in KNOWN_SUBPROCESS_HELPERS:
|
|
assert helper in HELPER_INTERNAL_CALLS, f"Missing HELPER_INTERNAL_CALLS entry for {helper}"
|
|
|
|
def test_run_tests_internal_calls_include_run_cmd(self) -> None:
|
|
assert "run_cmd" in HELPER_INTERNAL_CALLS["run_tests"]
|
|
|
|
def test_update_doc_versions_internal_calls_include_subprocess(self) -> None:
|
|
assert "subprocess" in HELPER_INTERNAL_CALLS["update_doc_versions"]
|
|
|
|
|
|
class TestIOFunctionChecks:
|
|
"""Tests for unpatched I/O function detection."""
|
|
|
|
def test_unpatched_get_pat_violation(self, tmp_path: Path) -> None:
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
from mymodule import get_pat
|
|
|
|
class TestExample:
|
|
def test_calls_get_pat(self) -> None:
|
|
result = get_pat("staging")
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert len(violations) == 1
|
|
assert violations[0].category == "unpatched-io"
|
|
assert "get_pat" in violations[0].message
|
|
|
|
def test_patched_get_pat_no_violation(self, tmp_path: Path) -> None:
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
from unittest.mock import patch
|
|
from mymodule import get_pat
|
|
|
|
class TestExample:
|
|
@patch("mymodule.get_pat", return_value="pat")
|
|
def test_patched(self, mock) -> None:
|
|
result = get_pat("staging")
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_unpatched_load_secrets_violation(self, tmp_path: Path) -> None:
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
from mymodule import load_secrets
|
|
|
|
class TestExample:
|
|
def test_calls_load_secrets(self) -> None:
|
|
result = load_secrets("staging")
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert len(violations) == 1
|
|
assert violations[0].category == "unpatched-io"
|
|
assert "load_secrets" in violations[0].message
|
|
|
|
def test_patched_load_secrets_no_violation(self, tmp_path: Path) -> None:
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
from unittest.mock import patch
|
|
from mymodule import load_secrets
|
|
|
|
class TestExample:
|
|
@patch("mymodule.load_secrets", return_value={})
|
|
def test_patched(self, mock) -> None:
|
|
result = load_secrets("staging")
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_unpatched_requests_get_violation(self, tmp_path: Path) -> None:
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import requests
|
|
|
|
class TestExample:
|
|
def test_calls_requests(self) -> None:
|
|
resp = requests.get("https://example.com")
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert len(violations) == 1
|
|
assert violations[0].category == "unpatched-io"
|
|
assert "requests.get" in violations[0].message or "get" in violations[0].message
|
|
|
|
def test_integration_marker_skips_subprocess(self, tmp_path: Path) -> None:
|
|
"""@pytest.mark.integration tests should not be flagged for subprocess.run."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import subprocess
|
|
import pytest
|
|
|
|
@pytest.mark.integration
|
|
def test_real_subprocess():
|
|
subprocess.run(["echo", "hello"])
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_integration_marker_skips_sleep(self, tmp_path: Path) -> None:
|
|
"""@pytest.mark.integration tests should not be flagged for time.sleep."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import time
|
|
import pytest
|
|
|
|
@pytest.mark.integration
|
|
def test_real_sleep():
|
|
time.sleep(1)
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_integration_marker_with_args_skips(self, tmp_path: Path) -> None:
|
|
"""@pytest.mark.integration(...) with args should also be skipped."""
|
|
file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import subprocess
|
|
import pytest
|
|
|
|
@pytest.mark.integration(scope="module")
|
|
def test_real_subprocess():
|
|
subprocess.run(["echo", "hello"])
|
|
""",
|
|
)
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
def test_integration_directory_skipped(self, tmp_path: Path) -> None:
|
|
"""Files in integration/ directories should be skipped entirely."""
|
|
integration_dir = tmp_path / "integration"
|
|
integration_dir.mkdir()
|
|
file = integration_dir / "test_real_io.py"
|
|
file.write_text("import subprocess\ndef test_real_subprocess():\n subprocess.run(['echo', 'hello'])\n")
|
|
violations = analyze_file(file)
|
|
assert violations == []
|
|
|
|
|
|
class TestCli:
|
|
"""Tests for the standalone CLI interface."""
|
|
|
|
def test_clean_directory_exits_zero(self, tmp_path: Path) -> None:
|
|
_write_test_file(
|
|
tmp_path,
|
|
"""
|
|
from unittest.mock import patch, MagicMock
|
|
class TestExample:
|
|
@patch("subprocess.run")
|
|
def test_ok(self, mock: MagicMock) -> None:
|
|
pass
|
|
""",
|
|
)
|
|
runner = CliRunner()
|
|
result = runner.invoke(cli, ["--test-path", str(tmp_path)])
|
|
assert result.exit_code == 0
|
|
assert "no violations" in result.output
|
|
|
|
def test_violations_exit_nonzero(self, tmp_path: Path) -> None:
|
|
_write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import subprocess
|
|
class TestExample:
|
|
def test_bad(self) -> None:
|
|
subprocess.run(["echo"])
|
|
""",
|
|
)
|
|
runner = CliRunner()
|
|
result = runner.invoke(cli, ["--test-path", str(tmp_path)])
|
|
assert result.exit_code == 1
|
|
assert "FAILED" in result.output
|
|
assert "unpatched-subprocess" in result.output
|
|
|
|
def test_strict_flag(self, tmp_path: Path) -> None:
|
|
_write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import subprocess
|
|
class TestExample:
|
|
def test_bad(self) -> None:
|
|
subprocess.run(["echo"])
|
|
""",
|
|
)
|
|
runner = CliRunner()
|
|
result = runner.invoke(cli, ["--test-path", str(tmp_path), "--strict"])
|
|
assert result.exit_code == 1
|
|
|
|
def test_category_filter(self, tmp_path: Path) -> None:
|
|
_write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import subprocess, time
|
|
class TestExample:
|
|
def test_bad(self) -> None:
|
|
subprocess.run(["echo"])
|
|
time.sleep(1)
|
|
""",
|
|
)
|
|
runner = CliRunner()
|
|
result = runner.invoke(cli, ["--test-path", str(tmp_path), "--categories", "unpatched-sleep"])
|
|
assert result.exit_code == 1
|
|
assert "unpatched-sleep" in result.output
|
|
assert "unpatched-subprocess" not in result.output
|
|
|
|
def test_max_loop_iterations_option(self, tmp_path: Path) -> None:
|
|
_write_test_file(
|
|
tmp_path,
|
|
"""
|
|
class TestExample:
|
|
def test_loop(self) -> None:
|
|
for _ in range(10):
|
|
assert True
|
|
""",
|
|
)
|
|
runner = CliRunner()
|
|
# With max=5, 10 iterations is a violation
|
|
result = runner.invoke(cli, ["--test-path", str(tmp_path), "--max-loop-iterations", "5"])
|
|
assert result.exit_code == 1
|
|
assert "excessive-iterations" in result.output
|
|
|
|
def test_no_test_files(self, tmp_path: Path) -> None:
|
|
runner = CliRunner()
|
|
result = runner.invoke(cli, ["--test-path", str(tmp_path)])
|
|
assert result.exit_code == 0
|
|
assert "no violations" in result.output
|
|
|
|
def test_strict_clean_directory_exits_zero(self, tmp_path: Path) -> None:
|
|
"""Strict mode with no violations should still exit 0."""
|
|
_write_test_file(
|
|
tmp_path,
|
|
"""
|
|
from unittest.mock import patch, MagicMock
|
|
class TestExample:
|
|
@patch("subprocess.run")
|
|
def test_ok(self, mock: MagicMock) -> None:
|
|
pass
|
|
""",
|
|
)
|
|
runner = CliRunner()
|
|
result = runner.invoke(cli, ["--test-path", str(tmp_path), "--strict"])
|
|
assert result.exit_code == 0
|
|
|
|
|
|
class TestPytestPlugin:
|
|
"""Tests for the pytest plugin hooks.
|
|
|
|
These hooks are marked with pragma: no cover because they're loaded
|
|
by pytest before coverage instrumentation starts. We test them via
|
|
direct calls to verify correctness.
|
|
"""
|
|
|
|
def test_pytest_addoption_registers_options(self) -> None:
|
|
"""Verify that pytest_addoption registers the expected options."""
|
|
from unittest.mock import MagicMock
|
|
|
|
from devx.tools.check_test_isolation import pytest_addoption
|
|
|
|
parser = MagicMock()
|
|
pytest_addoption(parser)
|
|
|
|
addoption_calls = parser.addoption.call_args_list
|
|
assert len(addoption_calls) >= 3
|
|
|
|
def test_pytest_collection_finish_noop_when_disabled(self) -> None:
|
|
"""Plugin should skip analysis when --no-test-isolation is set."""
|
|
from unittest.mock import MagicMock
|
|
|
|
from devx.tools.check_test_isolation import pytest_collection_finish
|
|
|
|
session = MagicMock()
|
|
session.config.getoption.side_effect = lambda opt: opt == "--no-test-isolation"
|
|
pytest_collection_finish(session)
|
|
|
|
def test_pytest_collection_finish_no_violations(self) -> None:
|
|
"""Plugin should not emit warnings when there are no violations."""
|
|
from unittest.mock import MagicMock
|
|
|
|
from devx.tools.check_test_isolation import pytest_collection_finish
|
|
|
|
session = MagicMock()
|
|
session.config.getoption.side_effect = lambda opt: False
|
|
session.items = []
|
|
pytest_collection_finish(session)
|
|
|
|
def test_pytest_collection_finish_with_violation(self, tmp_path: Path) -> None:
|
|
"""Plugin should emit warnings when violations are found."""
|
|
import warnings
|
|
from unittest.mock import MagicMock
|
|
|
|
from devx.tools.check_test_isolation import pytest_collection_finish
|
|
|
|
test_file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import subprocess
|
|
class TestExample:
|
|
def test_bad(self) -> None:
|
|
subprocess.run(["echo"])
|
|
""",
|
|
)
|
|
|
|
session = MagicMock()
|
|
session.config.getoption.side_effect = lambda opt: False
|
|
item = MagicMock()
|
|
item.fspath = str(test_file)
|
|
session.items = [item]
|
|
|
|
with warnings.catch_warnings(record=True) as w:
|
|
warnings.simplefilter("always")
|
|
pytest_collection_finish(session)
|
|
|
|
assert len(w) >= 1
|
|
assert any("Test isolation violation" in str(warning.message) for warning in w)
|
|
|
|
def test_pytest_collection_finish_strict_mode(self, tmp_path: Path) -> None:
|
|
"""Plugin should emit warnings and print summary in strict mode."""
|
|
import warnings
|
|
from unittest.mock import MagicMock
|
|
|
|
from devx.tools.check_test_isolation import pytest_collection_finish
|
|
|
|
test_file = _write_test_file(
|
|
tmp_path,
|
|
"""
|
|
import subprocess
|
|
class TestExample:
|
|
def test_bad(self) -> None:
|
|
subprocess.run(["echo"])
|
|
""",
|
|
)
|
|
|
|
session = MagicMock()
|
|
session.config.getoption.side_effect = lambda opt: {
|
|
"--no-test-isolation": False,
|
|
"--strict-test-isolation": True,
|
|
"--test-isolation-max-loop": 100,
|
|
}.get(opt, False)
|
|
item = MagicMock()
|
|
item.fspath = str(test_file)
|
|
session.items = [item]
|
|
|
|
with warnings.catch_warnings(record=True) as w:
|
|
warnings.simplefilter("always")
|
|
pytest_collection_finish(session)
|
|
|
|
assert len(w) >= 1
|
|
assert any("Test isolation violation" in str(warning.message) for warning in w)
|