Improve pytest execution (#4435)

Author: ssbarnea

.config/constraints.txt

@@ -74,13 +74,14 @@ pathspec==0.12.1          # via black, mkdocs, mkdocs-macros-plugin, yamllint, a
 pillow==11.0.0            # via cairosvg, mkdocs-ansible
 platformdirs==4.3.6       # via black, mkdocs-get-deps, mkdocstrings, pylint, tox, virtualenv
 pluggy==1.5.0             # via pytest, tox
-psutil==6.1.0             # via ansible-lint (pyproject.toml)
+psutil==6.1.0             # via pytest-xdist, ansible-lint (pyproject.toml)
 pycparser==2.22           # via cffi
 pygments==2.18.0          # via mkdocs-material, rich
-pylint==3.3.1             # via ansible-lint (pyproject.toml)
+pylint==3.3.2             # via ansible-lint (pyproject.toml)
 pymdown-extensions==10.12  # via markdown-exec, mkdocs-ansible, mkdocs-material, mkdocstrings
 pyproject-api==1.8.0      # via tox
-pytest==8.3.3             # via pytest-mock, pytest-plus, pytest-xdist, ansible-lint (pyproject.toml)
+pytest==8.3.4             # via pytest-instafail, pytest-mock, pytest-plus, pytest-xdist, ansible-lint (pyproject.toml)
+pytest-instafail==0.5.0   # via ansible-lint (pyproject.toml)
 pytest-mock==3.14.0       # via ansible-lint (pyproject.toml)
 pytest-plus==0.7.0        # via ansible-lint (pyproject.toml)
 pytest-xdist==3.6.1       # via ansible-lint (pyproject.toml)
@@ -95,6 +96,7 @@ rich==13.9.4              # via ansible-lint (pyproject.toml)
 rpds-py==0.22.1           # via jsonschema, referencing
 ruamel-yaml==0.18.6       # via ansible-lint (pyproject.toml)
 ruamel-yaml-clib==0.2.12  # via ruamel-yaml, ansible-lint (pyproject.toml)
+setproctitle==1.3.4       # via pytest-xdist
 six==1.16.0               # via python-dateutil
 soupsieve==2.6            # via beautifulsoup4
 subprocess-tee==0.4.2     # via ansible-compat, ansible-lint (pyproject.toml)

.config/requirements-test.in

@@ -11,7 +11,8 @@ pylint # IDE support
 pytest >= 7.2.2
 pytest-mock
 pytest-plus >= 0.6 # for PYTEST_REQPASS
-pytest-xdist >= 2.1.0
+pytest-xdist[psutil,setproctitle] >= 2.1.0
+pytest-instafail >= 0.5.0 # only for local development, via PYTEST_ADDOPTS=-edit
 ruamel.yaml>=0.17.31
 ruamel-yaml-clib  # needed for mypy
 tox >= 4.0.0

.pre-commit-config.yaml

@@ -205,7 +205,7 @@ repos:
           - wcmatch
           - yamllint
   - repo: https://github.com/pycqa/pylint
-    rev: v3.3.1
+    rev: v3.3.2
     hooks:
       - id: pylint
         args:

.vscode/settings.json

@@ -17,6 +17,9 @@
         "**/*.txt",
         "**/*.md"
     ],
+    "python.experiments.optInto": [
+        "pythonTestAdapter"
+    ],
     "python.terminal.activateEnvironment": true,
     "python.testing.pytestEnabled": true,
     "python.testing.unittestEnabled": false,

pyproject.toml

@@ -179,11 +179,7 @@ reportPossiblyUnboundVariable = false
 [tool.pytest.ini_options]
 # do not add options here as this will likely break either console runs or IDE
 # integration like vscode or pycharm
-addopts = "-p no:pytest_cov --failed-first"
-# https://code.visualstudio.com/docs/python/testing
-# coverage is re-enabled in `tox.ini`. That approach is safer than
-# `--no-cov` which prevents activation from tox.ini and which also fails
-# when plugin is effectively missing.
+addopts = "-p no:pytest_cov --durations=10 --failed-first"
 doctest_optionflags = ["ALLOW_UNICODE", "ELLIPSIS"]
 filterwarnings = [
   "error",
@@ -200,11 +196,15 @@ filterwarnings = [
   "ignore:Attribute s is deprecated and will be removed in Python 3.14; use value instead:DeprecationWarning"
 ]
 junit_duration_report = "call"
+junit_family = "xunit1"
 # Our github annotation parser from .github/workflows/tox.yml requires xunit1 format. Ref:
 # https://github.com/shyim/junit-report-annotations-action/issues/3#issuecomment-663241378
-junit_family = "xunit1"
 junit_suite_name = "ansible_lint_test_suite"
 minversion = "4.6.6"
+# https://code.visualstudio.com/docs/python/testing
+# coverage is re-enabled in `tox.ini`. That approach is safer than
+# `--no-cov` which prevents activation from tox.ini and which also fails
+# when plugin is effectively missing.
 norecursedirs = [
   "*.egg",
   ".cache",
@@ -214,12 +214,19 @@ norecursedirs = [
   ".github",
   ".mypy_cache",
   ".projects",
+  ".eggs",
   ".tox",
+  "__pycache__",
+  "ansible_collections",
   "build",
   "collections",
   "dist",
   "docs",
-  "src/ansible_lint.egg-info"
+  "examples",
+  "plugins",
+  "site",
+  "src/ansible_lint.egg-info",
+  "test/schemas"
 ]
 python_files = [
   "test_*.py",
@@ -333,6 +340,10 @@ in_place = true
 sort_inline_tables = true
 sort_table_keys = true
 
+[tool.uv]
+# incompatible with Windows
+environments = ["platform_system != 'Windows'"]
+
 [tool.uv.pip]
 annotation-style = "line"
 custom-compile-command = "tox run deps"

test/test_utils.py

@@ -312,6 +312,12 @@ def test_cli_auto_detect(capfd: CaptureFixture[str]) -> None:
         "-v",
         "-p",
         "--nocolor",
+        "--offline",
+        "--exclude=examples",
+        "--exclude=test",
+        "--exclude=src",
+        "--exclude=collections",
+        "--exclude=.github",
     ]
     result = subprocess.run(cmd, check=False).returncode
 
@@ -321,18 +327,12 @@ def test_cli_auto_detect(capfd: CaptureFixture[str]) -> None:
     out, err = capfd.readouterr()
 
     # An expected rule match from our examples
-    assert (
-        "examples/playbooks/empty_playbook.yml:1:1: "
-        "syntax-check[empty-playbook]: Empty playbook, nothing to do" in out
-    )
+    assert "playbook.yml:6: name[casing]" in out
     # assures that our ansible-lint config exclude was effective in excluding github files
     assert "Identified: .github/" not in out
     # assures that we can parse playbooks as playbooks
     assert "Identified: test/test/always-run-success.yml" not in err
-    assert (
-        "Executing syntax check on playbook examples/playbooks/mocked_dependency.yml"
-        in err
-    )
+    assert "Executing syntax check on playbook playbook.yml" in err
 
 
 def test_is_playbook() -> None:

tox.ini

@@ -157,6 +157,8 @@ setenv =
   # https://github.com/pawamoy/markdown-exec/issues/11
   NO_COLOR = 1
   TERM = dump
+  # https://squidfunk.github.io/mkdocs-material/plugins/requirements/image-processing/#troubleshooting
+  DYLD_FALLBACK_LIBRARY_PATH = /opt/homebrew/lib:{env:LD_LIBRARY_PATH}
 skip_install = false
 usedevelop = true
 commands_pre =