Test against Pytest 7.0.x and 7.1.x

.github/workflows/testing.yml

@@ -104,10 +104,13 @@ jobs:
           - "~=6.0.0"
           - "~=6.1.0"
           - "~=6.2.0"
-          - ">=7.0.0rc1"
+          - "~=7.0.0"
+          - "~=7.1.0"
         exclude:
           - {"python-version": "3.5", "pytest-version": "~=6.2.0"}
-          - {"python-version": "3.5", "pytest-version": ">=7.0.0rc1"}
+          - {"python-version": "3.5", "pytest-version": "~=7.0.0"}
+          - {"python-version": "3.5", "pytest-version": "~=7.1.0"}
+          - {"python-version": "3.6", "pytest-version": "~=7.1.0"}
           - {"python-version": "3.10", "pytest-version": "~=6.0.0"}
           - {"python-version": "3.10", "pytest-version": "~=6.1.0"}
 
@@ -281,8 +284,10 @@ jobs:
           - "3.10"
         pytest-version:
           - "~=6.2.0"
-          - ">=7.0.0rc1"
+          - "~=7.0.0"
+          - "~=7.1.0"
         exclude:
+          - {"python-version": "3.6", "pytest-version": "~=7.1.0"}
           - {"python-version": "3.10", "pytest-version": "~=6.0.0"}
           - {"python-version": "3.10", "pytest-version": "~=6.1.0"}
 
@@ -425,8 +430,10 @@ jobs:
           - "3.10"
         pytest-version:
           - "~=6.2.0"
-          - ">=7.0.0rc1"
+          - "~=7.0.0"
+          - "~=7.1.0"
         exclude:
+          - {"python-version": "3.6", "pytest-version": "~=7.1.0"}
           - {"python-version": "3.10", "pytest-version": "~=6.0.0"}
           - {"python-version": "3.10", "pytest-version": "~=6.1.0"}
 

.pylint-spelling-words

@@ -64,6 +64,7 @@ sids
 sitevars
 smartos
 str
+subtests
 sunos
 sys
 toml

noxfile.py

@@ -129,10 +129,18 @@ def tests(session):
         session.install(COVERAGE_VERSION_REQUIREMENT, silent=PIP_INSTALL_SILENT)
         pytest_version_requirement = os.environ.get("PYTEST_VERSION_REQUIREMENT") or None
         if pytest_version_requirement:
+            # Account that, under CI, when we test previous version of pytest
+            # we need to pin pytest-subtests to a lower version so that pytest
+            # itself does not get upgraded
+            pytest_requirements = []
             if not pytest_version_requirement.startswith("pytest"):
                 pytest_version_requirement = "pytest{}".format(pytest_version_requirement)
-            session.install(pytest_version_requirement, silent=PIP_INSTALL_SILENT)
+            pytest_requirements.append(pytest_version_requirement)
+            if pytest_version_requirement.startswith("pytest~=6"):
+                pytest_requirements.append("pytest-subtests<0.7.0")
+            session.install(*pytest_requirements, silent=PIP_INSTALL_SILENT)
         session.install("-e", ".", silent=PIP_INSTALL_SILENT)
+        session.install("-r", os.path.join("requirements", "base.txt"), silent=PIP_INSTALL_SILENT)
         session.install("-r", os.path.join("requirements", "tests.txt"), silent=PIP_INSTALL_SILENT)
 
         if EXTRA_REQUIREMENTS_INSTALL:
@@ -145,6 +153,13 @@ def tests(session):
             install_command += [req.strip() for req in EXTRA_REQUIREMENTS_INSTALL.split()]
             session.install(*install_command, silent=PIP_INSTALL_SILENT)
 
+    if IS_WINDOWS:
+        _python_version = python_version(session)
+        if _python_version > (3, 8) and _python_version < (3, 10):
+            # We skip a lot of tests, so lower the expected code coverage
+            global COVERAGE_FAIL_UNDER_PERCENT
+            COVERAGE_FAIL_UNDER_PERCENT = 50
+
     session.run("coverage", "erase")
     env.update(
         {

requirements/tests.txt

@@ -1,4 +1,3 @@
--r base.txt
 docker
 pytest-subtests
 pyfakefs==4.4.0; python_version == '3.5'

tests/functional/test_expensive_test.py

@@ -50,7 +50,7 @@ def test_two():
             assert True
         """
     )
-    res = pytester.runpytest("--run-destructive")
+    res = pytester.runpytest("--run-expensive")
     res.assert_outcomes(errors=2)
     res.stdout.no_fnmatch_line("*PytestUnknownMarkWarning*")
     res.stdout.fnmatch_lines(

tests/functional/test_requires_network.py

@@ -4,11 +4,23 @@
 """
 Test the ``@pytest.mark.requires_network`` marker.
 """
+import sys
 from unittest import mock
 
+import pytest
+
 from pytestskipmarkers.utils import ports
 from pytestskipmarkers.utils import socket
 
+pytestmark = [
+    pytest.mark.skipif(
+        sys.platform.startswith("win")
+        and sys.version_info >= (3, 8)
+        and sys.version_info < (3, 10),
+        reason="PyTest's capture and pytester.runpytest_inprocess looks broken on Windows and Py(>3.8,<3.10)",
+    ),
+]
+
 
 def test_has_local_network(pytester):
     pytester.makepyfile(

tests/functional/test_skip_if_binaries_missing.py

@@ -57,7 +57,7 @@ def test_one():
             python_binary
         )
     )
-    res = pytester.runpytest_inprocess("-ra", "-vv")
+    res = pytester.runpytest_subprocess("-ra", "-vv")
     res.assert_outcomes(passed=1)
     res.stdout.no_fnmatch_line("*PytestUnknownMarkWarning*")
 
@@ -74,6 +74,6 @@ def test_one():
             python_binary
         )
     )
-    res = pytester.runpytest_inprocess("-ra", "-vv")
+    res = pytester.runpytest_subprocess("-ra", "-vv")
     res.assert_outcomes(passed=1)
     res.stdout.no_fnmatch_line("*PytestUnknownMarkWarning*")

tests/functional/test_skip_if_not_root.py

@@ -7,6 +7,17 @@
 import sys
 from unittest import mock
 
+import pytest
+
+pytestmark = [
+    pytest.mark.skipif(
+        sys.platform.startswith("win")
+        and sys.version_info >= (3, 8)
+        and sys.version_info < (3, 10),
+        reason="PyTest's capture and pytester.runpytest_inprocess looks broken on Windows and Py(>3.8,<3.10)",
+    ),
+]
+
 
 def test_skip_if_not_root_skipped(pytester):
     pytester.makepyfile(
@@ -68,7 +79,7 @@ def test_two():
             assert True
         """
     )
-    res = pytester.runpytest("--run-destructive")
+    res = pytester.runpytest()
     res.assert_outcomes(errors=2)
     res.stdout.no_fnmatch_line("*PytestUnknownMarkWarning*")
     res.stdout.fnmatch_lines(

tests/functional/test_skip_on_aarch64.py

@@ -4,8 +4,20 @@
 """
 Test the ``@pytest.mark.skip_on_aarch64`` marker.
 """
+import sys
 from unittest import mock
 
+import pytest
+
+pytestmark = [
+    pytest.mark.skipif(
+        sys.platform.startswith("win")
+        and sys.version_info >= (3, 8)
+        and sys.version_info < (3, 10),
+        reason="PyTest's capture and pytester.runpytest_inprocess looks broken on Windows and Py(>3.8,<3.10)",
+    ),
+]
+
 
 def test_skipped(pytester):
     pytester.makepyfile(

tests/functional/test_skip_on_aix.py

@@ -4,8 +4,21 @@
 """
 Test the ``@pytest.mark.skip_on_aix`` marker.
 """
+import sys
 from unittest import mock
 
+import pytest
+
+
+pytestmark = [
+    pytest.mark.skipif(
+        sys.platform.startswith("win")
+        and sys.version_info >= (3, 8)
+        and sys.version_info < (3, 10),
+        reason="PyTest's capture and pytester.runpytest_inprocess looks broken on Windows and Py(>3.8,<3.10)",
+    ),
+]
+
 
 def test_skipped(pytester):
     pytester.makepyfile(

tests/functional/test_skip_on_darwin.py

@@ -4,8 +4,20 @@
 """
 Test the ``@pytest.mark.skip_on_darwin`` marker.
 """
+import sys
 from unittest import mock
 
+import pytest
+
+pytestmark = [
+    pytest.mark.skipif(
+        sys.platform.startswith("win")
+        and sys.version_info >= (3, 8)
+        and sys.version_info < (3, 10),
+        reason="PyTest's capture and pytester.runpytest_inprocess looks broken on Windows and Py(>3.8,<3.10)",
+    ),
+]
+
 
 def test_skipped(pytester):
     pytester.makepyfile(

tests/functional/test_skip_on_env.py

@@ -5,8 +5,20 @@
 Test the ``@pytest.mark.skip_on_env`` marker.
 """
 import os
+import sys
 from unittest import mock
 
+import pytest
+
+pytestmark = [
+    pytest.mark.skipif(
+        sys.platform.startswith("win")
+        and sys.version_info >= (3, 8)
+        and sys.version_info < (3, 10),
+        reason="PyTest's capture and pytester.runpytest_inprocess looks broken on Windows and Py(>3.8,<3.10)",
+    ),
+]
+
 
 def test_skipped(pytester):
     pytester.makepyfile(

tests/functional/test_skip_on_freebsd.py

@@ -4,8 +4,20 @@
 """
 Test the ``@pytest.mark.skip_on_freebsd`` marker.
 """
+import sys
 from unittest import mock
 
+import pytest
+
+pytestmark = [
+    pytest.mark.skipif(
+        sys.platform.startswith("win")
+        and sys.version_info >= (3, 8)
+        and sys.version_info < (3, 10),
+        reason="PyTest's capture and pytester.runpytest_inprocess looks broken on Windows and Py(>3.8,<3.10)",
+    ),
+]
+
 
 def test_skipped(pytester):
     pytester.makepyfile(

tests/functional/test_skip_on_linux.py

@@ -4,8 +4,20 @@
 """
 Test the ``@pytest.mark.skip_on_linux`` marker.
 """
+import sys
 from unittest import mock
 
+import pytest
+
+pytestmark = [
+    pytest.mark.skipif(
+        sys.platform.startswith("win")
+        and sys.version_info >= (3, 8)
+        and sys.version_info < (3, 10),
+        reason="PyTest's capture and pytester.runpytest_inprocess looks broken on Windows and Py(>3.8,<3.10)",
+    ),
+]
+
 
 def test_skipped(pytester):
     pytester.makepyfile(

tests/functional/test_skip_on_netbsd.py

@@ -4,8 +4,20 @@
 """
 Test the ``@pytest.mark.skip_on_netbsd`` marker.
 """
+import sys
 from unittest import mock
 
+import pytest
+
+pytestmark = [
+    pytest.mark.skipif(
+        sys.platform.startswith("win")
+        and sys.version_info >= (3, 8)
+        and sys.version_info < (3, 10),
+        reason="PyTest's capture and pytester.runpytest_inprocess looks broken on Windows and Py(>3.8,<3.10)",
+    ),
+]
+
 
 def test_skipped(pytester):
     pytester.makepyfile(

tests/functional/test_skip_on_openbsd.py

@@ -4,8 +4,20 @@
 """
 Test the ``@pytest.mark.skip_on_openbsd`` marker.
 """
+import sys
 from unittest import mock
 
+import pytest
+
+pytestmark = [
+    pytest.mark.skipif(
+        sys.platform.startswith("win")
+        and sys.version_info >= (3, 8)
+        and sys.version_info < (3, 10),
+        reason="PyTest's capture and pytester.runpytest_inprocess looks broken on Windows and Py(>3.8,<3.10)",
+    ),
+]
+
 
 def test_skipped(pytester):
     pytester.makepyfile(

tests/functional/test_skip_on_photonos.py

@@ -4,8 +4,20 @@
 """
 Test the ``@pytest.mark.skip_on_photonos`` marker.
 """
+import sys
 from unittest import mock
 
+import pytest
+
+pytestmark = [
+    pytest.mark.skipif(
+        sys.platform.startswith("win")
+        and sys.version_info >= (3, 8)
+        and sys.version_info < (3, 10),
+        reason="PyTest's capture and pytester.runpytest_inprocess looks broken on Windows and Py(>3.8,<3.10)",
+    ),
+]
+
 
 def test_skipped(pytester):
     pytester.makepyfile(

tests/functional/test_skip_on_platforms.py

@@ -4,10 +4,20 @@
 """
 Test the ``@pytest.mark.skip_on_platforms`` marker.
 """
+import sys
 from unittest import mock
 
 import pytest
 
+pytestmark = [
+    pytest.mark.skipif(
+        sys.platform.startswith("win")
+        and sys.version_info >= (3, 8)
+        and sys.version_info < (3, 10),
+        reason="PyTest's capture and pytester.runpytest_inprocess looks broken on Windows and Py(>3.8,<3.10)",
+    ),
+]
+
 
 @pytest.mark.parametrize(
     "platform",

tests/functional/test_skip_on_smartos.py

@@ -4,8 +4,20 @@
 """
 Test the ``@pytest.mark.skip_on_smartos`` marker.
 """
+import sys
 from unittest import mock
 
+import pytest
+
+pytestmark = [
+    pytest.mark.skipif(
+        sys.platform.startswith("win")
+        and sys.version_info >= (3, 8)
+        and sys.version_info < (3, 10),
+        reason="PyTest's capture and pytester.runpytest_inprocess looks broken on Windows and Py(>3.8,<3.10)",
+    ),
+]
+
 
 def test_skipped(pytester):
     pytester.makepyfile(