Restore compatibility with Python 3.9's legacy LL(1) parser yet again

.github/workflows/main.yml

@@ -30,6 +30,7 @@ jobs:
           - check-coverage
           - check-conjecture-coverage
           - check-py39-cover
+          - check-py39-oldparser
           - check-pypy39-cover
           - check-py310-cover
           - check-py310-nocover

hypothesis-python/RELEASE.rst

@@ -0,0 +1,8 @@
+RELEASE_TYPE: patch
+
+This patch restores compatibility when using `the legacy Python 3.9 LL(1)
+parser <https://docs.python.org/3/whatsnew/3.9.html#new-parser>`__ yet
+again, because the fix in :ref:`version 6.131.33 <v6.131.33>` was too
+brittle.
+
+Thanks to Marco Ricci for this fix!

hypothesis-python/src/_hypothesis_pytestplugin.py

@@ -305,6 +305,9 @@ def note_statistics(stats):
                 item.hypothesis_statistics = describe_statistics(stats)
 
             with collector.with_value(note_statistics):
+                # NOTE: For compatibility with Python 3.9's LL(1)
+                # parser, this is written as a nested with-statement,
+                # instead of a compound one.
                 with with_reporter(store):
                     with current_pytest_item.with_value(item):
                         yield

hypothesis-python/src/hypothesis/core.py

@@ -976,8 +976,12 @@ def execute_once(
 
             @proxies(self.test)
             def test(*args, **kwargs):
-                with unwrap_markers_from_group(), ensure_free_stackframes():
-                    return self.test(*args, **kwargs)
+                with unwrap_markers_from_group():
+                    # NOTE: For compatibility with Python 3.9's LL(1)
+                    # parser, this is written as a nested with-statement,
+                    # instead of a compound one.
+                    with ensure_free_stackframes():
+                        return self.test(*args, **kwargs)
 
         else:
 
@@ -988,8 +992,12 @@ def test(*args, **kwargs):
                 arg_gctime = gc_cumulative_time()
                 start = time.perf_counter()
                 try:
-                    with unwrap_markers_from_group(), ensure_free_stackframes():
-                        result = self.test(*args, **kwargs)
+                    with unwrap_markers_from_group():
+                        # NOTE: For compatibility with Python 3.9's LL(1)
+                        # parser, this is written as a nested with-statement,
+                        # instead of a compound one.
+                        with ensure_free_stackframes():
+                            result = self.test(*args, **kwargs)
                 finally:
                     finish = time.perf_counter()
                     in_drawtime = math.fsum(data.draw_times.values()) - arg_drawtime

hypothesis-python/src/hypothesis/internal/conjecture/engine.py

@@ -852,20 +852,21 @@ def on_observation(observation: Observation) -> None:
         return nullcontext()
 
     def run(self) -> None:
-        with (
-            local_settings(self.settings),
-            self.observe_for_provider(),
-        ):
-            try:
-                self._run()
-            except RunIsComplete:
-                pass
-            for v in self.interesting_examples.values():
-                self.debug_data(v)
-            self.debug(
-                "Run complete after %d examples (%d valid) and %d shrinks"
-                % (self.call_count, self.valid_examples, self.shrinks)
-            )
+        with local_settings(self.settings):
+            # NOTE: For compatibility with Python 3.9's LL(1)
+            # parser, this is written as a nested with-statement,
+            # instead of a compound one.
+            with self.observe_for_provider():
+                try:
+                    self._run()
+                except RunIsComplete:
+                    pass
+                for v in self.interesting_examples.values():
+                    self.debug_data(v)
+                self.debug(
+                    "Run complete after %d examples (%d valid) and %d shrinks"
+                    % (self.call_count, self.valid_examples, self.shrinks)
+                )
 
     @property
     def database(self) -> Optional[ExampleDatabase]:

hypothesis-python/src/hypothesis/vendor/pretty.py

@@ -692,6 +692,9 @@ def inner(obj: dict[object, object], p: RepresentationPrinter, cycle: bool) -> N
 
         if cycle:
             return p.text("{...}")
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
         with p.group(1, start, end):
             # If the dict contains both "" and b"" (empty string and empty bytes), we
             # ignore the BytesWarning raised by `python -bb` mode.  We can't use

hypothesis-python/tests/common/utils.py

@@ -115,6 +115,9 @@ def inverted_test(*arguments, **kwargs):
             # the `raises` context manager so that any problems in rigging the
             # PRNG don't accidentally count as the expected failure.
             with deterministic_PRNG():
+                # NOTE: For compatibility with Python 3.9's LL(1)
+                # parser, this is written as a nested with-statement,
+                # instead of a compound one.
                 with raises(e, match=match):
                     f(*arguments, **kwargs)
 
@@ -229,6 +232,9 @@ def temp_registered(type_, strat_or_factory):
 def raises_warning(expected_warning, match=None):
     """Use instead of pytest.warns to check that the raised warning is handled properly"""
     with raises(expected_warning, match=match) as r:
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
         with warnings.catch_warnings():
             warnings.simplefilter("error", category=expected_warning)
             yield r

hypothesis-python/tests/conjecture/test_engine.py

@@ -1065,16 +1065,20 @@ def test(data):
 def test_prefix_cannot_exceed_buffer_size(monkeypatch):
     buffer_size = 10
 
-    with deterministic_PRNG(), buffer_size_limit(buffer_size):
-
-        def test(data):
-            while data.draw_boolean():
+    with deterministic_PRNG():
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
+        with buffer_size_limit(buffer_size):
+
+            def test(data):
+                while data.draw_boolean():
+                    assert data.length <= buffer_size
                 assert data.length <= buffer_size
-            assert data.length <= buffer_size
 
-        runner = ConjectureRunner(test, settings=SMALL_COUNT_SETTINGS)
-        runner.run()
-        assert runner.valid_examples == buffer_size
+            runner = ConjectureRunner(test, settings=SMALL_COUNT_SETTINGS)
+            runner.run()
+            assert runner.valid_examples == buffer_size
 
 
 def test_does_not_shrink_multiple_bugs_when_told_not_to():

hypothesis-python/tests/conjecture/test_optimiser.py

@@ -210,6 +210,9 @@ def test(data):
 
 def test_optimiser_when_test_grows_buffer_to_overflow():
     with deterministic_PRNG():
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
         with buffer_size_limit(2):
 
             def test(data):

hypothesis-python/tests/conjecture/test_provider.py

@@ -303,6 +303,9 @@ class InvalidLifetime(TrivialProvider):
 
 def test_invalid_lifetime():
     with temp_register_backend("invalid_lifetime", InvalidLifetime):
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
         with pytest.raises(InvalidArgument):
             ConjectureRunner(
                 lambda: True, settings=settings(backend="invalid_lifetime")
@@ -649,17 +652,18 @@ def test_can_generate_from_all_available_providers(backend, strategy):
     def f(x):
         raise ValueError
 
-    with (
-        pytest.raises(ValueError),
-        (
+    with pytest.raises(ValueError):
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
+        with (
             pytest.warns(
                 HypothesisWarning, match="/dev/urandom is not available on windows"
             )
             if backend == "hypothesis-urandom" and WINDOWS
             else nullcontext()
-        ),
-    ):
-        f()
+        ):
+            f()
 
 
 def test_saves_on_fatal_error_with_backend():

hypothesis-python/tests/cover/test_control.py

@@ -75,13 +75,19 @@ def test_can_nest_build_context():
 
 def test_does_not_suppress_exceptions():
     with pytest.raises(AssertionError):
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
         with bc():
             raise AssertionError
     assert _current_build_context.value is None
 
 
 def test_suppresses_exceptions_in_teardown():
     with pytest.raises(ValueError) as exc_info:
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
         with bc():
 
             def foo():
@@ -96,6 +102,9 @@ def foo():
 
 def test_runs_multiple_cleanup_with_teardown():
     with pytest.raises(ExceptionGroup) as exc_info:
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
         with bc():
 
             def foo():
@@ -116,6 +125,9 @@ def bar():
 
 def test_raises_error_if_cleanup_fails_but_block_does_not():
     with pytest.raises(ValueError):
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
         with bc():
 
             def foo():
@@ -170,6 +182,9 @@ def test(x):
         assert x < 5
 
     with capture_out() as out:
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
         with reporting.with_reporter(reporting.default):
             with pytest.raises(AssertionError):
                 test()

hypothesis-python/tests/cover/test_database_backend.py

@@ -280,6 +280,9 @@ def test_ga_no_artifact(tmp_path):
 
 def test_ga_corrupted_artifact():
     """Tests that corrupted artifacts are properly detected and warned about."""
+    # NOTE: For compatibility with Python 3.9's LL(1)
+    # parser, this is written as a nested with-statement,
+    # instead of a compound one.
     with ga_empty_artifact() as (path, zip_path):
         # Corrupt the CRC of the zip file
         with open(zip_path, "rb+") as f:
@@ -296,6 +299,9 @@ def test_ga_deletes_old_artifacts():
     """Tests that old artifacts are automatically deleted."""
     now = datetime.now(timezone.utc)
     with ga_empty_artifact(date=now) as (path, file_now):
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
         with ga_empty_artifact(date=now - timedelta(hours=2), path=path) as (
             _,
             file_old,

hypothesis-python/tests/cover/test_debug_information.py

@@ -27,6 +27,9 @@ def test(i):
         assert i < 10
 
     with capture_out() as out:
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
         with pytest.raises(AssertionError):
             test()
 

hypothesis-python/tests/cover/test_explicit_examples.py

@@ -141,6 +141,9 @@ def test_positive(x):
         assert x > 0
 
     with reporting.with_reporter(reporting.default):
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
         with pytest.raises(AssertionError):
             with capture_out() as out:
                 test_positive()
@@ -192,6 +195,9 @@ def test(x):
         print(f"x -> {x}")
 
     with capture_out() as out:
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
         with reporting.with_reporter(reporting.default):
             test()
     ls = out.getvalue().splitlines()

hypothesis-python/tests/cover/test_lookup.py

@@ -1258,8 +1258,9 @@ def test_issue_4194_regression():
     inner = typing.Union[typing.Sequence["A"], MyProtocol]
     A = typing.Union[typing.Sequence[inner], MyProtocol]
 
-    with (
-        temp_registered(MyProtocol, st.just(b"")),
-        temp_registered(typing.ForwardRef("A"), st.integers()),
-    ):
-        find_any(st.from_type(A))
+    with temp_registered(MyProtocol, st.just(b"")):
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
+        with temp_registered(typing.ForwardRef("A"), st.integers()):
+            find_any(st.from_type(A))

hypothesis-python/tests/cover/test_monitoring.py

@@ -32,6 +32,9 @@ def test_monitoring_warns_on_registered_tool_id(warns_or_raises):
 
     # scrutineer can't run if something has already registered its tool id.
     with using_tool_id(MONITORING_TOOL_ID, "rogue"):
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
         with warns_or_raises(HypothesisWarning, match=r"already taken by tool rogue"):
 
             @given(st.integers())

hypothesis-python/tests/cover/test_numerics.py

@@ -169,6 +169,9 @@ def test_consistent_decimal_error():
     with pytest.raises(InvalidArgument) as excinfo:
         check_can_generate_examples(decimals(bad))
     with pytest.raises(InvalidArgument) as excinfo2:
+        # NOTE: For compatibility with Python 3.9's LL(1)
+        # parser, this is written as a nested with-statement,
+        # instead of a compound one.
         with decimal.localcontext(decimal.Context(traps=[])):
             check_can_generate_examples(decimals(bad))
     assert str(excinfo.value) == str(excinfo2.value)