add PrimitiveProvider.on_observation

Author: Liam-DeVoe

hypothesis-python/RELEASE.rst

@@ -0,0 +1,3 @@
+RELEASE_TYPE: patch
+
+Add |PrimitiveProvider.on_observation| to the internal :ref:`alternative backends <alternative-backends-internals>` interface.

hypothesis-python/docs/conf.py

@@ -276,14 +276,18 @@ def setup(app):
 .. |.example()| replace:: :func:`.example() <hypothesis.strategies.SearchStrategy.example>`
 
 .. |PrimitiveProvider| replace:: :class:`~hypothesis.internal.conjecture.providers.PrimitiveProvider`
-.. |PrimitiveProvider.realize| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.realize`
 .. |PrimitiveProvider.draw_integer| replace:: \
     :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.draw_integer`
 .. |PrimitiveProvider.draw_boolean| replace:: \
     :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.draw_boolean`
 .. |PrimitiveProvider.draw_float| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.draw_float`
 .. |PrimitiveProvider.draw_string| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.draw_string`
 .. |PrimitiveProvider.draw_bytes| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.draw_bytes`
+.. |PrimitiveProvider.realize| replace:: :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.realize`
+.. |PrimitiveProvider.on_observation| replace:: \
+    :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.on_observation`
+.. |PrimitiveProvider.per_test_case_context_manager| replace:: \
+    :func:`~hypothesis.internal.conjecture.providers.PrimitiveProvider.per_test_case_context_manager`
 
 .. |AVAILABLE_PROVIDERS| replace:: :data:`~hypothesis.internal.conjecture.providers.AVAILABLE_PROVIDERS`
 .. |TESTCASE_CALLBACKS| replace:: :data:`~hypothesis.internal.observability.TESTCASE_CALLBACKS`

hypothesis-python/src/hypothesis/internal/conjecture/engine.py

@@ -15,7 +15,7 @@
 import time
 from collections import defaultdict
 from collections.abc import Generator, Sequence
-from contextlib import contextmanager, suppress
+from contextlib import AbstractContextManager, contextmanager, nullcontext, suppress
 from dataclasses import dataclass, field
 from datetime import timedelta
 from enum import Enum
@@ -68,6 +68,7 @@
 from hypothesis.internal.conjecture.shrinker import Shrinker, ShrinkPredicateT, sort_key
 from hypothesis.internal.escalation import InterestingOrigin
 from hypothesis.internal.healthcheck import fail_health_check
+from hypothesis.internal.observability import Observation, with_observation_callback
 from hypothesis.reporting import base_report, report
 
 #: The maximum number of times the shrinker will reduce the complexity of a failing
@@ -830,8 +831,34 @@ def debug_data(self, data: Union[ConjectureData, ConjectureResult]) -> None:
             f"{', ' + data.output if data.output else ''}"
         )
 
+    def observe_for_provider(self) -> AbstractContextManager:
+        def on_observation(observation: Observation) -> None:
+            assert isinstance(self.provider, PrimitiveProvider)
+            # only fire if we actually used that provider to generate this observation
+            if not self._switch_to_hypothesis_provider:
+                self.provider.on_observation(observation)
+
+        # adding this callback enables observability. Use a nullcontext
+        # if the backend won't use observability.
+        return (
+            with_observation_callback(on_observation)
+            if (
+                self.settings.backend != "hypothesis"
+                # only for lifetime = "test_function" providers
+                and isinstance(self.provider, PrimitiveProvider)
+                # and the provider class overrode the default
+                # (see https://github.com/python/mypy/issues/14123 for type ignore)
+                and self.provider.on_observation.__func__  # type: ignore
+                is not PrimitiveProvider.on_observation
+            )
+            else nullcontext()
+        )
+
     def run(self) -> None:
-        with local_settings(self.settings):
+        with (
+            local_settings(self.settings),
+            self.observe_for_provider(),
+        ):
             try:
                 self._run()
             except RunIsComplete:

hypothesis-python/src/hypothesis/internal/conjecture/providers.py

@@ -62,7 +62,7 @@
     next_up,
 )
 from hypothesis.internal.intervalsets import IntervalSet
-from hypothesis.internal.observability import InfoObservationType
+from hypothesis.internal.observability import InfoObservationType, Observation
 
 if TYPE_CHECKING:
     from typing import TypeAlias
@@ -544,6 +544,37 @@ def observe_information_messages(
         assert lifetime in ("test_case", "test_function")
         yield from []
 
+    def on_observation(self, observation: Observation) -> None:  # noqa: B027
+        """
+        Called at the end of each test case which uses this provider, with the same
+        ``observation["type"] == "test_case"`` observation that is passed to
+        other callbacks in |TESTCASE_CALLBACKS|. This method is not called with
+        ``observation["type"] in {"info", "alert", "error"}`` observations.
+
+        Calls to this method are guaranteed to alternate with calls to
+        |PrimitiveProvider.per_test_case_context_manager|. For example:
+
+        .. code-block:: python
+
+            # test function starts
+            per_test_case_context_manager()
+            on_observation()
+            per_test_case_context_manager()
+            on_observation()
+            ...
+            # test function ends
+
+        Note that |PrimitiveProvider.on_observation| will not be called for test
+        cases which did not use this provider during generation, for example
+        during |Phase.reuse| or |Phase.shrink|, or because Hypothesis switched
+        to the standard Hypothesis backend after this backend raised too many
+        |BackendCannotProceed| exceptions.
+
+        By default, observability will not be enabled for backends which do not
+        override this method. By overriding this method, any test which sets
+        |settings.backend| to this provider will automatically enable observability.
+        """
+
     def span_start(self, label: int, /) -> None:  # noqa: B027  # non-abstract noop
         """Marks the beginning of a semantically meaningful span of choices.
 

hypothesis-python/src/hypothesis/internal/observability.py

@@ -15,6 +15,8 @@
 import sys
 import time
 import warnings
+from collections.abc import Generator
+from contextlib import contextmanager
 from dataclasses import dataclass
 from datetime import date, timedelta
 from functools import lru_cache
@@ -98,6 +100,17 @@ class TestCaseObservation(BaseObservation):
 TESTCASE_CALLBACKS: list[Callable[[Observation], None]] = []
 
 
+@contextmanager
+def with_observation_callback(
+    callback: Callable[[Observation], None],
+) -> Generator[None, None, None]:
+    TESTCASE_CALLBACKS.append(callback)
+    try:
+        yield
+    finally:
+        TESTCASE_CALLBACKS.remove(callback)
+
+
 def deliver_observation(observation: Observation) -> None:
     for callback in TESTCASE_CALLBACKS:
         callback(observation)

hypothesis-python/tests/conjecture/test_alt_backend.py

@@ -46,6 +46,7 @@
 )
 from hypothesis.internal.floats import SIGNALING_NAN
 from hypothesis.internal.intervalsets import IntervalSet
+from hypothesis.internal.observability import TESTCASE_CALLBACKS, Observation
 
 from tests.common.debug import minimal
 from tests.common.utils import (
@@ -718,3 +719,43 @@ def test_replay_choices():
     # trivial covering test
     provider = TrivialProvider(None)
     provider.replay_choices([1])
+
+
+class ObservationProvider(TrivialProvider):
+    def __init__(self, conjecturedata: "ConjectureData", /) -> None:
+        super().__init__(conjecturedata)
+        # calls to per_test_case_context_manager and on_observation alternate,
+        # starting with per_test_case_context_manager
+        self.expected = "per_test_case_context_manager"
+
+    @contextmanager
+    def per_test_case_context_manager(self):
+        assert self.expected == "per_test_case_context_manager"
+        self.expected = "on_observation"
+        yield
+
+    def on_observation(self, observation: Observation) -> None:
+        assert self.expected == "on_observation"
+        self.expected = "per_test_case_context_manager"
+
+
+def test_on_observation_alternates():
+    with temp_register_backend("observation", ObservationProvider):
+
+        @given(st.integers())
+        @settings(backend="observation")
+        def f(n):
+            pass
+
+        f()
+
+
+def test_on_observation_no_override():
+    with temp_register_backend("observation", TrivialProvider):
+
+        @given(st.integers())
+        @settings(backend="observation")
+        def f(n):
+            assert TESTCASE_CALLBACKS == []
+
+        f()