Sort And Relevance Cuffoff Part 3 - Add Relevance Cutoff Feature

src/marqo/core/constants.py

@@ -21,9 +21,10 @@
 MARQO_STRUCTURED_HYBRID_SEARCH_MINIMUM_VERSION = semver.VersionInfo.parse('2.10.0')
 MARQO_UNSTRUCTURED_HYBRID_SEARCH_MINIMUM_VERSION = semver.VersionInfo.parse('2.11.0')
 MARQO_CUSTOM_VECTOR_NORMALIZATION_MINIMUM_VERSION = semver.VersionInfo.parse('2.13.0')
+MARQO_SEMI_UNSTRUCTURED_INDEX_VERSION = semver.VersionInfo.parse('2.13.0')
 MARQO_GLOBAL_SCORE_MODIFIERS_MINIMUM_VERSION = semver.VersionInfo.parse('2.15.0')
 MARQO_RERANK_DEPTH_MINIMUM_VERSION = semver.VersionInfo.parse('2.15.0')
-MARQO_SORT_BY_MINIMUM_VERSION = semver.VersionInfo.parse('2.21.0')
+MARQO_SORT_BY_MINIMUM_VERSION = semver.VersionInfo.parse('2.22.0')
 MARQO_LANGUAGE_MINIMUM_VERSION = semver.VersionInfo.parse('2.16.0')
 MARQO_PARTIAL_UPDATE_MINIMUM_VERSION = semver.VersionInfo.parse('2.16.0')
 

src/marqo/core/models/marqo_index.py

@@ -640,7 +640,7 @@ def index_supports_language(self) -> bool:
     @property
     def index_supports_sorty_by(self) -> bool:
         """
-        Check if the index supports sort by.
+        Check if the index supports sort by or relevance cutoff.
         """
         return self._cache_or_get(
             'index_supports_sort_by',

src/marqo/core/search/hybrid_search.py

@@ -9,7 +9,8 @@
 from marqo.core import exceptions as core_exceptions
 from marqo.core.models.facets_parameters import FacetsParameters
 from marqo.core.models.hybrid_parameters import HybridParameters, RetrievalMethod, RankingMethod
-from marqo.core.models.marqo_index import UnstructuredMarqoIndex, StructuredMarqoIndex, SemiStructuredMarqoIndex
+from marqo.core.models.marqo_index import UnstructuredMarqoIndex, StructuredMarqoIndex, SemiStructuredMarqoIndex, \
+    IndexType
 from marqo.core.models.marqo_query import MarqoHybridQuery
 from marqo.core.semi_structured_vespa_index.semi_structured_vespa_index import SemiStructuredVespaIndex
 from marqo.core.vespa_index.vespa_index import for_marqo_index as vespa_index_factory
@@ -152,6 +153,21 @@ def search(
                 "'hybridParameters.queryLexical' is provided"
             )
 
+        if sort_by and (
+                marqo_index_version < constants.MARQO_SORT_BY_MINIMUM_VERSION or
+                not marqo_index.type == IndexType.SemiStructured
+        ):
+            raise core_exceptions.UnsupportedFeatureError(
+                f"The 'sortBy' features is only supported for unstructured indexes created "
+                f"with Marqo version {constants.MARQO_SORT_BY_MINIMUM_VERSION} or later "
+            )
+
+        if (relevance_cutoff and not marqo_index.type == IndexType.SemiStructured):
+            # Legacy unstructured indexes and structured indexes do not support relevance cutoff
+            raise core_exceptions.UnsupportedFeatureError(
+                f"The 'relevanceCutoff' feature is only supported for unstructured indexes created "
+                f"with Marqo version {constants.MARQO_SEMI_UNSTRUCTURED_INDEX_VERSION} or later "
+            )
 
         # Determine the text query prefix
         text_query_prefix = marqo_index.model.get_text_query_prefix(text_query_prefix)
@@ -319,7 +335,13 @@ def search(
             f"{total_results} results from Vespa."
         )
 
+        # Collect metadata for sort by
         if sort_by is not None:
             gathered_results["_sortCandidates"] = responses.root.fields.sort_candidates
 
+        # Collect metadata for relevance cutoff
+        if relevance_cutoff is not None:
+            gathered_results["_relevantCandidates"] = responses.root.fields.relevant_candidates
+            gathered_results["_probeCandidates"] = responses.root.fields.probe_candidates
+
         return gathered_results

src/marqo/core/structured_vespa_index/structured_vespa_index.py

@@ -665,7 +665,7 @@ def _to_vespa_hybrid_query(self, marqo_query: MarqoHybridQuery) -> Dict[str, Any
                 query["marqo__hybrid.relevanceCutoff.parameters.relativeScoreFactor"] = \
                     marqo_query.relevance_cutoff.parameters.relative_score_factor
             elif marqo_query.relevance_cutoff.method == RelevanceCutoffMethod.MeanStdDev:
-                query["marqo__hybrid.relevanceCutoff.parameters.meanStdDevFactor"] = \
+                query["marqo__hybrid.relevanceCutoff.parameters.stdDevFactor"] = \
                     marqo_query.relevance_cutoff.parameters.std_dev_factor
             else:
                 # No parameters for other methods
@@ -676,7 +676,7 @@ def _to_vespa_hybrid_query(self, marqo_query: MarqoHybridQuery) -> Dict[str, Any
         if marqo_query.sort_by:
             query["marqo__hybrid.sortBy.fields"] = [field.dict() for field in marqo_query.sort_by.fields]
             query["marqo__hybrid.sortBy.sortDepth"] = marqo_query.sort_by.sort_depth
-            query["marqo__hybrid.sortBy.sortCandidates"] = marqo_query.sort_by.sort_candidates
+            query["marqo__hybrid.sortBy.minSortCandidates"] = marqo_query.sort_by.min_sort_candidates
 
             query["query_features"]["marqo__sort_field_weights_0"] = {}
             query["query_features"]["marqo__sort_field_weights_1"] = {}

src/marqo/tensor_search/models/api_models.py

@@ -355,32 +355,33 @@ def _validate_sort_by_cannot_be_used_with_global_score_modifiers(cls, values):
         return values
 
     @root_validator(pre=False)
-    def _set_sort_by_sortCandidates_parameters(cls, values):
-        """Set the value for sortCandidates in sortBy if it is not provided.
+    def _validate_and_set_sort_by_min_sort_candidates_parameters(cls, values):
+        """validate the value for min_sort_candidates in sortBy.
+        If it is not provided and relevanceCutoff is None, this function will set it to a default value.
 
         Logics:
-        - If relevanceCutoff is provided, do not set sortCandidates, otherwise:
-        - If sortBy.sortCandidates is None, set it to the maximum of:
+        - If relevanceCutoff is provided, do not set min_sort_candidates, otherwise:
+        - If sortBy.min_sort_candidates is None, set it to the maximum of:
             - _DEFAULT_SORT_CANDIDATES_MULTIPLIER * limit
             - offset + limit
-        - If sortBy.sortCandidates is provided, ensure it is at least as large as offset + limit.
+        - If sortBy.min_sort_candidates is provided, ensure it is at least as large as offset + limit.
         """
         sort_by = values.get('sort_by')
         relevance_cutoff = values.get('relevance_cutoff')
         if sort_by is None or relevance_cutoff is not None:
             return values
 
-        if sort_by.sort_candidates is None:
-            sort_by.sort_candidates = max(
+        if sort_by.min_sort_candidates is None:
+            sort_by.min_sort_candidates = max(
                 cls._DEFAULT_SORT_CANDIDATES_MULTIPLIER * values.get('limit'),
                 values.get('offset') + values.get('limit')
             )
         else:
-            # If sortCandidates is provided, ensure it is at least as large as offset + limit
-            if sort_by.sort_candidates < (values.get('offset') + values.get('limit')):
+            # If min_sort_candidates is provided, ensure it is at least as large as offset + limit
+            if sort_by.min_sort_candidates < (values.get('offset') + values.get('limit')):
                 raise ValueError(
-                    f"sortCandidates must be at least as large as offset + limit. Received "
-                    f" sortCandidates={sort_by.sort_candidates}, limit={values.get('limit')}, "
+                    f" minSortCandidates must be at least as large as offset + limit. Received "
+                    f" minSortCandidates={sort_by.min_sort_candidates}, limit={values.get('limit')}, "
                     f" offset={values.get('offset')} "
                 )
         return values

src/marqo/tensor_search/models/relevance_cutoff_model.py

@@ -12,11 +12,11 @@ class RelevanceCutoffMethod(str, Enum):
 
 
 class RelativeMaxScoreParameters(StrictBaseModel):
-    relative_score_factor: float = Field(..., gt=0, le=1, alias="relativeScoreFactor")
+    relative_score_factor: float = Field(..., ge=0, le=1, alias="relativeScoreFactor")
 
 
 class MeanStdParameters(StrictBaseModel):
-    std_dev_factor: float = Field(..., gt=0, alias="stdDevFactor")
+    std_dev_factor: float = Field(..., alias="stdDevFactor")
 
 
 class RelevanceCutoffModel(StrictBaseModel):
@@ -26,11 +26,11 @@ class RelevanceCutoffModel(StrictBaseModel):
     Attributes:
         method (RelevanceCutoffMethod): The method to use for relevance cutoff.
         probe_depth (int): The number of documents to probe for relevance cutoff. Defaults to 1000. We use
-            a lexical search as a probe search. Check Vespa Customer Searcher for more details.
+            a lexical search as a probe search. Check Vespa Custom Searcher for more details.
         parameters (Union[RelativeMaxScoreParameters, MeanStdParameters]): The parameters for the relevance cutoff method.
             If the method is RelativeMaxScore, you must provide 'relativeScoreFactor' as a parameter.
             If the method is MeanStd, you must provide 'stdDevFactor' as a parameter.
-            Check Vespa Customer Searcher for more details.
+            Check Vespa Custom Searcher for more details.
     """
     method: RelevanceCutoffMethod
     probe_depth: int = Field(1000, ge=1, alias="probeDepth")

src/marqo/tensor_search/models/sort_by_model.py

@@ -49,10 +49,10 @@ class SortByModel(StrictBaseModel):
             Note that the order of fields in this list determines the order of sorting. Fields presented later will
             be used as tiebreakers for fields presented earlier.
         sort_depth (Optional[int]): The depth of sorting at the global phase.
-            Check Vespa Customer Searcher for more details.
-        sort_candidates (Optional[int]): The minimum number of candidates to be retrieved.
-            Check Vespa Customer Searcher for more details.
+            Check Vespa Custom Searcher for more details.
+        min_sort_candidates (Optional[int]): The minimum number of candidates to be retrieved.
+            Check Vespa Custom Searcher for more details.
     """
     fields: List[SortByField] = Field(..., min_items=1, max_items=3)
     sort_depth: Optional[int] = Field(None, ge=1, alias="sortDepth")
-    sort_candidates: Optional[int] = Field(None, ge=1, alias="sortCandidates")
+    min_sort_candidates: Optional[int] = Field(None, ge=1, alias="minSortCandidates")

src/marqo/tensor_search/tensor_search.py

@@ -422,21 +422,6 @@ def search(config: Config, index_name: str, text: Optional[Union[str, dict, Cust
             f"{str(constants.MARQO_RERANK_DEPTH_MINIMUM_VERSION)} or later. "
             f"This index was created with Marqo {marqo_index_version}."
         )
-
-    if sort_by:
-        if not isinstance(marqo_index, SemiStructuredMarqoIndex):
-            raise core_exceptions.UnsupportedFeatureError(
-                f"The 'sortBy' feature is only supported for unstructured indexes created with Marqo version "
-                f"{constants.MARQO_SORT_BY_MINIMUM_VERSION} or later. "
-                f"Your index is either a structured index or an old unstructured index"
-            )
-        if not marqo_index.index_supports_sorty_by:
-            raise core_exceptions.UnsupportedFeatureError(
-                f"The 'sortBy' feature is only supported for unstructured indexes created with Marqo version "
-                f"{constants.MARQO_SORT_BY_MINIMUM_VERSION} or later. "
-                f"This unstructured index was created with Marqo {marqo_index_version} "
-            )
-
     if search_method.upper() in {SearchMethod.TENSOR, SearchMethod.HYBRID}:
         # Default approximate and efSearch -- we can't set these at API-level since they're not a valid args
         # for lexical search

src/marqo/version.py

@@ -1,4 +1,4 @@
-__version__ = "2.21.0"
+__version__ = "2.22.0"
 
 def get_version() -> str:
     return f"{__version__}"

src/marqo/vespa/models/query_result.py

@@ -6,7 +6,9 @@
 # See https://docs.vespa.ai/en/reference/default-result-format.html
 class RootFields(BaseModel):
     total_count: Optional[int] = Field(None, alias='totalCount')
-    sort_candidates: Optional[int] = Field(None, alias='marqo__sortCandidates')
+    sort_candidates: Optional[int] = Field(None, alias='marqo__fields.sortCandidates')
+    relevant_candidates: Optional[int] = Field(None, alias='marqo__fields.relevantCandidates')
+    probe_candidates: Optional[int] = Field(None, alias='marqo__fields.probeCandidates')
 
 
 class Degraded(BaseModel):