Fix minscore propagation in text similarity reranker

docs/changelog/129223.yaml

@@ -0,0 +1,5 @@
+pr: 129223
+summary: Fix text similarity reranker does not propagate min score correctly
+area: Search
+type: bug
+issues: []

server/src/main/java/org/elasticsearch/search/retriever/CompoundRetrieverBuilder.java

@@ -195,7 +195,8 @@ public void onFailure(Exception e) {
         RankDocsRetrieverBuilder rankDocsRetrieverBuilder = new RankDocsRetrieverBuilder(
             rankWindowSize,
             newRetrievers.stream().map(s -> s.retriever).toList(),
-            results::get
+            results::get,
+            this.minScore
         );
         rankDocsRetrieverBuilder.retrieverName(retrieverName());
         return rankDocsRetrieverBuilder;

server/src/main/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilder.java

@@ -32,14 +32,16 @@ public class RankDocsRetrieverBuilder extends RetrieverBuilder {
     final int rankWindowSize;
     final List<RetrieverBuilder> sources;
     final Supplier<RankDoc[]> rankDocs;
+    final Float minScore;
 
-    public RankDocsRetrieverBuilder(int rankWindowSize, List<RetrieverBuilder> sources, Supplier<RankDoc[]> rankDocs) {
+    public RankDocsRetrieverBuilder(int rankWindowSize, List<RetrieverBuilder> sources, Supplier<RankDoc[]> rankDocs, Float minScore) {
         this.rankWindowSize = rankWindowSize;
         this.rankDocs = rankDocs;
         if (sources == null || sources.isEmpty()) {
             throw new IllegalArgumentException("sources must not be null or empty");
         }
         this.sources = sources;
+        this.minScore = minScore;
     }
 
     @Override
@@ -48,7 +50,7 @@ public String getName() {
     }
 
     private boolean sourceHasMinScore() {
-        return minScore != null || sources.stream().anyMatch(x -> x.minScore() != null);
+        return this.minScore != null || sources.stream().anyMatch(x -> x.minScore() != null);
     }
 
     private boolean sourceShouldRewrite(QueryRewriteContext ctx) throws IOException {
@@ -132,7 +134,7 @@ public void extractToSearchSourceBuilder(SearchSourceBuilder searchSourceBuilder
             searchSourceBuilder.size(rankWindowSize);
         }
         if (sourceHasMinScore()) {
-            searchSourceBuilder.minScore(this.minScore() == null ? Float.MIN_VALUE : this.minScore());
+            searchSourceBuilder.minScore(this.minScore == null ? Float.MIN_VALUE : this.minScore);
         }
         if (searchSourceBuilder.size() + searchSourceBuilder.from() > rankDocResults.length) {
             searchSourceBuilder.size(Math.max(0, rankDocResults.length - searchSourceBuilder.from()));
@@ -160,16 +162,21 @@ protected boolean doEquals(Object o) {
         RankDocsRetrieverBuilder other = (RankDocsRetrieverBuilder) o;
         return rankWindowSize == other.rankWindowSize
             && Arrays.equals(rankDocs.get(), other.rankDocs.get())
-            && sources.equals(other.sources);
+            && sources.equals(other.sources)
+            && Objects.equals(minScore, other.minScore);
     }
 
     @Override
     protected int doHashCode() {
-        return Objects.hash(super.hashCode(), rankWindowSize, Arrays.hashCode(rankDocs.get()), sources);
+        return Objects.hash(super.hashCode(), rankWindowSize, Arrays.hashCode(rankDocs.get()), sources, minScore);
     }
 
     @Override
     protected void doToXContent(XContentBuilder builder, Params params) throws IOException {
         throw new UnsupportedOperationException("toXContent() is not supported for " + this.getClass());
     }
+
+    public Float minScore() {
+        return minScore;
+    }
 }

server/src/test/java/org/elasticsearch/search/retriever/RankDocsRetrieverBuilderTests.java

@@ -97,7 +97,7 @@ private List<QueryBuilder> preFilters(QueryRewriteContext queryRewriteContext) t
     }
 
     private RankDocsRetrieverBuilder createRandomRankDocsRetrieverBuilder(QueryRewriteContext queryRewriteContext) throws IOException {
-        return new RankDocsRetrieverBuilder(randomIntBetween(1, 100), innerRetrievers(queryRewriteContext), rankDocsSupplier());
+        return new RankDocsRetrieverBuilder(randomIntBetween(1, 100), innerRetrievers(queryRewriteContext), rankDocsSupplier(), null);
     }
 
     public void testExtractToSearchSourceBuilder() throws IOException {

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/InferenceFeatures.java

@@ -52,6 +52,7 @@ public Set<NodeFeature> getTestFeatures() {
             SemanticInferenceMetadataFieldsMapper.EXPLICIT_NULL_FIXES,
             SEMANTIC_KNN_VECTOR_QUERY_REWRITE_INTERCEPTION_SUPPORTED,
             TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_ALIAS_HANDLING_FIX,
+            TextSimilarityRankRetrieverBuilder.TEXT_SIMILARITY_RERANKER_MINSCORE_FIX,
             SemanticInferenceMetadataFieldsMapper.INFERENCE_METADATA_FIELDS_ENABLED_BY_DEFAULT,
             SEMANTIC_TEXT_HIGHLIGHTER_DEFAULT,
             SEMANTIC_KNN_FILTER_FIX,

x-pack/plugin/inference/src/main/java/org/elasticsearch/xpack/inference/rank/textsimilarity/TextSimilarityRankRetrieverBuilder.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Objects;
 
@@ -39,6 +40,9 @@ public class TextSimilarityRankRetrieverBuilder extends CompoundRetrieverBuilder
     public static final NodeFeature TEXT_SIMILARITY_RERANKER_ALIAS_HANDLING_FIX = new NodeFeature(
         "text_similarity_reranker_alias_handling_fix"
     );
+    public static final NodeFeature TEXT_SIMILARITY_RERANKER_MINSCORE_FIX = new NodeFeature(
+        "text_similarity_reranker_minscore_fix"
+    );
 
     public static final ParseField RETRIEVER_FIELD = new ParseField("retriever");
     public static final ParseField INFERENCE_ID_FIELD = new ParseField("inference_id");
@@ -157,23 +161,21 @@ protected TextSimilarityRankRetrieverBuilder clone(
     protected RankDoc[] combineInnerRetrieverResults(List<ScoreDoc[]> rankResults, boolean explain) {
         assert rankResults.size() == 1;
         ScoreDoc[] scoreDocs = rankResults.getFirst();
-        TextSimilarityRankDoc[] textSimilarityRankDocs = new TextSimilarityRankDoc[scoreDocs.length];
+        List<TextSimilarityRankDoc> filteredDocs = new ArrayList<>();
+        // Filtering by min_score must be done here, after reranking.
+        // Applying min_score in the child retriever could prematurely exclude documents that would receive high scores from the reranker.
         for (int i = 0; i < scoreDocs.length; i++) {
             ScoreDoc scoreDoc = scoreDocs[i];
             assert scoreDoc.score >= 0;
-            if (explain) {
-                textSimilarityRankDocs[i] = new TextSimilarityRankDoc(
-                    scoreDoc.doc,
-                    scoreDoc.score,
-                    scoreDoc.shardIndex,
-                    inferenceId,
-                    field
-                );
-            } else {
-                textSimilarityRankDocs[i] = new TextSimilarityRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex);
+            if (minScore == null || scoreDoc.score >= minScore) {
+                if (explain) {
+                    filteredDocs.add(new TextSimilarityRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex, inferenceId, field));
+                } else {
+                    filteredDocs.add(new TextSimilarityRankDoc(scoreDoc.doc, scoreDoc.score, scoreDoc.shardIndex));
+                }
             }
         }
-        return textSimilarityRankDocs;
+        return filteredDocs.toArray(new TextSimilarityRankDoc[0]);
     }
 
     @Override

x-pack/plugin/inference/src/yamlRestTest/resources/rest-api-spec/test/inference/70_text_similarity_rank_retriever.yml

@@ -379,3 +379,111 @@ setup:
   - match: { hits.total.value: 1 }
   - length: { hits.hits: 1 }
   - match: { hits.hits.0._id: "doc_1" }
+
+---
+"Text similarity reranker respects min_score":
+
+  - requires:
+      cluster_features: "text_similarity_reranker_minscore_fix"
+      reason: test min score functionality
+
+  - do:
+      index:
+        index: test-index
+        id: doc_2
+        body:
+          text: "The phases of the Moon come from the position of the Moon relative to the Earth and Sun."
+          topic: [ "science" ]
+          subtopic: [ "astronomy" ]
+          inference_text_field: "10"
+        refresh: true
+
+  - do:
+      search:
+        index: test-index
+        body:
+          track_total_hits: true
+          fields: [ "text", "topic" ]
+          retriever:
+            text_similarity_reranker:
+              retriever:
+                standard:
+                  query:
+                    bool:
+                      should:
+                        - constant_score:
+                            filter:
+                              term: { subtopic: "technology" }
+                            boost: 10
+                        - constant_score:
+                            filter:
+                              term: { subtopic: "astronomy" }
+                            boost: 1
+              rank_window_size: 10
+              inference_id: my-rerank-model
+              inference_text: "How often does the moon hide the sun?"
+              field: inference_text_field
+              min_score: 10
+          size: 10
+
+  - match: { hits.total.value: 1 }
+  - length: { hits.hits: 1 }
+  - match: { hits.hits.0._id: "doc_2" }
+
+---
+"Text similarity reranker with min_score zero includes all docs":
+
+  - requires:
+      cluster_features: "text_similarity_reranker_minscore_fix"
+      reason: test min score functionality
+
+  - do:
+      search:
+        index: test-index
+        body:
+          track_total_hits: true
+          fields: [ "text", "topic" ]
+          retriever:
+            text_similarity_reranker:
+              retriever:
+                standard:
+                  query:
+                    match_all: {}
+              rank_window_size: 10
+              inference_id: my-rerank-model
+              inference_text: "How often does the moon hide the sun?"
+              field: inference_text_field
+              min_score: 0
+          size: 10
+
+  - match: { hits.total.value: 3 }
+  - length: { hits.hits: 3 }
+
+---
+"Text similarity reranker with high min_score excludes all docs":
+
+  - requires:
+      cluster_features: "text_similarity_reranker_minscore_fix"
+      reason: test min score functionality
+
+  - do:
+      search:
+        index: test-index
+        body:
+          track_total_hits: true
+          fields: [ "text", "topic" ]
+          retriever:
+            text_similarity_reranker:
+              retriever:
+                standard:
+                  query:
+                    match_all: {}
+              rank_window_size: 10
+              inference_id: my-rerank-model
+              inference_text: "How often does the moon hide the sun?"
+              field: inference_text_field
+              min_score: 1000
+          size: 10
+
+  - match: { hits.total.value: 0 }
+  - length: { hits.hits: 0 }