Merge branch 'main' into main

Signed-off-by: Tommy Shao <[email protected]>

Author: anntians

.github/benchmark-configs.json

@@ -256,5 +256,21 @@
       "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
     },
     "baseline_cluster_config": "x64-r5.xlarge-1-shard-0-replica-snapshot-baseline"
+  },
+  "id_16": {
+    "description": "Benchmarking config for NESTED workload, benchmarks nested queries with inner-hits",
+    "supported_major_versions": ["2", "3"],
+    "cluster-benchmark-configs": {
+      "SINGLE_NODE_CLUSTER": "true",
+      "MIN_DISTRIBUTION": "true",
+      "TEST_WORKLOAD": "nested",
+      "WORKLOAD_PARAMS": "{\"number_of_replicas\":\"0\",\"number_of_shards\":\"1\"}",
+      "CAPTURE_NODE_STAT": "true"
+    },
+    "cluster_configuration": {
+      "size": "Single-Node",
+      "data_instance_config": "4vCPU, 32G Mem, 16G Heap"
+    },
+    "baseline_cluster_config": "x64-r5.xlarge-single-node-1-shard-0-replica-baseline"
+  }
   }
-}

CHANGELOG.md

@@ -59,6 +59,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `com.microsoft.azure:msal4j` from 1.17.2 to 1.18.0 ([#16918](https://github.com/opensearch-project/OpenSearch/pull/16918))
 - Bump `org.apache.commons:commons-text` from 1.12.0 to 1.13.0 ([#16919](https://github.com/opensearch-project/OpenSearch/pull/16919))
 - Bump `ch.qos.logback:logback-core` from 1.5.12 to 1.5.16 ([#16951](https://github.com/opensearch-project/OpenSearch/pull/16951))
+- Bump `com.azure:azure-core-http-netty` from 1.15.5 to 1.15.7 ([#16952](https://github.com/opensearch-project/OpenSearch/pull/16952))
 
 ### Changed
 - Indexed IP field supports `terms_query` with more than 1025 IP masks [#16391](https://github.com/opensearch-project/OpenSearch/pull/16391)
@@ -86,8 +87,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Skip remote-repositories validations for node-joins when RepositoriesService is not in sync with cluster-state ([#16763](https://github.com/opensearch-project/OpenSearch/pull/16763))
 - Fix _list/shards API failing when closed indices are present ([#16606](https://github.com/opensearch-project/OpenSearch/pull/16606))
 - Fix remote shards balance ([#15335](https://github.com/opensearch-project/OpenSearch/pull/15335))
-- Fixing the bug to prevent updating the index.knn setting during restore snapshot(#16957)[https://github.com/opensearch-project/OpenSearch/pull/16957]
-
+- Always use `constant_score` query for `match_only_text` field ([#16964](https://github.com/opensearch-project/OpenSearch/pull/16964))
+- Fixing the bug to prevent updating the index.knn setting during restore snapshot ([#16957](https://github.com/opensearch-project/OpenSearch/pull/16957))
 
 ### Security
 

plugins/repository-azure/build.gradle

@@ -48,7 +48,7 @@ dependencies {
   api 'com.azure:azure-json:1.3.0'
   api 'com.azure:azure-xml:1.1.0'
   api 'com.azure:azure-storage-common:12.28.0'
-  api 'com.azure:azure-core-http-netty:1.15.5'
+  api 'com.azure:azure-core-http-netty:1.15.7'
   api "io.netty:netty-codec-dns:${versions.netty}"
   api "io.netty:netty-codec-socks:${versions.netty}"
   api "io.netty:netty-codec-http2:${versions.netty}"

plugins/repository-azure/licenses/azure-core-http-netty-1.15.5.jar.sha1

@@ -0,0 +1 @@
+a83247eeeb7f63f891e725228d54c3c24132c66a

plugins/repository-azure/licenses/azure-core-http-netty-1.15.7.jar.sha1

@@ -152,7 +152,7 @@ private static MetricStat validateStarTreeMetricSupport(
             MetricStat metricStat = ((MetricAggregatorFactory) aggregatorFactory).getMetricStat();
             field = ((MetricAggregatorFactory) aggregatorFactory).getField();
 
-            if (supportedMetrics.containsKey(field) && supportedMetrics.get(field).contains(metricStat)) {
+            if (field != null && supportedMetrics.containsKey(field) && supportedMetrics.get(field).contains(metricStat)) {
                 return metricStat;
             }
         }

server/src/main/java/org/opensearch/index/compositeindex/datacube/startree/utils/StarTreeQueryHelper.java

@@ -16,6 +16,7 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
@@ -290,6 +291,16 @@ public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions,
             return new SourceFieldMatchQuery(builder.build(), phrasePrefixQuery, this, context);
         }
 
+        @Override
+        public Query termQuery(Object value, QueryShardContext context) {
+            return new ConstantScoreQuery(super.termQuery(value, context));
+        }
+
+        @Override
+        public Query termQueryCaseInsensitive(Object value, QueryShardContext context) {
+            return new ConstantScoreQuery(super.termQueryCaseInsensitive(value, context));
+        }
+
         private List<List<Term>> getTermsFromTokenStream(TokenStream stream) throws IOException {
             final List<List<Term>> termArray = new ArrayList<>();
             TermToBytesRefAttribute termAtt = stream.getAttribute(TermToBytesRefAttribute.class);

server/src/main/java/org/opensearch/index/mapper/MatchOnlyTextFieldMapper.java

@@ -104,6 +104,6 @@ public String getStatsSubtype() {
     }
 
     public String getField() {
-        return config.fieldContext().field();
+        return config.fieldContext() != null ? config.fieldContext().field() : null;
     }
 }

server/src/main/java/org/opensearch/search/aggregations/support/ValuesSourceAggregatorFactory.java

@@ -15,11 +15,13 @@
 import org.apache.lucene.index.Term;
 import org.apache.lucene.search.BooleanClause;
 import org.apache.lucene.search.BooleanQuery;
+import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.MultiPhraseQuery;
 import org.apache.lucene.search.PhraseQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.tests.analysis.MockSynonymAnalyzer;
+import org.opensearch.common.lucene.search.AutomatonQueries;
 import org.opensearch.common.lucene.search.MultiPhrasePrefixQuery;
 import org.opensearch.core.common.Strings;
 import org.opensearch.core.xcontent.MediaTypeRegistry;
@@ -28,6 +30,7 @@
 import org.opensearch.index.query.MatchPhraseQueryBuilder;
 import org.opensearch.index.query.QueryShardContext;
 import org.opensearch.index.query.SourceFieldMatchQuery;
+import org.opensearch.index.query.TermQueryBuilder;
 import org.opensearch.index.search.MatchQuery;
 import org.junit.Before;
 
@@ -391,7 +394,7 @@ public void testPhraseQuery() throws IOException {
 
         assertThat(q, is(expectedQuery));
         Query q4 = new MatchPhraseQueryBuilder("field", "singleton").toQuery(queryShardContext);
-        assertThat(q4, is(new TermQuery(new Term("field", "singleton"))));
+        assertThat(q4, is(new ConstantScoreQuery(new TermQuery(new Term("field", "singleton")))));
 
         Query q2 = new MatchPhraseQueryBuilder("field", "three words here").toQuery(queryShardContext);
         expectedQuery = new SourceFieldMatchQuery(
@@ -447,4 +450,22 @@ public void testPhraseQuery() throws IOException {
         );
         assertThat(q6, is(expectedQuery));
     }
+
+    public void testTermQuery() throws Exception {
+        MapperService mapperService = createMapperService(mapping(b -> {
+            b.startObject("field");
+            {
+                b.field("type", textFieldName);
+                b.field("analyzer", "my_stop_analyzer"); // "standard" will be replaced with MockSynonymAnalyzer
+            }
+            b.endObject();
+        }));
+        QueryShardContext queryShardContext = createQueryShardContext(mapperService);
+
+        Query q = new TermQueryBuilder("field", "foo").rewrite(queryShardContext).toQuery(queryShardContext);
+        assertEquals(new ConstantScoreQuery(new TermQuery(new Term("field", "foo"))), q);
+
+        q = new TermQueryBuilder("field", "foo").caseInsensitive(true).rewrite(queryShardContext).toQuery(queryShardContext);
+        assertEquals(new ConstantScoreQuery(AutomatonQueries.caseInsensitiveTermQuery(new Term("field", "foo"))), q);
+    }
 }

server/src/test/java/org/opensearch/index/mapper/MatchOnlyTextFieldMapperTests.java

@@ -8,7 +8,11 @@
 
 package org.opensearch.index.mapper;
 
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.ConstantScoreQuery;
+import org.apache.lucene.search.TermQuery;
 import org.opensearch.common.lucene.Lucene;
+import org.opensearch.common.lucene.search.AutomatonQueries;
 
 public class MatchOnlyTextFieldTypeTests extends TextFieldTypeTests {
 
@@ -28,4 +32,18 @@ TextFieldMapper.TextFieldType createFieldType(boolean searchable) {
             ParametrizedFieldMapper.Parameter.metaParam().get()
         );
     }
+
+    @Override
+    public void testTermQuery() {
+        MappedFieldType ft = createFieldType(true);
+        assertEquals(new ConstantScoreQuery(new TermQuery(new Term("field", "foo"))), ft.termQuery("foo", null));
+        assertEquals(
+            new ConstantScoreQuery(AutomatonQueries.caseInsensitiveTermQuery(new Term("field", "fOo"))),
+            ft.termQueryCaseInsensitive("fOo", null)
+        );
+
+        MappedFieldType unsearchable = createFieldType(false);
+        IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery("bar", null));
+        assertEquals("Cannot search on field [field] since it is not indexed.", e.getMessage());
+    }
 }

server/src/test/java/org/opensearch/index/mapper/MatchOnlyTextFieldTypeTests.java

@@ -28,18 +28,27 @@
 import org.opensearch.common.lucene.Lucene;
 import org.opensearch.common.settings.Settings;
 import org.opensearch.common.util.FeatureFlags;
+import org.opensearch.common.util.MockBigArrays;
+import org.opensearch.common.util.MockPageCacheRecycler;
+import org.opensearch.core.indices.breaker.CircuitBreakerService;
+import org.opensearch.core.indices.breaker.NoneCircuitBreakerService;
 import org.opensearch.index.codec.composite.CompositeIndexFieldInfo;
 import org.opensearch.index.codec.composite.CompositeIndexReader;
 import org.opensearch.index.codec.composite.composite912.Composite912Codec;
 import org.opensearch.index.codec.composite912.datacube.startree.StarTreeDocValuesFormatTests;
 import org.opensearch.index.compositeindex.datacube.Dimension;
+import org.opensearch.index.compositeindex.datacube.Metric;
+import org.opensearch.index.compositeindex.datacube.MetricStat;
 import org.opensearch.index.compositeindex.datacube.NumericDimension;
 import org.opensearch.index.mapper.MappedFieldType;
 import org.opensearch.index.mapper.MapperService;
 import org.opensearch.index.mapper.NumberFieldMapper;
 import org.opensearch.index.query.QueryBuilder;
+import org.opensearch.index.query.QueryShardContext;
 import org.opensearch.index.query.TermQueryBuilder;
 import org.opensearch.search.aggregations.AggregationBuilder;
+import org.opensearch.search.aggregations.AggregatorFactories;
+import org.opensearch.search.aggregations.AggregatorFactory;
 import org.opensearch.search.aggregations.AggregatorTestCase;
 import org.opensearch.search.aggregations.InternalAggregation;
 import org.opensearch.search.aggregations.metrics.AvgAggregationBuilder;
@@ -49,14 +58,17 @@
 import org.opensearch.search.aggregations.metrics.InternalSum;
 import org.opensearch.search.aggregations.metrics.InternalValueCount;
 import org.opensearch.search.aggregations.metrics.MaxAggregationBuilder;
+import org.opensearch.search.aggregations.metrics.MetricAggregatorFactory;
 import org.opensearch.search.aggregations.metrics.MinAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.SumAggregationBuilder;
 import org.opensearch.search.aggregations.metrics.ValueCountAggregationBuilder;
+import org.opensearch.search.aggregations.support.ValuesSourceAggregatorFactory;
 import org.junit.After;
 import org.junit.Before;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Random;
@@ -69,6 +81,8 @@
 import static org.opensearch.search.aggregations.AggregationBuilders.min;
 import static org.opensearch.search.aggregations.AggregationBuilders.sum;
 import static org.opensearch.test.InternalAggregationTestCase.DEFAULT_MAX_BUCKETS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
 
 public class MetricAggregatorTests extends AggregatorTestCase {
 
@@ -267,6 +281,110 @@ public void testStarTreeDocValues() throws IOException {
             );
         }
 
+        CircuitBreakerService circuitBreakerService = new NoneCircuitBreakerService();
+
+        QueryShardContext queryShardContext = queryShardContextMock(
+            indexSearcher,
+            mapperServiceMock(),
+            createIndexSettings(),
+            circuitBreakerService,
+            new MockBigArrays(new MockPageCacheRecycler(Settings.EMPTY), circuitBreakerService).withCircuitBreaking()
+        );
+
+        MetricAggregatorFactory aggregatorFactory = mock(MetricAggregatorFactory.class);
+        when(aggregatorFactory.getSubFactories()).thenReturn(AggregatorFactories.EMPTY);
+        when(aggregatorFactory.getField()).thenReturn(FIELD_NAME);
+        when(aggregatorFactory.getMetricStat()).thenReturn(MetricStat.SUM);
+
+        // Case when field and metric type in aggregation are fully supported by star tree.
+        testCase(
+            indexSearcher,
+            query,
+            queryBuilder,
+            sumAggregationBuilder,
+            starTree,
+            supportedDimensions,
+            List.of(new Metric(FIELD_NAME, List.of(MetricStat.SUM, MetricStat.MAX, MetricStat.MIN, MetricStat.AVG))),
+            verifyAggregation(InternalSum::getValue),
+            aggregatorFactory,
+            true
+        );
+
+        // Case when the field is not supported by star tree
+        SumAggregationBuilder invalidFieldSumAggBuilder = sum("_name").field("hello");
+        testCase(
+            indexSearcher,
+            query,
+            queryBuilder,
+            invalidFieldSumAggBuilder,
+            starTree,
+            supportedDimensions,
+            Collections.emptyList(),
+            verifyAggregation(InternalSum::getValue),
+            invalidFieldSumAggBuilder.build(queryShardContext, null),
+            false // Invalid fields will return null StarTreeQueryContext which will not cause early termination by leaf collector
+        );
+
+        // Case when metric type in aggregation is not supported by star tree but the field is supported.
+        testCase(
+            indexSearcher,
+            query,
+            queryBuilder,
+            sumAggregationBuilder,
+            starTree,
+            supportedDimensions,
+            List.of(new Metric(FIELD_NAME, List.of(MetricStat.MAX, MetricStat.MIN, MetricStat.AVG))),
+            verifyAggregation(InternalSum::getValue),
+            aggregatorFactory,
+            false
+        );
+
+        // Case when field is not present in supported metrics
+        testCase(
+            indexSearcher,
+            query,
+            queryBuilder,
+            sumAggregationBuilder,
+            starTree,
+            supportedDimensions,
+            List.of(new Metric("hello", List.of(MetricStat.MAX, MetricStat.MIN, MetricStat.AVG))),
+            verifyAggregation(InternalSum::getValue),
+            aggregatorFactory,
+            false
+        );
+
+        AggregatorFactories aggregatorFactories = mock(AggregatorFactories.class);
+        when(aggregatorFactories.getFactories()).thenReturn(new AggregatorFactory[] { mock(MetricAggregatorFactory.class) });
+        when(aggregatorFactory.getSubFactories()).thenReturn(aggregatorFactories);
+
+        // Case when sub aggregations are present
+        testCase(
+            indexSearcher,
+            query,
+            queryBuilder,
+            sumAggregationBuilder,
+            starTree,
+            supportedDimensions,
+            List.of(new Metric("hello", List.of(MetricStat.MAX, MetricStat.MIN, MetricStat.AVG))),
+            verifyAggregation(InternalSum::getValue),
+            aggregatorFactory,
+            false
+        );
+
+        // Case when aggregation factory is not metric aggregation
+        testCase(
+            indexSearcher,
+            query,
+            queryBuilder,
+            sumAggregationBuilder,
+            starTree,
+            supportedDimensions,
+            List.of(new Metric("hello", List.of(MetricStat.MAX, MetricStat.MIN, MetricStat.AVG))),
+            verifyAggregation(InternalSum::getValue),
+            mock(ValuesSourceAggregatorFactory.class),
+            false
+        );
+
         ir.close();
         directory.close();
     }
@@ -287,6 +405,21 @@ private <T extends AggregationBuilder, V extends InternalAggregation> void testC
         CompositeIndexFieldInfo starTree,
         List<Dimension> supportedDimensions,
         BiConsumer<V, V> verify
+    ) throws IOException {
+        testCase(searcher, query, queryBuilder, aggBuilder, starTree, supportedDimensions, Collections.emptyList(), verify, null, true);
+    }
+
+    private <T extends AggregationBuilder, V extends InternalAggregation> void testCase(
+        IndexSearcher searcher,
+        Query query,
+        QueryBuilder queryBuilder,
+        T aggBuilder,
+        CompositeIndexFieldInfo starTree,
+        List<Dimension> supportedDimensions,
+        List<Metric> supportedMetrics,
+        BiConsumer<V, V> verify,
+        AggregatorFactory aggregatorFactory,
+        boolean assertCollectorEarlyTermination
     ) throws IOException {
         V starTreeAggregation = searchAndReduceStarTree(
             createIndexSettings(),
@@ -296,8 +429,11 @@ private <T extends AggregationBuilder, V extends InternalAggregation> void testC
             aggBuilder,
             starTree,
             supportedDimensions,
+            supportedMetrics,
             DEFAULT_MAX_BUCKETS,
             false,
+            aggregatorFactory,
+            assertCollectorEarlyTermination,
             DEFAULT_MAPPED_FIELD
         );
         V expectedAggregation = searchAndReduceStarTree(
@@ -308,8 +444,11 @@ private <T extends AggregationBuilder, V extends InternalAggregation> void testC
             aggBuilder,
             null,
             null,
+            null,
             DEFAULT_MAX_BUCKETS,
             false,
+            aggregatorFactory,
+            assertCollectorEarlyTermination,
             DEFAULT_MAPPED_FIELD
         );
         verify.accept(expectedAggregation, starTreeAggregation);