Merge remote-tracking branch 'upstream/main' into fix_multisearch

Author: gaobinlong

.github/workflows/precommit.yml

@@ -7,7 +7,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        java: [ 11, 17, 21 ]
+        java: [ 11, 17, 21, 23 ]
         os: [ubuntu-latest, windows-latest, macos-latest, macos-13]
     steps:
       - uses: actions/checkout@v4

CHANGELOG.md

@@ -7,8 +7,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 ### Added
 - Add support for async deletion in S3BlobContainer ([#15621](https://github.com/opensearch-project/OpenSearch/pull/15621))
 - MultiTermQueries in keyword fields now default to `indexed` approach and gated behind cluster setting ([#15637](https://github.com/opensearch-project/OpenSearch/pull/15637))
+- [Workload Management] Add Integration Tests for Workload Management CRUD APIs ([#15955](https://github.com/opensearch-project/OpenSearch/pull/15955))
 - [Workload Management] QueryGroup resource cancellation framework changes ([#15651](https://github.com/opensearch-project/OpenSearch/pull/15651))
 - [Workload Management] Add orchestrator for wlm resiliency (QueryGroupService) ([#15925](https://github.com/opensearch-project/OpenSearch/pull/15925))
+- [Workload Management] Add QueryGroup Stats API Logic ([15777](https://github.com/opensearch-project/OpenSearch/pull/15777))
 - Fallback to Remote cluster-state on Term-Version check mismatch - ([#15424](https://github.com/opensearch-project/OpenSearch/pull/15424))
 - Implement WithFieldName interface in ValuesSourceAggregationBuilder & FieldSortBuilder ([#15916](https://github.com/opensearch-project/OpenSearch/pull/15916))
 - Add successfulSearchShardIndices in searchRequestContext ([#15967](https://github.com/opensearch-project/OpenSearch/pull/15967), [#16110](https://github.com/opensearch-project/OpenSearch/pull/16110))
@@ -52,6 +54,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Remove identity-related feature flagged code from the RestController ([#15430](https://github.com/opensearch-project/OpenSearch/pull/15430))
 - Remove Identity FeatureFlag ([#16024](https://github.com/opensearch-project/OpenSearch/pull/16024))
 - Ensure RestHandler.Wrapper delegates all implementations to the wrapped handler ([#16154](https://github.com/opensearch-project/OpenSearch/pull/16154))
+- Code cleanup: Remove ApproximateIndexOrDocValuesQuery ([#16273](https://github.com/opensearch-project/OpenSearch/pull/16273))
 
 
 ### Deprecated
@@ -65,6 +68,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Fix search_as_you_type not supporting multi-fields ([#15988](https://github.com/opensearch-project/OpenSearch/pull/15988))
 - Avoid infinite loop when `flat_object` field contains invalid token ([#15985](https://github.com/opensearch-project/OpenSearch/pull/15985))
 - Fix infinite loop in nested agg ([#15931](https://github.com/opensearch-project/OpenSearch/pull/15931))
+- Fix update settings with null replica not honoring cluster setting bug ([#14948](https://github.com/opensearch-project/OpenSearch/pull/14948))
 - Fix race condition in node-join and node-left ([#15521](https://github.com/opensearch-project/OpenSearch/pull/15521))
 - Streaming bulk request hangs ([#16158](https://github.com/opensearch-project/OpenSearch/pull/16158))
 - Fix warnings from SLF4J on startup when repository-s3 is installed ([#16194](https://github.com/opensearch-project/OpenSearch/pull/16194))

buildSrc/build.gradle

@@ -114,7 +114,7 @@ dependencies {
   api 'com.github.johnrengelman:shadow:8.1.1'
   api 'org.jdom:jdom2:2.0.6.1'
   api "org.jetbrains.kotlin:kotlin-stdlib-jdk8:${props.getProperty('kotlin')}"
-  api 'de.thetaphi:forbiddenapis:3.6'
+  api 'de.thetaphi:forbiddenapis:3.8'
   api 'com.avast.gradle:gradle-docker-compose-plugin:0.17.6'
   api "org.yaml:snakeyaml:${props.getProperty('snakeyaml')}"
   api 'org.apache.maven:maven-model:3.9.6'

buildSrc/src/main/java/org/opensearch/gradle/precommit/ThirdPartyAuditPrecommitPlugin.java

@@ -51,7 +51,7 @@ public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
     public TaskProvider<? extends Task> createTask(Project project) {
         project.getPlugins().apply(CompileOnlyResolvePlugin.class);
         project.getConfigurations().create("forbiddenApisCliJar");
-        project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.5.1");
+        project.getDependencies().add("forbiddenApisCliJar", "de.thetaphi:forbiddenapis:3.8");
 
         Configuration jdkJarHellConfig = project.getConfigurations().create(JDK_JAR_HELL_CONFIG_NAME);
         if (BuildParams.isInternal() && project.getPath().equals(":libs:opensearch-core") == false) {

buildSrc/src/main/java/org/opensearch/gradle/test/DistroTestPlugin.java

@@ -77,9 +77,9 @@
 import java.util.stream.Stream;
 
 public class DistroTestPlugin implements Plugin<Project> {
-    private static final String SYSTEM_JDK_VERSION = "21.0.4+7";
+    private static final String SYSTEM_JDK_VERSION = "23+37";
     private static final String SYSTEM_JDK_VENDOR = "adoptium";
-    private static final String GRADLE_JDK_VERSION = "21.0.4+7";
+    private static final String GRADLE_JDK_VERSION = "23+37";
     private static final String GRADLE_JDK_VENDOR = "adoptium";
 
     // all distributions used by distro tests. this is temporary until tests are per distribution

buildSrc/version.properties

@@ -2,7 +2,7 @@ opensearch        = 3.0.0
 lucene            = 9.12.0
 
 bundled_jdk_vendor = adoptium
-bundled_jdk = 21.0.4+7
+bundled_jdk = 23+37
 
 # optional dependencies
 spatial4j         = 0.7
@@ -60,9 +60,9 @@ bouncycastle=1.78
 randomizedrunner  = 2.7.1
 junit             = 4.13.2
 hamcrest          = 2.1
-mockito           = 5.12.0
+mockito           = 5.14.1
 objenesis         = 3.2
-bytebuddy         = 1.14.9
+bytebuddy         = 1.15.4
 
 # benchmark dependencies
 jmh               = 1.35

gradle/code-coverage.gradle

@@ -19,7 +19,7 @@ repositories {
 
 allprojects {
   plugins.withId('jacoco') {
-    jacoco.toolVersion = '0.8.10'
+    jacoco.toolVersion = '0.8.12'
   }
 }
 

libs/grok/src/main/java/org/opensearch/grok/GrokCaptureConfig.java

@@ -114,8 +114,8 @@ public GrokCaptureExtracter forBoolean(Function<Consumer<Boolean>, GrokCaptureEx
     /**
      * Build an extract that has access to the "native" type of the extracter
      * match. This means that patterns like {@code %{NUMBER:bytes:float}} has
-     * access to an actual {@link float}. Extracters returned from this method
-     * should be stateless stateless and can be reused. Pathological implementations
+     * access to an actual float. Extracters returned from this method
+     * should be stateless and can be reused. Pathological implementations
      * of the {@code map} parameter could violate this, but the caller should
      * take care to stay sane.
      * <p>
@@ -144,27 +144,27 @@ public interface NativeExtracterMap<T> {
         T forString(Function<Consumer<String>, GrokCaptureExtracter> buildExtracter);
 
         /**
-         * Called when the native type is an {@link int}.
+         * Called when the native type is an int.
          */
         T forInt(Function<IntConsumer, GrokCaptureExtracter> buildExtracter);
 
         /**
-         * Called when the native type is an {@link long}.
+         * Called when the native type is an long.
          */
         T forLong(Function<LongConsumer, GrokCaptureExtracter> buildExtracter);
 
         /**
-         * Called when the native type is an {@link float}.
+         * Called when the native type is an float.
          */
         T forFloat(Function<FloatConsumer, GrokCaptureExtracter> buildExtracter);
 
         /**
-         * Called when the native type is an {@link double}.
+         * Called when the native type is an double.
          */
         T forDouble(Function<DoubleConsumer, GrokCaptureExtracter> buildExtracter);
 
         /**
-         * Called when the native type is an {@link boolean}.
+         * Called when the native type is an boolean.
          */
         T forBoolean(Function<Consumer<Boolean>, GrokCaptureExtracter> buildExtracter);
     }

plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2DiscoveryTests.java

@@ -175,6 +175,7 @@ protected List<TransportAddress> buildDynamicHosts(Settings nodeSettings, int no
                                 exchange.getResponseHeaders().set("Content-Type", "text/xml; charset=UTF-8");
                                 exchange.sendResponseHeaders(HttpStatus.SC_OK, responseBody.length);
                                 exchange.getResponseBody().write(responseBody);
+                                exchange.getResponseBody().flush();
                                 return;
                             }
                         }

plugins/discovery-ec2/src/test/java/org/opensearch/discovery/ec2/Ec2RetriesTests.java

@@ -125,6 +125,7 @@ public void testEC2DiscoveryRetriesOnRateLimiting() throws IOException {
                     exchange.getResponseHeaders().set("Content-Type", "text/xml; charset=UTF-8");
                     exchange.sendResponseHeaders(HttpStatus.SC_OK, responseBody.length);
                     exchange.getResponseBody().write(responseBody);
+                    exchange.getResponseBody().flush();
                     return;
                 }
             }

plugins/repository-hdfs/build.gradle

@@ -146,6 +146,10 @@ for (String fixtureName : ['hdfsFixture', 'haHdfsFixture', 'secureHdfsFixture',
     }
     final List<String> miniHDFSArgs = []
 
+    if (BuildParams.runtimeJavaVersion >= JavaVersion.VERSION_23) {
+      miniHDFSArgs.add('-Djava.security.manager=allow')
+    }
+
     // If it's a secure fixture, then depend on Kerberos Fixture and principals + add the krb5conf to the JVM options
     if (fixtureName.equals('secureHdfsFixture') || fixtureName.equals('secureHaHdfsFixture')) {
       miniHDFSArgs.add("-Djava.security.krb5.conf=${project(':test:fixtures:krb5kdc-fixture').ext.krb5Conf("hdfs")}");

plugins/repository-s3/src/main/java/org/opensearch/repositories/s3/S3RepositoryPlugin.java

@@ -188,7 +188,7 @@ private static int allocatedProcessors(Settings settings) {
     }
 
     private static int urgentPoolCount(Settings settings) {
-        return boundedBy((allocatedProcessors(settings) + 7) / 8, 1, 2);
+        return boundedBy((allocatedProcessors(settings) + 1) / 2, 1, 2);
     }
 
     private static int priorityPoolCount(Settings settings) {

plugins/repository-s3/src/test/java/org/opensearch/repositories/s3/S3RepositoryPluginTests.java

@@ -0,0 +1,81 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.repositories.s3;
+
+import org.opensearch.common.settings.Settings;
+import org.opensearch.common.unit.SizeUnit;
+import org.opensearch.common.unit.SizeValue;
+import org.opensearch.common.util.concurrent.OpenSearchThreadPoolExecutor;
+import org.opensearch.test.OpenSearchTestCase;
+import org.opensearch.threadpool.ExecutorBuilder;
+import org.opensearch.threadpool.ThreadPool;
+import org.opensearch.threadpool.ThreadPool.ThreadPoolType;
+
+import java.io.IOException;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.concurrent.Executor;
+
+import static org.hamcrest.CoreMatchers.instanceOf;
+import static org.hamcrest.Matchers.equalTo;
+import static org.hamcrest.Matchers.notNullValue;
+
+public class S3RepositoryPluginTests extends OpenSearchTestCase {
+
+    private static final String URGENT_FUTURE_COMPLETION = "urgent_future_completion";
+
+    public void testGetExecutorBuilders() throws IOException {
+        final int processors = randomIntBetween(1, 64);
+        Settings settings = Settings.builder().put("node.name", "test").put("node.processors", processors).build();
+        Path configPath = createTempDir();
+        ThreadPool threadPool = null;
+        try (S3RepositoryPlugin plugin = new S3RepositoryPlugin(settings, configPath)) {
+            List<ExecutorBuilder<?>> executorBuilders = plugin.getExecutorBuilders(settings);
+            assertNotNull(executorBuilders);
+            assertFalse(executorBuilders.isEmpty());
+            threadPool = new ThreadPool(settings, executorBuilders.toArray(new ExecutorBuilder<?>[0]));
+            final Executor executor = threadPool.executor(URGENT_FUTURE_COMPLETION);
+            assertNotNull(executor);
+            assertThat(executor, instanceOf(OpenSearchThreadPoolExecutor.class));
+            final OpenSearchThreadPoolExecutor openSearchThreadPoolExecutor = (OpenSearchThreadPoolExecutor) executor;
+            final ThreadPool.Info info = threadPool.info(URGENT_FUTURE_COMPLETION);
+            int size = boundedBy((processors + 1) / 2, 1, 2);
+            assertThat(info.getName(), equalTo(URGENT_FUTURE_COMPLETION));
+            assertThat(info.getThreadPoolType(), equalTo(ThreadPoolType.FIXED));
+            assertThat(info.getQueueSize(), notNullValue());
+            assertThat(info.getQueueSize(), equalTo(new SizeValue(10, SizeUnit.KILO)));
+            assertThat(openSearchThreadPoolExecutor.getQueue().remainingCapacity(), equalTo(10_000));
+
+            assertThat(info.getMin(), equalTo(size));
+            assertThat(openSearchThreadPoolExecutor.getCorePoolSize(), equalTo(size));
+            assertThat(info.getMax(), equalTo(size));
+            assertThat(openSearchThreadPoolExecutor.getMaximumPoolSize(), equalTo(size));
+
+            final int availableProcessors = Runtime.getRuntime().availableProcessors();
+            if (processors > availableProcessors) {
+                assertWarnings(
+                    "setting [node.processors] to value ["
+                        + processors
+                        + "] which is more than available processors ["
+                        + availableProcessors
+                        + "] is deprecated"
+                );
+            }
+        } finally {
+            if (threadPool != null) {
+                terminate(threadPool);
+            }
+        }
+    }
+
+    private static int boundedBy(int value, int min, int max) {
+        return Math.min(max, Math.max(min, value));
+    }
+
+}

plugins/workload-management/build.gradle

@@ -10,6 +10,7 @@
  */
 
 apply plugin: 'opensearch.yaml-rest-test'
+apply plugin: 'opensearch.java-rest-test'
 apply plugin: 'opensearch.internal-cluster-test'
 
 opensearchplugin {