Merge branch 'main' into rw/removefeatureflag-1

Signed-off-by: Vinay Krishna Pudyodu <[email protected]>

Author: vinaykpud

.github/workflows/gradle-check.yml

@@ -24,7 +24,7 @@ jobs:
     - uses: actions/checkout@v4
     - name: Get changed files
       id: changed-files-specific
-      uses: tj-actions/[email protected].4
+      uses: tj-actions/[email protected].5
       with:
         files_ignore: |
           release-notes/*.md

CHANGELOG.md

@@ -36,6 +36,10 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add update and delete support in pull-based ingestion ([#17822](https://github.com/opensearch-project/OpenSearch/pull/17822))
 - Allow maxPollSize and pollTimeout in IngestionSource to be configurable ([#17863](https://github.com/opensearch-project/OpenSearch/pull/17863))
 - [Star Tree] [Search] Add query changes to support unsigned-long in star tree ([#17275](https://github.com/opensearch-project/OpenSearch/pull/17275))
+- Add `ApproximateMatchAllQuery` that targets match_all queries and approximates sorts ([#17772](https://github.com/opensearch-project/OpenSearch/pull/17772))
+- Add TermsQuery support to Search GRPC endpoint ([#17888](https://github.com/opensearch-project/OpenSearch/pull/17888))
+- Support sub agg in filter rewrite optimization ([#17447](https://github.com/opensearch-project/OpenSearch/pull/17447)
+- Disable scoring of keyword term search by default, fallback logic with new use_similarity:true parameter ([#17889](https://github.com/opensearch-project/OpenSearch/pull/17889))
 
 ### Changed
 - Migrate BC libs to their FIPS counterparts ([#14912](https://github.com/opensearch-project/OpenSearch/pull/14912))
@@ -46,6 +50,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Avoid skewed segment replication lag metric ([#17831](https://github.com/opensearch-project/OpenSearch/pull/17831))
 - Increase the default segment counter step size when replica promoting ([#17568](https://github.com/opensearch-project/OpenSearch/pull/17568))
 - [WLM] Rename QueryGroup to WorkloadGroup ([#17901](https://github.com/opensearch-project/OpenSearch/pull/17901))
+- Relaxes jarHell check for optionally extended plugins([#17893](https://github.com/opensearch-project/OpenSearch/pull/17893)))
 
 ### Dependencies
 - Bump `com.nimbusds:nimbus-jose-jwt` from 9.41.1 to 10.0.2 ([#17607](https://github.com/opensearch-project/OpenSearch/pull/17607), [#17669](https://github.com/opensearch-project/OpenSearch/pull/17669))
@@ -55,8 +60,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `dangoslen/dependabot-changelog-helper` from 3 to 4 ([#17498](https://github.com/opensearch-project/OpenSearch/pull/17498))
 - Bump `com.google.api:gax` from 2.35.0 to 2.63.1 ([#17465](https://github.com/opensearch-project/OpenSearch/pull/17465))
 - Bump `com.azure:azure-storage-blob` from 12.29.1 to 12.30.0 ([#17667](https://github.com/opensearch-project/OpenSearch/pull/17667))
-- Bump `tj-actions/changed-files` from 46.0.1 to 46.0.4 ([#17666](https://github.com/opensearch-project/OpenSearch/pull/17666), [#17813](https://github.com/opensearch-project/OpenSearch/pull/17813))
-- Bump `com.google.code.gson:gson` from 2.11.0 to 2.12.1 ([#17668](https://github.com/opensearch-project/OpenSearch/pull/17668))
+- Bump `tj-actions/changed-files` from 46.0.1 to 46.0.5 ([#17666](https://github.com/opensearch-project/OpenSearch/pull/17666), [#17813](https://github.com/opensearch-project/OpenSearch/pull/17813), [#17920](https://github.com/opensearch-project/OpenSearch/pull/17920))
+- Bump `com.google.code.gson:gson` from 2.11.0 to 2.13.0 ([#17668](https://github.com/opensearch-project/OpenSearch/pull/17668), [#17921](https://github.com/opensearch-project/OpenSearch/pull/17921)), [#17926](https://github.com/opensearch-project/OpenSearch/pull/17926))
 - Bump `com.github.luben:zstd-jni` from 1.5.5-1 to 1.5.6-1 ([#17674](https://github.com/opensearch-project/OpenSearch/pull/17674))
 - Bump `lycheeverse/lychee-action` from 2.3.0 to 2.4.0 ([#17731](https://github.com/opensearch-project/OpenSearch/pull/17731))
 - Bump `com.netflix.nebula.ospackage-base` from 11.11.1 to 11.11.2 ([#17734](https://github.com/opensearch-project/OpenSearch/pull/17734))

buildSrc/src/main/java/org/opensearch/gradle/agent/JavaAgent.java

@@ -0,0 +1,71 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gradle.agent;
+
+import org.gradle.api.Plugin;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.tasks.Copy;
+import org.gradle.api.tasks.TaskProvider;
+import org.gradle.api.tasks.testing.Test;
+
+import java.io.File;
+import java.util.Objects;
+
+/**
+ * Gradle plugin to automatically configure the OpenSearch Java agent
+ * for test tasks in OpenSearch plugin projects.
+ */
+public class JavaAgent implements Plugin<Project> {
+
+    /**
+     * Plugin implementation that sets up java agent configuration and applies it to test tasks.
+     */
+    @Override
+    public void apply(Project project) {
+        Configuration agentConfiguration = project.getConfigurations().findByName("agent");
+        if (agentConfiguration == null) {
+            agentConfiguration = project.getConfigurations().create("agent");
+        }
+
+        project.afterEvaluate(p -> {
+            String opensearchVersion = getOpensearchVersion(p);
+            p.getDependencies().add("agent", "org.opensearch:opensearch-agent-bootstrap:" + opensearchVersion);
+            p.getDependencies().add("agent", "org.opensearch:opensearch-agent:" + opensearchVersion);
+        });
+
+        Configuration finalAgentConfiguration = agentConfiguration;
+        TaskProvider<Copy> prepareJavaAgent = project.getTasks().register("prepareJavaAgent", Copy.class, task -> {
+            task.from(finalAgentConfiguration);
+            task.into(new File(project.getBuildDir(), "agent"));
+        });
+
+        project.getTasks().withType(Test.class).configureEach(testTask -> {
+            testTask.dependsOn(prepareJavaAgent);
+
+            final String opensearchVersion = getOpensearchVersion(project);
+
+            testTask.doFirst(task -> {
+                File agentJar = new File(project.getBuildDir(), "agent/opensearch-agent-" + opensearchVersion + ".jar");
+
+                testTask.jvmArgs("-javaagent:" + agentJar.getAbsolutePath());
+            });
+        });
+    }
+
+    /**
+     * Gets the OpenSearch version from project properties, with a fallback default.
+     *
+     * @param project The Gradle project
+     * @return The OpenSearch version to use
+     */
+    private String getOpensearchVersion(Project project) {
+        return Objects.requireNonNull(project.property("opensearch_version")).toString();
+    }
+}

buildSrc/src/main/resources/META-INF/gradle-plugins/opensearch.java-agent.properties

@@ -0,0 +1,9 @@
+#
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
+#
+
+implementation-class=org.opensearch.gradle.agent.JavaAgent

buildSrc/src/test/java/org/opensearch/gradle/agent/JavaAgentTests.java

@@ -0,0 +1,66 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ */
+
+package org.opensearch.gradle.agent;
+
+import org.opensearch.gradle.test.GradleUnitTestCase;
+import org.gradle.api.Project;
+import org.gradle.api.artifacts.Configuration;
+import org.gradle.api.tasks.Copy;
+import org.gradle.testfixtures.ProjectBuilder;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.IOException;
+
+public class JavaAgentTests extends GradleUnitTestCase {
+    private TemporaryFolder projectDir;
+    private final String PREPARE_JAVA_AGENT_TASK = "prepareJavaAgent";
+
+    @Before
+    public void setUp() throws IOException {
+        projectDir = new TemporaryFolder();
+        projectDir.create();
+    }
+
+    @After
+    public void tearDown() {
+        projectDir.delete();
+    }
+
+    /**
+     * This test is used to verify that adding the 'opensearch.java-agent' to the project
+     * creates the necessary agent configuration and tasks. This is basically
+     * a behavioral test of the {@link JavaAgent#apply(Project)} method.
+     */
+    @Test
+    public void applyJavaAgentPlugin() {
+        // Create an empty project and apply the JavaAgent plugin
+        Project project = ProjectBuilder.builder().build();
+        project.getPluginManager().apply(JavaAgent.class);
+
+        // Verify the agent configuration was created
+        Configuration agentConfig = project.getConfigurations().findByName("agent");
+        assertNotNull("Agent configuration should be created", agentConfig);
+
+        // Verify the prepareJavaAgent task was created and is of the right type
+        assertNotNull("prepareJavaAgent task should be created", project.getTasks().findByName(PREPARE_JAVA_AGENT_TASK));
+        assertTrue("prepareJavaAgent task should be of type Copy", project.getTasks().findByName(PREPARE_JAVA_AGENT_TASK) instanceof Copy);
+
+        // Verify the destination directory of the Copy task
+        Copy prepareTask = (Copy) project.getTasks().findByName(PREPARE_JAVA_AGENT_TASK);
+        assertEquals(
+            "Destination directory should be build/agent",
+            new File(project.getBuildDir(), "agent"),
+            prepareTask.getDestinationDir()
+        );
+    }
+}

distribution/packages/build.gradle

@@ -175,6 +175,9 @@ Closure commonPackageConfig(String type, boolean jdk, String architecture) {
           with jdkFiles(project, 'linux', architecture)
         }
       }
+      into('agent') {
+        with agentFiles()
+      }
       // we need to specify every intermediate directory in these paths so the package managers know they are explicitly
       // intended to manage them; otherwise they may be left behind on uninstallation. duplicate calls of the same
       // directory are fine

distribution/src/config/jvm.options

@@ -85,3 +85,4 @@ ${error.file}
 23:-XX:CompileCommand=dontinline,java/lang/invoke/MethodHandle.asTypeUncached
 
 21-:-javaagent:agent/opensearch-agent.jar
+21-:--add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED

gradle/ide.gradle

@@ -49,10 +49,6 @@ if (System.getProperty('idea.active') == 'true') {
     }
   }
 
-  buildScan {
-    server = 'https://127.0.0.1'
-  }
-
   idea {
     project {
       vcs = 'Git'

gradle/run.gradle

@@ -43,9 +43,17 @@ testClusters {
       installedPlugins = Eval.me(installedPlugins)
       for (String p : installedPlugins) {
         plugin('plugins:'.concat(p))
+        if (p.equals("arrow-flight-rpc")) {
+          // Add system properties for Netty configuration
+          systemProperty 'io.netty.allocator.numDirectArenas', '1'
+          systemProperty 'io.netty.noUnsafe', 'false'
+          systemProperty 'io.netty.tryUnsafe', 'true'
+          systemProperty 'io.netty.tryReflectionSetAccessible', 'true'
+        }
       }
     }
   }
+
 }
 
 tasks.register("run", RunTask) {

libs/arrow-spi/build.gradle

@@ -38,6 +38,8 @@
 import org.apache.lucene.index.IndexWriterConfig;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.NoMergePolicy;
+import org.apache.lucene.index.Term;
+import org.apache.lucene.search.ConstantScoreQuery;
 import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TermQuery;
 import org.apache.lucene.store.Directory;
@@ -121,9 +123,13 @@ public void testStoringQueryBuilders() throws IOException {
                 CheckedFunction<Integer, Query, IOException> queries = queryStore.getQueries(leafContext);
                 assertEquals(queryBuilders.length, leafContext.reader().numDocs());
                 for (int i = 0; i < queryBuilders.length; i++) {
-                    TermQuery query = (TermQuery) queries.apply(i);
-                    assertEquals(queryBuilders[i].fieldName(), query.getTerm().field());
-                    assertEquals(queryBuilders[i].value(), query.getTerm().text());
+                    Query query = queries.apply(i);
+                    if (query instanceof ConstantScoreQuery constantScoreQuery) {
+                        query = constantScoreQuery.getQuery();
+                    }
+                    Term term = ((TermQuery) query).getTerm();
+                    assertEquals(queryBuilders[i].fieldName(), term.field());
+                    assertEquals(queryBuilders[i].value(), term.text());
                 }
             }
         }