Merge branch 'main' into ehcache_disk_integ

Signed-off-by: Sagar Upadhyaya <[email protected]>

Author: sgup432

.ci/bwcVersions

@@ -28,4 +28,5 @@ BWC_VERSION:
   - "2.11.1"
   - "2.11.2"
   - "2.12.0"
+  - "2.12.1"
   - "2.13.0"

.github/ISSUE_TEMPLATE/meta.yml

@@ -23,6 +23,36 @@ body:
       placeholder: Ex. - [ ] https://github.com/opensearch-project/security/issues/3888 Add views to the cluster metadata schema
     validations:
       required: true
-
-
-
+  - type: dropdown
+    attributes:
+      label: Related component
+      description: Choose a specific OpenSearch component your project belongs to.  If you are unsure of which component to select or if the component is not present, select "Other".
+      multiple: false
+      options:
+        - # Empty first option to force selection
+        - Build
+        - Clients
+        - Cluster Manager
+        - Extensions
+        - Indexing:Performance
+        - Indexing:Replication
+        - Indexing
+        - Libraries
+        - Other
+        - Plugins
+        - Search:Aggregations
+        - Search:Performance
+        - Search:Query Capabilities
+        - Search:Query Insights
+        - Search:Relevance
+        - Search:Remote Search
+        - Search:Resiliency
+        - Search:Searchable Snapshots
+        - Search
+        - Storage:Durability
+        - Storage:Performance
+        - Storage:Remote
+        - Storage:Snapshots
+        - Storage
+    validations:
+      required: true

.github/workflows/check-compatibility.yml

@@ -61,7 +61,7 @@ jobs:
           body-includes: 'Compatibility status:'
 
       - name: Add comment on the PR
-        uses: peter-evans/create-or-update-comment@v3
+        uses: peter-evans/create-or-update-comment@v4
         with:
           comment-id: ${{ steps.fc.outputs.comment-id }}
           issue-number: ${{ github.event.number }}

.github/workflows/gradle-check.yml

@@ -78,7 +78,7 @@ jobs:
 
       - name: Create Comment Success
         if: ${{ github.event_name == 'pull_request_target' && success() && env.result == 'SUCCESS' }}
-        uses: peter-evans/create-or-update-comment@v3
+        uses: peter-evans/create-or-update-comment@v4
         with:
           issue-number: ${{ env.pr_number }}
           body: |
@@ -101,7 +101,7 @@ jobs:
 
       - name: Create Comment Flaky
         if: ${{ github.event_name == 'pull_request_target' && success() && env.result != 'SUCCESS' }}
-        uses: peter-evans/create-or-update-comment@v3
+        uses: peter-evans/create-or-update-comment@v4
         with:
           issue-number: ${{ env.pr_number }}
           body: |
@@ -111,7 +111,7 @@ jobs:
 
       - name: Create Comment Failure
         if: ${{ github.event_name == 'pull_request_target' && failure() }}
-        uses: peter-evans/create-or-update-comment@v3
+        uses: peter-evans/create-or-update-comment@v4
         with:
           issue-number: ${{ env.pr_number }}
           body: |

.github/workflows/poc-checklist.yml

@@ -11,7 +11,7 @@ jobs:
       issues: write
     steps:
       - name: Add comment
-        uses: peter-evans/create-or-update-comment@v3
+        uses: peter-evans/create-or-update-comment@v4
         with:
           issue-number: ${{ github.event.issue.number }}
           body: |

.github/workflows/pull-request-checks.yml

@@ -17,7 +17,7 @@ jobs:
     name: Verify Description Checklist
     runs-on: ubuntu-latest
     steps:
-      - uses: peternied/check-pull-request-description-checklist@v1
+      - uses: peternied/check-pull-request-description-checklist@v1.1
         with:
           checklist-items: |
             New functionality includes testing.

.github/workflows/wrapper.yml

@@ -8,4 +8,4 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: gradle/wrapper-validation-action@v1
+      - uses: gradle/wrapper-validation-action@v2

CHANGELOG.md

@@ -16,6 +16,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Allow to pass the list settings through environment variables (like [], ["a", "b", "c"], ...) ([#10625](https://github.com/opensearch-project/OpenSearch/pull/10625))
 - [Admission Control] Integrate CPU AC with ResourceUsageCollector and add CPU AC stats to nodes/stats ([#10887](https://github.com/opensearch-project/OpenSearch/pull/10887))
 - [S3 Repository] Add setting to control connection count for sync client ([#12028](https://github.com/opensearch-project/OpenSearch/pull/12028))
+- Views, simplify data access and manipulation by providing a virtual layer over one or more indices ([#11957](https://github.com/opensearch-project/OpenSearch/pull/11957))
+- Add Remote Store Migration Experimental flag and allow mixed mode clusters under same ([#11986](https://github.com/opensearch-project/OpenSearch/pull/11986))
 
 ### Dependencies
 - Bump `log4j-core` from 2.18.0 to 2.19.0
@@ -48,6 +50,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `org.eclipse.jgit` from 6.5.0 to 6.7.0 ([#10147](https://github.com/opensearch-project/OpenSearch/pull/10147))
 - Bump OpenTelemetry from 1.30.1 to 1.31.0 ([#10617](https://github.com/opensearch-project/OpenSearch/pull/10617))
 - Bump OpenTelemetry from 1.31.0 to 1.32.0 and OpenTelemetry Semconv from 1.21.0-alpha to 1.23.1-alpha ([#11305](https://github.com/opensearch-project/OpenSearch/pull/11305))
+- Bump `org.bouncycastle:bcprov-jdk15to18` to `org.bouncycastle:bcprov-jdk18on` version 1.77 ([#12317](https://github.com/opensearch-project/OpenSearch/pull/12317))
+- Bump `org.bouncycastle:bcmail-jdk15to18` to `org.bouncycastle:bcmail-jdk18on` version 1.77 ([#12317](https://github.com/opensearch-project/OpenSearch/pull/12317))
+- Bump `org.bouncycastle:bcpkix-jdk15to18` to `org.bouncycastle:bcpkix-jdk18on` version 1.77 ([#12317](https://github.com/opensearch-project/OpenSearch/pull/12317))
 
 ### Changed
 - [CCR] Add getHistoryOperationsFromTranslog method to fetch the history snapshot from translogs ([#3948](https://github.com/opensearch-project/OpenSearch/pull/3948))
@@ -87,6 +92,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Update supported version for must_exist parameter in update aliases API ([#11872](https://github.com/opensearch-project/OpenSearch/pull/11872))
 - [Bug] Check phase name before SearchRequestOperationsListener onPhaseStart ([#12035](https://github.com/opensearch-project/OpenSearch/pull/12035))
 - Fix Span operation names generated from RestActions ([#12005](https://github.com/opensearch-project/OpenSearch/pull/12005))
+- Fix error in RemoteSegmentStoreDirectory when debug logging is enabled ([#12328](https://github.com/opensearch-project/OpenSearch/pull/12328))
 
 ### Security
 
@@ -97,11 +103,21 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Add community_id ingest processor ([#12121](https://github.com/opensearch-project/OpenSearch/pull/12121))
 - Introduce query level setting `index.query.max_nested_depth` limiting nested queries ([#3268](https://github.com/opensearch-project/OpenSearch/issues/3268)
 - Add toString methods to MultiSearchRequest, MultiGetRequest and CreateIndexRequest ([#12163](https://github.com/opensearch-project/OpenSearch/pull/12163))
+- Support for returning scores in matched queries ([#11626](https://github.com/opensearch-project/OpenSearch/pull/11626))
+- Add shard id property to SearchLookup for use in field types provided by plugins ([#1063](https://github.com/opensearch-project/OpenSearch/pull/1063))
 
 ### Dependencies
 - Bump `peter-evans/find-comment` from 2 to 3 ([#12288](https://github.com/opensearch-project/OpenSearch/pull/12288))
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.25.1 to 2.33.0 ([#12289](https://github.com/opensearch-project/OpenSearch/pull/12289))
 - Bump `com.squareup.okio:okio` from 3.7.0 to 3.8.0 ([#12290](https://github.com/opensearch-project/OpenSearch/pull/12290))
+- Bump `gradle/wrapper-validation-action` from 1 to 2 ([#12367](https://github.com/opensearch-project/OpenSearch/pull/12367))
+- Bump `netty` from 4.1.106.Final to 4.1.107.Final ([#12372](https://github.com/opensearch-project/OpenSearch/pull/12372))
+- Bump `opentelemetry` from 1.34.1 to 1.35.0 ([#12388](https://github.com/opensearch-project/OpenSearch/pull/12388))
+- Bump Apache Lucene from 9.9.2 to 9.10.0 ([#12392](https://github.com/opensearch-project/OpenSearch/pull/12392))
+- Bump `org.apache.logging.log4j:log4j-core` from 2.22.1 to 2.23.0 ([#12464](https://github.com/opensearch-project/OpenSearch/pull/12464))
+- Bump `antlr4` from 4.11.1 to 4.13.1 ([#12445](https://github.com/opensearch-project/OpenSearch/pull/12445))
+- Bump `com.netflix.nebula.ospackage-base` from 11.8.0 to 11.8.1 ([#12461](https://github.com/opensearch-project/OpenSearch/pull/12461))
+- Bump `peter-evans/create-or-update-comment` from 3 to 4 ([#12462](https://github.com/opensearch-project/OpenSearch/pull/12462))
 
 ### Changed
 - Allow composite aggregation to run under a parent filter aggregation ([#11499](https://github.com/opensearch-project/OpenSearch/pull/11499))
@@ -112,7 +128,13 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 
 ### Fixed
 - Fix for deserilization bug in weighted round-robin metadata ([#11679](https://github.com/opensearch-project/OpenSearch/pull/11679))
+- [Revert] [Bug] Check phase name before SearchRequestOperationsListener onPhaseStart ([#12035](https://github.com/opensearch-project/OpenSearch/pull/12035))
+- Add support of special WrappingSearchAsyncActionPhase so the onPhaseStart() will always be followed by onPhaseEnd() within AbstractSearchAsyncAction ([#12293](https://github.com/opensearch-project/OpenSearch/pull/12293))
 - Add a system property to configure YamlParser codepoint limits ([#12298](https://github.com/opensearch-project/OpenSearch/pull/12298))
+- Prevent read beyond slice boundary in ByteArrayIndexInput ([#10481](https://github.com/opensearch-project/OpenSearch/issues/10481))
+- Fix the "highlight.max_analyzer_offset" request parameter with "plain" highlighter ([#10919](https://github.com/opensearch-project/OpenSearch/pull/10919))
+- Warn about deprecated and ignored index.mapper.dynamic index setting ([#11193](https://github.com/opensearch-project/OpenSearch/pull/11193))
+- Fix get task API does not refresh resource stats ([#11531](https://github.com/opensearch-project/OpenSearch/pull/11531))
 
 ### Security
 

TRIAGING.md

@@ -68,7 +68,7 @@ Yes, there are several labels that are used to identify the 'state' of issues fi
 | Outcome      | Label            | Description                                                                                                                                                                                      | Canned Response                                                                                                                                                           |
 |--------------|------------------|--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------------|
 | Accepted     | `-untriaged`     | The issue has the details needed to be directed towards area owners.                                                                                                                            | "Thanks for filing this issue, please feel free to submit a pull request."                                                                                                 |
-| Rejected     | N/A              | The issue will be closed with a reason for why it was rejected. Reasons might include lack of details, or being outside the scope of the project.                                               | "Thanks for creating this issue; however, it isn't being accepted due to {REASON}. Please feel free to re-open after addressing the reason."                                |
+| Rejected     | N/A              | The issue will be closed with a reason for why it was rejected. Reasons might include lack of details, or being outside the scope of the project.                                               | "Thanks for creating this issue; however, it isn't being accepted due to {REASON}. Please feel free to open a new issue after addressing the reason."                                |
 | Area Triage  | `+{AREALABEL}`  | OpenSearch has many different areas. If it's unclear whether an issue should be accepted, it will be labeled with the area and an owner will be @mentioned for follow-up.                        | "Thanks for creating this issue; the triage meeting was unsure if this issue should be accepted, @{PERSON} or someone from the area please review and then accept or reject this issue?" |
 | Transfer     | N/A              | If the issue applies to another repository within the OpenSearch Project, it will be transferred accordingly.                                                                                    | "@opensearch-project/triage, can you please transfer this issue to project {REPOSITORY}." Or, if someone at the meeting has permissions, they can start the transfer.        |
 

build.gradle

@@ -516,7 +516,6 @@ subprojects {
         includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexClusterDefaultDocRep")
         includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexIT")
         includeClasses.add("org.opensearch.remotestore.CreateRemoteIndexTranslogDisabledIT")
-        includeClasses.add("org.opensearch.remotestore.RemoteIndexPrimaryRelocationIT")
         includeClasses.add("org.opensearch.remotestore.RemoteStoreBackpressureIT")
         includeClasses.add("org.opensearch.remotestore.RemoteStoreIT")
         includeClasses.add("org.opensearch.remotestore.RemoteStoreRefreshListenerIT")

buildSrc/src/testKit/thirdPartyAudit/sample_jars/build.gradle

@@ -17,7 +17,7 @@ repositories {
 }
 
 dependencies {
-  implementation "org.apache.logging.log4j:log4j-core:2.22.1"
+  implementation "org.apache.logging.log4j:log4j-core:2.23.0"
 }
 
 ["0.0.1", "0.0.2"].forEach { v ->

buildSrc/version.properties

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.9.2
+lucene            = 9.11.0-snapshot-8a555eb
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 21.0.2+13
@@ -18,15 +18,15 @@ asm               = 9.6
 jettison          = 1.5.4
 woodstox          = 6.4.0
 kotlin            = 1.7.10
-antlr4            = 4.11.1
+antlr4            = 4.13.1
 guava             = 32.1.1-jre
 protobuf          = 3.22.3
 jakarta_annotation = 1.3.5
 
 # when updating the JNA version, also update the version in buildSrc/build.gradle
 jna               = 5.13.0
 
-netty             = 4.1.106.Final
+netty             = 4.1.107.Final
 joda              = 2.12.2
 
 # project reactor
@@ -50,7 +50,7 @@ reactivestreams   = 1.0.4
 # when updating this version, you need to ensure compatibility with:
 #  - plugins/ingest-attachment (transitive dependency, check the upstream POM)
 #  - distribution/tools/plugin-cli
-bouncycastle=1.76
+bouncycastle=1.77
 # test dependencies
 randomizedrunner  = 2.7.1
 junit             = 4.13.2
@@ -70,5 +70,5 @@ jzlib             = 1.1.3
 resteasy          = 6.2.4.Final
 
 # opentelemetry dependencies
-opentelemetry         = 1.34.1
+opentelemetry         = 1.35.0
 opentelemetrysemconv  = 1.23.1-alpha

distribution/packages/build.gradle

@@ -63,7 +63,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "com.netflix.nebula.ospackage-base" version "11.8.0"
+  id "com.netflix.nebula.ospackage-base" version "11.8.1"
 }
 
 void addProcessFilesTask(String type, boolean jdk) {

gradle/ide.gradle

@@ -82,6 +82,9 @@ if (System.getProperty('idea.active') == 'true') {
         runConfigurations {
           defaults(JUnit) {
             vmParameters = '-ea -Djava.locale.providers=SPI,COMPAT'
+            if (BuildParams.runtimeJavaVersion > JavaVersion.VERSION_17) {
+              vmParameters += ' -Djava.security.manager=allow'
+            }
           }
         }
         copyright {

libs/core/licenses/lucene-core-9.11.0-snapshot-8a555eb.jar.sha1

@@ -0,0 +1 @@
+fe1cf5663be8bdb6aa757fd4101bc551684c90fb

libs/core/licenses/lucene-core-9.9.2.jar.sha1

@@ -99,8 +99,9 @@ public class Version implements Comparable<Version>, ToXContentFragment {
     public static final Version V_2_11_1 = new Version(2110199, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version V_2_11_2 = new Version(2110299, org.apache.lucene.util.Version.LUCENE_9_7_0);
     public static final Version V_2_12_0 = new Version(2120099, org.apache.lucene.util.Version.LUCENE_9_9_2);
-    public static final Version V_2_13_0 = new Version(2130099, org.apache.lucene.util.Version.LUCENE_9_9_2);
-    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_9_2);
+    public static final Version V_2_12_1 = new Version(2120199, org.apache.lucene.util.Version.LUCENE_9_9_2);
+    public static final Version V_2_13_0 = new Version(2130099, org.apache.lucene.util.Version.LUCENE_9_10_0);
+    public static final Version V_3_0_0 = new Version(3000099, org.apache.lucene.util.Version.LUCENE_9_11_0);
     public static final Version CURRENT = V_3_0_0;
 
     public static Version fromId(int id) {

libs/core/src/main/java/org/opensearch/Version.java

@@ -85,7 +85,8 @@ public class XContentParserTests extends OpenSearchTestCase {
         () -> randomAlphaOfLengthBetween(1, SmileXContent.DEFAULT_MAX_STRING_LEN),
         /* YAML parser limitation */
         XContentType.YAML,
-        () -> randomRealisticUnicodeOfCodepointLengthBetween(1, YamlXContent.DEFAULT_CODEPOINT_LIMIT)
+        /* use 75% of the limit, difficult to get the exact size of the content right */
+        () -> randomRealisticUnicodeOfCodepointLengthBetween(1, (int) (YamlXContent.DEFAULT_CODEPOINT_LIMIT * 0.75))
     );
 
     private static final Map<XContentType, Supplier<String>> OFF_LIMIT_GENERATORS = Map.of(

libs/x-content/src/test/java/org/opensearch/common/xcontent/XContentParserTests.java

@@ -0,0 +1 @@
+17125bae1d965624e265ef49552f6465a2bfa307

modules/lang-expression/licenses/antlr4-runtime-4.11.1.jar.sha1

@@ -0,0 +1 @@
+00759eaff8f62b38ba66a05f26ab784c268908d3

modules/lang-expression/licenses/antlr4-runtime-4.13.1.jar.sha1

@@ -77,7 +77,7 @@ public void setUp() throws Exception {
         when(fieldData.load(any())).thenReturn(atomicFieldData);
 
         service = new ExpressionScriptEngine();
-        lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData);
+        lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData, SearchLookup.UNKNOWN_SHARD_ID);
     }
 
     private FieldScript.LeafFactory compile(String expression) {

modules/lang-expression/licenses/lucene-expressions-9.11.0-snapshot-8a555eb.jar.sha1

@@ -77,7 +77,7 @@ public void setUp() throws Exception {
         when(fieldData.load(any())).thenReturn(atomicFieldData);
 
         service = new ExpressionScriptEngine();
-        lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData);
+        lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData, SearchLookup.UNKNOWN_SHARD_ID);
     }
 
     private NumberSortScript.LeafFactory compile(String expression) {

modules/lang-expression/licenses/lucene-expressions-9.9.2.jar.sha1

@@ -77,7 +77,7 @@ public void setUp() throws Exception {
         when(fieldData.load(any())).thenReturn(atomicFieldData);
 
         service = new ExpressionScriptEngine();
-        lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData);
+        lookup = new SearchLookup(mapperService, (ignored, lookup) -> fieldData, SearchLookup.UNKNOWN_SHARD_ID);
     }
 
     private TermsSetQueryScript.LeafFactory compile(String expression) {

modules/lang-expression/src/test/java/org/opensearch/script/expression/ExpressionFieldScriptTests.java

@@ -0,0 +1 @@
+17125bae1d965624e265ef49552f6465a2bfa307

modules/lang-expression/src/test/java/org/opensearch/script/expression/ExpressionNumberSortScriptTests.java

@@ -45,10 +45,10 @@
 import org.antlr.v4.runtime.atn.PredictionContextCache;
 import org.antlr.v4.runtime.dfa.DFA;
 
-@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue" })
+@SuppressWarnings({ "all", "warnings", "unchecked", "unused", "cast", "CheckReturnValue", "this-escape" })
 abstract class PainlessLexer extends Lexer {
     static {
-        RuntimeMetaData.checkVersion("4.11.1", RuntimeMetaData.VERSION);
+        RuntimeMetaData.checkVersion("4.13.1", RuntimeMetaData.VERSION);
     }
 
     protected static final DFA[] _decisionToDFA;