Merge branch 'main' into node-stats

Signed-off-by: Vinay Krishna Pudyodu <[email protected]>

Author: vinaykpud

CHANGELOG-3.0.md

@@ -44,6 +44,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Refactor the `:server` module `org.opensearch.client` to `org.opensearch.transport.client` to eliminate top level split packages for JPMS support ([#17272](https://github.com/opensearch-project/OpenSearch/pull/17272))
 - Use Lucene `BM25Similarity` as default since the `LegacyBM25Similarity` is marked as deprecated ([#17306](https://github.com/opensearch-project/OpenSearch/pull/17306))
 - Wildcard field index only 3gram of the input data [#17349](https://github.com/opensearch-project/OpenSearch/pull/17349)
+- Use BC libraries to parse PEM files, increase key length, allow general use of known cryptographic binary extensions, remove unused BC dependencies ([#3420](https://github.com/opensearch-project/OpenSearch/pull/14912))
 
 ### Deprecated
 

CHANGELOG.md

@@ -25,6 +25,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 - Bump `me.champeau.gradle.japicmp` from 0.4.5 to 0.4.6 ([#17375](https://github.com/opensearch-project/OpenSearch/pull/17375))
 - Bump `com.google.api.grpc:proto-google-common-protos` from 2.37.1 to 2.52.0 ([#17379](https://github.com/opensearch-project/OpenSearch/pull/17379))
 - Bump `net.minidev:json-smart` from 2.5.1 to 2.5.2 ([#17378](https://github.com/opensearch-project/OpenSearch/pull/17378))
+- Bump `com.netflix.nebula.ospackage-base` from 11.10.1 to 11.11.1 ([#17374](https://github.com/opensearch-project/OpenSearch/pull/17374))
 
 ### Changed
 - Convert transport-reactor-netty4 to use gradle version catalog [#17233](https://github.com/opensearch-project/OpenSearch/pull/17233)

buildSrc/build.gradle

@@ -229,12 +229,8 @@ if (project != rootProject) {
 
   forbiddenPatterns {
     exclude '**/*.wav'
-    exclude '**/*.p12'
-    exclude '**/*.jks'
-    exclude '**/*.crt'
     // the file that actually defines nocommit
     exclude '**/ForbiddenPatternsTask.java'
-    exclude '**/*.bcfks'
   }
 
   testingConventions {

buildSrc/src/main/java/org/opensearch/gradle/precommit/ForbiddenPatternsTask.java

@@ -83,8 +83,13 @@ public class ForbiddenPatternsTask extends DefaultTask {
         .exclude("**/*.ico")
         .exclude("**/*.jar")
         .exclude("**/*.zip")
+        .exclude("**/*.p12")
         .exclude("**/*.jks")
         .exclude("**/*.crt")
+        .exclude("**/*.der")
+        .exclude("**/*.pem")
+        .exclude("**/*.key")
+        .exclude("**/*.bcfks")
         .exclude("**/*.keystore")
         .exclude("**/*.png");
 

buildSrc/src/main/java/org/opensearch/gradle/testclusters/OpenSearchNode.java

@@ -548,15 +548,15 @@ public synchronized void start() {
 
         logToProcessStdout("Creating opensearch keystore with password set to [" + keystorePassword + "]");
         if (keystorePassword.length() > 0) {
-            runOpenSearchBinScriptWithInput(keystorePassword + "\n" + keystorePassword, "opensearch-keystore", "create", "-p");
+            runOpenSearchBinScriptWithInput(keystorePassword + "\n" + keystorePassword + "\n", "opensearch-keystore", "create", "-p");
         } else {
             runOpenSearchBinScript("opensearch-keystore", "-v", "create");
         }
 
         if (keystoreSettings.isEmpty() == false || keystoreFiles.isEmpty() == false) {
             logToProcessStdout("Adding " + keystoreSettings.size() + " keystore settings and " + keystoreFiles.size() + " keystore files");
 
-            keystoreSettings.forEach((key, value) -> runKeystoreCommandWithPassword(keystorePassword, value.toString(), "add", "-x", key));
+            keystoreSettings.forEach((key, value) -> runKeystoreCommandWithPassword(keystorePassword, value.toString(), "add", key));
 
             for (Map.Entry<String, File> entry : keystoreFiles.entrySet()) {
                 File file = entry.getValue();
@@ -738,7 +738,12 @@ private void runOpenSearchBinScriptWithInput(String input, String tool, CharSequ
     }
 
     private void runKeystoreCommandWithPassword(String keystorePassword, String input, CharSequence... args) {
-        final String actualInput = keystorePassword.length() > 0 ? keystorePassword + "\n" + input : input;
+        final String actualInput;
+        if (keystorePassword.length() > 0) {
+            actualInput = keystorePassword + "\n" + input + "\n" + input;
+        } else {
+            actualInput = input + "\n" + input;
+        }
         runOpenSearchBinScriptWithInput(actualInput, "opensearch-keystore", args);
     }
 

client/rest/build.gradle

@@ -75,10 +75,6 @@ tasks.withType(CheckForbiddenApis).configureEach {
   replaceSignatureFiles('jdk-signatures', 'http-signatures')
 }
 
-forbiddenPatterns {
-  exclude '**/*.der'
-}
-
 tasks.named('forbiddenApisTest').configure {
   //we are using jdk-internal instead of jdk-non-portable to allow for com.sun.net.httpserver.* usage
   bundledSignatures -= 'jdk-non-portable'

distribution/packages/build.gradle

@@ -63,7 +63,7 @@ import java.util.regex.Pattern
  */
 
 plugins {
-  id "com.netflix.nebula.ospackage-base" version "11.10.1"
+  id "com.netflix.nebula.ospackage-base" version "11.11.1"
 }
 
 void addProcessFilesTask(String type, boolean jdk) {

distribution/tools/plugin-cli/build.gradle

@@ -39,7 +39,9 @@ dependencies {
   compileOnly project(":libs:opensearch-cli")
   api "org.bouncycastle:bcpg-fips:2.0.9"
   api "org.bouncycastle:bc-fips:2.0.0"
-  testImplementation project(":test:framework")
+  testImplementation(project(":test:framework")) {
+    exclude group: 'org.bouncycastle'
+  }
   testImplementation 'com.google.jimfs:jimfs:1.3.0'
   testRuntimeOnly("com.google.guava:guava:${versions.guava}") {
     transitive = false

libs/ssl-config/build.gradle

@@ -34,6 +34,9 @@ apply plugin: "opensearch.publish"
 
 dependencies {
   api project(':libs:opensearch-common')
+  api "org.bouncycastle:bcprov-jdk18on:${versions.bouncycastle}"
+  api "org.bouncycastle:bcpkix-jdk18on:${versions.bouncycastle}"
+  runtimeOnly "org.bouncycastle:bcutil-jdk18on:${versions.bouncycastle}"
 
   testImplementation(project(":test:framework")) {
     exclude group: 'org.opensearch', module: 'opensearch-ssl-config'
@@ -44,16 +47,12 @@ dependencies {
   testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
 }
 
-
 tasks.named('forbiddenApisMain').configure {
   replaceSignatureFiles 'jdk-signatures'
 }
 
-forbiddenPatterns {
-  exclude '**/*.key'
-  exclude '**/*.pem'
-  exclude '**/*.p12'
-  exclude '**/*.jks'
+tasks.named("dependencyLicenses").configure {
+    mapping from: /bc.*/, to: 'bouncycastle'
 }
 
 tasks.test {

plugins/ingest-attachment/licenses/bcpkix-jdk18on-1.78.jar.sha1 renamed to libs/ssl-config/licenses/bcpkix-jdk18on-1.78.jar.sha1

@@ -0,0 +1 @@
+81c1f5e06f206be5dad137d563609dbe66c81d31

plugins/ingest-attachment/licenses/bcprov-jdk18on-1.78.jar.sha1 renamed to libs/ssl-config/licenses/bcprov-jdk18on-1.78.jar.sha1

@@ -0,0 +1,14 @@
+Copyright (c) 2000 - 2023 The Legion of the Bouncy Castle Inc. (https://www.bouncycastle.org)
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+documentation files (the "Software"), to deal in the Software without restriction, including without limitation
+the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software,
+and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

libs/ssl-config/licenses/bcutil-jdk18on-1.78.jar.sha1

@@ -0,0 +1 @@
+

libs/ssl-config/licenses/bouncycastle-LICENSE.txt

@@ -32,6 +32,8 @@
 
 package org.opensearch.common.ssl;
 
+import org.bouncycastle.pkcs.PKCSException;
+
 import javax.net.ssl.KeyManagerFactory;
 import javax.net.ssl.X509ExtendedKeyManager;
 
@@ -91,7 +93,7 @@ private PrivateKey getPrivateKey() {
             throw new SslConfigException("the configured ssl private key file [" + key.toAbsolutePath() + "] does not exist", e);
         } catch (IOException e) {
             throw new SslConfigException("the configured ssl private key file [" + key.toAbsolutePath() + "] cannot be read", e);
-        } catch (GeneralSecurityException e) {
+        } catch (PKCSException e) {
             throw new SslConfigException("cannot load ssl private key file [" + key.toAbsolutePath() + "]", e);
         }
     }