Merge branch 'opensearch-project:main' into issueSA

Author: stephen-crawford

.github/workflows/auto-release.yml

@@ -14,7 +14,7 @@ jobs:
     steps:
       - name: GitHub App token
         id: github_app_token
-        uses: tibdex/github-app-token@v1.5.0
+        uses: tibdex/github-app-token@v2.1.0
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.APP_PRIVATE_KEY }}

.github/workflows/backport.yml

@@ -26,7 +26,7 @@ jobs:
     steps:
       - name: GitHub App token
         id: github_app_token
-        uses: tibdex/github-app-token@v1.5.0
+        uses: tibdex/github-app-token@v2.1.0
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.APP_PRIVATE_KEY }}

.github/workflows/create-documentation-issue.yml

@@ -14,7 +14,7 @@ jobs:
     steps:
       - name: GitHub App token
         id: github_app_token
-        uses: tibdex/github-app-token@v1.5.0
+        uses: tibdex/github-app-token@v2.1.0
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.APP_PRIVATE_KEY }}

.github/workflows/dependabot_pr.yml

@@ -11,7 +11,7 @@ jobs:
     steps:
       - name: GitHub App token
         id: github_app_token
-        uses: tibdex/github-app-token@v1.5.0
+        uses: tibdex/github-app-token@v2.1.0
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.APP_PRIVATE_KEY }}

.github/workflows/precommit.yml

@@ -22,10 +22,6 @@ jobs:
       - name: Setup docker (missing on MacOS)
         if: runner.os == 'macos'
         run: |
-          # Workaround for https://github.com/actions/runner-images/issues/8104
-          brew remove --ignore-dependencies qemu
-          curl -o ./qemu.rb https://raw.githubusercontent.com/Homebrew/homebrew-core/f88e30b3a23ef3735580f9b05535ce5a0a03c9e3/Formula/qemu.rb
-          brew install ./qemu.rb
           brew install docker
           colima start
           sudo ln -sf $HOME/.colima/default/docker.sock /var/run/docker.sock

.github/workflows/stalled.yml

@@ -13,7 +13,7 @@ jobs:
     steps:
       - name: GitHub App token
         id: github_app_token
-        uses: tibdex/github-app-token@v1.5.0
+        uses: tibdex/github-app-token@v2.1.0
         with:
           app_id: ${{ secrets.APP_ID }}
           private_key: ${{ secrets.APP_PRIVATE_KEY }}

.github/workflows/version.yml

@@ -128,11 +128,15 @@ dependencies {
   testFixturesApi "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${props.getProperty('randomizedrunner')}"
   testFixturesApi gradleApi()
   testFixturesApi gradleTestKit()
-  testImplementation 'com.github.tomakehurst:wiremock-jre8-standalone:2.35.0'
+  testImplementation 'org.wiremock:wiremock-standalone:3.1.0'
   testImplementation "org.mockito:mockito-core:${props.getProperty('mockito')}"
   integTestImplementation('org.spockframework:spock-core:2.3-groovy-3.0') {
     exclude module: "groovy"
   }
+  implementation('org.ajoberstar.grgit:grgit-core:5.2.0') {
+    exclude group: 'org.eclipse.jgit', module: 'org.eclipse.jgit'
+  }
+  implementation 'org.eclipse.jgit:org.eclipse.jgit:6.7.0.202309050840-r'
 }
 
 configurations.all {

buildSrc/build.gradle

@@ -1,5 +1,5 @@
 opensearch        = 3.0.0
-lucene            = 9.8.0-snapshot-4373c3b
+lucene            = 9.8.0-snapshot-95cdd2e
 
 bundled_jdk_vendor = adoptium
 bundled_jdk = 20.0.2+9

buildSrc/version.properties

@@ -41,6 +41,15 @@
  * @opensearch.internal
  */
 public class Iterators {
+
+    /**
+     * Concat iterators
+     *
+     * @param iterators the iterators to concat
+     * @param <T> the type of iterator
+     * @return a new {@link ConcatenatedIterator}
+     * @throws NullPointerException if iterators is null
+     */
     public static <T> Iterator<T> concat(Iterator<? extends T>... iterators) {
         if (iterators == null) {
             throw new NullPointerException("iterators");
@@ -71,6 +80,11 @@ static class ConcatenatedIterator<T> implements Iterator<T> {
             this.iterators = iterators;
         }
 
+        /**
+         * Returns {@code true} if the iteration has more elements. (In other words, returns {@code true} if {@link #next} would return an
+         * element rather than throwing an exception.)
+         * @return {@code true} if the iteration has more elements
+         */
         @Override
         public boolean hasNext() {
             boolean hasNext = false;
@@ -81,6 +95,11 @@ public boolean hasNext() {
             return hasNext;
         }
 
+        /**
+         * Returns the next element in the iteration.
+         * @return the next element in the iteration
+         * @throws NoSuchElementException if the iteration has no more elements
+         */
         @Override
         public T next() {
             if (!hasNext()) {

libs/common/src/main/java/org/opensearch/common/collect/Iterators.java

@@ -30,10 +30,13 @@
  * @opensearch.experimental - class methods might change
  */
 public class ZstdCompressor implements Compressor {
-    // An arbitrary header that we use to identify compressed streams
-    // It needs to be different from other compressors and to not be specific
-    // enough so that no stream starting with these bytes could be detected as
-    // a XContent
+
+    /**
+     * An arbitrary header that we use to identify compressed streams
+     * It needs to be different from other compressors and to not be specific
+     * enough so that no stream starting with these bytes could be detected as
+     * a XContent
+     * */
     private static final byte[] HEADER = new byte[] { 'Z', 'S', 'T', 'D', '\0' };
 
     /**
@@ -44,10 +47,20 @@ public class ZstdCompressor implements Compressor {
     @PublicApi(since = "2.10.0")
     public static final String NAME = "ZSTD";
 
+    /**
+     * The compression level for {@link ZstdOutputStreamNoFinalizer}
+     */
     private static final int LEVEL = 3;
 
+    /** The buffer size for {@link BufferedInputStream} and {@link BufferedOutputStream}
+     */
     private static final int BUFFER_SIZE = 4096;
 
+    /**
+     * Compares the given bytes with the {@link ZstdCompressor#HEADER} of a compressed stream
+     * @param bytes the bytes to compare to ({@link ZstdCompressor#HEADER})
+     * @return true if the bytes are the {@link ZstdCompressor#HEADER}, false otherwise
+     */
     @Override
     public boolean isCompressed(BytesReference bytes) {
         if (bytes.length() < HEADER.length) {
@@ -61,11 +74,22 @@ public boolean isCompressed(BytesReference bytes) {
         return true;
     }
 
+    /**
+     * Returns the length of the {@link ZstdCompressor#HEADER}
+     * @return the {@link ZstdCompressor#HEADER} length
+     */
     @Override
     public int headerLength() {
         return HEADER.length;
     }
 
+    /**
+     * Returns a new {@link ZstdInputStreamNoFinalizer} from the given compressed {@link InputStream}
+     * @param in the compressed {@link InputStream}
+     * @return a new {@link ZstdInputStreamNoFinalizer} from the given compressed {@link InputStream}
+     * @throws IOException if an I/O error occurs
+     * @throws IllegalArgumentException if the input stream is not compressed with ZSTD
+     */
     @Override
     public InputStream threadLocalInputStream(InputStream in) throws IOException {
         final byte[] header = in.readNBytes(HEADER.length);
@@ -75,17 +99,36 @@ public InputStream threadLocalInputStream(InputStream in) throws IOException {
         return new ZstdInputStreamNoFinalizer(new BufferedInputStream(in, BUFFER_SIZE), RecyclingBufferPool.INSTANCE);
     }
 
+    /**
+     * Returns a new {@link ZstdOutputStreamNoFinalizer} from the given {@link OutputStream}
+     * @param out the {@link OutputStream}
+     * @return a new {@link ZstdOutputStreamNoFinalizer} from the given {@link OutputStream}
+     * @throws IOException if an I/O error occurs
+     */
     @Override
     public OutputStream threadLocalOutputStream(OutputStream out) throws IOException {
         out.write(HEADER);
         return new ZstdOutputStreamNoFinalizer(new BufferedOutputStream(out, BUFFER_SIZE), RecyclingBufferPool.INSTANCE, LEVEL);
     }
 
+    /**
+     * Always throws an {@link UnsupportedOperationException} as ZSTD compression is supported only for snapshotting
+     * @param bytesReference a reference to the bytes to uncompress
+     * @return always throws an exception
+     * @throws UnsupportedOperationException if the method is called
+     * @throws IOException is never thrown
+     */
     @Override
     public BytesReference uncompress(BytesReference bytesReference) throws IOException {
         throw new UnsupportedOperationException("ZSTD compression is supported only for snapshotting");
     }
 
+    /**
+     * Always throws an {@link UnsupportedOperationException} as ZSTD compression is supported only for snapshotting
+     * @param bytesReference a reference to the bytes to compress
+     * @return always throws an exception
+     * @throws UnsupportedOperationException if the method is called
+     */
     @Override
     public BytesReference compress(BytesReference bytesReference) throws IOException {
         throw new UnsupportedOperationException("ZSTD compression is supported only for snapshotting");

libs/compress/src/main/java/org/opensearch/compress/ZstdCompressor.java

@@ -0,0 +1 @@
+d2f7fbc5b2c49ca777a169d579f41082a9a57cc7

libs/core/licenses/lucene-core-9.8.0-snapshot-4373c3b.jar.sha1

@@ -71,17 +71,23 @@ public interface CircuitBreaker {
 
     /**
      * The type of breaker
-     *
+     * can be {@link #MEMORY}, {@link #PARENT}, or {@link #NOOP}
      * @opensearch.internal
      */
     enum Type {
-        // A regular or ChildMemoryCircuitBreaker
+        /** A regular or ChildMemoryCircuitBreaker */
         MEMORY,
-        // A special parent-type for the hierarchy breaker service
+        /** A special parent-type for the hierarchy breaker service */
         PARENT,
-        // A breaker where every action is a noop, it never breaks
+        /** A breaker where every action is a noop, it never breaks */
         NOOP;
 
+        /**
+         * Converts string (case-insensitive) to breaker {@link Type}
+         * @param value "noop", "parent", or "memory" (case-insensitive)
+         * @return the breaker {@link Type}
+         * @throws IllegalArgumentException if value is not "noop", "parent", or "memory"
+         */
         public static Type parseValue(String value) {
             switch (value.toLowerCase(Locale.ROOT)) {
                 case "noop":
@@ -98,13 +104,13 @@ public static Type parseValue(String value) {
 
     /**
      * The breaker durability
-     *
+     * can be {@link #TRANSIENT} or {@link #PERMANENT}
      * @opensearch.internal
      */
     enum Durability {
-        // The condition that tripped the circuit breaker fixes itself eventually.
+        /** The condition that tripped the circuit breaker fixes itself eventually. */
         TRANSIENT,
-        // The condition that tripped the circuit breaker requires manual intervention.
+        /** The condition that tripped the circuit breaker requires manual intervention. */
         PERMANENT
     }
 
@@ -120,11 +126,14 @@ enum Durability {
      * @param bytes number of bytes to add
      * @param label string label describing the bytes being added
      * @return the number of "used" bytes for the circuit breaker
+     * @throws CircuitBreakingException if the breaker tripped
      */
     double addEstimateBytesAndMaybeBreak(long bytes, String label) throws CircuitBreakingException;
 
     /**
      * Adjust the circuit breaker without tripping
+     * @param bytes number of bytes to add
+     * @return the number of "used" bytes for the circuit breaker
      */
     long addWithoutBreaking(long bytes);
 
@@ -154,7 +163,10 @@ enum Durability {
     String getName();
 
     /**
-     * @return whether a tripped circuit breaker will reset itself (transient) or requires manual intervention (permanent).
+     * Returns the {@link Durability} of this breaker
+     * @return whether a tripped circuit breaker will
+     * reset itself ({@link Durability#TRANSIENT})
+     * or requires manual intervention ({@link Durability#PERMANENT}).
      */
     Durability getDurability();
 

libs/core/licenses/lucene-core-9.8.0-snapshot-95cdd2e.jar.sha1

@@ -46,8 +46,11 @@
  */
 public class CircuitBreakingException extends OpenSearchException {
 
+    /** The number of bytes wanted */
     private final long bytesWanted;
+    /** The circuit breaker limit */
     private final long byteLimit;
+    /** The {@link CircuitBreaker.Durability} of the circuit breaker */
     private final CircuitBreaker.Durability durability;
 
     public CircuitBreakingException(StreamInput in) throws IOException {
@@ -88,6 +91,7 @@ public CircuitBreaker.Durability getDurability() {
         return durability;
     }
 
+    /** Always returns {@link RestStatus#TOO_MANY_REQUESTS} */
     @Override
     public RestStatus status() {
         return RestStatus.TOO_MANY_REQUESTS;