Merge branch 'main' into feature/94-comply-with-latest-delta-sharing-protocol

Author: agilelab-tmnd1991

.github/workflows/build_doc.yaml

@@ -8,7 +8,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           cache: 'gradle'
           distribution: temurin
@@ -18,13 +18,13 @@ jobs:
       - name: Execute npm run build
         run:  |
               cp -r protocol docsite/static/protocol
-              ./gradlew docsite:npm_run_build --no-daemon
+              ./gradlew docsite:npm_install docsite:npm_run_build --no-daemon
               chmod -c -R +rX "docsite/build" | while read line; do
                 echo "::warning title=Invalid file permissions automatically fixed::$line"
               done
         shell: bash
       - name: Upload gh-pages artifact
-        uses: actions/upload-pages-artifact@v2
+        uses: actions/upload-pages-artifact@v3
         with:
           path: docsite/build
   deploy:
@@ -43,5 +43,5 @@ jobs:
     steps:
       - name: Deploy to GitHub Pages
         id: deployment
-        uses: actions/deploy-pages@v2
+        uses: actions/deploy-pages@v4
 

.github/workflows/compile.yaml

@@ -16,7 +16,7 @@ jobs:
         with:
           fetch-depth: 0
           ref: ${{ github.event.pull_request.head.sha }}
-      - uses: actions/setup-java@v3
+      - uses: actions/setup-java@v4
         with:
           cache: 'gradle'
           distribution: temurin

.gitignore

@@ -12,4 +12,8 @@ build/
 .gradle
 .env
 client/bin
-server/bin
+server/bin
+.terraform
+.terraform.*
+*.tfstate
+*.tfstate.backup

build.gradle.kts

@@ -1,4 +1,4 @@
 group = "io.whitefox"
 plugins {
-    id("co.uzzu.dotenv.gradle").version("3.0.0")
+    id("co.uzzu.dotenv.gradle").version("4.0.0")
 }

buildSrc/build.gradle.kts

@@ -8,6 +8,6 @@ repositories {
 
 dependencies {
     implementation("org.openapi.generator:org.openapi.generator.gradle.plugin:6.6.0")
-    implementation("com.diffplug.spotless:spotless-plugin-gradle:6.23.1")
+    implementation("com.diffplug.spotless:spotless-plugin-gradle:6.23.3")
     implementation("com.palantir.gradle.gitversion:gradle-git-version:3.0.0")
 }

client-spark/build.gradle.kts

@@ -19,15 +19,14 @@ dependencies {
 
     // DELTA
     testImplementation("org.apache.hadoop:hadoop-common:3.3.6")
-    testImplementation("io.delta:delta-sharing-spark_2.12:1.0.2")
+    testImplementation("io.delta:delta-sharing-spark_2.13:1.0.3")
 
     //SPARK
-    testImplementation("org.apache.spark:spark-core_2.12:3.3.2")
-    testImplementation("org.apache.spark:spark-sql_2.12:3.3.2")
-    testImplementation("com.github.mrpowers:spark-fast-tests_2.12:1.3.0")
+    testImplementation("org.apache.spark:spark-sql_2.13:3.5.0")
+    testImplementation("com.github.mrpowers:spark-fast-tests_2.13:1.3.0")
 
     //JUNIT
-    testImplementation("org.junit.jupiter:junit-jupiter:5.8.1")
+    testImplementation("org.junit.jupiter:junit-jupiter:5.10.1")
 }
 
 

client-spark/src/test/java/io/whitefox/api/client/ITDeltaSharingClient.java

@@ -4,34 +4,31 @@
 
 import com.github.mrpowers.spark.fast.tests.DatasetComparer;
 import io.whitefox.api.client.model.CreateMetastore;
-import io.whitefox.api.client.model.Metastore;
+import io.whitefox.api.client.model.Provider;
 import io.whitefox.api.models.MrFoxDeltaTableSchema;
-import io.whitefox.api.utils.SparkUtil;
+import io.whitefox.api.utils.ScalaUtils;
 import io.whitefox.api.utils.StorageManagerInitializer;
 import io.whitefox.api.utils.TablePath;
+import io.whitefox.api.utils.TestSparkSession;
 import java.util.List;
-import org.apache.spark.sql.SparkSession;
 import org.apache.spark.sql.types.DataType;
 import org.apache.spark.sql.types.Metadata;
 import org.apache.spark.sql.types.StructField;
 import org.apache.spark.sql.types.StructType;
 import org.junit.jupiter.api.BeforeAll;
 import org.junit.jupiter.api.Tag;
 import org.junit.jupiter.api.Test;
-import scala.collection.GenMap;
 
 @Tag("clientSparkTest")
-public class ITDeltaSharingClient implements DatasetComparer, SparkUtil {
+public class ITDeltaSharingClient implements DatasetComparer, ScalaUtils {
 
   private final StorageManagerInitializer storageManagerInitializer;
   private final String deltaTablePath;
-  private final SparkSession spark;
 
   public ITDeltaSharingClient() {
     this.storageManagerInitializer = new StorageManagerInitializer();
     this.deltaTablePath =
         TablePath.getDeltaTablePath(getClass().getClassLoader().getResource("MrFoxProfile.json"));
-    this.spark = newSparkSession();
   }
 
   @BeforeAll
@@ -41,10 +38,11 @@ static void initStorageManager() {
 
   @Test
   void showS3Table1withQueryTableApi() {
+    var spark = TestSparkSession.newSparkSession();
     storageManagerInitializer.createS3DeltaTable();
     var ds = spark.read().format("deltaSharing").load(deltaTablePath);
     var expectedSchema = new StructType(new StructField[] {
-      new StructField("id", DataType.fromDDL("long"), true, new Metadata(GenMap.empty()))
+      new StructField("id", DataType.fromDDL("long"), true, new Metadata(emptyScalaMap()))
     });
     var expectedData = spark
         .createDataFrame(
@@ -57,15 +55,16 @@ void showS3Table1withQueryTableApi() {
             MrFoxDeltaTableSchema.class)
         .toDF();
 
-    assertEquals(expectedSchema.json(), ds.schema().json());
+    assertEquals(expectedSchema, ds.schema());
     assertEquals(5, ds.count());
     assertSmallDatasetEquality(ds, expectedData, true, false, false, 500);
   }
 
   @Test
-  void createGlueMetastore() {
-    Metastore metastore = storageManagerInitializer.createGlueMetastore();
-    assertEquals(metastore.getName(), "MrFoxMetastore");
-    assertEquals(metastore.getType(), CreateMetastore.TypeEnum.GLUE.getValue());
+  void createProviderWithGlueMetastore() {
+    Provider provider = storageManagerInitializer.createProviderWithGlueMetastore();
+    assertEquals(provider.getStorage().getName(), "MrFoxStorage");
+    assertEquals(provider.getMetastore().getName(), "MrFoxMetastore");
+    assertEquals(provider.getMetastore().getType(), CreateMetastore.TypeEnum.GLUE.getValue());
   }
 }

client-spark/src/test/java/io/whitefox/api/utils/ScalaUtils.java

@@ -0,0 +1,7 @@
+package io.whitefox.api.utils;
+
+public interface ScalaUtils {
+  default <K, V> scala.collection.immutable.Map<K, V> emptyScalaMap() {
+    return scala.collection.immutable.Map$.MODULE$.empty();
+  }
+}

client-spark/src/test/java/io/whitefox/api/utils/SparkUtil.java

@@ -49,11 +49,15 @@ public void createS3DeltaTable() {
         addTableToSchemaRequest(providerRequest.getName(), createTableRequest.getName())));
   }
 
-  public Metastore createGlueMetastore() {
+  public Provider createProviderWithGlueMetastore() {
     var metastoreRequest = createMetastoreRequest(s3TestConfig, CreateMetastore.TypeEnum.GLUE);
-    return ApiUtils.recoverConflictLazy(
+    var metastore = ApiUtils.recoverConflictLazy(
         () -> metastoreV1Api.createMetastore(metastoreRequest),
         () -> metastoreV1Api.describeMetastore(metastoreRequest.getName()));
+    var providerRequest = addProviderRequest(Optional.of(metastore.getName()), TableFormat.iceberg);
+    return ApiUtils.recoverConflictLazy(
+        () -> providerV1Api.addProvider(providerRequest),
+        () -> providerV1Api.getProvider(providerRequest.getName()));
   }
 
   private String createSchemaRequest(TableFormat tableFormat) {

client-spark/src/test/java/io/whitefox/api/utils/StorageManagerInitializer.java

@@ -0,0 +1,18 @@
+package io.whitefox.api.utils;
+
+import org.apache.spark.sql.SparkSession;
+
+public class TestSparkSession {
+
+  private static final class SparkHolder {
+    private static final SparkSession spark = SparkSession.builder()
+        .appName("delta sharing client test")
+        .config("spark.driver.host", "localhost")
+        .master("local[1, 4]")
+        .getOrCreate();
+  }
+
+  public static SparkSession newSparkSession() {
+    return SparkHolder.spark.newSession();
+  }
+}

client-spark/src/test/java/io/whitefox/api/utils/TestSparkSession.java

@@ -21,8 +21,8 @@ not run properly on newer versions of the JVM.
 
 As soon as you clone the project you should verify that you are able to build and test locally, to do so you need to 
 run the `check` command of Gradle, you can achieve that using either `gradlew` script in the project root (`.
-/gradlew check`) or run the same [gradle task from intellij](https://www.jetbrains.com/help/idea/work-with-gradle-tasks.
-html). If you're default jvm is not version 11, you can run `gradlew` passing another java home as follows: 
+/gradlew check`) or run the same [gradle task from intellij](https://www.jetbrains.com/help/idea/work-with-gradle-tasks.html).
+If you're default jvm is not version 11, you can run `gradlew` passing another java home as follows: 
 `./gradlew -Dorg.gradle.java.home=<PATH_TO_JAVA_HOME> build`.
 
 Sometimes IntelliJ will tell you have build errors, especially when moving from one branch to the other. The problem