diff --git a/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt b/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt index 04b04225e7..d4d412a30f 100644 --- a/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt +++ b/build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt @@ -133,6 +133,11 @@ constructor(private val softwareComponentFactory: SoftwareComponentFactory) : Pl suppressPomMetadataWarningsFor("testFixturesApiElements") suppressPomMetadataWarningsFor("testFixturesRuntimeElements") + + if (project.tasks.findByName("createPolarisSparkJar") != null) { + // if the project contains spark client jar, also publish the jar to maven + artifact(project.tasks.named("createPolarisSparkJar").get()) + } } if ( diff --git a/plugins/spark/README.md b/plugins/spark/README.md index ae4a566cd9..c7d6bc876b 100644 --- a/plugins/spark/README.md +++ b/plugins/spark/README.md @@ -30,8 +30,8 @@ and depends on iceberg-spark-runtime 1.9.0. # Build Plugin Jar A task createPolarisSparkJar is added to build a jar for the Polaris Spark plugin, the jar is named as: -`polaris-iceberg--spark-runtime-_-.jar`. For example: -`polaris-iceberg-1.9.0-spark-runtime-3.5_2.12-0.10.0-beta-incubating-SNAPSHOT.jar`. +`polaris-spark-_--bundle.jar`. For example: +`polaris-spark-3.5_2.12-0.11.0-beta-incubating-SNAPSHOT-bundle.jar`. - `./gradlew :polaris-spark-3.5_2.12:createPolarisSparkJar` -- build jar for Spark 3.5 with Scala version 2.12. - `./gradlew :polaris-spark-3.5_2.13:createPolarisSparkJar` -- build jar for Spark 3.5 with Scala version 2.13. @@ -67,12 +67,12 @@ bin/spark-shell \ ``` Assume the path to the built Spark client jar is -`/polaris/plugins/spark/v3.5/spark/build/2.12/libs/polaris-iceberg-1.9.0-spark-runtime-3.5_2.12-0.10.0-beta-incubating-SNAPSHOT.jar` +`/polaris/plugins/spark/v3.5/spark/build/2.12/libs/polaris-spark-3.5_2.12-0.11.0-beta-incubating-SNAPSHOT-bundle.jar` and the name of the catalog is `polaris`. The cli command will look like following: ```shell bin/spark-shell \ ---jars /polaris/plugins/spark/v3.5/spark/build/2.12/libs/polaris-iceberg-1.9.0-spark-runtime-3.5_2.12-0.10.0-beta-incubating-SNAPSHOT.jar \ +--jars /polaris/plugins/spark/v3.5/spark/build/2.12/libs/polaris-spark-3.5_2.12-0.11.0-beta-incubating-SNAPSHOT-bundle.jar \ --packages org.apache.iceberg:iceberg-aws-bundle:1.9.0,io.delta:delta-spark_2.12:3.3.1 \ --conf spark.sql.extensions=org.apache.iceberg.spark.extensions.IcebergSparkSessionExtensions,io.delta.sql.DeltaSparkSessionExtension \ --conf spark.sql.catalog.spark_catalog=org.apache.spark.sql.delta.catalog.DeltaCatalog \ diff --git a/plugins/spark/v3.5/getting-started/notebooks/SparkPolaris.ipynb b/plugins/spark/v3.5/getting-started/notebooks/SparkPolaris.ipynb index 2b3b9cc46e..f4459b0714 100644 --- a/plugins/spark/v3.5/getting-started/notebooks/SparkPolaris.ipynb +++ b/plugins/spark/v3.5/getting-started/notebooks/SparkPolaris.ipynb @@ -266,7 +266,7 @@ "from pyspark.sql import SparkSession\n", "\n", "spark = (SparkSession.builder\n", - " .config(\"spark.jars\", \"../polaris_libs/polaris-iceberg-1.9.0-spark-runtime-3.5_2.12-0.11.0-beta-incubating-SNAPSHOT.jar\")\n", + " .config(\"spark.jars\", \"../polaris_libs/polaris-spark-3.5_2.12-0.11.0-beta-incubating-SNAPSHOT-bundle.jar\")\n", " .config(\"spark.jars.packages\", \"org.apache.iceberg:iceberg-aws-bundle:1.9.0,io.delta:delta-spark_2.12:3.2.1\")\n", " .config(\"spark.sql.catalog.spark_catalog\", \"org.apache.spark.sql.delta.catalog.DeltaCatalog\")\n", " .config('spark.sql.iceberg.vectorization.enabled', 'false')\n", diff --git a/plugins/spark/v3.5/regtests/run.sh b/plugins/spark/v3.5/regtests/run.sh index d850a4465f..acf5d1a906 100755 --- a/plugins/spark/v3.5/regtests/run.sh +++ b/plugins/spark/v3.5/regtests/run.sh @@ -72,7 +72,7 @@ for SCALA_VERSION in "${SCALA_VERSIONS[@]}"; do echo "RUN REGRESSION TEST FOR SPARK_MAJOR_VERSION=${SPARK_MAJOR_VERSION}, SPARK_VERSION=${SPARK_VERSION}, SCALA_VERSION=${SCALA_VERSION}" # find the project jar SPARK_DIR=${SPARK_ROOT_DIR}/spark - JAR_PATH=$(find ${SPARK_DIR} -name "polaris-iceberg-*.*-spark-runtime-${SPARK_MAJOR_VERSION}_${SCALA_VERSION}-*.jar" -print -quit) + JAR_PATH=$(find ${SPARK_DIR} -name "polaris-spark-${SPARK_MAJOR_VERSION}_${SCALA_VERSION}-*.*-bundle.jar" -print -quit) echo "find jar ${JAR_PATH}" SPARK_EXISTS="TRUE" diff --git a/plugins/spark/v3.5/spark/LICENSE b/plugins/spark/v3.5/spark/LICENSE new file mode 100644 index 0000000000..f1e099caa3 --- /dev/null +++ b/plugins/spark/v3.5/spark/LICENSE @@ -0,0 +1,325 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------------------------------------- + +This product includes a gradle wrapper. + +* gradlew +* gradle/wrapper/gradle-wrapper.properties + +Copyright: 2010-2019 Gradle Authors. +Home page: https://github.com/gradle/gradle +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from Apache Iceberg. + +* spec/iceberg-rest-catalog-open-api.yaml +* spec/polaris-catalog-apis/oauth-tokens-api.yaml +* integration-tests/src/main/java/org/apache/polaris/service/it/test/PolarisRestCatalogIntegrationTest.java +* service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/IcebergCatalog.java +* service/common/src/main/java/org/apache/polaris/service/catalog/iceberg/CatalogHandlerUtils.java +* plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/PolarisSparkCatalog.java +* plugins/spark/v3.5/spark/src/test/java/org/apache/polaris/spark/PolarisInMemoryCatalog.java +* plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/PolarisRESTCatalog.java +* plugins/spark/v3.5/spark/src/main/java/org/apache/polaris/spark/SparkCatalog.java + +Copyright: Copyright 2017-2025 The Apache Software Foundation +Home page: https://iceberg.apache.org +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from OpenAPITool openapi-generator + +* server-templates/formParams.mustache +* server-templates/apiService.mustache +* server-templates/bodyParams.mustache +* server-templates/pojo.mustache +* server-templates/headerParams.mustache +* server-templates/apiServiceImpl.mustache +* server-templates/queryParams.mustache +* server-templates/api.mustache + +Copyright: Copyright 2018 OpenAPI-Generator Contributors (https://openapi-generator.tech) + Copyright 2018 SmartBear Software +Home page: https://openapi-generator.tech/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from Google Docsy. + +* site/layouts/community/list.html +* site/layouts/docs/baseof.html +* site/layouts/partials/community_links.html +* site/layouts/partials/head.html +* site/layouts/partials/navbar.html +* site/layouts/shortcodes/redoc-polaris.html + +Home page: https://www.docsy.dev/ +License: https://www.apache.org/licenses/LICENSE-2.0 + +-------------------------------------------------------------------------------- + +This product includes code from Project Nessie. + +* build-logic/src/main/kotlin/LicenseFileValidation.kt +* build-logic/src/main/kotlin/copiedcode/CopiedCodeCheckerPlugin.kt +* build-logic/src/main/kotlin/copiedcode/CopiedCodeCheckerExtension.kt +* build-logic/src/main/kotlin/publishing/PublishingHelperPlugin.kt +* build-logic/src/main/kotlin/Utilities.kt +* build-logic/src/main/kotlin/polaris-shadow-jar.gradle.kts +* build-logic/src/main/kotlin/polaris-quarkus.gradle.kts +* tools/config-docs/annotations/src/main/java/org/apache/polaris/docs/ConfigDocs.java +* tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/DocGenDoclet.java +* tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/MarkdownFormatter.java +* tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/MarkdownPropertyFormatter.java +* tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/MarkdownTypeFormatter.java +* tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/PropertiesConfigItem.java +* tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/PropertiesConfigPageGroup.java +* tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/PropertiesConfigs.java +* tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/PropertyInfo.java +* tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/SmallRyeConfigMappingInfo.java +* tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/SmallRyeConfigPropertyInfo.java +* tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/SmallRyeConfigs.java +* tools/config-docs/generator/src/main/java/org/apache/polaris/docs/generator/ReferenceConfigDocsGenerator.java +* tools/config-docs/generator/src/test/java/org/apache/polaris/docs/generator/TestDocGenTool.java +* tools/config-docs/generator/src/test/java/tests/properties/ConfigProps.java +* tools/config-docs/generator/src/test/java/tests/properties/MoreProps.java +* tools/config-docs/generator/src/test/java/tests/smallrye/AllTypes.java +* tools/config-docs/generator/src/test/java/tests/smallrye/ExtremelyNested.java +* tools/config-docs/generator/src/test/java/tests/smallrye/InterfaceOne.java +* tools/config-docs/generator/src/test/java/tests/smallrye/InterfaceThree.java +* tools/config-docs/generator/src/test/java/tests/smallrye/InterfaceTwo.java +* tools/config-docs/generator/src/test/java/tests/smallrye/MappedA.java +* tools/config-docs/generator/src/test/java/tests/smallrye/NestedA.java +* tools/config-docs/generator/src/test/java/tests/smallrye/NestedA1.java +* tools/config-docs/generator/src/test/java/tests/smallrye/NestedA11.java +* tools/config-docs/generator/src/test/java/tests/smallrye/NestedA2.java +* tools/config-docs/generator/src/test/java/tests/smallrye/NestedB.java +* tools/config-docs/generator/src/test/java/tests/smallrye/NestedB1.java +* tools/config-docs/generator/src/test/java/tests/smallrye/NestedB12.java +* tools/config-docs/generator/src/test/java/tests/smallrye/NestedSectionA.java +* tools/config-docs/generator/src/test/java/tests/smallrye/NestedSectionB.java +* tools/config-docs/generator/src/test/java/tests/smallrye/NestedSectionC.java +* tools/config-docs/generator/src/test/java/tests/smallrye/NestedSectionTypeA.java +* tools/config-docs/generator/src/test/java/tests/smallrye/NestedSectionTypeB.java +* tools/config-docs/generator/src/test/java/tests/smallrye/NestedSectionsRoot.java +* tools/config-docs/generator/src/test/java/tests/smallrye/OtherMapped.java +* tools/config-docs/generator/src/test/java/tests/smallrye/SomeEnum.java +* tools/config-docs/generator/src/test/java/tests/smallrye/VeryNested.java +* tools/container-spec-helper/src/main/java/org/apache/polaris/containerspec/ContainerSpecHelper.java +* quarkus/admin/src/main/java/org/apache/polaris/admintool/PolarisAdminTool.java +* helm/polaris/tests/logging_storage_test.yaml +* helm/polaris/tests/quantity_test.yaml +* helm/polaris/tests/service_monitor_test.yaml +* helm/polaris/templates/_helpers.tpl +* helm/polaris/templates/serviceaccount.yaml +* helm/polaris/templates/servicemonitor.yaml +* helm/polaris/templates/storage.yaml + +Copyright: Copyright 2015-2025 Dremio Corporation +Home page: https://projectnessie.org/ +License: https://www.apache.org/licenses/LICENSE-2.0 diff --git a/plugins/spark/v3.5/spark/NOTICE b/plugins/spark/v3.5/spark/NOTICE new file mode 100644 index 0000000000..8c392caff7 --- /dev/null +++ b/plugins/spark/v3.5/spark/NOTICE @@ -0,0 +1,16 @@ +Apache Polaris (incubating) +Copyright 2025 The Apache Software Foundation + +This product includes software developed at +The Apache Software Foundation (http://www.apache.org/). + +The initial code for the Polaris project was donated +to the ASF by Snowflake Inc. (https://www.snowflake.com/) copyright 2024. + +-------------------------------------------------------------------------------- + +This project includes code from Project Nessie, developed at Dremio, +with the following copyright notice: + +| Nessie +| Copyright 2015-2025 Dremio Corporation diff --git a/plugins/spark/v3.5/spark/build.gradle.kts b/plugins/spark/v3.5/spark/build.gradle.kts index 600d150f4f..8e2f3a354f 100644 --- a/plugins/spark/v3.5/spark/build.gradle.kts +++ b/plugins/spark/v3.5/spark/build.gradle.kts @@ -151,13 +151,16 @@ tasks.register("checkNoDisallowedImports") { tasks.named("check") { dependsOn("checkNoDisallowedImports") } tasks.register("createPolarisSparkJar") { - archiveClassifier = null - archiveBaseName = - "polaris-iceberg-${icebergVersion}-spark-runtime-${sparkMajorVersion}_${scalaVersion}" + archiveClassifier = "bundle" isZip64 = true - // pack both the source code and dependencies + // include the LICENSE and NOTICE files for the shadow Jar + from(projectDir) { + include("LICENSE") + include("NOTICE") + } + // pack both the source code and dependencies from(sourceSets.main.get().output) configurations = listOf(project.configurations.runtimeClasspath.get()) diff --git a/site/content/in-dev/unreleased/polaris-spark-client.md b/site/content/in-dev/unreleased/polaris-spark-client.md index 712082e273..4ceb536a9c 100644 --- a/site/content/in-dev/unreleased/polaris-spark-client.md +++ b/site/content/in-dev/unreleased/polaris-spark-client.md @@ -71,7 +71,7 @@ bin/spark-shell \ --conf spark.sql.catalog..scope='PRINCIPAL_ROLE:ALL' \ --conf spark.sql.catalog..token-refresh-enabled=true ``` -Assume the released Polaris Spark client you want to use is `org.apache.polaris:polaris-iceberg-1.8.1-spark-runtime-3.5_2.12:1.0.0`, +Assume the released Polaris Spark client you want to use is `org.apache.polaris:polaris-spark-3.5_2.12:1.0.0`, replace the `polaris-spark-client-package` field with the release. The `spark-catalog-name` is the catalog name you will use with Spark, and `polaris-catalog-name` is the catalog name used