|
23 | 23 | from multiprocessing.pool import ThreadPool |
24 | 24 | from typing import List, Set, Dict, Optional, Any, Callable, Union, Tuple |
25 | 25 |
|
| 26 | +from py4j.protocol import Py4JJavaError |
26 | 27 | from pyspark.errors.exceptions.base import UnsupportedOperationException |
27 | 28 | from pyspark.sql import DataFrame, Row |
28 | 29 | from pyspark.sql.functions import col, lit, expr, floor |
@@ -1187,6 +1188,42 @@ def test_protocolUpgrade(self) -> None: |
1187 | 1188 | with self.assertRaisesRegex(ValueError, "writerVersion"): |
1188 | 1189 | dt.upgradeTableProtocol(1, {}) # type: ignore[arg-type] |
1189 | 1190 |
|
| 1191 | + def test_addFeatureSupport(self) -> None: |
| 1192 | + try: |
| 1193 | + self.spark.conf.set('spark.databricks.delta.minReaderVersion', '1') |
| 1194 | + self.spark.conf.set('spark.databricks.delta.minWriterVersion', '2') |
| 1195 | + self.__writeDeltaTable([('a', 1), ('b', 2), ('c', 3), ('d', 4)]) |
| 1196 | + dt = DeltaTable.forPath(self.spark, self.tempFile) |
| 1197 | + finally: |
| 1198 | + self.spark.conf.unset('spark.databricks.delta.minReaderVersion') |
| 1199 | + self.spark.conf.unset('spark.databricks.delta.minWriterVersion') |
| 1200 | + |
| 1201 | + # bad args |
| 1202 | + with self.assertRaisesRegex(Py4JJavaError, "DELTA_UNSUPPORTED_FEATURES_IN_CONFIG"): |
| 1203 | + dt.addFeatureSupport("abc") |
| 1204 | + with self.assertRaisesRegex(ValueError, "featureName needs to be a string"): |
| 1205 | + dt.addFeatureSupport(12345) # type: ignore[arg-type] |
| 1206 | + with self.assertRaisesRegex(ValueError, "featureName needs to be a string"): |
| 1207 | + dt.addFeatureSupport([12345]) # type: ignore[arg-type] |
| 1208 | + with self.assertRaisesRegex(ValueError, "featureName needs to be a string"): |
| 1209 | + dt.addFeatureSupport({}) # type: ignore[arg-type] |
| 1210 | + with self.assertRaisesRegex(ValueError, "featureName needs to be a string"): |
| 1211 | + dt.addFeatureSupport([]) # type: ignore[arg-type] |
| 1212 | + |
| 1213 | + # good args |
| 1214 | + dt.addFeatureSupport("appendOnly") |
| 1215 | + dt_details = dt.detail().collect()[0].asDict() |
| 1216 | + self.assertTrue(dt_details["minReaderVersion"] == 1, "The upgrade should be a no-op") |
| 1217 | + self.assertTrue(dt_details["minWriterVersion"] == 2, "The upgrade should be a no-op") |
| 1218 | + self.assertEqual(sorted(dt_details["tableFeatures"]), ["appendOnly", "invariants"]) |
| 1219 | + |
| 1220 | + dt.addFeatureSupport("deletionVectors") |
| 1221 | + dt_details = dt.detail().collect()[0].asDict() |
| 1222 | + self.assertTrue(dt_details["minReaderVersion"] == 3, "DV requires reader version 3") |
| 1223 | + self.assertTrue(dt_details["minWriterVersion"] == 7, "DV requires writer version 7") |
| 1224 | + self.assertEqual(sorted(dt_details["tableFeatures"]), |
| 1225 | + ["appendOnly", "deletionVectors", "invariants"]) |
| 1226 | + |
1190 | 1227 | def test_restore_to_version(self) -> None: |
1191 | 1228 | self.__writeDeltaTable([('a', 1), ('b', 2)]) |
1192 | 1229 | self.__overwriteDeltaTable([('a', 3), ('b', 2)], |
|
0 commit comments