From 879182606874966432736be737801e786fcd05d2 Mon Sep 17 00:00:00 2001 From: Huang-Hsiang Cheng Date: Tue, 21 Jul 2026 17:26:28 -0700 Subject: [PATCH 1/3] Disalbe ORC testing parameter and ORC-related tests in Iceberg Spark --- dev/diffs/iceberg/1.11.0.diff | 630 ++++++++++++++++++++++++++++++++-- 1 file changed, 608 insertions(+), 22 deletions(-) diff --git a/dev/diffs/iceberg/1.11.0.diff b/dev/diffs/iceberg/1.11.0.diff index a404ded20a..959294e8ad 100644 --- a/dev/diffs/iceberg/1.11.0.diff +++ b/dev/diffs/iceberg/1.11.0.diff @@ -57,6 +57,54 @@ index f766fbb79a..c5e31185a9 100644 .config(TestBase.DISABLE_UI) .enableHiveSupport() .getOrCreate(); +diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java +index b5d6415763..226e747c0c 100644 +--- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java ++++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/SparkRowLevelOperationsTestBase.java +@@ -113,20 +113,6 @@ public abstract class SparkRowLevelOperationsTestBase extends ExtensionsTestBase + + " fanout = {6}, branch = {7}, planningMode = {8}, formatVersion = {9}") + public static Object[][] parameters() { + return new Object[][] { +- { +- "testhive", +- SparkCatalog.class.getName(), +- ImmutableMap.of( +- "type", "hive", +- "default-namespace", "default"), +- FileFormat.ORC, +- true, +- WRITE_DISTRIBUTION_MODE_NONE, +- true, +- SnapshotRef.MAIN_BRANCH, +- LOCAL, +- 2 +- }, + { + "testhive", + SparkCatalog.class.getName(), +diff --git a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRowLevelOperationsWithLineage.java b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRowLevelOperationsWithLineage.java +index f38178a8e8..7730368523 100644 +--- a/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRowLevelOperationsWithLineage.java ++++ b/spark/v4.1/spark-extensions/src/test/java/org/apache/iceberg/spark/extensions/TestRowLevelOperationsWithLineage.java +@@ -95,18 +95,6 @@ public abstract class TestRowLevelOperationsWithLineage extends SparkRowLevelOpe + + " fanout = {6}, branch = {7}, planningMode = {8}, formatVersion = {9}") + public static Object[][] parameters() { + return new Object[][] { +- { +- "testhadoop", +- SparkCatalog.class.getName(), +- ImmutableMap.of("type", "hadoop"), +- FileFormat.ORC, +- false, +- WRITE_DISTRIBUTION_MODE_HASH, +- true, +- null, +- LOCAL, +- 3 +- }, + { + "testhadoop", + SparkCatalog.class.getName(), diff --git a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java b/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java index 3fd84553f0..350b4a7561 100644 --- a/spark/v4.1/spark/src/jmh/java/org/apache/iceberg/spark/action/DeleteOrphanFilesBenchmark.java @@ -251,6 +299,273 @@ index 507d7b313b..3b73dcc014 100644 .config("spark.ui.enabled", "false") .config(DISABLE_UI) .enableHiveSupport() +diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkCompressionUtil.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkCompressionUtil.java +index aa329efbba..6d42d4517a 100644 +--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkCompressionUtil.java ++++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkCompressionUtil.java +@@ -20,7 +20,6 @@ package org.apache.iceberg.spark; + + import static org.apache.iceberg.FileFormat.AVRO; + import static org.apache.iceberg.FileFormat.METADATA; +-import static org.apache.iceberg.FileFormat.ORC; + import static org.apache.iceberg.FileFormat.PARQUET; + import static org.assertj.core.api.Assertions.assertThat; + import static org.mockito.ArgumentMatchers.anyBoolean; +@@ -67,17 +66,6 @@ public class TestSparkCompressionUtil { + assertThat(ratio3).isEqualTo(2.0); + } + +- @Test +- public void testOrcCompressionRatios() { +- configureShuffle("lz4", true); +- +- double ratio1 = shuffleCompressionRatio(ORC, "zlib"); +- assertThat(ratio1).isEqualTo(3.0); +- +- double ratio2 = shuffleCompressionRatio(ORC, "lz4"); +- assertThat(ratio2).isEqualTo(2.0); +- } +- + @Test + public void testAvroCompressionRatios() { + configureShuffle("lz4", true); +@@ -103,9 +91,6 @@ public class TestSparkCompressionUtil { + double ratio1 = shuffleCompressionRatio(PARQUET, "SOME_PARQUET_CODEC"); + assertThat(ratio1).isEqualTo(2.0); + +- double ratio2 = shuffleCompressionRatio(ORC, "SOME_ORC_CODEC"); +- assertThat(ratio2).isEqualTo(2.0); +- + double ratio3 = shuffleCompressionRatio(AVRO, "SOME_AVRO_CODEC"); + assertThat(ratio3).isEqualTo(1.0); + } +@@ -124,9 +109,6 @@ public class TestSparkCompressionUtil { + double ratio1 = shuffleCompressionRatio(PARQUET, null); + assertThat(ratio1).isEqualTo(2.0); + +- double ratio2 = shuffleCompressionRatio(ORC, null); +- assertThat(ratio2).isEqualTo(2.0); +- + double ratio3 = shuffleCompressionRatio(AVRO, null); + assertThat(ratio3).isEqualTo(1.0); + } +@@ -138,9 +120,6 @@ public class TestSparkCompressionUtil { + double ratio1 = shuffleCompressionRatio(PARQUET, "zstd"); + assertThat(ratio1).isEqualTo(4.0); + +- double ratio2 = shuffleCompressionRatio(ORC, "zlib"); +- assertThat(ratio2).isEqualTo(4.0); +- + double ratio3 = shuffleCompressionRatio(AVRO, "gzip"); + assertThat(ratio3).isEqualTo(2.0); + } +diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkWriteConf.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkWriteConf.java +index 336067c312..0cf6a020d4 100644 +--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkWriteConf.java ++++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/TestSparkWriteConf.java +@@ -25,13 +25,9 @@ import static org.apache.iceberg.TableProperties.DELETE_AVRO_COMPRESSION; + import static org.apache.iceberg.TableProperties.DELETE_AVRO_COMPRESSION_LEVEL; + import static org.apache.iceberg.TableProperties.DELETE_DEFAULT_FILE_FORMAT; + import static org.apache.iceberg.TableProperties.DELETE_DISTRIBUTION_MODE; +-import static org.apache.iceberg.TableProperties.DELETE_ORC_COMPRESSION; +-import static org.apache.iceberg.TableProperties.DELETE_ORC_COMPRESSION_STRATEGY; + import static org.apache.iceberg.TableProperties.DELETE_PARQUET_COMPRESSION; + import static org.apache.iceberg.TableProperties.DELETE_PARQUET_COMPRESSION_LEVEL; + import static org.apache.iceberg.TableProperties.MERGE_DISTRIBUTION_MODE; +-import static org.apache.iceberg.TableProperties.ORC_COMPRESSION; +-import static org.apache.iceberg.TableProperties.ORC_COMPRESSION_STRATEGY; + import static org.apache.iceberg.TableProperties.PARQUET_COMPRESSION; + import static org.apache.iceberg.TableProperties.PARQUET_COMPRESSION_LEVEL; + import static org.apache.iceberg.TableProperties.PARQUET_SHRED_VARIANTS; +@@ -356,30 +352,6 @@ public class TestSparkWriteConf extends TestBaseWithCatalog { + "3", + DELETE_PARQUET_COMPRESSION_LEVEL, + "3")), +- Lists.newArrayList( +- ImmutableMap.of( +- COMPRESSION_CODEC, +- "zstd", +- SparkSQLProperties.COMPRESSION_STRATEGY, +- "compression"), +- ImmutableMap.of( +- DEFAULT_FILE_FORMAT, +- "orc", +- DELETE_DEFAULT_FILE_FORMAT, +- "orc", +- ORC_COMPRESSION, +- "zlib", +- DELETE_ORC_COMPRESSION, +- "snappy"), +- ImmutableMap.of( +- DELETE_ORC_COMPRESSION, +- "zstd", +- ORC_COMPRESSION, +- "zstd", +- DELETE_ORC_COMPRESSION_STRATEGY, +- "compression", +- ORC_COMPRESSION_STRATEGY, +- "compression")), + Lists.newArrayList( + ImmutableMap.of(COMPRESSION_CODEC, "zstd", COMPRESSION_LEVEL, "9"), + ImmutableMap.of( +@@ -480,26 +452,6 @@ public class TestSparkWriteConf extends TestBaseWithCatalog { + "5", + DELETE_PARQUET_COMPRESSION_LEVEL, + "5")), +- Lists.newArrayList( +- ImmutableMap.of(), +- ImmutableMap.of( +- DEFAULT_FILE_FORMAT, +- "orc", +- DELETE_DEFAULT_FILE_FORMAT, +- "orc", +- ORC_COMPRESSION, +- "snappy", +- ORC_COMPRESSION_STRATEGY, +- "speed"), +- ImmutableMap.of( +- DELETE_ORC_COMPRESSION, +- "snappy", +- ORC_COMPRESSION, +- "snappy", +- ORC_COMPRESSION_STRATEGY, +- "speed", +- DELETE_ORC_COMPRESSION_STRATEGY, +- "speed")), + Lists.newArrayList( + ImmutableMap.of(), + ImmutableMap.of( +@@ -553,30 +505,6 @@ public class TestSparkWriteConf extends TestBaseWithCatalog { + "5", + DELETE_PARQUET_COMPRESSION_LEVEL, + "6")), +- Lists.newArrayList( +- ImmutableMap.of(), +- ImmutableMap.of( +- DEFAULT_FILE_FORMAT, +- "orc", +- DELETE_DEFAULT_FILE_FORMAT, +- "orc", +- ORC_COMPRESSION, +- "snappy", +- ORC_COMPRESSION_STRATEGY, +- "speed", +- DELETE_ORC_COMPRESSION, +- "zstd", +- DELETE_ORC_COMPRESSION_STRATEGY, +- "compression"), +- ImmutableMap.of( +- DELETE_ORC_COMPRESSION, +- "zstd", +- ORC_COMPRESSION, +- "snappy", +- ORC_COMPRESSION_STRATEGY, +- "speed", +- DELETE_ORC_COMPRESSION_STRATEGY, +- "compression")), + Lists.newArrayList( + ImmutableMap.of(), + ImmutableMap.of( +diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java +index dae721b1d7..dd9f5b70d0 100644 +--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java ++++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/actions/TestRewriteTablePathsAction.java +@@ -88,6 +88,7 @@ import org.apache.spark.storage.BlockManager; + import org.apache.spark.storage.BroadcastBlockId; + import org.junit.jupiter.api.AfterEach; + import org.junit.jupiter.api.BeforeEach; ++import org.junit.jupiter.api.Disabled; + import org.junit.jupiter.api.TestTemplate; + import org.junit.jupiter.api.extension.ExtendWith; + import org.junit.jupiter.api.io.TempDir; +@@ -421,7 +422,7 @@ public class TestRewriteTablePathsAction extends TestBase { + runPositionDeletesTest("avro"); + } + +- @TestTemplate ++ @Disabled + public void testPositionDeletesOrc() throws Exception { + runPositionDeletesTest("orc"); + } +diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestOrcWrite.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestOrcWrite.java +index e149e57e81..59b1e3879d 100644 +--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestOrcWrite.java ++++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestOrcWrite.java +@@ -30,9 +30,11 @@ import org.apache.iceberg.io.FileAppender; + import org.apache.iceberg.orc.ORC; + import org.apache.iceberg.types.Types; + import org.apache.spark.sql.catalyst.InternalRow; ++import org.junit.jupiter.api.Disabled; + import org.junit.jupiter.api.Test; + import org.junit.jupiter.api.io.TempDir; + ++@Disabled + public class TestOrcWrite { + @TempDir private Path temp; + +diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReadMetadataColumns.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReadMetadataColumns.java +index 13acaa1e3a..600ea6c7ef 100644 +--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReadMetadataColumns.java ++++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReadMetadataColumns.java +@@ -60,10 +60,12 @@ import org.apache.spark.sql.catalyst.expressions.GenericInternalRow; + import org.apache.spark.sql.vectorized.ColumnarBatch; + import org.apache.spark.unsafe.types.UTF8String; + import org.junit.jupiter.api.BeforeEach; ++import org.junit.jupiter.api.Disabled; + import org.junit.jupiter.api.TestTemplate; + import org.junit.jupiter.api.extension.ExtendWith; + import org.junit.jupiter.api.io.TempDir; + ++@Disabled + @ExtendWith(ParameterizedTestExtension.class) + public class TestSparkOrcReadMetadataColumns { + private static final Schema DATA_SCHEMA = +diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReader.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReader.java +index 3fcfe6845c..cae495eb26 100644 +--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReader.java ++++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkOrcReader.java +@@ -39,8 +39,10 @@ import org.apache.iceberg.spark.data.vectorized.VectorizedSparkOrcReaders; + import org.apache.iceberg.types.Types; + import org.apache.spark.sql.catalyst.InternalRow; + import org.apache.spark.sql.vectorized.ColumnarBatch; ++import org.junit.jupiter.api.Disabled; + import org.junit.jupiter.api.Test; + ++@Disabled + public class TestSparkOrcReader extends AvroDataTestBase { + @Override + protected void writeAndValidate(Schema schema) throws IOException { +diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkRecordOrcReaderWriter.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkRecordOrcReaderWriter.java +index 634327a81d..09ae9a4364 100644 +--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkRecordOrcReaderWriter.java ++++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestSparkRecordOrcReaderWriter.java +@@ -40,8 +40,10 @@ import org.apache.iceberg.orc.ORC; + import org.apache.iceberg.relocated.com.google.common.collect.Lists; + import org.apache.iceberg.types.Types; + import org.apache.spark.sql.catalyst.InternalRow; ++import org.junit.jupiter.api.Disabled; + import org.junit.jupiter.api.Test; + ++@Disabled + public class TestSparkRecordOrcReaderWriter extends AvroDataTestBase { + private static final int NUM_RECORDS = 200; + +diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestVectorizedOrcDataReader.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestVectorizedOrcDataReader.java +index 891cb9481c..60e2bb52f8 100644 +--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestVectorizedOrcDataReader.java ++++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/TestVectorizedOrcDataReader.java +@@ -46,9 +46,11 @@ import org.apache.orc.OrcConf; + import org.apache.spark.sql.catalyst.InternalRow; + import org.apache.spark.sql.vectorized.ColumnarBatch; + import org.junit.jupiter.api.BeforeAll; ++import org.junit.jupiter.api.Disabled; + import org.junit.jupiter.api.Test; + import org.junit.jupiter.api.io.TempDir; + ++@Disabled + public class TestVectorizedOrcDataReader { + @TempDir public static Path temp; + diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java index b61ecfa2f4..d696e85139 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/data/vectorized/parquet/TestParquetDictionaryEncodedVectorizedReads.java @@ -290,10 +605,50 @@ index da9cd63921..fae1357a40 100644 .config(TestBase.DISABLE_UI) .getOrCreate(); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -index d381822483..043d6596c0 100644 +index d381822483..7a3b4602f2 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestCompressionSettings.java -@@ -144,7 +144,18 @@ public class TestCompressionSettings extends CatalogTestBase { +@@ -19,7 +19,6 @@ + package org.apache.iceberg.spark.source; + + import static org.apache.iceberg.FileFormat.AVRO; +-import static org.apache.iceberg.FileFormat.ORC; + import static org.apache.iceberg.FileFormat.PARQUET; + import static org.apache.iceberg.RowLevelOperationMode.MERGE_ON_READ; + import static org.apache.iceberg.TableProperties.AVRO_COMPRESSION; +@@ -27,10 +26,8 @@ import static org.apache.iceberg.TableProperties.DEFAULT_FILE_FORMAT; + import static org.apache.iceberg.TableProperties.DELETE_AVRO_COMPRESSION; + import static org.apache.iceberg.TableProperties.DELETE_DEFAULT_FILE_FORMAT; + import static org.apache.iceberg.TableProperties.DELETE_MODE; +-import static org.apache.iceberg.TableProperties.DELETE_ORC_COMPRESSION; + import static org.apache.iceberg.TableProperties.DELETE_PARQUET_COMPRESSION; + import static org.apache.iceberg.TableProperties.FORMAT_VERSION; +-import static org.apache.iceberg.TableProperties.ORC_COMPRESSION; + import static org.apache.iceberg.TableProperties.PARQUET_COMPRESSION; + import static org.apache.iceberg.spark.SparkSQLProperties.COMPRESSION_CODEC; + import static org.apache.iceberg.spark.SparkSQLProperties.COMPRESSION_LEVEL; +@@ -118,20 +115,6 @@ public class TestCompressionSettings extends CatalogTestBase { + PARQUET, + ImmutableMap.of(COMPRESSION_CODEC, "gzip") + }, +- { +- SparkCatalogConfig.SPARK_SESSION.catalogName(), +- SparkCatalogConfig.SPARK_SESSION.implementation(), +- SparkCatalogConfig.SPARK_SESSION.properties(), +- ORC, +- ImmutableMap.of(COMPRESSION_CODEC, "zstd", COMPRESSION_STRATEGY, "speed") +- }, +- { +- SparkCatalogConfig.SPARK_SESSION.catalogName(), +- SparkCatalogConfig.SPARK_SESSION.implementation(), +- SparkCatalogConfig.SPARK_SESSION.properties(), +- ORC, +- ImmutableMap.of(COMPRESSION_CODEC, "zstd", COMPRESSION_STRATEGY, "compression") +- }, + { + SparkCatalogConfig.SPARK_SESSION.catalogName(), + SparkCatalogConfig.SPARK_SESSION.implementation(), +@@ -144,7 +127,18 @@ public class TestCompressionSettings extends CatalogTestBase { @BeforeAll public static void startSpark() { @@ -313,6 +668,32 @@ index d381822483..043d6596c0 100644 } @BeforeEach +@@ -172,10 +166,8 @@ public class TestCompressionSettings extends CatalogTestBase { + Map tableProperties = Maps.newHashMap(); + tableProperties.put(PARQUET_COMPRESSION, "gzip"); + tableProperties.put(AVRO_COMPRESSION, "gzip"); +- tableProperties.put(ORC_COMPRESSION, "zlib"); + tableProperties.put(DELETE_PARQUET_COMPRESSION, "gzip"); + tableProperties.put(DELETE_AVRO_COMPRESSION, "gzip"); +- tableProperties.put(DELETE_ORC_COMPRESSION, "zlib"); + tableProperties.put(DELETE_MODE, MERGE_ON_READ.modeName()); + tableProperties.put(FORMAT_VERSION, "2"); + sql("ALTER TABLE %s SET TBLPROPERTIES ('%s' '%s')", TABLE_NAME, DEFAULT_FILE_FORMAT, format); +diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2Coercion.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2Coercion.java +index f51a06853a..cccd5cc8f2 100644 +--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2Coercion.java ++++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataFrameWriterV2Coercion.java +@@ -38,10 +38,8 @@ public class TestDataFrameWriterV2Coercion extends TestBaseWithCatalog { + public static Object[][] parameters() { + return new Object[][] { + parameter(FileFormat.AVRO, "byte"), +- parameter(FileFormat.ORC, "byte"), + parameter(FileFormat.PARQUET, "byte"), + parameter(FileFormat.AVRO, "short"), +- parameter(FileFormat.ORC, "short"), + parameter(FileFormat.PARQUET, "short") + }; + } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java index e67ec5fd62..ed47a307c3 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestDataSourceOptions.java @@ -338,7 +719,7 @@ index e67ec5fd62..ed47a307c3 100644 @AfterAll diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java -index 24fecf4eb2..abab7798bc 100644 +index 24fecf4eb2..7b13f1aa00 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java @@ -117,6 +117,14 @@ public class TestFilteredScan { @@ -356,6 +737,15 @@ index 24fecf4eb2..abab7798bc 100644 .config(TestBase.DISABLE_UI) .getOrCreate(); } +@@ -145,8 +153,6 @@ public class TestFilteredScan { + {FileFormat.PARQUET, false, LOCAL}, + {FileFormat.PARQUET, true, DISTRIBUTED}, + {FileFormat.AVRO, false, LOCAL}, +- {FileFormat.ORC, false, DISTRIBUTED}, +- {FileFormat.ORC, true, LOCAL} + }; + } + diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java index d0103ff46e..62a7bdc04d 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestForwardCompatibility.java @@ -394,11 +784,99 @@ index a637b975fe..26304e8894 100644 .config(TestBase.DISABLE_UI) .getOrCreate(); } +diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData.java +index bf3bcacbfb..c535fb680a 100644 +--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData.java ++++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestIdentityPartitionData.java +@@ -85,22 +85,6 @@ public class TestIdentityPartitionData extends TestBase { + TableProperties.DATA_PLANNING_MODE, LOCAL.modeName(), + TableProperties.DELETE_PLANNING_MODE, LOCAL.modeName()) + }, +- { +- FileFormat.ORC, +- false, +- ImmutableMap.of( +- TableProperties.DEFAULT_FILE_FORMAT, "orc", +- TableProperties.DATA_PLANNING_MODE, DISTRIBUTED.modeName(), +- TableProperties.DELETE_PLANNING_MODE, DISTRIBUTED.modeName()) +- }, +- { +- FileFormat.ORC, +- true, +- ImmutableMap.of( +- TableProperties.DEFAULT_FILE_FORMAT, "orc", +- TableProperties.DATA_PLANNING_MODE, LOCAL.modeName(), +- TableProperties.DELETE_PLANNING_MODE, LOCAL.modeName()) +- }, + }; + } + +diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +index a417454b45..72724ef70d 100644 +--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java ++++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestMetadataTablesWithPartitionEvolution.java +@@ -19,7 +19,6 @@ + package org.apache.iceberg.spark.source; + + import static org.apache.iceberg.FileFormat.AVRO; +-import static org.apache.iceberg.FileFormat.ORC; + import static org.apache.iceberg.FileFormat.PARQUET; + import static org.apache.iceberg.MetadataTableType.ALL_DATA_FILES; + import static org.apache.iceberg.MetadataTableType.ALL_ENTRIES; +@@ -68,24 +67,6 @@ public class TestMetadataTablesWithPartitionEvolution extends CatalogTestBase { + @Parameters(name = "catalog = {0}, impl = {1}, conf = {2}, fileFormat = {3}, formatVersion = {4}") + public static Object[][] parameters() { + return new Object[][] { +- { +- "testhive", +- SparkCatalog.class.getName(), +- ImmutableMap.of( +- "type", "hive", +- "default-namespace", "default"), +- ORC, +- 1 +- }, +- { +- "testhive", +- SparkCatalog.class.getName(), +- ImmutableMap.of( +- "type", "hive", +- "default-namespace", "default"), +- ORC, +- 2 +- }, + {"testhadoop", SparkCatalog.class.getName(), ImmutableMap.of("type", "hadoop"), PARQUET, 1}, + {"testhadoop", SparkCatalog.class.getName(), ImmutableMap.of("type", "hadoop"), PARQUET, 2}, + { +diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestORCDataFrameWrite.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestORCDataFrameWrite.java +index 892e260f66..2b982e076e 100644 +--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestORCDataFrameWrite.java ++++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestORCDataFrameWrite.java +@@ -24,8 +24,10 @@ import org.apache.iceberg.FileFormat; + import org.apache.iceberg.Table; + import org.apache.iceberg.TableProperties; + import org.apache.spark.SparkException; ++import org.junit.jupiter.api.Disabled; + import org.junit.jupiter.api.Test; + ++@Disabled + public class TestORCDataFrameWrite extends DataFrameWriteTestBase { + @Override + protected void configureTable(Table table) { diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -index 8098db81f9..0e83b02f45 100644 +index 8098db81f9..17be9b602b 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java -@@ -120,6 +120,14 @@ public class TestPartitionPruning { +@@ -90,8 +90,6 @@ public class TestPartitionPruning { + {FileFormat.PARQUET, false, DISTRIBUTED}, + {FileFormat.PARQUET, true, LOCAL}, + {FileFormat.AVRO, false, DISTRIBUTED}, +- {FileFormat.ORC, false, LOCAL}, +- {FileFormat.ORC, true, DISTRIBUTED} + }; + } + +@@ -120,6 +118,14 @@ public class TestPartitionPruning { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -414,10 +892,19 @@ index 8098db81f9..0e83b02f45 100644 .getOrCreate(); TestPartitionPruning.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -index 9b5b22a73f..5d41bbeb5e 100644 +index 9b5b22a73f..0042b1a270 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java -@@ -113,6 +113,14 @@ public class TestPartitionValues { +@@ -75,8 +75,6 @@ public class TestPartitionValues { + {FileFormat.PARQUET, false}, + {FileFormat.PARQUET, true}, + {FileFormat.AVRO, false}, +- {FileFormat.ORC, false}, +- {FileFormat.ORC, true} + }; + } + +@@ -113,6 +111,14 @@ public class TestPartitionValues { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -432,6 +919,24 @@ index 9b5b22a73f..5d41bbeb5e 100644 .config(TestBase.DISABLE_UI) .getOrCreate(); } +diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java +index 14ad107e50..0f2979185d 100644 +--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java ++++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestPositionDeletesTable.java +@@ -132,13 +132,6 @@ public class TestPositionDeletesTable extends CatalogTestBase { + FileFormat.AVRO, + 2 + }, +- { +- SparkCatalogConfig.HIVE.catalogName(), +- SparkCatalogConfig.HIVE.implementation(), +- CATALOG_PROPS, +- FileFormat.ORC, +- 2 +- }, + { + SparkCatalogConfig.HIVE.catalogName(), + SparkCatalogConfig.HIVE.implementation(), diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java index 3004e8fa5c..38b6a1f6a4 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSnapshotSelection.java @@ -471,10 +976,18 @@ index d719ca6751..eba566120e 100644 .getOrCreate(); TestSparkDataFile.sparkContext = JavaSparkContext.fromSparkContext(spark.sparkContext()); diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -index c2f5afef0e..0dac36653c 100644 +index c2f5afef0e..01e1af730c 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkDataWrite.java -@@ -104,6 +104,14 @@ public class TestSparkDataWrite { +@@ -94,7 +94,6 @@ public class TestSparkDataWrite { + new Object[] {FileFormat.PARQUET, SnapshotRef.MAIN_BRANCH}, + new Object[] {FileFormat.PARQUET, "testBranch"}, + new Object[] {FileFormat.AVRO, null}, +- new Object[] {FileFormat.ORC, "testBranch"} + }; + } + +@@ -104,6 +103,14 @@ public class TestSparkDataWrite { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -489,7 +1002,7 @@ index c2f5afef0e..0dac36653c 100644 .config(TestBase.DISABLE_UI) .getOrCreate(); } -@@ -149,7 +157,7 @@ public class TestSparkDataWrite { +@@ -149,7 +156,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -498,7 +1011,7 @@ index c2f5afef0e..0dac36653c 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); for (ManifestFile manifest : SnapshotUtil.latestSnapshot(table, branch).allManifests(table.io())) { -@@ -219,7 +227,7 @@ public class TestSparkDataWrite { +@@ -219,7 +226,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -507,7 +1020,7 @@ index c2f5afef0e..0dac36653c 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -264,7 +272,7 @@ public class TestSparkDataWrite { +@@ -264,7 +271,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -516,7 +1029,7 @@ index c2f5afef0e..0dac36653c 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -316,7 +324,7 @@ public class TestSparkDataWrite { +@@ -316,7 +323,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -525,7 +1038,7 @@ index c2f5afef0e..0dac36653c 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -358,7 +366,7 @@ public class TestSparkDataWrite { +@@ -358,7 +365,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -534,7 +1047,7 @@ index c2f5afef0e..0dac36653c 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -397,7 +405,7 @@ public class TestSparkDataWrite { +@@ -397,7 +404,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -543,7 +1056,7 @@ index c2f5afef0e..0dac36653c 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); List files = Lists.newArrayList(); -@@ -461,7 +469,7 @@ public class TestSparkDataWrite { +@@ -461,7 +468,7 @@ public class TestSparkDataWrite { Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -552,7 +1065,7 @@ index c2f5afef0e..0dac36653c 100644 assertThat(actual).hasSameSizeAs(expected).isEqualTo(expected); } -@@ -814,7 +822,7 @@ public class TestSparkDataWrite { +@@ -814,7 +821,7 @@ public class TestSparkDataWrite { // Since write and commit succeeded, the rows should be readable Dataset result = spark.read().format("iceberg").load(targetLocation); List actual = @@ -561,11 +1074,33 @@ index c2f5afef0e..0dac36653c 100644 assertThat(actual) .hasSize(records.size() + records2.size()) .containsExactlyInAnyOrder( +diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +index 7498f129b2..0423a5e9d4 100644 +--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java ++++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkMetadataColumns.java +@@ -95,8 +95,6 @@ public class TestSparkMetadataColumns extends TestBase { + parameters.add(new Object[] {FileFormat.PARQUET, false, version}); + parameters.add(new Object[] {FileFormat.PARQUET, true, version}); + parameters.add(new Object[] {FileFormat.AVRO, false, version}); +- parameters.add(new Object[] {FileFormat.ORC, false, version}); +- parameters.add(new Object[] {FileFormat.ORC, true, version}); + } + return parameters.toArray(new Object[0][]); + } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -index de6a5e5902..e220f0dd31 100644 +index de6a5e5902..a4bd27681c 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReadProjection.java -@@ -89,6 +89,14 @@ public class TestSparkReadProjection extends TestReadProjection { +@@ -72,8 +72,6 @@ public class TestSparkReadProjection extends TestReadProjection { + {FileFormat.PARQUET, false, LOCAL}, + {FileFormat.PARQUET, true, DISTRIBUTED}, + {FileFormat.AVRO, false, LOCAL}, +- {FileFormat.ORC, false, DISTRIBUTED}, +- {FileFormat.ORC, true, LOCAL} + }; + } + +@@ -89,6 +87,14 @@ public class TestSparkReadProjection extends TestReadProjection { SparkSession.builder() .master("local[2]") .config("spark.driver.host", InetAddress.getLoopbackAddress().getHostAddress()) @@ -581,10 +1116,18 @@ index de6a5e5902..e220f0dd31 100644 .getOrCreate(); ImmutableMap config = diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -index 0d61930571..108d31b965 100644 +index 0d61930571..dd6d30d018 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java +++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkReaderDeletes.java -@@ -138,6 +138,14 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -118,7 +118,6 @@ public class TestSparkReaderDeletes extends DeleteReadTests { + parameters.add(new Object[] {FileFormat.PARQUET, version, false, PlanningMode.DISTRIBUTED}); + parameters.add(new Object[] {FileFormat.PARQUET, version, true, PlanningMode.LOCAL}); + if (version == 2) { +- parameters.add(new Object[] {FileFormat.ORC, version, false, PlanningMode.DISTRIBUTED}); + parameters.add(new Object[] {FileFormat.AVRO, version, false, PlanningMode.LOCAL}); + } + } +@@ -138,6 +137,14 @@ public class TestSparkReaderDeletes extends DeleteReadTests { .config("spark.ui.liveUpdate.period", 0) .config(SQLConf.PARTITION_OVERWRITE_MODE().key(), "dynamic") .config("spark.hadoop." + METASTOREURIS.varname, hiveConf.get(METASTOREURIS.varname)) @@ -599,7 +1142,7 @@ index 0d61930571..108d31b965 100644 .config(TestBase.DISABLE_UI) .enableHiveSupport() .getOrCreate(); -@@ -210,7 +218,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { +@@ -210,7 +217,8 @@ public class TestSparkReaderDeletes extends DeleteReadTests { } protected boolean countDeletes() { @@ -628,6 +1171,23 @@ index cb2f866fab..a161d541cf 100644 .config(TestBase.DISABLE_UI) .enableHiveSupport() .getOrCreate(); +diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java +index fd133de4ba..a4a042474e 100644 +--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java ++++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestSparkScan.java +@@ -103,12 +103,6 @@ public class TestSparkScan extends TestBaseWithCatalog { + SparkCatalogConfig.HADOOP.implementation(), + SparkCatalogConfig.HADOOP.properties(), + "avro" +- }, +- { +- SparkCatalogConfig.HADOOP.catalogName(), +- SparkCatalogConfig.HADOOP.implementation(), +- SparkCatalogConfig.HADOOP.properties(), +- "orc" + } + }; + } diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java index 5e900ea0ba..c9299d1ebf 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreaming.java @@ -647,6 +1207,32 @@ index 5e900ea0ba..c9299d1ebf 100644 .config(TestBase.DISABLE_UI) .getOrCreate(); } +diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java +index 3957872be7..d464956572 100644 +--- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java ++++ b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestStructuredStreamingRead3.java +@@ -842,20 +842,15 @@ public final class TestStructuredStreamingRead3 extends CatalogTestBase { + Lists.newArrayList( + new SimpleRecord(1, "one"), new SimpleRecord(2, "two"), new SimpleRecord(3, "three")); + +- List orcFileRecords = +- Lists.newArrayList(new SimpleRecord(4, "four"), new SimpleRecord(5, "five")); +- + List avroFileRecords = + Lists.newArrayList(new SimpleRecord(6, "six"), new SimpleRecord(7, "seven")); + + appendData(parquetFileRecords); +- appendData(orcFileRecords, "orc"); + appendData(avroFileRecords, "avro"); + + StreamingQuery query = startStream(); + assertThat(rowsAvailable(query)) +- .containsExactlyInAnyOrderElementsOf( +- Iterables.concat(parquetFileRecords, orcFileRecords, avroFileRecords)); ++ .containsExactlyInAnyOrderElementsOf(Iterables.concat(parquetFileRecords, avroFileRecords)); + } + + @TestTemplate diff --git a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java b/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java index 79781a8fc3..748b0a8327 100644 --- a/spark/v4.1/spark/src/test/java/org/apache/iceberg/spark/source/TestTimestampWithoutZone.java From f13f24d3f017e7a9a779a0e2c481a35f126edd04 Mon Sep 17 00:00:00 2001 From: Huang-Hsiang Cheng Date: Wed, 22 Jul 2026 09:59:44 -0700 Subject: [PATCH 2/3] Make sure Comet falls back on ORC data file format --- .../comet/CometIcebergNativeSuite.scala | 33 +++++++++++++++++++ 1 file changed, 33 insertions(+) diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala index 7b9bd6b3d6..5b2f3abab9 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala @@ -4400,4 +4400,37 @@ class CometIcebergNativeSuite } } } + + test("Iceberg table with ORC data file format falls back to Spark") { + assume(icebergAvailable, "Iceberg not available in classpath") + withTempIcebergDir { warehouseDir => + withSQLConf( + "spark.sql.catalog.test_cat" -> "org.apache.iceberg.spark.SparkCatalog", + "spark.sql.catalog.test_cat.type" -> "hadoop", + "spark.sql.catalog.test_cat.warehouse" -> warehouseDir.getAbsolutePath, + CometConf.COMET_ENABLED.key -> "true", + CometConf.COMET_EXEC_ENABLED.key -> "true", + CometConf.COMET_ICEBERG_NATIVE_ENABLED.key -> "true") { + + try { + spark.sql(""" + CREATE TABLE test_cat.db.orc_table( + id INT, + name STRING + ) + USING iceberg + TBLPROPERTIES('write.format.default' = 'orc') + """) + + spark.sql("""INSERT INTO test_cat.db.orc_table VALUES (1, 'Alice'), (2, 'Bob')""") + + checkIcebergNativeScanFallback( + "SELECT * FROM test_cat.db.orc_table", + "Comet doesn't support accelerated Iceberg ORC scans") + } finally { + spark.sql(s"DROP TABLE test_cat.db.orc_table") + } + } + } + } } From 6574e3dc3db2e522abbc46fb05cba53440a19780 Mon Sep 17 00:00:00 2001 From: hsiang-c <137842490+hsiang-c@users.noreply.github.com> Date: Wed, 22 Jul 2026 13:35:24 -0700 Subject: [PATCH 3/3] Scalafix --- .../test/scala/org/apache/comet/CometIcebergNativeSuite.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala index 5b2f3abab9..59320ed21b 100644 --- a/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala +++ b/spark/src/test/scala/org/apache/comet/CometIcebergNativeSuite.scala @@ -4428,7 +4428,7 @@ class CometIcebergNativeSuite "SELECT * FROM test_cat.db.orc_table", "Comet doesn't support accelerated Iceberg ORC scans") } finally { - spark.sql(s"DROP TABLE test_cat.db.orc_table") + spark.sql("DROP TABLE test_cat.db.orc_table") } } }