I upgraded PySpark from 3.5.5 to 3.5.6, and now all unit tests with an overwrite operation are failing with this error:
pyspark.errors.exceptions.captured.AnalysisException: Table does not support truncate in batch mode.
I tried to look into the documentation, but I didn't find any hint as to why this is happening or how to fix it.
Minimal reproducible example:
# Local Spark session
session_builder = (
SparkSession.builder.appName("PySpark Automated Testing")
.config(
"spark.sql.warehouse.dir",
"/path/to/tmp/folder",
)
.config(
"spark.sql.execution.arrow.pyspark.enabled",
"true",
)
.config(
"spark.sql.extensions",
"io.delta.sql.DeltaSparkSessionExtension",
)
.config(
"spark.sql.catalog.spark_catalog",
"org.apache.spark.sql.delta.catalog.DeltaCatalog",
)
.config(
"spark.databricks.delta.schema.autoMerge.enabled",
"true",
)
.config(
"spark.sql.shuffle.partitions",
10,
)
.config(
"spark.databricks.delta.snapshotPartitions",
"2",
)
.config(
"spark.ui.enabled",
"false",
)
.config(
"spark.ui.showConsoleProgress",
"false",
)
.config(
"spark.databricks.delta.properties.defaults.appendOnly",
"false",
)
)
spark = configure_spark_with_delta_pip(session_builder).getOrCreate()
# Table creation
spark_catalog = spark.catalog.currentCatalog()
spark_database = spark.catalog.currentDatabase()
table_name = "test_overwrite_table"
DeltaTable.createIfNotExists(spark).tableName(f"{spark_database}.{table_name}").execute()
df = spark.createDataFrame(
data=[
Row(pk="id1", col_1="a"),
Row(pk="id2", col_1="b"),
],
)
# Table overwrite
df.write.format("delta").mode("overwrite").saveAsTable(f"{spark_catalog}.{spark_database}.{table_name}")