From be81d13844fd81774889ffe846bb2bdb0219b602 Mon Sep 17 00:00:00 2001 From: Micah Wylde Date: Fri, 27 Dec 2024 14:44:11 -0800 Subject: [PATCH] Fix regression with delta writes of invariant columns (#816) --- crates/arroyo-connectors/src/filesystem/sink/delta.rs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/crates/arroyo-connectors/src/filesystem/sink/delta.rs b/crates/arroyo-connectors/src/filesystem/sink/delta.rs index 4adbad80b..3dd3eff7b 100644 --- a/crates/arroyo-connectors/src/filesystem/sink/delta.rs +++ b/crates/arroyo-connectors/src/filesystem/sink/delta.rs @@ -3,6 +3,7 @@ use anyhow::{Context, Result}; use arrow::datatypes::{Schema, SchemaRef}; use arroyo_storage::{get_current_credentials, StorageProvider}; use arroyo_types::to_millis; +use deltalake::TableProperty::{MinReaderVersion, MinWriterVersion}; use deltalake::{ aws::storage::s3_constants::AWS_S3_ALLOW_UNSAFE_RENAME, kernel::{Action, Add}, @@ -86,6 +87,8 @@ async fn create_new_table( CreateBuilder::new() .with_log_store(delta_object_store) .with_columns(delta_schema.fields().cloned()) + .with_configuration_property(MinReaderVersion, Some("3")) + .with_configuration_property(MinWriterVersion, Some("7")) .await .map_err(Into::into) }