diff --git a/crates/common/src/knobs.rs b/crates/common/src/knobs.rs index 72100467..1cb6bf7a 100644 --- a/crates/common/src/knobs.rs +++ b/crates/common/src/knobs.rs @@ -1242,3 +1242,16 @@ pub static DISABLE_FUZZY_TEXT_SEARCH: LazyLock = /// Conductors that are colocated. pub static INSTANCE_LOADER_CONCURRENCY: LazyLock = LazyLock::new(|| env_config("INSTANCE_LOADER_CONCURRENCY", 16)); + +/// The maximum number of bytes to buffer in an multipart upload. +/// There may be stricter limits imposed by the storage provider, but this is +/// the target max size for the buffer to protect against memory exhaustion. +/// The maximum number of parts is 10000, so this imposes a max file size, which +/// defaults to 10000 * 100MiB = 1TB. When reducing this knob, make sure the +/// maximum file size can fit a snapshot export of the instance. +/// Storage uploads can run in parallel, and the buffers get cloned during +/// upload, so make sure this knob times ~8 can fit in memory. +/// +/// Defaults to 100MiB. +pub static STORAGE_MAX_INTERMEDIATE_PART_SIZE: LazyLock = + LazyLock::new(|| env_config("STORAGE_MAX_INTERMEDIATE_PART_SIZE", 100 * (1 << 20)));