From c3bb0362bea271fdbc744ec2a968ab11fe20f965 Mon Sep 17 00:00:00 2001 From: Xuanwo Date: Tue, 7 Jan 2025 18:34:37 +0800 Subject: [PATCH] docs: Polish docs for Operator, Reader and Writer Signed-off-by: Xuanwo --- core/src/types/operator/operator.rs | 1246 +++++++++++++----- core/src/types/operator/operator_futures.rs | 1258 +++++++++++++++++-- 2 files changed, 2073 insertions(+), 431 deletions(-) diff --git a/core/src/types/operator/operator.rs b/core/src/types/operator/operator.rs index de4fffe66822..11485cb57cbd 100644 --- a/core/src/types/operator/operator.rs +++ b/core/src/types/operator/operator.rs @@ -29,30 +29,81 @@ use crate::raw::*; use crate::types::delete::Deleter; use crate::*; -/// Operator is the entry for all public async APIs. +/// The `Operator` serves as the entry point for all public asynchronous APIs. /// -/// Developer should manipulate the data from storage service through Operator only by right. +/// For more details about the `Operator`, refer to the [`concepts`][crate::docs::concepts] section. /// -/// We will usually do some general checks and data transformations in this layer, -/// like normalizing path from input, checking whether the path refers to one file or one directory, -/// and so on. +/// ## Build /// -/// Read [`concepts`][crate::docs::concepts] for more about [`Operator`]. +/// Users can initialize an `Operator` through the following methods: /// -/// # Examples +/// - [`Operator::new`]: Creates an operator using a [`services`] builder, such as [`services::S3`]. +/// - [`Operator::from_config`]: Creates an operator using a [`services`] configuration, such as [`services::S3Config`]. +/// - [`Operator::from_iter`]: Creates an operator from an iterator of configuration key-value pairs. /// -/// Read more backend init examples in [`services`] +/// ``` +/// # use anyhow::Result; +/// use opendal::services::Memory; +/// use opendal::Operator; +/// async fn test() -> Result<()> { +/// // Build an `Operator` to start operating the storage. +/// let _: Operator = Operator::new(Memory::default())?.finish(); +/// +/// Ok(()) +/// } +/// ``` +/// +/// ## Layer +/// +/// After the operator is built, users can add the layers they need on top of it. +/// +/// OpenDAL offers various layers for users to choose from, such as `RetryLayer`, `LoggingLayer`, and more. Visit [`layers`] for further details. /// /// ``` /// # use anyhow::Result; -/// use opendal::services::Fs; +/// use opendal::layers::RetryLayer; +/// use opendal::services::Memory; /// use opendal::Operator; /// async fn test() -> Result<()> { -/// // Create fs backend builder. -/// let mut builder = Fs::default().root("/tmp"); +/// let op: Operator = Operator::new(Memory::default())?.finish(); /// -/// // Build an `Operator` to start operating the storage. -/// let _: Operator = Operator::new(builder)?.finish(); +/// // OpenDAL will retry failed operations now. +/// let op = op.layer(RetryLayer::default()); +/// +/// Ok(()) +/// } +/// ``` +/// +/// ## Operate +/// +/// After the operator is built and the layers are added, users can start operating the storage. +/// +/// The operator is `Send`, `Sync`, and `Clone`. It has no internal state, and all APIs only take +/// a `&self` reference, making it safe to share the operator across threads. +/// +/// Operator provides a consistent API pattern for data operations. For reading operations, it exposes: +/// +/// - [`Operator::read`]: Basic operation that reads entire content into memory +/// - [`Operator::read_with`]: Enhanced read operation with additional options (range, if_match, if_none_match) +/// - [`Operator::reader`]: Creates a lazy reader for on-demand data streaming +/// - [`Operator::reader_with`]: Creates a configurable reader with conditional options (if_match, if_none_match) +/// +/// The [`Reader`] created by [`Operator`] supports custom read control methods and can be converted +/// into `futures::AsyncRead` for broader ecosystem compatibility. +/// +/// ``` +/// # use anyhow::Result; +/// use opendal::layers::RetryLayer; +/// use opendal::services::Memory; +/// use opendal::Operator; +/// async fn test() -> Result<()> { +/// let op: Operator = Operator::new(Memory::default())?.finish(); +/// +/// // OpenDAL will retry failed operations now. +/// let op = op.layer(RetryLayer::default()); +/// +/// // Read all data into memory. +/// let data = op.read("path/to/file").await?; /// /// Ok(()) /// } @@ -209,7 +260,7 @@ impl Operator { /// If file exists and it's etag doesn't match, an error with kind [`ErrorKind::ConditionNotMatch`] /// will be returned. /// - /// ```no_run + /// ``` /// # use opendal::Result; /// use opendal::Operator; /// @@ -228,7 +279,7 @@ impl Operator { /// If file exists and it's etag match, an error with kind [`ErrorKind::ConditionNotMatch`] /// will be returned. /// - /// ```no_run + /// ``` /// # use opendal::Result; /// use opendal::Operator; /// @@ -246,7 +297,7 @@ impl Operator { /// /// If the version doesn't exist, an error with kind [`ErrorKind::NotFound`] will be returned. /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// @@ -437,7 +488,7 @@ impl Operator { /// /// # Examples /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// # use futures::TryStreamExt; @@ -471,11 +522,10 @@ impl Operator { /// If we have a file with size `n`. /// /// - `..` means read bytes in range `[0, n)` of file. - /// - `0..1024` means read bytes in range `[0, 1024)` of file + /// - `0..1024` and `..1024` means read bytes in range `[0, 1024)` of file /// - `1024..` means read bytes in range `[1024, n)` of file - /// - `..1024` means read bytes in range `(n - 1024, n)` of file /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// # use futures::TryStreamExt; @@ -485,6 +535,61 @@ impl Operator { /// # } /// ``` /// + /// ## `concurrent` + /// + /// Set `concurrent` for the reader. + /// + /// OpenDAL by default to write file without concurrent. This is not efficient for cases when users + /// read large chunks of data. By setting `concurrent`, opendal will read files concurrently + /// on support storage services. + /// + /// By setting `concurrent`, opendal will fetch chunks concurrently with + /// the given chunk size. + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use opendal::Scheme; + /// # async fn test(op: Operator) -> Result<()> { + /// let r = op.read_with("path/to/file").concurrent(8).await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ## `chunk` + /// + /// OpenDAL will use services' preferred chunk size by default. Users can set chunk based on their own needs. + /// + /// This following example will make opendal read data in 4MiB chunks: + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use opendal::Scheme; + /// # async fn test(op: Operator) -> Result<()> { + /// let r = op.read_with("path/to/file").chunk(4 * 1024 * 1024).await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ## `version` + /// + /// Set `version` for this `read` request. + /// + /// This feature can be used to retrieve the data of a specified version of the given path. + /// + /// If the version doesn't exist, an error with kind [`ErrorKind::NotFound`] will be returned. + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// + /// # async fn test(op: Operator, version: &str) -> Result<()> { + /// let mut bs = op.read_with("path/to/file").version(version).await?; + /// # Ok(()) + /// # } + /// ``` + /// /// ## `if_match` /// /// Set `if_match` for this `read` request. @@ -494,7 +599,7 @@ impl Operator { /// If file exists and it's etag doesn't match, an error with kind [`ErrorKind::ConditionNotMatch`] /// will be returned. /// - /// ```no_run + /// ``` /// # use opendal::Result; /// use opendal::Operator; /// # async fn test(op: Operator, etag: &str) -> Result<()> { @@ -512,7 +617,7 @@ impl Operator { /// If file exists and it's etag match, an error with kind [`ErrorKind::ConditionNotMatch`] /// will be returned. /// - /// ```no_run + /// ``` /// # use opendal::Result; /// use opendal::Operator; /// # async fn test(op: Operator, etag: &str) -> Result<()> { @@ -521,57 +626,42 @@ impl Operator { /// # } /// ``` /// - /// ## `concurrent` + /// ## `if_modified_since` /// - /// Set `concurrent` for the reader. + /// Set `if_modified_since` for this `read` request. /// - /// OpenDAL by default to write file without concurrent. This is not efficient for cases when users - /// read large chunks of data. By setting `concurrent`, opendal will read files concurrently - /// on support storage services. + /// This feature can be used to check if the file has been modified since the given timestamp. /// - /// By setting `concurrent`, opendal will fetch chunks concurrently with - /// the given chunk size. + /// If file exists and it hasn't been modified since the specified time, an error with kind + /// [`ErrorKind::ConditionNotMatch`] will be returned. /// - /// ```no_run - /// # use opendal::Result; - /// # use opendal::Operator; - /// # use opendal::Scheme; - /// # async fn test(op: Operator) -> Result<()> { - /// let r = op.read_with("path/to/file").concurrent(8).await?; - /// # Ok(()) - /// # } /// ``` - /// - /// ## `chunk` - /// - /// OpenDAL will use services' preferred chunk size by default. Users can set chunk based on their own needs. - /// - /// This following example will make opendal read data in 4MiB chunks: - /// - /// ```no_run /// # use opendal::Result; - /// # use opendal::Operator; - /// # use opendal::Scheme; - /// # async fn test(op: Operator) -> Result<()> { - /// let r = op.read_with("path/to/file").chunk(4 * 1024 * 1024).await?; + /// use opendal::Operator; + /// use chrono::DateTime; + /// use chrono::Utc; + /// # async fn test(op: Operator, time: DateTime) -> Result<()> { + /// let mut metadata = op.read_with("path/to/file").if_modified_since(time).await?; /// # Ok(()) /// # } /// ``` /// - /// ## `version` + /// ## `if_unmodified_since` /// - /// Set `version` for this `read` request. + /// Set `if_unmodified_since` for this `read` request. /// - /// This feature can be used to retrieve the data of a specified version of the given path. + /// This feature can be used to check if the file hasn't been modified since the given timestamp. /// - /// If the version doesn't exist, an error with kind [`ErrorKind::NotFound`] will be returned. + /// If file exists and it has been modified since the specified time, an error with kind + /// [`ErrorKind::ConditionNotMatch`] will be returned. /// - /// ```no_run + /// ``` /// # use opendal::Result; - /// # use opendal::Operator; - /// - /// # async fn test(op: Operator, version: &str) -> Result<()> { - /// let mut bs = op.read_with("path/to/file").version(version).await?; + /// use opendal::Operator; + /// use chrono::DateTime; + /// use chrono::Utc; + /// # async fn test(op: Operator, time: DateTime) -> Result<()> { + /// let mut metadata = op.read_with("path/to/file").if_unmodified_since(time).await?; /// # Ok(()) /// # } /// ``` @@ -580,7 +670,7 @@ impl Operator { /// /// Read the whole path into a bytes. /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// # use futures::TryStreamExt; @@ -630,7 +720,7 @@ impl Operator { /// /// # Examples /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// # use futures::TryStreamExt; @@ -666,7 +756,7 @@ impl Operator { /// By setting `concurrent``, opendal will fetch chunks concurrently with /// the give chunk size. /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// # use opendal::Scheme; @@ -682,7 +772,36 @@ impl Operator { /// /// This following example will make opendal read data in 4MiB chunks: /// - /// ```no_run + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use opendal::Scheme; + /// # async fn test(op: Operator) -> Result<()> { + /// let r = op + /// .reader_with("path/to/file") + /// .chunk(4 * 1024 * 1024) + /// .await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ## `gap` + /// + /// Controls the optimization strategy for range reads in [`Reader::fetch`]. + /// + /// When performing range reads, if the gap between two requested ranges is smaller than + /// the configured `gap` size, OpenDAL will merge these ranges into a single read request + /// and discard the unrequested data in between. This helps reduce the number of API calls + /// to remote storage services. + /// + /// This optimization is particularly useful when performing multiple small range reads + /// that are close to each other, as it reduces the overhead of multiple network requests + /// at the cost of transferring some additional data. + /// + /// In this example, if two requested ranges are separated by less than 1MiB, + /// they will be merged into a single read request: + /// + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// # use opendal::Scheme; @@ -690,6 +809,7 @@ impl Operator { /// let r = op /// .reader_with("path/to/file") /// .chunk(4 * 1024 * 1024) + /// .gap(1024 * 1024) // 1MiB gap /// .await?; /// # Ok(()) /// # } @@ -703,19 +823,95 @@ impl Operator { /// /// If the version doesn't exist, an error with kind [`ErrorKind::NotFound`] will be returned. /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// /// # async fn test(op: Operator, version: &str) -> Result<()> { - /// let mut bs = op.reader_with("path/to/file").version(version).await?; + /// let mut r = op.reader_with("path/to/file").version(version).await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ## `if_match` + /// + /// Set `if-match` for this `read` request. + /// + /// This feature can be used to check if the file's `ETag` matches the given `ETag`. + /// + /// If file exists and it's etag doesn't match, an error with kind [`ErrorKind::ConditionNotMatch`] + /// will be returned. + /// + /// ``` + /// # use opendal::Result; + /// use opendal::Operator; + /// # async fn test(op: Operator, etag: &str) -> Result<()> { + /// let mut r = op.reader_with("path/to/file").if_match(etag).await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ## `if_none_match` + /// + /// Set `if-none-match` for this `read` request. + /// + /// This feature can be used to check if the file's `ETag` doesn't match the given `ETag`. + /// + /// If file exists and it's etag match, an error with kind [`ErrorKind::ConditionNotMatch`] + /// will be returned. + /// + /// ``` + /// # use opendal::Result; + /// use opendal::Operator; + /// # async fn test(op: Operator, etag: &str) -> Result<()> { + /// let mut r = op.reader_with("path/to/file").if_none_match(etag).await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ## `if_modified_since` + /// + /// Set `if-modified-since` for this `read` request. + /// + /// This feature can be used to check if the file has been modified since the given timestamp. + /// + /// If file exists and it hasn't been modified since the specified time, an error with kind + /// [`ErrorKind::ConditionNotMatch`] will be returned. + /// + /// ``` + /// # use opendal::Result; + /// use opendal::Operator; + /// use chrono::DateTime; + /// use chrono::Utc; + /// # async fn test(op: Operator, time: DateTime) -> Result<()> { + /// let mut r = op.reader_with("path/to/file").if_modified_since(time).await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ## `if_unmodified_since` + /// + /// Set `if-unmodified-since` for this `read` request. + /// + /// This feature can be used to check if the file hasn't been modified since the given timestamp. + /// + /// If file exists and it has been modified since the specified time, an error with kind + /// [`ErrorKind::ConditionNotMatch`] will be returned. + /// + /// ``` + /// # use opendal::Result; + /// use opendal::Operator; + /// use chrono::DateTime; + /// use chrono::Utc; + /// # async fn test(op: Operator, time: DateTime) -> Result<()> { + /// let mut r = op.reader_with("path/to/file").if_unmodified_since(time).await?; /// # Ok(()) /// # } /// ``` /// /// # Examples /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// # use opendal::Scheme; @@ -772,7 +968,7 @@ impl Operator { /// /// # Examples /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// # use futures::StreamExt; @@ -800,7 +996,7 @@ impl Operator { /// /// # Examples /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// @@ -857,7 +1053,7 @@ impl Operator { /// /// # Examples /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// @@ -935,7 +1131,7 @@ impl Operator { /// /// # Examples /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// use bytes::Bytes; @@ -952,7 +1148,527 @@ impl Operator { self.writer_with(path).await } - /// Create a writer for streaming data to the given path with more options. + /// Create a writer for streaming data to the given path with more options. + /// + /// # Usages + /// + /// ## `append` + /// + /// Sets append mode for this write request. + /// + /// ### Capability + /// + /// Check [`Capability::write_can_append`] before using this feature. + /// + /// ### Behavior + /// + /// - By default, write operations overwrite existing files + /// - When append is set to true: + /// - New data will be appended to the end of existing file + /// - If file doesn't exist, it will be created + /// - If not supported, will return an error + /// + /// This operation allows adding data to existing files instead of overwriting them. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let mut w = op.writer_with("path/to/file").append(true).await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ## `chunk` + /// + /// Sets chunk size for buffered writes. + /// + /// ### Capability + /// + /// Check [`Capability::write_multi_min_size`] and [`Capability::write_multi_max_size`] for size limits. + /// + /// ### Behavior + /// + /// - By default, OpenDAL sets optimal chunk size based on service capabilities + /// - When chunk size is set: + /// - Data will be buffered until reaching chunk size + /// - One API call will be made per chunk + /// - Last chunk may be smaller than chunk size + /// - Important considerations: + /// - Some services require minimum chunk sizes (e.g. S3's EntityTooSmall error) + /// - Smaller chunks increase API calls and costs + /// - Larger chunks increase memory usage, but improve performance and reduce costs + /// + /// ### Performance Impact + /// + /// Setting appropriate chunk size can: + /// - Reduce number of API calls + /// - Improve overall throughput + /// - Lower operation costs + /// - Better utilize network bandwidth + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// // Set 8MiB chunk size - data will be sent in one API call at close + /// let mut w = op + /// .writer_with("path/to/file") + /// .chunk(8 * 1024 * 1024) + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// # `concurrent` + /// + /// Sets concurrent write operations for this writer. + /// + /// ## Behavior + /// + /// - By default, OpenDAL writes files sequentially + /// - When concurrent is set: + /// - Multiple write operations can execute in parallel + /// - Write operations return immediately without waiting if tasks space are available + /// - Close operation ensures all writes complete in order + /// - Memory usage increases with concurrency level + /// - If not supported, falls back to sequential writes + /// + /// This feature significantly improves performance when: + /// - Writing large files + /// - Network latency is high + /// - Storage service supports concurrent uploads like multipart uploads + /// + /// ## Performance Impact + /// + /// Setting appropriate concurrency can: + /// - Increase write throughput + /// - Reduce total write time + /// - Better utilize available bandwidth + /// - Trade memory for performance + /// + /// ## Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// // Enable concurrent writes with 8 parallel operations + /// let mut w = op.writer_with("path/to/file").concurrent(8).await?; + /// + /// // First write starts immediately + /// w.write(vec![0; 4096]).await?; + /// + /// // Second write runs concurrently with first + /// w.write(vec![1; 4096]).await?; + /// + /// // Ensures all writes complete successfully and in order + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ## `cache_control` + /// + /// Sets Cache-Control header for this write operation. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_cache_control`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, sets Cache-Control as system metadata on the target file + /// - The value should follow HTTP Cache-Control header format + /// - If not supported, the value will be ignored + /// + /// This operation allows controlling caching behavior for the written content. + /// + /// ### Use Cases + /// + /// - Setting browser cache duration + /// - Configuring CDN behavior + /// - Optimizing content delivery + /// - Managing cache invalidation + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// // Cache content for 7 days (604800 seconds) + /// let mut w = op + /// .writer_with("path/to/file") + /// .cache_control("max-age=604800") + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ### References + /// + /// - [MDN Cache-Control](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control) + /// - [RFC 7234 Section 5.2](https://tools.ietf.org/html/rfc7234#section-5.2) + /// + /// ## `content_type` + /// + /// Sets `Content-Type` header for this write operation. + /// + /// ## Capability + /// + /// Check [`Capability::write_with_content_type`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, sets Content-Type as system metadata on the target file + /// - The value should follow MIME type format (e.g. "text/plain", "image/jpeg") + /// - If not supported, the value will be ignored + /// + /// This operation allows specifying the media type of the content being written. + /// + /// ## Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// // Set content type for plain text file + /// let mut w = op + /// .writer_with("path/to/file") + /// .content_type("text/plain") + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ## `content_disposition` + /// + /// Sets Content-Disposition header for this write request. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_content_disposition`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, sets Content-Disposition as system metadata on the target file + /// - The value should follow HTTP Content-Disposition header format + /// - Common values include: + /// - `inline` - Content displayed within browser + /// - `attachment` - Content downloaded as file + /// - `attachment; filename="example.jpg"` - Downloaded with specified filename + /// - If not supported, the value will be ignored + /// + /// This operation allows controlling how the content should be displayed or downloaded. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let mut w = op + /// .writer_with("path/to/file") + /// .content_disposition("attachment; filename=\"filename.jpg\"") + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ## `content_encoding` + /// + /// Sets Content-Encoding header for this write request. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_content_encoding`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, sets Content-Encoding as system metadata on the target file + /// - The value should follow HTTP Content-Encoding header format + /// - Common values include: + /// - `gzip` - Content encoded using gzip compression + /// - `deflate` - Content encoded using deflate compression + /// - `br` - Content encoded using Brotli compression + /// - `identity` - No encoding applied (default value) + /// - If not supported, the value will be ignored + /// + /// This operation allows specifying the encoding applied to the content being written. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let mut w = op + /// .writer_with("path/to/file") + /// .content_encoding("gzip") + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ## `if_match` + /// + /// Sets If-Match header for this write request. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_if_match`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, the write operation will only succeed if the target's ETag matches the specified value + /// - The value should be a valid ETag string + /// - Common values include: + /// - A specific ETag value like `"686897696a7c876b7e"` + /// - `*` - Matches any existing resource + /// - If not supported, the value will be ignored + /// + /// This operation provides conditional write functionality based on ETag matching, + /// helping prevent unintended overwrites in concurrent scenarios. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let mut w = op + /// .writer_with("path/to/file") + /// .if_match("\"686897696a7c876b7e\"") + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ## `if_none_match` + /// + /// Sets If-None-Match header for this write request. + /// + /// Note: Certain services, like `s3`, support `if_not_exists` but not `if_none_match`. + /// Use `if_not_exists` if you only want to check whether a file exists. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_if_none_match`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, the write operation will only succeed if the target's ETag does not match the specified value + /// - The value should be a valid ETag string + /// - Common values include: + /// - A specific ETag value like `"686897696a7c876b7e"` + /// - `*` - Matches if the resource does not exist + /// - If not supported, the value will be ignored + /// + /// This operation provides conditional write functionality based on ETag non-matching, + /// useful for preventing overwriting existing resources or ensuring unique writes. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let mut w = op + /// .writer_with("path/to/file") + /// .if_none_match("\"686897696a7c876b7e\"") + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ## `if_not_exists` + /// + /// Sets the condition that write operation will succeed only if target does not exist. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_if_not_exists`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, the write operation will only succeed if the target path does not exist + /// - Equivalent to setting `if_none_match("*")` if available + /// - Will return error if target already exists + /// - If not supported, the value will be ignored + /// + /// This operation provides a way to ensure write operations only create new resources + /// without overwriting existing ones, useful for implementing "create if not exists" logic. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let mut w = op + /// .writer_with("path/to/file") + /// .if_not_exists(true) + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ## `user_metadata` + /// + /// Sets user metadata for this write request. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_user_metadata`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, the user metadata will be attached to the object during write + /// - Accepts key-value pairs where both key and value are strings + /// - Keys are case-insensitive in most services + /// - Services may have limitations for user metadata, for example: + /// - Key length is typically limited (e.g., 1024 bytes) + /// - Value length is typically limited (e.g., 4096 bytes) + /// - Total metadata size might be limited + /// - Some characters might be forbidden in keys + /// - If not supported, the metadata will be ignored + /// + /// User metadata provides a way to attach custom metadata to objects during write operations. + /// This metadata can be retrieved later when reading the object. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let mut w = op + /// .writer_with("path/to/file") + /// .user_metadata([ + /// ("language".to_string(), "rust".to_string()), + /// ("author".to_string(), "OpenDAL".to_string()), + /// ]) + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` + pub fn writer_with(&self, path: &str) -> FutureWriter>> { + let path = normalize_path(path); + + OperatorFuture::new( + self.inner().clone(), + path, + ( + OpWrite::default().merge_executor(self.default_executor.clone()), + OpWriter::default(), + ), + |inner, path, (args, options)| async move { + if !validate_path(&path, EntryMode::FILE) { + return Err( + Error::new(ErrorKind::IsADirectory, "write path is a directory") + .with_operation("Operator::writer") + .with_context("service", inner.info().scheme().into_static()) + .with_context("path", &path), + ); + } + + let context = WriteContext::new(inner, path, args, options); + let w = Writer::new(context).await?; + Ok(w) + }, + ) + } + + /// Write data with extra options. + /// + /// # Notes + /// + /// ## Streaming Write + /// + /// This method performs a single bulk write operation for all bytes. For finer-grained + /// memory control or lazy writing, consider using [`Operator::writer_with`] instead. + /// + /// ## Multipart Uploads + /// + /// OpenDAL handles multipart uploads through the [`Writer`] abstraction, managing all + /// the upload details automatically. You can customize the upload behavior by configuring + /// `chunk` size and `concurrent` operations via [`Operator::writer_with`]. /// /// # Usages /// @@ -976,7 +1692,7 @@ impl Operator { /// /// ### Example /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// # use futures::StreamExt; @@ -984,10 +1700,7 @@ impl Operator { /// use bytes::Bytes; /// /// # async fn test(op: Operator) -> Result<()> { - /// let mut w = op.writer_with("path/to/file").append(true).await?; - /// w.write(vec![0; 4096]).await?; - /// w.write(vec![1; 4096]).await?; - /// w.close().await?; + /// let _ = op.write_with("path/to/file", vec![0; 4096]).append(true).await?; /// # Ok(()) /// # } /// ``` @@ -1022,7 +1735,7 @@ impl Operator { /// /// ### Example /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// # use futures::StreamExt; @@ -1031,13 +1744,10 @@ impl Operator { /// /// # async fn test(op: Operator) -> Result<()> { /// // Set 8MiB chunk size - data will be sent in one API call at close - /// let mut w = op - /// .writer_with("path/to/file") + /// let _ = op + /// .write_with("path/to/file", vec![0; 4096]) /// .chunk(8 * 1024 * 1024) /// .await?; - /// w.write(vec![0; 4096]).await?; - /// w.write(vec![1; 4096]).await?; - /// w.close().await?; /// # Ok(()) /// # } /// ``` @@ -1071,7 +1781,7 @@ impl Operator { /// /// ## Example /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// # use futures::StreamExt; @@ -1079,17 +1789,8 @@ impl Operator { /// use bytes::Bytes; /// /// # async fn test(op: Operator) -> Result<()> { - /// // Enable concurrent writes with 8 parallel operations - /// let mut w = op.writer_with("path/to/file").concurrent(8).await?; - /// - /// // First write starts immediately - /// w.write(vec![0; 4096]).await?; - /// - /// // Second write runs concurrently with first - /// w.write(vec![1; 4096]).await?; - /// - /// // Ensures all writes complete successfully and in order - /// w.close().await?; + /// // Enable concurrent writes with 8 parallel operations at 128B chunk. + /// let _ = op.write_with("path/to/file", vec![0; 4096]).chunk(128).concurrent(8).await?; /// # Ok(()) /// # } /// ``` @@ -1100,10 +1801,6 @@ impl Operator { /// /// ### Capability /// - /// Sets `Cache-Control` header for this write request. - /// - /// ### Capability - /// /// Check [`Capability::write_with_cache_control`] before using this feature. /// /// ### Behavior @@ -1123,7 +1820,7 @@ impl Operator { /// /// ### Example /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// # use futures::StreamExt; @@ -1132,13 +1829,10 @@ impl Operator { /// /// # async fn test(op: Operator) -> Result<()> { /// // Cache content for 7 days (604800 seconds) - /// let mut w = op - /// .writer_with("path/to/file") + /// let _ = op + /// .write_with("path/to/file", vec![0; 4096]) /// .cache_control("max-age=604800") /// .await?; - /// w.write(vec![0; 4096]).await?; - /// w.write(vec![1; 4096]).await?; - /// w.close().await?; /// # Ok(()) /// # } /// ``` @@ -1166,20 +1860,17 @@ impl Operator { /// /// ## Example /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// use bytes::Bytes; /// /// # async fn test(op: Operator) -> Result<()> { /// // Set content type for plain text file - /// let mut w = op - /// .writer_with("path/to/file") + /// let _ = op + /// .write_with("path/to/file", vec![0; 4096]) /// .content_type("text/plain") /// .await?; - /// w.write(vec![0; 4096]).await?; - /// w.write(vec![1; 4096]).await?; - /// w.close().await?; /// # Ok(()) /// # } /// ``` @@ -1214,320 +1905,211 @@ impl Operator { /// use bytes::Bytes; /// /// # async fn test(op: Operator) -> Result<()> { - /// let mut w = op - /// .writer_with("path/to/file") + /// let _ = op + /// .write_with("path/to/file", vec![0; 4096]) /// .content_disposition("attachment; filename=\"filename.jpg\"") /// .await?; - /// w.write(vec![0; 4096]).await?; - /// w.write(vec![1; 4096]).await?; - /// w.close().await?; /// # Ok(()) /// # } /// ``` - pub fn writer_with(&self, path: &str) -> FutureWriter>> { - let path = normalize_path(path); - - OperatorFuture::new( - self.inner().clone(), - path, - ( - OpWrite::default().merge_executor(self.default_executor.clone()), - OpWriter::default(), - ), - |inner, path, (args, options)| async move { - if !validate_path(&path, EntryMode::FILE) { - return Err( - Error::new(ErrorKind::IsADirectory, "write path is a directory") - .with_operation("Operator::writer") - .with_context("service", inner.info().scheme().into_static()) - .with_context("path", &path), - ); - } - - let context = WriteContext::new(inner, path, args, options); - let w = Writer::new(context).await?; - Ok(w) - }, - ) - } - - /// Write data with extra options. - /// - /// # Notes - /// - /// ## Streaming Write - /// - /// This method performs a single bulk write operation for all bytes. For finer-grained - /// memory control or lazy writing, consider using [`Operator::writer_with`] instead. - /// - /// ## Multipart Uploads - /// - /// OpenDAL handles multipart uploads through the [`Writer`] abstraction, managing all - /// the upload details automatically. You can customize the upload behavior by configuring - /// `chunk` size and `concurrent` operations via [`Operator::writer_with`]. - /// - /// # Usages /// - /// ## `append` + /// ## `content_encoding` /// - /// Sets `append` mode for this write request. + /// Sets Content-Encoding header for this write request. /// /// ### Capability /// - /// Check [`Capability::write_with_append`] before using this feature. + /// Check [`Capability::write_with_content_encoding`] before using this feature. /// /// ### Behavior /// - /// - If append is true, data will be appended to the end of existing file - /// - If append is false (default), existing file will be overwritten + /// - If supported, sets Content-Encoding as system metadata on the target file + /// - The value should follow HTTP Content-Encoding header format + /// - Common values include: + /// - `gzip` - Content encoded using gzip compression + /// - `deflate` - Content encoded using deflate compression + /// - `br` - Content encoded using Brotli compression + /// - `identity` - No encoding applied (default value) + /// - If not supported, the value will be ignored /// - /// This operation allows appending data to existing files instead of overwriting them. + /// This operation allows specifying the encoding applied to the content being written. /// /// ### Example /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; /// use bytes::Bytes; /// /// # async fn test(op: Operator) -> Result<()> { - /// let bs = b"hello, world!".to_vec(); - /// let _ = op.write_with("path/to/file", bs).append(true).await?; + /// let _ = op + /// .write_with("path/to/file", vec![0; 4096]) + /// .content_encoding("gzip") + /// .await?; /// # Ok(()) /// # } /// ``` /// - /// ## `cache_control` + /// ## `if_match` /// - /// Sets `Cache-Control` header for this write request. + /// Sets If-Match header for this write request. /// /// ### Capability /// - /// Check [`Capability::write_with_cache_control`] before using this feature. + /// Check [`Capability::write_with_if_match`] before using this feature. /// /// ### Behavior /// - /// - If supported, sets Cache-Control as system metadata on the target file - /// - The value should follow HTTP Cache-Control header format + /// - If supported, the write operation will only succeed if the target's ETag matches the specified value + /// - The value should be a valid ETag string + /// - Common values include: + /// - A specific ETag value like `"686897696a7c876b7e"` + /// - `*` - Matches any existing resource /// - If not supported, the value will be ignored /// - /// This operation allows controlling caching behavior for the written content. - /// - /// ## Use Cases - /// - /// - Setting browser cache duration - /// - Configuring CDN behavior - /// - Optimizing content delivery - /// - Managing cache invalidation + /// This operation provides conditional write functionality based on ETag matching, + /// helping prevent unintended overwrites in concurrent scenarios. /// /// ### Example /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; /// use bytes::Bytes; /// /// # async fn test(op: Operator) -> Result<()> { - /// let bs = b"hello, world!".to_vec(); /// let _ = op - /// .write_with("path/to/file", bs) - /// .cache_control("max-age=604800") + /// .write_with("path/to/file", vec![0; 4096]) + /// .if_match("\"686897696a7c876b7e\"") /// .await?; /// # Ok(()) /// # } /// ``` /// - /// ## `content_type` + /// ## `if_none_match` /// - /// Sets Content-Type header for this write request. + /// Sets If-None-Match header for this write request. + /// + /// Note: Certain services, like `s3`, support `if_not_exists` but not `if_none_match`. + /// Use `if_not_exists` if you only want to check whether a file exists. /// /// ### Capability /// - /// Check [`Capability::write_with_content_type`] before using this feature. + /// Check [`Capability::write_with_if_none_match`] before using this feature. /// /// ### Behavior /// - /// - If supported, sets Content-Type as system metadata on the target file - /// - The value should follow MIME type format (e.g. "text/plain", "image/jpeg") + /// - If supported, the write operation will only succeed if the target's ETag does not match the specified value + /// - The value should be a valid ETag string + /// - Common values include: + /// - A specific ETag value like `"686897696a7c876b7e"` + /// - `*` - Matches if the resource does not exist /// - If not supported, the value will be ignored /// - /// This operation allows specifying the media type of the content being written. + /// This operation provides conditional write functionality based on ETag non-matching, + /// useful for preventing overwriting existing resources or ensuring unique writes. /// /// ### Example /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; /// use bytes::Bytes; /// /// # async fn test(op: Operator) -> Result<()> { - /// let bs = b"hello, world!".to_vec(); /// let _ = op - /// .write_with("path/to/file", bs) - /// .content_type("text/plain") + /// .write_with("path/to/file", vec![0; 4096]) + /// .if_none_match("\"686897696a7c876b7e\"") /// .await?; /// # Ok(()) /// # } /// ``` /// - /// ## `content_disposition` + /// ## `if_not_exists` /// - /// Sets Content-Disposition header for this write request. + /// Sets the condition that write operation will succeed only if target does not exist. /// /// ### Capability /// - /// Check [`Capability::write_with_content_disposition`] before using this feature. + /// Check [`Capability::write_with_if_not_exists`] before using this feature. /// /// ### Behavior /// - /// - If supported, sets Content-Disposition as system metadata on the target file - /// - The value should follow HTTP Content-Disposition header format - /// - Common values include: - /// - `inline` - Content displayed within browser - /// - `attachment` - Content downloaded as file - /// - `attachment; filename="example.jpg"` - Downloaded with specified filename + /// - If supported, the write operation will only succeed if the target path does not exist + /// - Equivalent to setting `if_none_match("*")` if available + /// - Will return error if target already exists /// - If not supported, the value will be ignored /// - /// This operation allows controlling how the content should be displayed or downloaded. + /// This operation provides a way to ensure write operations only create new resources + /// without overwriting existing ones, useful for implementing "create if not exists" logic. /// /// ### Example /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; /// use bytes::Bytes; /// /// # async fn test(op: Operator) -> Result<()> { - /// let bs = b"hello, world!".to_vec(); /// let _ = op - /// .write_with("path/to/file", bs) - /// .content_disposition("attachment; filename=\"filename.jpg\"") + /// .write_with("path/to/file", vec![0; 4096]) + /// .if_not_exists(true) /// .await?; /// # Ok(()) /// # } /// ``` /// - /// ## `content_encoding` + /// ## `user_metadata` /// - /// Sets Content-Encoding header for this write request. + /// Sets user metadata for this write request. /// /// ### Capability /// - /// Check [`Capability::write_with_content_encoding`] before using this feature. + /// Check [`Capability::write_with_user_metadata`] before using this feature. /// /// ### Behavior /// - /// - If supported, sets Content-Encoding as system metadata on the target file - /// - The value should follow HTTP Content-Encoding header format - /// - If not supported, the value will be ignored + /// - If supported, the user metadata will be attached to the object during write + /// - Accepts key-value pairs where both key and value are strings + /// - Keys are case-insensitive in most services + /// - Services may have limitations for user metadata, for example: + /// - Key length is typically limited (e.g., 1024 bytes) + /// - Value length is typically limited (e.g., 4096 bytes) + /// - Total metadata size might be limited + /// - Some characters might be forbidden in keys + /// - If not supported, the metadata will be ignored /// - /// This operation allows specifying the content encoding for the written content. + /// User metadata provides a way to attach custom metadata to objects during write operations. + /// This metadata can be retrieved later when reading the object. /// - /// ## Example + /// ### Example /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; /// use bytes::Bytes; + /// /// # async fn test(op: Operator) -> Result<()> { - /// let bs = b"hello, world!".to_vec(); /// let _ = op - /// .write_with("path/to/file", bs) - /// .content_encoding("br") + /// .write_with("path/to/file", vec![0; 4096]) + /// .user_metadata([ + /// ("language".to_string(), "rust".to_string()), + /// ("author".to_string(), "OpenDAL".to_string()), + /// ]) /// .await?; /// # Ok(()) /// # } /// ``` - /// - /// ## `if_none_match` - /// - /// Sets an `if none match` condition with specified ETag for this write request. - /// - /// ### Capability - /// - /// Check [`Capability::write_with_if_none_match`] before using this feature. - /// - /// ### Behavior - /// - /// - If the target file's ETag equals the specified one, returns [`ErrorKind::ConditionNotMatch`] - /// - If the target file's ETag differs from the specified one, proceeds with the write operation - /// - /// This operation will succeed when the target's ETag is different from the specified one, - /// providing a way for concurrency control. - /// - /// ### Example - /// - /// ```no_run - /// # use opendal::{ErrorKind, Result}; - /// use opendal::Operator; - /// # async fn test(op: Operator, etag: &str) -> Result<()> { - /// let bs = b"hello, world!".to_vec(); - /// let res = op.write_with("path/to/file", bs).if_none_match(etag).await; - /// assert!(res.is_err()); - /// assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); - /// # Ok(()) - /// # } - /// ``` - /// - /// ## `if_not_exists` - /// - /// Sets an `if not exists` condition for this write request. - /// - /// ### Capability - /// - /// Check [`Capability::write_with_if_not_exists`] before using this feature. - /// - /// ### Behavior - /// - /// - If the target file exists, returns [`ErrorKind::ConditionNotMatch`] - /// - If the target file doesn't exist, proceeds with the write operation - /// - /// This operation provides atomic file creation that is concurrency-safe. - /// Only one write operation will succeed while others will fail. - /// - /// ### Example - /// - /// ```no_run - /// # use opendal::{ErrorKind, Result}; - /// use opendal::Operator; - /// # async fn test(op: Operator, etag: &str) -> Result<()> { - /// let bs = b"hello, world!".to_vec(); - /// let res = op.write_with("path/to/file", bs).if_not_exists(true).await; - /// assert!(res.is_err()); - /// assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); - /// # Ok(()) - /// # } - /// ``` - /// - /// ## `if_match` - /// - /// Sets an `if match` condition with specified ETag for this write request. - /// - /// ### Capability - /// - /// Check [`Capability::write_with_if_match`] before using this feature. - /// - /// ### Behavior - /// - /// - If the target file's ETag matches the specified one, proceeds with the write operation - /// - If the target file's ETag does not match the specified one, returns [`ErrorKind::ConditionNotMatch`] - /// - /// This operation will succeed when the target's ETag matches the specified one, - /// providing a way for conditional writes. - /// - /// ### Example - /// - /// ```no_run - /// # use opendal::{ErrorKind, Result}; - /// use opendal::Operator; - /// # async fn test(op: Operator, incorrect_etag: &str) -> Result<()> { - /// let bs = b"hello, world!".to_vec(); - /// let res = op.write_with("path/to/file", bs).if_match(incorrect_etag).await; - /// assert!(res.is_err()); - /// assert_eq!(res.unwrap_err().kind(), ErrorKind::ConditionNotMatch); - /// # Ok(()) - /// # } - /// ``` pub fn write_with( &self, path: &str, @@ -1600,7 +2182,7 @@ impl Operator { /// /// If the version doesn't exist, OpenDAL will not return errors. /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// @@ -1849,7 +2431,7 @@ impl Operator { /// /// This example will list all entries under the dir `path/to/dir/`. /// - /// ```no_run + /// ``` /// # use anyhow::Result; /// use opendal::EntryMode; /// use opendal::Operator; @@ -1878,7 +2460,7 @@ impl Operator { /// `path/to/prefix/`, `path/to/prefix_1` and so on. If you do want to list a dir, please /// make sure the path is end with `/`. /// - /// ```no_run + /// ``` /// # use anyhow::Result; /// use opendal::EntryMode; /// use opendal::Operator; @@ -1925,7 +2507,7 @@ impl Operator { /// /// The following example will resume the list operation from the `breakpoint`. /// - /// ```no_run + /// ``` /// # use opendal::Result; /// use opendal::Operator; /// # async fn test(op: Operator) -> Result<()> { @@ -1944,7 +2526,7 @@ impl Operator { /// If `recursive` is set to `true`, we will list all entries recursively. If not, we'll only /// list the entries in the specified dir. /// - /// ```no_run + /// ``` /// # use opendal::Result; /// use opendal::Operator; /// # async fn test(op: Operator) -> Result<()> { @@ -1960,7 +2542,7 @@ impl Operator { /// if `version` is enabled, all file versions will be returned; otherwise, /// only the current files will be returned. /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// # async fn test(op: Operator) -> Result<()> { @@ -1975,7 +2557,7 @@ impl Operator { /// /// This example will list all entries under the dir `path/to/dir/` /// - /// ```no_run + /// ``` /// # use anyhow::Result; /// use opendal::EntryMode; /// use opendal::Operator; @@ -2000,7 +2582,7 @@ impl Operator { /// /// This example will list all entries starts with prefix `path/to/prefix` /// - /// ```no_run + /// ``` /// # use anyhow::Result; /// use opendal::EntryMode; /// use opendal::Operator; @@ -2050,7 +2632,7 @@ impl Operator { /// /// # Examples /// - /// ```no_run + /// ``` /// # use anyhow::Result; /// # use futures::io; /// use futures::TryStreamExt; @@ -2091,7 +2673,7 @@ impl Operator { /// /// The following example will resume the list operation from the `breakpoint`. /// - /// ```no_run + /// ``` /// # use opendal::Result; /// use opendal::Operator; /// # async fn test(op: Operator) -> Result<()> { @@ -2110,7 +2692,7 @@ impl Operator { /// If `recursive` is set to `true`, we will list all entries recursively. If not, we'll only /// list the entries in the specified dir. /// - /// ```no_run + /// ``` /// # use opendal::Result; /// use opendal::Operator; /// # async fn test(op: Operator) -> Result<()> { @@ -2126,7 +2708,7 @@ impl Operator { /// if `version` is enabled, all file versions will be returned; otherwise, /// only the current files will be returned. /// - /// ```no_run + /// ``` /// # use opendal::Result; /// # use opendal::Operator; /// # async fn test(op: Operator) -> Result<()> { @@ -2139,7 +2721,7 @@ impl Operator { /// /// ## List all files recursively /// - /// ```no_run + /// ``` /// # use anyhow::Result; /// use futures::TryStreamExt; /// use opendal::EntryMode; @@ -2178,7 +2760,7 @@ impl Operator { /// /// # Example /// - /// ```no_run + /// ``` /// use anyhow::Result; /// use futures::io; /// use opendal::Operator; @@ -2207,7 +2789,7 @@ impl Operator { /// /// # Example /// - /// ```no_run + /// ``` /// use anyhow::Result; /// use futures::io; /// use opendal::Operator; @@ -2249,7 +2831,7 @@ impl Operator { /// /// # Example /// - /// ```no_run + /// ``` /// use anyhow::Result; /// use futures::io; /// use opendal::Operator; @@ -2287,7 +2869,7 @@ impl Operator { /// /// Override the [`content-disposition`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition) header returned by storage services. /// - /// ```no_run + /// ``` /// use std::time::Duration; /// /// use anyhow::Result; @@ -2306,7 +2888,7 @@ impl Operator { /// /// Override the [`cache-control`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control) header returned by storage services. /// - /// ```no_run + /// ``` /// use std::time::Duration; /// /// use anyhow::Result; @@ -2325,7 +2907,7 @@ impl Operator { /// /// Override the [`content-type`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type) header returned by storage services. /// - /// ```no_run + /// ``` /// use std::time::Duration; /// /// use anyhow::Result; @@ -2370,7 +2952,7 @@ impl Operator { /// /// # Example /// - /// ```no_run + /// ``` /// use std::time::Duration; /// /// use anyhow::Result; @@ -2405,7 +2987,7 @@ impl Operator { /// /// Set the [`content-type`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Type) header returned by storage services. /// - /// ```no_run + /// ``` /// use std::time::Duration; /// /// use anyhow::Result; @@ -2429,7 +3011,7 @@ impl Operator { /// /// Set the [`content-disposition`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Content-Disposition) header returned by storage services. /// - /// ```no_run + /// ``` /// use std::time::Duration; /// /// use anyhow::Result; @@ -2453,7 +3035,7 @@ impl Operator { /// /// Set the [`cache-control`](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control) header returned by storage services. /// - /// ```no_run + /// ``` /// use std::time::Duration; /// /// use anyhow::Result; diff --git a/core/src/types/operator/operator_futures.rs b/core/src/types/operator/operator_futures.rs index 5d4ee97a372c..bbfa02657db7 100644 --- a/core/src/types/operator/operator_futures.rs +++ b/core/src/types/operator/operator_futures.rs @@ -202,49 +202,170 @@ impl>> FuturePresignWrite { pub type FutureRead = OperatorFuture<(OpRead, OpReader), Buffer, F>; impl>> FutureRead { - /// Set the If-Match for this operation. - pub fn if_match(self, v: &str) -> Self { - self.map(|(args, op_reader)| (args.with_if_match(v), op_reader)) + /// Set the executor for this operation. + pub fn executor(self, executor: Executor) -> Self { + self.map(|(args, op_reader)| (args.with_executor(executor), op_reader)) } - /// Set the If-None-Match for this operation. - pub fn if_none_match(self, v: &str) -> Self { - self.map(|(args, op_reader)| (args.with_if_none_match(v), op_reader)) + /// Set `range` for this `read` request. + /// + /// If we have a file with size `n`. + /// + /// - `..` means read bytes in range `[0, n)` of file. + /// - `0..1024` and `..1024` means read bytes in range `[0, 1024)` of file + /// - `1024..` means read bytes in range `[1024, n)` of file + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::TryStreamExt; + /// # async fn test(op: Operator) -> Result<()> { + /// let bs = op.read_with("path/to/file").range(0..1024).await?; + /// # Ok(()) + /// # } + /// ``` + pub fn range(self, range: impl RangeBounds) -> Self { + self.map(|(args, op_reader)| (args.with_range(range.into()), op_reader)) } - /// Set the If-Modified-Since for this operation. - pub fn if_modified_since(self, v: DateTime) -> Self { - self.map(|(args, op_reader)| (args.with_if_modified_since(v), op_reader)) + /// Set `concurrent` for the reader. + /// + /// OpenDAL by default to write file without concurrent. This is not efficient for cases when users + /// read large chunks of data. By setting `concurrent`, opendal will read files concurrently + /// on support storage services. + /// + /// By setting `concurrent`, opendal will fetch chunks concurrently with + /// the given chunk size. + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use opendal::Scheme; + /// # async fn test(op: Operator) -> Result<()> { + /// let r = op.read_with("path/to/file").concurrent(8).await?; + /// # Ok(()) + /// # } + /// ``` + pub fn concurrent(self, concurrent: usize) -> Self { + self.map(|(args, op_reader)| (args, op_reader.with_concurrent(concurrent))) } - /// Set the If-Unmodified-Since for this operation. - pub fn if_unmodified_since(self, v: DateTime) -> Self { - self.map(|(args, op_reader)| (args.with_if_unmodified_since(v), op_reader)) + /// OpenDAL will use services' preferred chunk size by default. Users can set chunk based on their own needs. + /// + /// This following example will make opendal read data in 4MiB chunks: + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use opendal::Scheme; + /// # async fn test(op: Operator) -> Result<()> { + /// let r = op.read_with("path/to/file").chunk(4 * 1024 * 1024).await?; + /// # Ok(()) + /// # } + /// ``` + pub fn chunk(self, chunk_size: usize) -> Self { + self.map(|(args, op_reader)| (args, op_reader.with_chunk(chunk_size))) } - /// Set the version for this operation. + /// Set `version` for this `read` request. + /// + /// This feature can be used to retrieve the data of a specified version of the given path. + /// + /// If the version doesn't exist, an error with kind [`ErrorKind::NotFound`] will be returned. + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// + /// # async fn test(op: Operator, version: &str) -> Result<()> { + /// let mut bs = op.read_with("path/to/file").version(version).await?; + /// # Ok(()) + /// # } + /// ``` pub fn version(self, v: &str) -> Self { self.map(|(args, op_reader)| (args.with_version(v), op_reader)) } - /// Set the executor for this operation. - pub fn executor(self, executor: Executor) -> Self { - self.map(|(args, op_reader)| (args.with_executor(executor), op_reader)) + /// Set `if_match` for this `read` request. + /// + /// This feature can be used to check if the file's `ETag` matches the given `ETag`. + /// + /// If file exists and it's etag doesn't match, an error with kind [`ErrorKind::ConditionNotMatch`] + /// will be returned. + /// + /// ``` + /// # use opendal::Result; + /// use opendal::Operator; + /// # async fn test(op: Operator, etag: &str) -> Result<()> { + /// let mut metadata = op.read_with("path/to/file").if_match(etag).await?; + /// # Ok(()) + /// # } + /// ``` + pub fn if_match(self, v: &str) -> Self { + self.map(|(args, op_reader)| (args.with_if_match(v), op_reader)) } - /// Set the range header for this operation. - pub fn range(self, range: impl RangeBounds) -> Self { - self.map(|(args, op_reader)| (args.with_range(range.into()), op_reader)) + /// Set `if_none_match` for this `read` request. + /// + /// This feature can be used to check if the file's `ETag` doesn't match the given `ETag`. + /// + /// If file exists and it's etag match, an error with kind [`ErrorKind::ConditionNotMatch`] + /// will be returned. + /// + /// ``` + /// # use opendal::Result; + /// use opendal::Operator; + /// # async fn test(op: Operator, etag: &str) -> Result<()> { + /// let mut metadata = op.read_with("path/to/file").if_none_match(etag).await?; + /// # Ok(()) + /// # } + /// ``` + pub fn if_none_match(self, v: &str) -> Self { + self.map(|(args, op_reader)| (args.with_if_none_match(v), op_reader)) } - /// Set the concurrent read task amount. - pub fn concurrent(self, concurrent: usize) -> Self { - self.map(|(args, op_reader)| (args, op_reader.with_concurrent(concurrent))) + /// ## `if_modified_since` + /// + /// Set `if_modified_since` for this `read` request. + /// + /// This feature can be used to check if the file has been modified since the given timestamp. + /// + /// If file exists and it hasn't been modified since the specified time, an error with kind + /// [`ErrorKind::ConditionNotMatch`] will be returned. + /// + /// ``` + /// # use opendal::Result; + /// use opendal::Operator; + /// use chrono::DateTime; + /// use chrono::Utc; + /// # async fn test(op: Operator, time: DateTime) -> Result<()> { + /// let mut metadata = op.read_with("path/to/file").if_modified_since(time).await?; + /// # Ok(()) + /// # } + /// ``` + pub fn if_modified_since(self, v: DateTime) -> Self { + self.map(|(args, op_reader)| (args.with_if_modified_since(v), op_reader)) } - /// Set the chunk size for this operation. - pub fn chunk(self, chunk_size: usize) -> Self { - self.map(|(args, op_reader)| (args, op_reader.with_chunk(chunk_size))) + /// Set `if_unmodified_since` for this `read` request. + /// + /// This feature can be used to check if the file hasn't been modified since the given timestamp. + /// + /// If file exists and it has been modified since the specified time, an error with kind + /// [`ErrorKind::ConditionNotMatch`] will be returned. + /// + /// ``` + /// # use opendal::Result; + /// use opendal::Operator; + /// use chrono::DateTime; + /// use chrono::Utc; + /// # async fn test(op: Operator, time: DateTime) -> Result<()> { + /// let mut metadata = op.read_with("path/to/file").if_unmodified_since(time).await?; + /// # Ok(()) + /// # } + /// ``` + pub fn if_unmodified_since(self, v: DateTime) -> Self { + self.map(|(args, op_reader)| (args.with_if_unmodified_since(v), op_reader)) } } @@ -258,45 +379,177 @@ impl>> FutureRead { pub type FutureReader = OperatorFuture<(OpRead, OpReader), Reader, F>; impl>> FutureReader { - /// Set the If-Match for this operation. - pub fn if_match(self, etag: &str) -> Self { - self.map(|(op_read, op_reader)| (op_read.with_if_match(etag), op_reader)) - } - - /// Set the If-None-Match for this operation. - pub fn if_none_match(self, etag: &str) -> Self { - self.map(|(op_read, op_reader)| (op_read.with_if_none_match(etag), op_reader)) - } - - /// Set the If-Modified-Since for this operation. - pub fn if_modified_since(self, v: DateTime) -> Self { - self.map(|(op_read, op_reader)| (op_read.with_if_modified_since(v), op_reader)) - } - - /// Set the If-Unmodified-Since for this operation. - pub fn if_unmodified_since(self, v: DateTime) -> Self { - self.map(|(op_read, op_reader)| (op_read.with_if_unmodified_since(v), op_reader)) - } - - /// Set the version for this operation. + /// Set `version` for this `reader`. + /// + /// This feature can be used to retrieve the data of a specified version of the given path. + /// + /// If the version doesn't exist, an error with kind [`ErrorKind::NotFound`] will be returned. + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// + /// # async fn test(op: Operator, version: &str) -> Result<()> { + /// let mut r = op.reader_with("path/to/file").version(version).await?; + /// # Ok(()) + /// # } + /// ``` pub fn version(self, v: &str) -> Self { self.map(|(op_read, op_reader)| (op_read.with_version(v), op_reader)) } - /// Set the concurrent read task amount. + /// Set `concurrent` for the reader. + /// + /// OpenDAL by default to write file without concurrent. This is not efficient for cases when users + /// read large chunks of data. By setting `concurrent`, opendal will reading files concurrently + /// on support storage services. + /// + /// By setting `concurrent``, opendal will fetch chunks concurrently with + /// the give chunk size. + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use opendal::Scheme; + /// # async fn test(op: Operator) -> Result<()> { + /// let r = op.reader_with("path/to/file").concurrent(8).await?; + /// # Ok(()) + /// # } + /// ``` pub fn concurrent(self, concurrent: usize) -> Self { self.map(|(op_read, op_reader)| (op_read, op_reader.with_concurrent(concurrent))) } - /// Set the chunk size for this reader. + /// OpenDAL will use services' preferred chunk size by default. Users can set chunk based on their own needs. + /// + /// This following example will make opendal read data in 4MiB chunks: + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use opendal::Scheme; + /// # async fn test(op: Operator) -> Result<()> { + /// let r = op + /// .reader_with("path/to/file") + /// .chunk(4 * 1024 * 1024) + /// .await?; + /// # Ok(()) + /// # } + /// ``` pub fn chunk(self, chunk_size: usize) -> Self { self.map(|(op_read, op_reader)| (op_read, op_reader.with_chunk(chunk_size))) } - /// Set the gap size for this reader. + /// Controls the optimization strategy for range reads in [`Reader::fetch`]. + /// + /// When performing range reads, if the gap between two requested ranges is smaller than + /// the configured `gap` size, OpenDAL will merge these ranges into a single read request + /// and discard the unrequested data in between. This helps reduce the number of API calls + /// to remote storage services. + /// + /// This optimization is particularly useful when performing multiple small range reads + /// that are close to each other, as it reduces the overhead of multiple network requests + /// at the cost of transferring some additional data. + /// + /// In this example, if two requested ranges are separated by less than 1MiB, + /// they will be merged into a single read request: + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use opendal::Scheme; + /// # async fn test(op: Operator) -> Result<()> { + /// let r = op + /// .reader_with("path/to/file") + /// .chunk(4 * 1024 * 1024) + /// .gap(1024 * 1024) // 1MiB gap + /// .await?; + /// # Ok(()) + /// # } + /// ``` pub fn gap(self, gap_size: usize) -> Self { self.map(|(op_read, op_reader)| (op_read, op_reader.with_gap(gap_size))) } + + /// Set `if-match` for this `read` request. + /// + /// This feature can be used to check if the file's `ETag` matches the given `ETag`. + /// + /// If file exists and it's etag doesn't match, an error with kind [`ErrorKind::ConditionNotMatch`] + /// will be returned. + /// + /// ``` + /// # use opendal::Result; + /// use opendal::Operator; + /// # async fn test(op: Operator, etag: &str) -> Result<()> { + /// let mut r = op.reader_with("path/to/file").if_match(etag).await?; + /// # Ok(()) + /// # } + /// ``` + pub fn if_match(self, etag: &str) -> Self { + self.map(|(op_read, op_reader)| (op_read.with_if_match(etag), op_reader)) + } + + /// Set `if-none-match` for this `read` request. + /// + /// This feature can be used to check if the file's `ETag` doesn't match the given `ETag`. + /// + /// If file exists and it's etag match, an error with kind [`ErrorKind::ConditionNotMatch`] + /// will be returned. + /// + /// ``` + /// # use opendal::Result; + /// use opendal::Operator; + /// # async fn test(op: Operator, etag: &str) -> Result<()> { + /// let mut r = op.reader_with("path/to/file").if_none_match(etag).await?; + /// # Ok(()) + /// # } + /// ``` + pub fn if_none_match(self, etag: &str) -> Self { + self.map(|(op_read, op_reader)| (op_read.with_if_none_match(etag), op_reader)) + } + + /// Set `if-modified-since` for this `read` request. + /// + /// This feature can be used to check if the file has been modified since the given timestamp. + /// + /// If file exists and it hasn't been modified since the specified time, an error with kind + /// [`ErrorKind::ConditionNotMatch`] will be returned. + /// + /// ``` + /// # use opendal::Result; + /// use opendal::Operator; + /// use chrono::DateTime; + /// use chrono::Utc; + /// # async fn test(op: Operator, time: DateTime) -> Result<()> { + /// let mut r = op.reader_with("path/to/file").if_modified_since(time).await?; + /// # Ok(()) + /// # } + /// ``` + pub fn if_modified_since(self, v: DateTime) -> Self { + self.map(|(op_read, op_reader)| (op_read.with_if_modified_since(v), op_reader)) + } + + /// Set `if-unmodified-since` for this `read` request. + /// + /// This feature can be used to check if the file hasn't been modified since the given timestamp. + /// + /// If file exists and it has been modified since the specified time, an error with kind + /// [`ErrorKind::ConditionNotMatch`] will be returned. + /// + /// ``` + /// # use opendal::Result; + /// use opendal::Operator; + /// use chrono::DateTime; + /// use chrono::Utc; + /// # async fn test(op: Operator, time: DateTime) -> Result<()> { + /// let mut r = op.reader_with("path/to/file").if_unmodified_since(time).await?; + /// # Ok(()) + /// # } + /// ``` + pub fn if_unmodified_since(self, v: DateTime) -> Self { + self.map(|(op_read, op_reader)| (op_read.with_if_unmodified_since(v), op_reader)) + } } /// Future that generated by [`Operator::write_with`]. @@ -305,80 +558,458 @@ impl>> FutureReader { pub type FutureWrite = OperatorFuture<(OpWrite, OpWriter, Buffer), (), F>; impl>> FutureWrite { - /// Set the append mode of op. + /// Set the executor for this operation. + pub fn executor(self, executor: Executor) -> Self { + self.map(|(args, options, bs)| (args.with_executor(executor), options, bs)) + } + + /// Sets append mode for this write request. + /// + /// ### Capability + /// + /// Check [`Capability::write_can_append`] before using this feature. + /// + /// ### Behavior + /// + /// - By default, write operations overwrite existing files + /// - When append is set to true: + /// - New data will be appended to the end of existing file + /// - If file doesn't exist, it will be created + /// - If not supported, will return an error /// - /// If the append mode is set, the data will be appended to the end of the file. + /// This operation allows adding data to existing files instead of overwriting them. /// - /// # Notes + /// ### Example /// - /// Service could return `Unsupported` if the underlying storage does not support append. + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let _ = op.write_with("path/to/file", vec![0; 4096]).append(true).await?; + /// # Ok(()) + /// # } + /// ``` pub fn append(self, v: bool) -> Self { self.map(|(args, options, bs)| (args.with_append(v), options, bs)) } - /// Set the buffer size of op. + /// Sets chunk size for buffered writes. + /// + /// ### Capability + /// + /// Check [`Capability::write_multi_min_size`] and [`Capability::write_multi_max_size`] for size limits. + /// + /// ### Behavior + /// + /// - By default, OpenDAL sets optimal chunk size based on service capabilities + /// - When chunk size is set: + /// - Data will be buffered until reaching chunk size + /// - One API call will be made per chunk + /// - Last chunk may be smaller than chunk size + /// - Important considerations: + /// - Some services require minimum chunk sizes (e.g. S3's EntityTooSmall error) + /// - Smaller chunks increase API calls and costs + /// - Larger chunks increase memory usage, but improve performance and reduce costs /// - /// If buffer size is set, the data will be buffered by the underlying writer. + /// ### Performance Impact /// - /// ## NOTE + /// Setting appropriate chunk size can: + /// - Reduce number of API calls + /// - Improve overall throughput + /// - Lower operation costs + /// - Better utilize network bandwidth /// - /// Service could have their own minimum buffer size while perform write operations like - /// multipart uploads. So the buffer size may be larger than the given buffer size. + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// // Set 8MiB chunk size - data will be sent in one API call at close + /// let _ = op + /// .write_with("path/to/file", vec![0; 4096]) + /// .chunk(8 * 1024 * 1024) + /// .await?; + /// # Ok(()) + /// # } + /// ``` pub fn chunk(self, v: usize) -> Self { self.map(|(args, options, bs)| (args, options.with_chunk(v), bs)) } - /// Set the maximum concurrent write task amount. + /// Sets concurrent write operations for this writer. + /// + /// ## Behavior + /// + /// - By default, OpenDAL writes files sequentially + /// - When concurrent is set: + /// - Multiple write operations can execute in parallel + /// - Write operations return immediately without waiting if tasks space are available + /// - Close operation ensures all writes complete in order + /// - Memory usage increases with concurrency level + /// - If not supported, falls back to sequential writes + /// + /// This feature significantly improves performance when: + /// - Writing large files + /// - Network latency is high + /// - Storage service supports concurrent uploads like multipart uploads + /// + /// ## Performance Impact + /// + /// Setting appropriate concurrency can: + /// - Increase write throughput + /// - Reduce total write time + /// - Better utilize available bandwidth + /// - Trade memory for performance + /// + /// ## Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// // Enable concurrent writes with 8 parallel operations at 128B chunk. + /// let _ = op.write_with("path/to/file", vec![0; 4096]).chunk(128).concurrent(8).await?; + /// # Ok(()) + /// # } + /// ``` pub fn concurrent(self, v: usize) -> Self { self.map(|(args, options, bs)| (args.with_concurrent(v), options, bs)) } - /// Set the content type of option + /// Sets Cache-Control header for this write operation. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_cache_control`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, sets Cache-Control as system metadata on the target file + /// - The value should follow HTTP Cache-Control header format + /// - If not supported, the value will be ignored + /// + /// This operation allows controlling caching behavior for the written content. + /// + /// ### Use Cases + /// + /// - Setting browser cache duration + /// - Configuring CDN behavior + /// - Optimizing content delivery + /// - Managing cache invalidation + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// // Cache content for 7 days (604800 seconds) + /// let _ = op + /// .write_with("path/to/file", vec![0; 4096]) + /// .cache_control("max-age=604800") + /// .await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ### References + /// + /// - [MDN Cache-Control](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control) + /// - [RFC 7234 Section 5.2](https://tools.ietf.org/html/rfc7234#section-5.2) pub fn cache_control(self, v: &str) -> Self { self.map(|(args, options, bs)| (args.with_cache_control(v), options, bs)) } - /// Set the content type of option + /// Sets `Content-Type` header for this write operation. + /// + /// ## Capability + /// + /// Check [`Capability::write_with_content_type`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, sets Content-Type as system metadata on the target file + /// - The value should follow MIME type format (e.g. "text/plain", "image/jpeg") + /// - If not supported, the value will be ignored + /// + /// This operation allows specifying the media type of the content being written. + /// + /// ## Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// // Set content type for plain text file + /// let _ = op + /// .write_with("path/to/file", vec![0; 4096]) + /// .content_type("text/plain") + /// .await?; + /// # Ok(()) + /// # } + /// ``` pub fn content_type(self, v: &str) -> Self { self.map(|(args, options, bs)| (args.with_content_type(v), options, bs)) } - /// Set the content disposition of option + /// ## `content_disposition` + /// + /// Sets Content-Disposition header for this write request. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_content_disposition`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, sets Content-Disposition as system metadata on the target file + /// - The value should follow HTTP Content-Disposition header format + /// - Common values include: + /// - `inline` - Content displayed within browser + /// - `attachment` - Content downloaded as file + /// - `attachment; filename="example.jpg"` - Downloaded with specified filename + /// - If not supported, the value will be ignored + /// + /// This operation allows controlling how the content should be displayed or downloaded. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let _ = op + /// .write_with("path/to/file", vec![0; 4096]) + /// .content_disposition("attachment; filename=\"filename.jpg\"") + /// .await?; + /// # Ok(()) + /// # } + /// ``` pub fn content_disposition(self, v: &str) -> Self { self.map(|(args, options, bs)| (args.with_content_disposition(v), options, bs)) } - /// Set the content encoding of option + /// Sets Content-Encoding header for this write request. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_content_encoding`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, sets Content-Encoding as system metadata on the target file + /// - The value should follow HTTP Content-Encoding header format + /// - Common values include: + /// - `gzip` - Content encoded using gzip compression + /// - `deflate` - Content encoded using deflate compression + /// - `br` - Content encoded using Brotli compression + /// - `identity` - No encoding applied (default value) + /// - If not supported, the value will be ignored + /// + /// This operation allows specifying the encoding applied to the content being written. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let _ = op + /// .write_with("path/to/file", vec![0; 4096]) + /// .content_encoding("gzip") + /// .await?; + /// # Ok(()) + /// # } + /// ``` pub fn content_encoding(self, v: &str) -> Self { self.map(|(args, options, bs)| (args.with_content_encoding(v), options, bs)) } - /// Set the executor for this operation. - pub fn executor(self, executor: Executor) -> Self { - self.map(|(args, options, bs)| (args.with_executor(executor), options, bs)) - } - - /// Set the If-Match for this operation. + /// Sets If-Match header for this write request. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_if_match`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, the write operation will only succeed if the target's ETag matches the specified value + /// - The value should be a valid ETag string + /// - Common values include: + /// - A specific ETag value like `"686897696a7c876b7e"` + /// - `*` - Matches any existing resource + /// - If not supported, the value will be ignored + /// + /// This operation provides conditional write functionality based on ETag matching, + /// helping prevent unintended overwrites in concurrent scenarios. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let _ = op + /// .write_with("path/to/file", vec![0; 4096]) + /// .if_match("\"686897696a7c876b7e\"") + /// .await?; + /// # Ok(()) + /// # } + /// ``` pub fn if_match(self, s: &str) -> Self { self.map(|(args, options, bs)| (args.with_if_match(s), options, bs)) } - /// Set the If-None-Match for this operation. + /// Sets If-None-Match header for this write request. + /// + /// Note: Certain services, like `s3`, support `if_not_exists` but not `if_none_match`. + /// Use `if_not_exists` if you only want to check whether a file exists. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_if_none_match`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, the write operation will only succeed if the target's ETag does not match the specified value + /// - The value should be a valid ETag string + /// - Common values include: + /// - A specific ETag value like `"686897696a7c876b7e"` + /// - `*` - Matches if the resource does not exist + /// - If not supported, the value will be ignored + /// + /// This operation provides conditional write functionality based on ETag non-matching, + /// useful for preventing overwriting existing resources or ensuring unique writes. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let _ = op + /// .write_with("path/to/file", vec![0; 4096]) + /// .if_none_match("\"686897696a7c876b7e\"") + /// .await?; + /// # Ok(()) + /// # } + /// ``` pub fn if_none_match(self, s: &str) -> Self { self.map(|(args, options, bs)| (args.with_if_none_match(s), options, bs)) } - /// Set the If-Not-Exist for this operation. + /// Sets the condition that write operation will succeed only if target does not exist. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_if_not_exists`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, the write operation will only succeed if the target path does not exist + /// - Will return error if target already exists + /// - If not supported, the value will be ignored + /// + /// This operation provides a way to ensure write operations only create new resources + /// without overwriting existing ones, useful for implementing "create if not exists" logic. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let _ = op + /// .write_with("path/to/file", vec![0; 4096]) + /// .if_not_exists(true) + /// .await?; + /// # Ok(()) + /// # } + /// ``` pub fn if_not_exists(self, b: bool) -> Self { self.map(|(args, options, bs)| (args.with_if_not_exists(b), options, bs)) } - /// Set the user defined metadata of the op + /// Sets user metadata for this write request. + /// + /// ### Capability /// - /// ## Notes + /// Check [`Capability::write_with_user_metadata`] before using this feature. /// - /// we don't need to include the user defined metadata prefix in the key - /// every service will handle it internally + /// ### Behavior + /// + /// - If supported, the user metadata will be attached to the object during write + /// - Accepts key-value pairs where both key and value are strings + /// - Keys are case-insensitive in most services + /// - Services may have limitations for user metadata, for example: + /// - Key length is typically limited (e.g., 1024 bytes) + /// - Value length is typically limited (e.g., 4096 bytes) + /// - Total metadata size might be limited + /// - Some characters might be forbidden in keys + /// - If not supported, the metadata will be ignored + /// + /// User metadata provides a way to attach custom metadata to objects during write operations. + /// This metadata can be retrieved later when reading the object. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let _ = op + /// .write_with("path/to/file", vec![0; 4096]) + /// .user_metadata([ + /// ("language".to_string(), "rust".to_string()), + /// ("author".to_string(), "OpenDAL".to_string()), + /// ]) + /// .await?; + /// # Ok(()) + /// # } + /// ``` pub fn user_metadata(self, data: impl IntoIterator) -> Self { self.map(|(args, options, bs)| { ( @@ -396,67 +1027,496 @@ impl>> FutureWrite { pub type FutureWriter = OperatorFuture<(OpWrite, OpWriter), Writer, F>; impl>> FutureWriter { - /// Set the append mode of op. + /// Set the executor for this operation. + pub fn executor(self, executor: Executor) -> Self { + self.map(|(args, options)| (args.with_executor(executor), options)) + } + + /// Sets append mode for this write request. /// - /// If the append mode is set, the data will be appended to the end of the file. + /// ### Capability /// - /// ## Notes + /// Check [`Capability::write_can_append`] before using this feature. /// - /// Service could return `Unsupported` if the underlying storage does not support append. + /// ### Behavior + /// + /// - By default, write operations overwrite existing files + /// - When append is set to true: + /// - New data will be appended to the end of existing file + /// - If file doesn't exist, it will be created + /// - If not supported, will return an error + /// + /// This operation allows adding data to existing files instead of overwriting them. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let mut w = op.writer_with("path/to/file").append(true).await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` pub fn append(self, v: bool) -> Self { self.map(|(args, options)| (args.with_append(v), options)) } - /// Set the chunk size of op. + /// Sets chunk size for buffered writes. /// - /// If chunk size is set, the data will be chunked by the underlying writer. + /// ### Capability /// - /// ## NOTE + /// Check [`Capability::write_multi_min_size`] and [`Capability::write_multi_max_size`] for size limits. /// - /// Service could have their own limitation for chunk size. It's possible that chunk size - /// is not equal to the given chunk size. + /// ### Behavior /// - /// For example: + /// - By default, OpenDAL sets optimal chunk size based on service capabilities + /// - When chunk size is set: + /// - Data will be buffered until reaching chunk size + /// - One API call will be made per chunk + /// - Last chunk may be smaller than chunk size + /// - Important considerations: + /// - Some services require minimum chunk sizes (e.g. S3's EntityTooSmall error) + /// - Smaller chunks increase API calls and costs + /// - Larger chunks increase memory usage, but improve performance and reduce costs /// - /// - AWS S3 requires the part size to be in [5MiB, 5GiB]. - /// - GCS requires the part size to be aligned with 256 KiB. + /// ### Performance Impact /// - /// The services will alter the chunk size to meet their requirements. + /// Setting appropriate chunk size can: + /// - Reduce number of API calls + /// - Improve overall throughput + /// - Lower operation costs + /// - Better utilize network bandwidth + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// // Set 8MiB chunk size - data will be sent in one API call at close + /// let mut w = op + /// .writer_with("path/to/file") + /// .chunk(8 * 1024 * 1024) + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` pub fn chunk(self, v: usize) -> Self { self.map(|(args, options)| (args, options.with_chunk(v))) } - /// Set the maximum concurrent write task amount. + /// Sets concurrent write operations for this writer. + /// + /// ## Behavior + /// + /// - By default, OpenDAL writes files sequentially + /// - When concurrent is set: + /// - Multiple write operations can execute in parallel + /// - Write operations return immediately without waiting if tasks space are available + /// - Close operation ensures all writes complete in order + /// - Memory usage increases with concurrency level + /// - If not supported, falls back to sequential writes + /// + /// This feature significantly improves performance when: + /// - Writing large files + /// - Network latency is high + /// - Storage service supports concurrent uploads like multipart uploads + /// + /// ## Performance Impact + /// + /// Setting appropriate concurrency can: + /// - Increase write throughput + /// - Reduce total write time + /// - Better utilize available bandwidth + /// - Trade memory for performance + /// + /// ## Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// // Enable concurrent writes with 8 parallel operations + /// let mut w = op.writer_with("path/to/file").concurrent(8).await?; + /// + /// // First write starts immediately + /// w.write(vec![0; 4096]).await?; + /// + /// // Second write runs concurrently with first + /// w.write(vec![1; 4096]).await?; + /// + /// // Ensures all writes complete successfully and in order + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` pub fn concurrent(self, v: usize) -> Self { self.map(|(args, options)| (args.with_concurrent(v), options)) } - /// Set the content type of option + /// Sets Cache-Control header for this write operation. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_cache_control`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, sets Cache-Control as system metadata on the target file + /// - The value should follow HTTP Cache-Control header format + /// - If not supported, the value will be ignored + /// + /// This operation allows controlling caching behavior for the written content. + /// + /// ### Use Cases + /// + /// - Setting browser cache duration + /// - Configuring CDN behavior + /// - Optimizing content delivery + /// - Managing cache invalidation + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// // Cache content for 7 days (604800 seconds) + /// let mut w = op + /// .writer_with("path/to/file") + /// .cache_control("max-age=604800") + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` + /// + /// ### References + /// + /// - [MDN Cache-Control](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control) + /// - [RFC 7234 Section 5.2](https://tools.ietf.org/html/rfc7234#section-5.2) pub fn cache_control(self, v: &str) -> Self { self.map(|(args, options)| (args.with_cache_control(v), options)) } - /// Set the content type of option + /// Sets `Content-Type` header for this write operation. + /// + /// ## Capability + /// + /// Check [`Capability::write_with_content_type`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, sets Content-Type as system metadata on the target file + /// - The value should follow MIME type format (e.g. "text/plain", "image/jpeg") + /// - If not supported, the value will be ignored + /// + /// This operation allows specifying the media type of the content being written. + /// + /// ## Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// // Set content type for plain text file + /// let mut w = op + /// .writer_with("path/to/file") + /// .content_type("text/plain") + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` pub fn content_type(self, v: &str) -> Self { self.map(|(args, options)| (args.with_content_type(v), options)) } - /// Set the content disposition of option + /// ## `content_disposition` + /// + /// Sets Content-Disposition header for this write request. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_content_disposition`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, sets Content-Disposition as system metadata on the target file + /// - The value should follow HTTP Content-Disposition header format + /// - Common values include: + /// - `inline` - Content displayed within browser + /// - `attachment` - Content downloaded as file + /// - `attachment; filename="example.jpg"` - Downloaded with specified filename + /// - If not supported, the value will be ignored + /// + /// This operation allows controlling how the content should be displayed or downloaded. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let mut w = op + /// .writer_with("path/to/file") + /// .content_disposition("attachment; filename=\"filename.jpg\"") + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` pub fn content_disposition(self, v: &str) -> Self { self.map(|(args, options)| (args.with_content_disposition(v), options)) } - /// Set the executor for this operation. - pub fn executor(self, executor: Executor) -> Self { - self.map(|(args, options)| (args.with_executor(executor), options)) + /// Sets Content-Encoding header for this write request. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_content_encoding`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, sets Content-Encoding as system metadata on the target file + /// - The value should follow HTTP Content-Encoding header format + /// - Common values include: + /// - `gzip` - Content encoded using gzip compression + /// - `deflate` - Content encoded using deflate compression + /// - `br` - Content encoded using Brotli compression + /// - `identity` - No encoding applied (default value) + /// - If not supported, the value will be ignored + /// + /// This operation allows specifying the encoding applied to the content being written. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let mut w = op + /// .writer_with("path/to/file") + /// .content_encoding("gzip") + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` + pub fn content_encoding(self, v: &str) -> Self { + self.map(|(args, options)| (args.with_content_encoding(v), options)) } - /// Set the user defined metadata of the op + /// Sets If-Match header for this write request. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_if_match`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, the write operation will only succeed if the target's ETag matches the specified value + /// - The value should be a valid ETag string + /// - Common values include: + /// - A specific ETag value like `"686897696a7c876b7e"` + /// - `*` - Matches any existing resource + /// - If not supported, the value will be ignored + /// + /// This operation provides conditional write functionality based on ETag matching, + /// helping prevent unintended overwrites in concurrent scenarios. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let mut w = op + /// .writer_with("path/to/file") + /// .if_match("\"686897696a7c876b7e\"") + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` + pub fn if_match(self, s: &str) -> Self { + self.map(|(args, options)| (args.with_if_match(s), options)) + } + + /// Sets If-None-Match header for this write request. + /// + /// Note: Certain services, like `s3`, support `if_not_exists` but not `if_none_match`. + /// Use `if_not_exists` if you only want to check whether a file exists. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_if_none_match`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, the write operation will only succeed if the target's ETag does not match the specified value + /// - The value should be a valid ETag string + /// - Common values include: + /// - A specific ETag value like `"686897696a7c876b7e"` + /// - `*` - Matches if the resource does not exist + /// - If not supported, the value will be ignored + /// + /// This operation provides conditional write functionality based on ETag non-matching, + /// useful for preventing overwriting existing resources or ensuring unique writes. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let mut w = op + /// .writer_with("path/to/file") + /// .if_none_match("\"686897696a7c876b7e\"") + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` + pub fn if_none_match(self, s: &str) -> Self { + self.map(|(args, options)| (args.with_if_none_match(s), options)) + } + + /// Sets the condition that write operation will succeed only if target does not exist. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_if_not_exists`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, the write operation will only succeed if the target path does not exist + /// - Will return error if target already exists + /// - If not supported, the value will be ignored + /// + /// This operation provides a way to ensure write operations only create new resources + /// without overwriting existing ones, useful for implementing "create if not exists" logic. + /// + /// ### Example + /// + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; + /// + /// # async fn test(op: Operator) -> Result<()> { + /// let mut w = op + /// .writer_with("path/to/file") + /// .if_not_exists(true) + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.write(vec![1; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` + pub fn if_not_exists(self, b: bool) -> Self { + self.map(|(args, options)| (args.with_if_not_exists(b), options)) + } + + /// Sets user metadata for this write request. + /// + /// ### Capability + /// + /// Check [`Capability::write_with_user_metadata`] before using this feature. + /// + /// ### Behavior + /// + /// - If supported, the user metadata will be attached to the object during write + /// - Accepts key-value pairs where both key and value are strings + /// - Keys are case-insensitive in most services + /// - Services may have limitations for user metadata, for example: + /// - Key length is typically limited (e.g., 1024 bytes) + /// - Value length is typically limited (e.g., 4096 bytes) + /// - Total metadata size might be limited + /// - Some characters might be forbidden in keys + /// - If not supported, the metadata will be ignored + /// + /// User metadata provides a way to attach custom metadata to objects during write operations. + /// This metadata can be retrieved later when reading the object. + /// + /// ### Example /// - /// ## Notes + /// ``` + /// # use opendal::Result; + /// # use opendal::Operator; + /// # use futures::StreamExt; + /// # use futures::SinkExt; + /// use bytes::Bytes; /// - /// we don't need to include the user defined metadata prefix in the key. - /// every service will handle it internally + /// # async fn test(op: Operator) -> Result<()> { + /// let mut w = op + /// .writer_with("path/to/file") + /// .user_metadata([ + /// ("content-type".to_string(), "text/plain".to_string()), + /// ("author".to_string(), "OpenDAL".to_string()), + /// ]) + /// .await?; + /// w.write(vec![0; 4096]).await?; + /// w.close().await?; + /// # Ok(()) + /// # } + /// ``` pub fn user_metadata(self, data: impl IntoIterator) -> Self { self.map(|(args, options)| (args.with_user_metadata(HashMap::from_iter(data)), options)) }