Azure storage: Allowed either content_md5 or content_crc64 in respons…

…es (#304) * fixed put_block * prepared for release
MindFlavor · Jun 27, 2020 · 1036adc · 1036adc
1 parent 1995aac
commit 1036adc
Show file tree

Hide file tree

Showing 14 changed files with 122 additions and 28 deletions.
diff --git a/README.md b/README.md
@@ -5,7 +5,7 @@
 
 [![Build Status](https://travis-ci.org/MindFlavor/AzureSDKForRust.svg?branch=master)](https://travis-ci.org/MindFlavor/AzureSDKForRust) [![Coverage Status](https://coveralls.io/repos/MindFlavor/AzureSDKForRust/badge.svg?branch=master&service=github)](https://coveralls.io/github/MindFlavor/AzureSDKForRust?branch=master) ![stability-unstable](https://img.shields.io/badge/stability-unstable-yellow.svg)
 
-[![tag](https://img.shields.io/github/tag/mindflavor/AzureSDKForRust.svg)](https://github.com/MindFlavor/AzureSDKForRust/tree/aad_0.46.0) [![release](https://img.shields.io/github/release/mindflavor/AzureSDKForRust.svg)](https://github.com/MindFlavor/AzureSDKForRust/releases/tag/aad_0.46.0) [![commitssince](https://img.shields.io/github/commits-since/mindflavor/AzureSDKForRust/aad_0.46.0)](https://github.com/MindFlavor/AzureSDKForRust/commits/master)
+[![tag](https://img.shields.io/github/tag/mindflavor/AzureSDKForRust.svg)](https://github.com/MindFlavor/AzureSDKForRust/tree/storage_blob_0.44.3) [![release](https://img.shields.io/github/release/mindflavor/AzureSDKForRust.svg)](https://github.com/MindFlavor/AzureSDKForRust/releases/tag/storage_blob_0.44.3) [![commitssince](https://img.shields.io/github/commits-since/mindflavor/AzureSDKForRust/storage_blob_0.44.3)](https://github.com/MindFlavor/AzureSDKForRust/commits/master)
 
 [![GitHub contributors](https://img.shields.io/github/contributors/MindFlavor/AzureSDKForRust.svg)](https://github.com/MindFlavor/AzureSDKForRust/graphs/contributors)
 

diff --git a/azure_sdk_auth_aad/Cargo.toml b/azure_sdk_auth_aad/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name               = "azure_sdk_auth_aad"
-version            = "0.46.0"
+version            = "0.46.1"
 description        = "Rust wrappers around Microsoft Azure REST APIs - Azure OAuth2 helper crate"
 readme             = "README.md"
 authors            = ["Francesco Cogno <[email protected]>"]
@@ -15,8 +15,8 @@ categories         = ["api-bindings"]
 edition            = "2018"
 
 [dependencies]
-azure_sdk_core       = { path = "../azure_sdk_core", version = "0.43.3" }
-oauth2               = { version = "3.0.0-alpha.9", features = ["reqwest-010", "futures-03"], default-features = false}
+azure_sdk_core       = { path = "../azure_sdk_core", version = "0.43.5" }
+oauth2               = { version = "3.0.0", features = ["reqwest-010", "futures-03"], default-features = false}
 url                  = "2.1"
 futures              = "0.3"
 serde                = "1.0"

diff --git a/azure_sdk_core/Cargo.toml b/azure_sdk_core/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name          = "azure_sdk_core"
-version       = "0.43.4"
+version       = "0.43.5"
 description   = "Rust wrappers around Microsoft Azure REST APIs - Core crate"
 readme        = "README.md"
 authors       = ["Francesco Cogno <[email protected]>", "Max Gortman <[email protected]>", "Dong Liu <[email protected]>"]

diff --git a/azure_sdk_core/src/errors.rs b/azure_sdk_core/src/errors.rs
@@ -210,6 +210,9 @@ quick_error! {
         HeaderNotFound(msg: String) {
             display("Header not found: {}", msg)
         }
+        HeadersNotFound(headers: Vec<String>) {
+            display("At least one of these headers must be present: {:?}", headers)
+        }
         ResponseParsingError(err: TraversingError){
             from()
             display("Traversing error: {}", err)

diff --git a/azure_sdk_core/src/lib.rs b/azure_sdk_core/src/lib.rs
@@ -88,6 +88,12 @@ impl NotAssigned for No {}
 
 create_enum!(DeleteSnapshotsMethod, (Include, "include"), (Only, "only"));
 
+#[derive(Debug, Clone, PartialEq)]
+pub enum Consistency {
+    Md5([u8; 16]),
+    Crc64([u8; 8]),
+}
+
 pub trait TimeoutSupport {
     type O;
     fn with_timeout(self, timeout: u64) -> Self::O;
@@ -959,6 +965,16 @@ pub fn content_md5_from_headers(headers: &HeaderMap) -> Result<[u8; 16], AzureEr
     Ok(content_md5)
 }
 
+pub fn content_crc64_from_headers_optional(
+    headers: &HeaderMap,
+) -> Result<Option<[u8; 8]>, AzureError> {
+    if headers.contains_key(CONTENT_CRC64) {
+        Ok(Some(content_crc64_from_headers(headers)?))
+    } else {
+        Ok(None)
+    }
+}
+
 pub fn content_crc64_from_headers(headers: &HeaderMap) -> Result<[u8; 8], AzureError> {
     let content_crc64 = headers
         .get(CONTENT_CRC64)
@@ -979,6 +995,21 @@ pub fn content_crc64_from_headers(headers: &HeaderMap) -> Result<[u8; 8], AzureE
     Ok(content_crc64)
 }
 
+pub fn consistency_from_headers(headers: &HeaderMap) -> Result<Consistency, AzureError> {
+    if let Some(content_crc64) = content_crc64_from_headers_optional(headers)? {
+        return Ok(Consistency::Crc64(content_crc64));
+    } else {
+        if let Some(content_md5) = content_md5_from_headers_optional(headers)? {
+            return Ok(Consistency::Md5(content_md5));
+        }
+    }
+
+    Err(AzureError::HeadersNotFound(vec![
+        CONTENT_CRC64.to_owned(),
+        CONTENT_MD5.to_owned(),
+    ]))
+}
+
 pub fn last_modified_from_headers_optional(
     headers: &HeaderMap,
 ) -> Result<Option<DateTime<Utc>>, AzureError> {

diff --git a/azure_sdk_cosmos/Cargo.toml b/azure_sdk_cosmos/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name          = "azure_sdk_cosmos"
-version       = "0.100.2"
+version       = "0.100.3"
 description   = "Rust wrappers around Microsoft Azure REST APIs - Azure Cosmos DB crate"
 readme        = "README.md"
 authors       = ["Francesco Cogno <[email protected]>", "Max Gortman <[email protected]>"]
@@ -15,7 +15,7 @@ categories    = ["api-bindings"]
 edition       = "2018"
 
 [dependencies]
-azure_sdk_core           = { path = "../azure_sdk_core", version = "0.43.4" }
+azure_sdk_core           = { path = "../azure_sdk_core", version = "0.43.5" }
 ring                     = "0.16"
 base64                   = "0.12"
 chrono                   = "0.4"

diff --git a/azure_sdk_service_bus/Cargo.toml b/azure_sdk_service_bus/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name          = "azure_sdk_service_bus"
-version       = "0.44.1"
+version       = "0.44.2"
 description   = "Rust wrappers around Microsoft Azure REST APIs - Service Bus crate"
 readme        = "README.md"
 authors       = ["Francesco Cogno <[email protected]>", "Max Gortman <[email protected]>", "Dong Liu <[email protected]>"]
@@ -15,7 +15,7 @@ categories    = ["api-bindings"]
 edition       = "2018"
 
 [dependencies]
-azure_sdk_core       = { path = "../azure_sdk_core", version = "0.43.3" }
+azure_sdk_core       = { path = "../azure_sdk_core", version = "0.43.5" }
 ring                 = "0.16"
 base64               = "0.12"
 chrono               = "0.4"

diff --git a/azure_sdk_storage_account/Cargo.toml b/azure_sdk_storage_account/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name          = "azure_sdk_storage_account"
-version       = "0.41.1"
+version       = "0.41.2"
 description   = "Rust wrappers around Microsoft Azure REST APIs - Blob storage account crate"
 readme        = "README.md"
 authors       = ["Francesco Cogno <[email protected]>", "Max Gortman <[email protected]>"]
@@ -15,8 +15,8 @@ categories    = ["api-bindings"]
 edition       = "2018"
 
 [dependencies]
-azure_sdk_core           = { path = "../azure_sdk_core", version = "0.43.4" }
-azure_sdk_storage_core   = { path = "../azure_sdk_storage_core", version = "0.44.1" }
+azure_sdk_core           = { path = "../azure_sdk_core", version = "0.43.5" }
+azure_sdk_storage_core   = { path = "../azure_sdk_storage_core", version = "0.44.2" }
 chrono                   = "0.4"
 http                     = "0.2"
 hyper                    = "0.13"

diff --git a/azure_sdk_storage_blob/Cargo.toml b/azure_sdk_storage_blob/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name          = "azure_sdk_storage_blob"
-version       = "0.44.2"
+version       = "0.44.3"
 description   = "Rust wrappers around Microsoft Azure REST APIs - Blob storage crate"
 readme        = "README.md"
 authors       = ["Francesco Cogno <[email protected]>", "Max Gortman <[email protected]>", "Dong Liu <[email protected]>"]
@@ -15,8 +15,8 @@ categories    = ["api-bindings"]
 edition       = "2018"
 
 [dependencies]
-azure_sdk_core          = { path = "../azure_sdk_core", version = "0.43.4" }
-azure_sdk_storage_core  = { path = "../azure_sdk_storage_core", version = "0.44.1" }
+azure_sdk_core          = { path = "../azure_sdk_core", version = "0.43.5" }
+azure_sdk_storage_core  = { path = "../azure_sdk_storage_core", version = "0.44.2" }
 md5                     = "0.7"
 RustyXML                = "0.3"
 base64                  = "0.12"

diff --git a/azure_sdk_storage_blob/examples/blob_04.rs b/azure_sdk_storage_blob/examples/blob_04.rs
@@ -0,0 +1,47 @@
+use azure_sdk_core::prelude::*;
+use azure_sdk_storage_blob::prelude::*;
+use azure_sdk_storage_core::prelude::*;
+use std::error::Error;
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn Error>> {
+    env_logger::init();
+
+    // First we retrieve the account name and master key from environment variables.
+    let account =
+        std::env::var("STORAGE_ACCOUNT").expect("Set env variable STORAGE_ACCOUNT first!");
+    let master_key =
+        std::env::var("STORAGE_MASTER_KEY").expect("Set env variable STORAGE_MASTER_KEY first!");
+
+    let container_name = std::env::args()
+        .nth(1)
+        .expect("please specify container name as command line parameter");
+
+    let client = client::with_access_key(&account, &master_key);
+
+    let data = b"1337 azure blob test";
+    let blob = "test1";
+    let mut block_ids = Vec::new();
+    for (i, block) in data.chunks(64 * 1024 * 1024 /* 64 MiB */).enumerate() {
+        block_ids.push(i.to_be_bytes());
+        let digest = md5::compute(block);
+        let put_block_response = client
+            .put_block()
+            .with_container_name(&container_name)
+            .with_blob_name(blob)
+            .with_body(block)
+            .with_block_id(&i.to_be_bytes()[..])
+            .with_content_md5(&digest[..])
+            .finalize()
+            .await?;
+
+        println!("put_block_response == {:#?}", put_block_response);
+    }
+
+    let mut block_list = BlockList::default();
+    for id in block_ids.iter() {
+        block_list.blocks.push(BlobBlockType::Uncommitted(&id[..]));
+    }
+
+    Ok(())
+}
diff --git a/azure_sdk_storage_blob/src/blob/responses/put_block_response.rs b/azure_sdk_storage_blob/src/blob/responses/put_block_response.rs
@@ -1,14 +1,14 @@
 use azure_sdk_core::errors::AzureError;
 use azure_sdk_core::{
-    content_crc64_from_headers, date_from_headers, request_id_from_headers,
-    request_server_encrypted_from_headers, RequestId,
+    consistency_from_headers, date_from_headers, request_id_from_headers,
+    request_server_encrypted_from_headers, Consistency, RequestId,
 };
 use chrono::{DateTime, Utc};
 use http::HeaderMap;
 
 #[derive(Debug, Clone, PartialEq)]
 pub struct PutBlockResponse {
-    pub content_crc64: [u8; 8],
+    pub consistency: Consistency,
     pub request_id: RequestId,
     pub date: DateTime<Utc>,
     pub request_server_encrypted: bool,
@@ -18,13 +18,13 @@ impl PutBlockResponse {
     pub(crate) fn from_headers(headers: &HeaderMap) -> Result<PutBlockResponse, AzureError> {
         debug!("{:#?}", headers);
 
-        let content_crc64 = content_crc64_from_headers(headers)?;
+        let consistency = consistency_from_headers(headers)?;
         let request_id = request_id_from_headers(headers)?;
         let date = date_from_headers(headers)?;
         let request_server_encrypted = request_server_encrypted_from_headers(headers)?;
 
         Ok(PutBlockResponse {
-            content_crc64,
+            consistency,
             request_id,
             date,
             request_server_encrypted,

diff --git a/azure_sdk_storage_blob/tests/blob.rs b/azure_sdk_storage_blob/tests/blob.rs
@@ -3,7 +3,7 @@
 extern crate log;
 
 use azure_sdk_core::prelude::*;
-use azure_sdk_core::DeleteSnapshotsMethod;
+use azure_sdk_core::{Consistency, DeleteSnapshotsMethod};
 use azure_sdk_storage_blob::{
     blob::BlockListType,
     container::{Container, PublicAccess, PublicAccessSupport},
@@ -146,7 +146,9 @@ async fn put_and_get_block_list() {
     let contents2 = "BBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB";
     let contents3 = "CCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCCC";
 
-    client
+    let digest3 = md5::compute(contents3);
+
+    let put_block_response = client
         .put_block()
         .with_container_name(&container.name)
         .with_blob_name(name)
@@ -156,6 +158,11 @@ async fn put_and_get_block_list() {
         .await
         .unwrap();
 
+    match &put_block_response.consistency {
+        Consistency::Crc64(_) => {}
+        _ => panic!("must receive a content_crc64 header"),
+    }
+
     client
         .put_block()
         .with_container_name(&container.name)
@@ -166,16 +173,22 @@ async fn put_and_get_block_list() {
         .await
         .unwrap();
 
-    client
+    let put_block_response = client
         .put_block()
         .with_container_name(&container.name)
         .with_blob_name(name)
         .with_body(&contents3.as_bytes())
         .with_block_id(b"block3")
+        .with_content_md5(&digest3[..])
         .finalize()
         .await
         .unwrap();
 
+    match &put_block_response.consistency {
+        Consistency::Md5(_) => {}
+        _ => panic!("must receive a content_md5 header"),
+    }
+
     let received_block_list = client
         .get_block_list()
         .with_container_name(&container.name)

diff --git a/azure_sdk_storage_core/Cargo.toml b/azure_sdk_storage_core/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name          = "azure_sdk_storage_core"
-version       = "0.44.1"
+version       = "0.44.2"
 description   = "Rust wrappers around Microsoft Azure REST APIs - Core storage crate"
 readme        = "README.md"
 authors       = ["Francesco Cogno <[email protected]>", "Max Gortman <[email protected]>", "Dong Liu <[email protected]>"]
@@ -15,7 +15,7 @@ categories    = ["api-bindings"]
 edition       = "2018"
 
 [dependencies]
-azure_sdk_core          = { path = "../azure_sdk_core", version = "0.43.4" }
+azure_sdk_core          = { path = "../azure_sdk_core", version = "0.43.5" }
 ring                    = "0.16"
 base64                  = "0.12"
 chrono                  = "0.4"

diff --git a/azure_sdk_storage_table/Cargo.toml b/azure_sdk_storage_table/Cargo.toml
@@ -1,6 +1,6 @@
 [package]
 name          = "azure_sdk_storage_table"
-version       = "0.41.1"
+version       = "0.41.2"
 description   = "Rust wrappers around Microsoft Azure REST APIs - Table storage crate"
 readme        = "README.md"
 authors       = ["Francesco Cogno <[email protected]>", "gzp-crey", "Max Gortman <[email protected]>", "Dong Liu <[email protected]>"]
@@ -15,8 +15,8 @@ categories    = ["api-bindings"]
 edition       = "2018"
 
 [dependencies]
-azure_sdk_core         = { path = "../azure_sdk_core", version = "0.43.4" }
-azure_sdk_storage_core = { path = "../azure_sdk_storage_core", version = "0.44.1" }
+azure_sdk_core         = { path = "../azure_sdk_core", version = "0.43.5" }
+azure_sdk_storage_core = { path = "../azure_sdk_storage_core", version = "0.44.2" }
 chrono                 = "0.4"
 http                   = "0.2"
 hyper                  = "0.13"