From 5d63c89f8b284f2c79776f7b56b636248fa35aa1 Mon Sep 17 00:00:00 2001 From: 3cL1p5e7 <3cL1p5e7@gmail.com> Date: Sun, 27 Feb 2022 23:30:51 +0300 Subject: [PATCH 1/8] added body decoding before validation --- Cargo.lock | 29 +++++++++++++++++++++++++++++ Cargo.toml | 1 + src/main.rs | 39 ++++++++++++++++++++++++++++++++++----- 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index b1c5a8d..740a173 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,12 @@ # It is not intended for manual editing. version = 3 +[[package]] +name = "adler" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe" + [[package]] name = "aho-corasick" version = "0.7.18" @@ -417,6 +423,18 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37ab347416e802de484e4d03c7316c48f1ecb56574dfd4a46a80f173ce1de04d" +[[package]] +name = "flate2" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6988e897c1c9c485f43b47a529cef42fde0547f9d8d41a7062518f1d8fc53f" +dependencies = [ + "cfg-if", + "crc32fast", + "libc", + "miniz_oxide", +] + [[package]] name = "fnv" version = "1.0.7" @@ -757,6 +775,7 @@ dependencies = [ "base64", "candid", "clap", + "flate2", "garcon", "hex", "hyper", @@ -974,6 +993,16 @@ version = "0.3.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d" +[[package]] +name = "miniz_oxide" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a92518e98c078586bc6c934028adcca4c92a53d6a958196de835170a01d84e4b" +dependencies = [ + "adler", + "autocfg", +] + [[package]] name = "mio" version = "0.7.14" diff --git a/Cargo.toml b/Cargo.toml index 3d0dab1..f335dd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ anyhow = "1.0.34" base64 = "0.13" candid = { version = "0.7.11", features = ["mute_warnings"] } clap = { version = "3", features = ["cargo", "derive"] } +flate2 = "1.0.0" garcon = { version = "0.2.3", features = ["async"] } hex = "0.4.3" hyper = { version = "0.14.13", features = ["full"] } diff --git a/src/main.rs b/src/main.rs index ec46be6..7120ea8 100644 --- a/src/main.rs +++ b/src/main.rs @@ -35,6 +35,8 @@ use std::{ Arc, Mutex, }, }; +use std::io::prelude::{Read}; +use flate2::read::{GzDecoder, DeflateDecoder}; mod config; mod logging; @@ -282,6 +284,7 @@ async fn forward_request( let mut certificate: Option, ()>> = None; let mut tree: Option, ()>> = None; + let mut encoding: Option = None; let mut builder = Response::builder().status(StatusCode::from_u16(http_response.status_code)?); for HeaderField(name, value) in http_response.headers { @@ -338,6 +341,9 @@ async fn forward_request( } } } + } else if name.eq_ignore_ascii_case("CONTENT-ENCODING") { + let enc = value.trim().to_string(); + encoding = Some(enc); } builder = builder.header(&name, value); @@ -348,6 +354,7 @@ async fn forward_request( } else { None }; + let decoded_body = decode_body(&http_response.body, encoding); let is_streaming = http_response.streaming_strategy.is_some(); let response = if let Some(streaming_strategy) = http_response.streaming_strategy { let (mut sender, body) = body::Body::channel(); @@ -407,7 +414,7 @@ async fn forward_request( &canister_id, &agent, &uri, - &http_response.body, + &decoded_body, logger.clone(), ) { Ok(valid) => valid, @@ -467,6 +474,25 @@ async fn forward_request( Ok(response) } +fn decode_body(body: &[u8], encoding: Option) -> Vec { + match encoding { + Some(enc) => match enc.as_str() { + "gzip" => { + let decoded: &mut Vec = &mut vec![]; + GzDecoder::new(body).read_to_end(decoded).unwrap(); + decoded.to_vec() + }, + "deflate" => { + let decoded: &mut Vec = &mut vec![]; + DeflateDecoder::new(body).read_to_end(decoded).unwrap(); + decoded.to_vec() + }, + _ => body.to_vec(), + }, + _ => body.to_vec(), + } +} + fn validate_body( certificate: &[u8], tree: &[u8], @@ -515,8 +541,11 @@ fn validate_body( } let path = ["http_assets".into(), uri.path().into()]; + println!("START {}", uri.path()); let tree_sha = match tree.lookup_path(&path) { - LookupResult::Found(v) => v, + LookupResult::Found(v) => { + v + }, _ => match tree.lookup_path(&["http_assets".into(), "/index.html".into()]) { LookupResult::Found(v) => v, _ => { @@ -532,9 +561,9 @@ fn validate_body( let mut sha256 = Sha256::new(); sha256.update(response_body); - let body_sha = sha256.finalize(); - - Ok(&body_sha[..] == tree_sha) + let body_sha: [u8; 32] = sha256.finalize().into(); + println!("CHECK {} =? {}({})", body_sha[0], tree_sha[0], tree_sha[tree_sha.len() - 1]); + Ok(body_sha == tree_sha) } fn is_hop_header(name: &str) -> bool { From cf04a016a0b672ab855bb7e6c0bc323b6a9b2db2 Mon Sep 17 00:00:00 2001 From: 3cL1p5e7 <3cL1p5e7@gmail.com> Date: Sun, 27 Feb 2022 23:37:49 +0300 Subject: [PATCH 2/8] cleanup --- src/main.rs | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main.rs b/src/main.rs index 7120ea8..9a5d063 100644 --- a/src/main.rs +++ b/src/main.rs @@ -541,7 +541,6 @@ fn validate_body( } let path = ["http_assets".into(), uri.path().into()]; - println!("START {}", uri.path()); let tree_sha = match tree.lookup_path(&path) { LookupResult::Found(v) => { v @@ -562,7 +561,6 @@ fn validate_body( let mut sha256 = Sha256::new(); sha256.update(response_body); let body_sha: [u8; 32] = sha256.finalize().into(); - println!("CHECK {} =? {}({})", body_sha[0], tree_sha[0], tree_sha[tree_sha.len() - 1]); Ok(body_sha == tree_sha) } From d158dff32af560fd69207c5d9ed97bc6bc78a59e Mon Sep 17 00:00:00 2001 From: 3cL1p5e7 <3cL1p5e7@gmail.com> Date: Mon, 28 Feb 2022 11:53:15 +0300 Subject: [PATCH 3/8] check of chunk certificates + while streaming --- src/main.rs | 157 ++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 122 insertions(+), 35 deletions(-) diff --git a/src/main.rs b/src/main.rs index 9a5d063..47a9531 100644 --- a/src/main.rs +++ b/src/main.rs @@ -284,6 +284,8 @@ async fn forward_request( let mut certificate: Option, ()>> = None; let mut tree: Option, ()>> = None; + let mut chunk_tree: Option> = None; + let mut chunk_index: String = String::from("0"); let mut encoding: Option = None; let mut builder = Response::builder().status(StatusCode::from_u16(http_response.status_code)?); @@ -292,6 +294,10 @@ async fn forward_request( for field in value.split(',') { if let Some((_, name, b64_value)) = regex_captures!("^(.*)=:(.*):$", field.trim()) { slog::trace!(logger, ">> certificate {}: {}", name, b64_value); + if name == "chunk_index" { + chunk_index = b64_value.to_string(); + continue; + } let bytes = base64::decode(b64_value).map_err(|e| { slog::warn!( logger, @@ -338,6 +344,18 @@ async fn forward_request( bytes } }); + } else if name == "chunk_tree" { + chunk_tree = match (chunk_tree, bytes) { + (None, bytes) => bytes.ok(), + (Some(chunk_tree), Ok(bytes)) => { + slog::warn!(logger, "duplicate chunk_tree field: {:?}", bytes); + Some(chunk_tree) + }, + (Some(chunk_tree), Err(_)) => { + slog::warn!(logger, "duplicate chunk_tree field (failed to decode)"); + Some(chunk_tree) + }, + }; } } } @@ -354,7 +372,40 @@ async fn forward_request( } else { None }; - let decoded_body = decode_body(&http_response.body, encoding); + + let decoded_body = decode_body(&http_response.body, encoding.clone()); + let body_valid = match (certificate.clone(), tree.clone()) { + (Some(Ok(certificate)), Some(Ok(tree))) => match validate_body( + &certificate, + &tree, + chunk_tree, + chunk_index, + &canister_id, + &agent, + &uri, + &decoded_body, + logger.clone(), + ) { + Ok(valid) => valid, + Err(e) => { + return Ok(Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body(format!("Certificate validation failed: {}", e).into()) + .unwrap()); + } + }, + (Some(_), _) | (_, Some(_)) => false, + // Canisters don't have to provide certified variables + (None, None) => true, + }; + + if !body_valid && !cfg!(feature = "skip_body_verification") { + return Ok(Response::builder() + .status(StatusCode::INTERNAL_SERVER_ERROR) + .body("Body does not pass verification".into()) + .unwrap()); + } + let is_streaming = http_response.streaming_strategy.is_some(); let response = if let Some(streaming_strategy) = http_response.streaming_strategy { let (mut sender, body) = body::Body::channel(); @@ -366,6 +417,7 @@ async fn forward_request( let streaming_canister_id_id = callback.callback.principal; let method_name = callback.callback.method; let mut callback_token = callback.token; + let chunk_index = callback_token.index.0.to_str_radix(10); let logger = logger.clone(); tokio::spawn(async move { let canister = HttpRequestCanister::create(&agent, streaming_canister_id_id); @@ -383,7 +435,41 @@ async fn forward_request( .call() .await { - Ok((StreamingCallbackHttpResponse { body, token },)) => { + Ok((StreamingCallbackHttpResponse { body, token, chunk_tree },)) => { + let body_valid = match (certificate.clone(), tree.clone(), chunk_tree) { + (Some(Ok(certificate)), Some(Ok(tree)), Some(chunk_tree)) => { + let decoded_chunk_tree = base64::decode(chunk_tree).map_err(|e| { + slog::warn!( + logger, + "Unable to decode chunk_tree from base64: {}", + e + ); + }).ok(); + + + let decoded_body = decode_body(&body, encoding.clone()); + validate_body( + &certificate, + &tree, + decoded_chunk_tree, + chunk_index.clone(), + &canister_id, + &agent, + &uri, + &decoded_body, + logger.clone(), + ) + .ok() + .unwrap_or(false) + }, + _ => false, + }; + + if !body_valid && !cfg!(feature = "skip_body_verification") { + sender.abort(); + break; + } + if sender.send_data(Bytes::from(body)).await.is_err() { sender.abort(); break; @@ -407,35 +493,6 @@ async fn forward_request( builder.body(body)? } else { - let body_valid = match (certificate, tree) { - (Some(Ok(certificate)), Some(Ok(tree))) => match validate_body( - &certificate, - &tree, - &canister_id, - &agent, - &uri, - &decoded_body, - logger.clone(), - ) { - Ok(valid) => valid, - Err(e) => { - return Ok(Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .body(format!("Certificate validation failed: {}", e).into()) - .unwrap()); - } - }, - (Some(_), _) | (_, Some(_)) => false, - // Canisters don't have to provide certified variables - (None, None) => true, - }; - - if !body_valid && !cfg!(feature = "skip_body_verification") { - return Ok(Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .body("Body does not pass verification".into()) - .unwrap()); - } builder.body(http_response.body.into())? }; @@ -496,6 +553,8 @@ fn decode_body(body: &[u8], encoding: Option) -> Vec { fn validate_body( certificate: &[u8], tree: &[u8], + chunk_tree: Option>, + chunk_index: String, canister_id: &Principal, agent: &Agent, uri: &Uri, @@ -542,9 +601,7 @@ fn validate_body( let path = ["http_assets".into(), uri.path().into()]; let tree_sha = match tree.lookup_path(&path) { - LookupResult::Found(v) => { - v - }, + LookupResult::Found(v) => v, _ => match tree.lookup_path(&["http_assets".into(), "/index.html".into()]) { LookupResult::Found(v) => v, _ => { @@ -561,7 +618,37 @@ fn validate_body( let mut sha256 = Sha256::new(); sha256.update(response_body); let body_sha: [u8; 32] = sha256.finalize().into(); - Ok(body_sha == tree_sha) + + if let Some(tree) = chunk_tree { + let chunk_tree: HashTree = serde_cbor::from_slice(&tree).map_err(AgentError::InvalidCborData)?; + + let chunk_tree_digest = chunk_tree.digest(); + + if chunk_tree_digest != tree_sha { + slog::trace!( + logger, + ">> Invalid chunk_tree in the header. Digest does not equal tree_sha", + ); + return Ok(false); + } + + let index_path = [chunk_index.into()]; + let chunk_sha = match chunk_tree.lookup_path(&index_path) { + LookupResult::Found(v) => v, + _ => { + slog::trace!( + logger, + ">> Invalid Tree in the header. Does not contain path {:?}", + path + ); + return Ok(false); + } + }; + + Ok(body_sha == chunk_sha) + } else { + Ok(body_sha == tree_sha) + } } fn is_hop_header(name: &str) -> bool { From 439f55554ef05c02cb10bea6e37839d61f5bb08d Mon Sep 17 00:00:00 2001 From: 3cL1p5e7 <3cL1p5e7@gmail.com> Date: Tue, 1 Mar 2022 13:07:47 +0300 Subject: [PATCH 4/8] review fixes for chunks validation --- src/main.rs | 244 +++++++++++++++++++++++++++++----------------------- 1 file changed, 138 insertions(+), 106 deletions(-) diff --git a/src/main.rs b/src/main.rs index 47a9531..1587fe6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -47,6 +47,9 @@ static MAX_HTTP_REQUEST_STREAM_CALLBACK_CALL_COUNT: i32 = 1000; // The maximum length of a body we should log as tracing. static MAX_LOG_BODY_SIZE: usize = 100; +// The limit of a buffer we should decompress ~10mb. +static MAX_BYTES_SIZE_TO_DECOMPRESS: u64 = 10_000_000; + #[derive(Parser)] #[clap( version = crate_version!(), @@ -171,6 +174,105 @@ fn resolve_canister_id( None } +struct HeadersData { + certificate: Option, ()>>, + tree: Option, ()>>, + chunk_tree: Option>, + chunk_index: String, + encoding: Option, +} + +fn extract_headers_data( + headers: &Vec, + logger: &slog::Logger +) -> HeadersData { + let mut headers_data = HeadersData { + certificate: None, + tree: None, + chunk_tree: None, + chunk_index: String::from("0"), + encoding: None, + }; + + for HeaderField(name, value) in headers { + if name.eq_ignore_ascii_case("IC-CERTIFICATE") { + for field in value.split(',') { + if let Some((_, name, b64_value)) = regex_captures!("^(.*)=:(.*):$", field.trim()) { + slog::trace!(logger, ">> certificate {}: {}", name, b64_value); + if name == "chunk_index" { + headers_data.chunk_index = b64_value.to_string(); + continue; + } + let bytes = base64::decode(b64_value).map_err(|e| { + slog::warn!( + logger, + "Unable to decode {} in ic-certificate from base64: {}", + name, + e + ); + }); + if name == "certificate" { + headers_data.certificate = Some(match (headers_data.certificate, bytes) { + (None, bytes) => bytes, + (Some(Ok(certificate)), Ok(bytes)) => { + slog::warn!(logger, "duplicate certificate field: {:?}", bytes); + Ok(certificate) + } + (Some(Ok(certificate)), Err(_)) => { + slog::warn!( + logger, + "duplicate certificate field (failed to decode)" + ); + Ok(certificate) + } + (Some(Err(_)), bytes) => { + slog::warn!( + logger, + "duplicate certificate field (failed to decode)" + ); + bytes + } + }); + } else if name == "tree" { + headers_data.tree = Some(match (headers_data.tree, bytes) { + (None, bytes) => bytes, + (Some(Ok(tree)), Ok(bytes)) => { + slog::warn!(logger, "duplicate tree field: {:?}", bytes); + Ok(tree) + } + (Some(Ok(tree)), Err(_)) => { + slog::warn!(logger, "duplicate tree field (failed to decode)"); + Ok(tree) + } + (Some(Err(_)), bytes) => { + slog::warn!(logger, "duplicate tree field (failed to decode)"); + bytes + } + }); + } else if name == "chunk_tree" { + headers_data.chunk_tree = match (headers_data.chunk_tree, bytes) { + (None, bytes) => bytes.ok(), + (Some(chunk_tree), Ok(bytes)) => { + slog::warn!(logger, "duplicate chunk_tree field: {:?}", bytes); + Some(chunk_tree) + }, + (Some(chunk_tree), Err(_)) => { + slog::warn!(logger, "duplicate chunk_tree field (failed to decode)"); + Some(chunk_tree) + }, + }; + } + } + } + } else if name.eq_ignore_ascii_case("CONTENT-ENCODING") { + let enc = value.trim().to_string(); + headers_data.encoding = Some(enc); + } + } + + headers_data +} + async fn forward_request( request: Request, agent: Arc, @@ -282,104 +384,27 @@ async fn forward_request( http_response }; - let mut certificate: Option, ()>> = None; - let mut tree: Option, ()>> = None; - let mut chunk_tree: Option> = None; - let mut chunk_index: String = String::from("0"); - let mut encoding: Option = None; - let mut builder = Response::builder().status(StatusCode::from_u16(http_response.status_code)?); - for HeaderField(name, value) in http_response.headers { - if name.eq_ignore_ascii_case("IC-CERTIFICATE") { - for field in value.split(',') { - if let Some((_, name, b64_value)) = regex_captures!("^(.*)=:(.*):$", field.trim()) { - slog::trace!(logger, ">> certificate {}: {}", name, b64_value); - if name == "chunk_index" { - chunk_index = b64_value.to_string(); - continue; - } - let bytes = base64::decode(b64_value).map_err(|e| { - slog::warn!( - logger, - "Unable to decode {} in ic-certificate from base64: {}", - name, - e - ); - }); - if name == "certificate" { - certificate = Some(match (certificate, bytes) { - (None, bytes) => bytes, - (Some(Ok(certificate)), Ok(bytes)) => { - slog::warn!(logger, "duplicate certificate field: {:?}", bytes); - Ok(certificate) - } - (Some(Ok(certificate)), Err(_)) => { - slog::warn!( - logger, - "duplicate certificate field (failed to decode)" - ); - Ok(certificate) - } - (Some(Err(_)), bytes) => { - slog::warn!( - logger, - "duplicate certificate field (failed to decode)" - ); - bytes - } - }); - } else if name == "tree" { - tree = Some(match (tree, bytes) { - (None, bytes) => bytes, - (Some(Ok(tree)), Ok(bytes)) => { - slog::warn!(logger, "duplicate tree field: {:?}", bytes); - Ok(tree) - } - (Some(Ok(tree)), Err(_)) => { - slog::warn!(logger, "duplicate tree field (failed to decode)"); - Ok(tree) - } - (Some(Err(_)), bytes) => { - slog::warn!(logger, "duplicate tree field (failed to decode)"); - bytes - } - }); - } else if name == "chunk_tree" { - chunk_tree = match (chunk_tree, bytes) { - (None, bytes) => bytes.ok(), - (Some(chunk_tree), Ok(bytes)) => { - slog::warn!(logger, "duplicate chunk_tree field: {:?}", bytes); - Some(chunk_tree) - }, - (Some(chunk_tree), Err(_)) => { - slog::warn!(logger, "duplicate chunk_tree field (failed to decode)"); - Some(chunk_tree) - }, - }; - } - } - } - } else if name.eq_ignore_ascii_case("CONTENT-ENCODING") { - let enc = value.trim().to_string(); - encoding = Some(enc); - } - - builder = builder.header(&name, value); + for HeaderField(name, value) in &http_response.headers { + builder = builder.header(name, value); } - + + let headers_data = extract_headers_data(&http_response.headers, &logger); let body = if logger.is_trace_enabled() { Some(http_response.body.clone()) } else { None }; - let decoded_body = decode_body(&http_response.body, encoding.clone()); - let body_valid = match (certificate.clone(), tree.clone()) { + let decoded_body = decode_body(&http_response.body, headers_data.encoding.clone()); + let body_valid = match (headers_data.certificate.clone(), headers_data.tree.clone()) { (Some(Ok(certificate)), Some(Ok(tree))) => match validate_body( - &certificate, - &tree, - chunk_tree, - chunk_index, + Certificates { + certificate, + tree, + chunk_tree: headers_data.chunk_tree.clone(), + chunk_index: headers_data.chunk_index.clone(), + }, &canister_id, &agent, &uri, @@ -436,7 +461,7 @@ async fn forward_request( .await { Ok((StreamingCallbackHttpResponse { body, token, chunk_tree },)) => { - let body_valid = match (certificate.clone(), tree.clone(), chunk_tree) { + let body_valid = match (headers_data.certificate.clone(), headers_data.tree.clone(), chunk_tree) { (Some(Ok(certificate)), Some(Ok(tree)), Some(chunk_tree)) => { let decoded_chunk_tree = base64::decode(chunk_tree).map_err(|e| { slog::warn!( @@ -446,13 +471,14 @@ async fn forward_request( ); }).ok(); - - let decoded_body = decode_body(&body, encoding.clone()); + let decoded_body = decode_body(&body, headers_data.encoding.clone()); validate_body( - &certificate, - &tree, - decoded_chunk_tree, - chunk_index.clone(), + Certificates { + certificate, + tree, + chunk_tree: decoded_chunk_tree, + chunk_index: chunk_index.clone(), + }, &canister_id, &agent, &uri, @@ -536,12 +562,14 @@ fn decode_body(body: &[u8], encoding: Option) -> Vec { Some(enc) => match enc.as_str() { "gzip" => { let decoded: &mut Vec = &mut vec![]; - GzDecoder::new(body).read_to_end(decoded).unwrap(); + let decoder = GzDecoder::new(body); + decoder.take(MAX_BYTES_SIZE_TO_DECOMPRESS).read_to_end(decoded).unwrap(); decoded.to_vec() }, "deflate" => { - let decoded: &mut Vec = &mut vec![]; - DeflateDecoder::new(body).read_to_end(decoded).unwrap(); + let decoded: &mut Vec = &mut vec![]; + let decoder = DeflateDecoder::new(body); + decoder.take(MAX_BYTES_SIZE_TO_DECOMPRESS).read_to_end(decoded).unwrap(); decoded.to_vec() }, _ => body.to_vec(), @@ -550,11 +578,15 @@ fn decode_body(body: &[u8], encoding: Option) -> Vec { } } -fn validate_body( - certificate: &[u8], - tree: &[u8], +struct Certificates { + certificate: Vec, + tree: Vec, chunk_tree: Option>, chunk_index: String, +} + +fn validate_body( + certificates: Certificates, canister_id: &Principal, agent: &Agent, uri: &Uri, @@ -562,8 +594,8 @@ fn validate_body( logger: slog::Logger, ) -> anyhow::Result { let cert: Certificate = - serde_cbor::from_slice(certificate).map_err(AgentError::InvalidCborData)?; - let tree: HashTree = serde_cbor::from_slice(tree).map_err(AgentError::InvalidCborData)?; + serde_cbor::from_slice(&certificates.certificate).map_err(AgentError::InvalidCborData)?; + let tree: HashTree = serde_cbor::from_slice(&certificates.tree).map_err(AgentError::InvalidCborData)?; if let Err(e) = agent.verify(&cert) { slog::trace!(logger, ">> certificate failed verification: {}", e); @@ -619,7 +651,7 @@ fn validate_body( sha256.update(response_body); let body_sha: [u8; 32] = sha256.finalize().into(); - if let Some(tree) = chunk_tree { + if let Some(tree) = certificates.chunk_tree { let chunk_tree: HashTree = serde_cbor::from_slice(&tree).map_err(AgentError::InvalidCborData)?; let chunk_tree_digest = chunk_tree.digest(); @@ -632,7 +664,7 @@ fn validate_body( return Ok(false); } - let index_path = [chunk_index.into()]; + let index_path = [certificates.chunk_index.into()]; let chunk_sha = match chunk_tree.lookup_path(&index_path) { LookupResult::Found(v) => v, _ => { From cbe66b935c35f43b83ab49ac376e32477f0201c7 Mon Sep 17 00:00:00 2001 From: 3cL1p5e7 <3cL1p5e7@gmail.com> Date: Tue, 1 Mar 2022 15:33:44 +0300 Subject: [PATCH 5/8] chunk certification: make backward compatibility --- src/main.rs | 170 ++++++++++++++++++++++++++++++---------------------- 1 file changed, 97 insertions(+), 73 deletions(-) diff --git a/src/main.rs b/src/main.rs index 1587fe6..d831655 100644 --- a/src/main.rs +++ b/src/main.rs @@ -174,6 +174,25 @@ fn resolve_canister_id( None } +fn decode_hash_tree( + name: &str, + value: Option, + logger: &slog::Logger +) -> Result, ()> { + match value { + Some(tree) => base64::decode(tree) + .map_err(|e| { + slog::warn!( + logger, + "Unable to decode {} from base64: {}", + name, + e + ); + }), + _ => Err(()), + } +} + struct HeadersData { certificate: Option, ()>>, tree: Option, ()>>, @@ -203,14 +222,7 @@ fn extract_headers_data( headers_data.chunk_index = b64_value.to_string(); continue; } - let bytes = base64::decode(b64_value).map_err(|e| { - slog::warn!( - logger, - "Unable to decode {} in ic-certificate from base64: {}", - name, - e - ); - }); + let bytes = decode_hash_tree(name, Some(b64_value.to_string()), logger); if name == "certificate" { headers_data.certificate = Some(match (headers_data.certificate, bytes) { (None, bytes) => bytes, @@ -396,42 +408,23 @@ async fn forward_request( None }; - let decoded_body = decode_body(&http_response.body, headers_data.encoding.clone()); - let body_valid = match (headers_data.certificate.clone(), headers_data.tree.clone()) { - (Some(Ok(certificate)), Some(Ok(tree))) => match validate_body( - Certificates { - certificate, - tree, - chunk_tree: headers_data.chunk_tree.clone(), - chunk_index: headers_data.chunk_index.clone(), - }, - &canister_id, - &agent, - &uri, - &decoded_body, - logger.clone(), - ) { - Ok(valid) => valid, - Err(e) => { - return Ok(Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .body(format!("Certificate validation failed: {}", e).into()) - .unwrap()); - } - }, - (Some(_), _) | (_, Some(_)) => false, - // Canisters don't have to provide certified variables - (None, None) => true, - }; - - if !body_valid && !cfg!(feature = "skip_body_verification") { + let is_streaming = http_response.streaming_strategy.is_some(); + let body_valid = validate( + &headers_data, + &canister_id, + &agent, + &uri, + &http_response.body, + is_streaming, + logger.clone(), + ); + if body_valid.is_err() { return Ok(Response::builder() .status(StatusCode::INTERNAL_SERVER_ERROR) - .body("Body does not pass verification".into()) + .body(body_valid.unwrap_err().into()) .unwrap()); } - let is_streaming = http_response.streaming_strategy.is_some(); let response = if let Some(streaming_strategy) = http_response.streaming_strategy { let (mut sender, body) = body::Body::channel(); let agent = agent.as_ref().clone(); @@ -461,42 +454,25 @@ async fn forward_request( .await { Ok((StreamingCallbackHttpResponse { body, token, chunk_tree },)) => { - let body_valid = match (headers_data.certificate.clone(), headers_data.tree.clone(), chunk_tree) { - (Some(Ok(certificate)), Some(Ok(tree)), Some(chunk_tree)) => { - let decoded_chunk_tree = base64::decode(chunk_tree).map_err(|e| { - slog::warn!( - logger, - "Unable to decode chunk_tree from base64: {}", - e - ); - }).ok(); - - let decoded_body = decode_body(&body, headers_data.encoding.clone()); - validate_body( - Certificates { - certificate, - tree, - chunk_tree: decoded_chunk_tree, - chunk_index: chunk_index.clone(), - }, - &canister_id, - &agent, - &uri, - &decoded_body, - logger.clone(), - ) - .ok() - .unwrap_or(false) - }, - _ => false, + let decoded_chunk_tree = decode_hash_tree("chunk_tree", chunk_tree, &logger); + let chunk_headers_data = HeadersData { + certificate: headers_data.certificate.clone(), + tree: headers_data.tree.clone(), + encoding: headers_data.encoding.clone(), + chunk_tree: decoded_chunk_tree.ok(), + chunk_index: chunk_index.clone(), }; - - if !body_valid && !cfg!(feature = "skip_body_verification") { - sender.abort(); - break; - } + let body_valid = validate( + &chunk_headers_data, + &canister_id, + &agent, + &uri, + &body, + is_streaming, + logger.clone(), + ); - if sender.send_data(Bytes::from(body)).await.is_err() { + if body_valid.is_err() || sender.send_data(Bytes::from(body)).await.is_err() { sender.abort(); break; } @@ -557,6 +533,54 @@ async fn forward_request( Ok(response) } +fn validate( + headers_data: &HeadersData, + canister_id: &Principal, + agent: &Agent, + uri: &Uri, + response_body: &[u8], + is_streaming: bool, + logger: slog::Logger, +) -> Result<(), String> { + let decoded_body = decode_body(response_body, headers_data.encoding.clone()); + let body_valid = match (headers_data.certificate.clone(), headers_data.tree.clone()) { + (Some(Ok(certificate)), Some(Ok(tree))) => match validate_body( + Certificates { + certificate, + tree, + chunk_tree: headers_data.chunk_tree.clone(), + chunk_index: headers_data.chunk_index.clone(), + }, + &canister_id, + &agent, + &uri, + &decoded_body, + logger.clone(), + ) { + Ok(valid) => if valid { + Ok(()) + } else { + Err("Body does not pass verification".to_string()) + }, + Err(e) => Err(format!("Certificate validation failed: {}", e)), + }, + (Some(_), _) | (_, Some(_)) => Err("Body does not pass verification".to_string()), + // Canisters don't have to provide certified variables + (None, None) => Ok(()), + }; + + if body_valid.is_err() && !cfg!(feature = "skip_body_verification") { + match (is_streaming, headers_data.chunk_tree.is_some()) { + (true, false) => {}, // backward compatibility. Headers could not contain chunk_tree witness for streaming + _ => { + return Err(body_valid.unwrap_err()); + }, + } + } + + Ok(()) +} + fn decode_body(body: &[u8], encoding: Option) -> Vec { match encoding { Some(enc) => match enc.as_str() { From 346fbdd2691731f89e6eaf7cbaea0e399db61136 Mon Sep 17 00:00:00 2001 From: 3cL1p5e7 <3cL1p5e7@gmail.com> Date: Wed, 2 Mar 2022 15:14:30 +0300 Subject: [PATCH 6/8] lint + fmt + now decode_body returns body sha --- src/main.rs | 126 ++++++++++++++++++++++++++++------------------------ 1 file changed, 69 insertions(+), 57 deletions(-) diff --git a/src/main.rs b/src/main.rs index d831655..7dc6cd6 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ use crate::config::dns_canister_config::DnsCanisterConfig; use clap::{crate_authors, crate_version, AppSettings, Parser}; +use flate2::read::{DeflateDecoder, GzDecoder}; use hyper::{ body, body::Bytes, @@ -24,6 +25,7 @@ use ic_utils::{ use lazy_regex::regex_captures; use sha2::{Digest, Sha256}; use slog::Drain; +use std::io::prelude::Read; use std::{ convert::Infallible, error::Error, @@ -35,8 +37,6 @@ use std::{ Arc, Mutex, }, }; -use std::io::prelude::{Read}; -use flate2::read::{GzDecoder, DeflateDecoder}; mod config; mod logging; @@ -177,18 +177,12 @@ fn resolve_canister_id( fn decode_hash_tree( name: &str, value: Option, - logger: &slog::Logger + logger: &slog::Logger, ) -> Result, ()> { match value { - Some(tree) => base64::decode(tree) - .map_err(|e| { - slog::warn!( - logger, - "Unable to decode {} from base64: {}", - name, - e - ); - }), + Some(tree) => base64::decode(tree).map_err(|e| { + slog::warn!(logger, "Unable to decode {} from base64: {}", name, e); + }), _ => Err(()), } } @@ -201,10 +195,7 @@ struct HeadersData { encoding: Option, } -fn extract_headers_data( - headers: &Vec, - logger: &slog::Logger -) -> HeadersData { +fn extract_headers_data(headers: &[HeaderField], logger: &slog::Logger) -> HeadersData { let mut headers_data = HeadersData { certificate: None, tree: None, @@ -267,11 +258,14 @@ fn extract_headers_data( (Some(chunk_tree), Ok(bytes)) => { slog::warn!(logger, "duplicate chunk_tree field: {:?}", bytes); Some(chunk_tree) - }, + } (Some(chunk_tree), Err(_)) => { - slog::warn!(logger, "duplicate chunk_tree field (failed to decode)"); + slog::warn!( + logger, + "duplicate chunk_tree field (failed to decode)" + ); Some(chunk_tree) - }, + } }; } } @@ -400,7 +394,7 @@ async fn forward_request( for HeaderField(name, value) in &http_response.headers { builder = builder.header(name, value); } - + let headers_data = extract_headers_data(&http_response.headers, &logger); let body = if logger.is_trace_enabled() { Some(http_response.body.clone()) @@ -408,7 +402,9 @@ async fn forward_request( None }; - let is_streaming = http_response.streaming_strategy.is_some(); + // No need to stream when get 206 HTTP partial response + let is_streaming = + http_response.streaming_strategy.is_some() && http_response.status_code != 206; let body_valid = validate( &headers_data, &canister_id, @@ -425,7 +421,8 @@ async fn forward_request( .unwrap()); } - let response = if let Some(streaming_strategy) = http_response.streaming_strategy { + let response = if is_streaming { + let streaming_strategy = http_response.streaming_strategy.unwrap(); let (mut sender, body) = body::Body::channel(); let agent = agent.as_ref().clone(); sender.send_data(Bytes::from(http_response.body)).await?; @@ -453,8 +450,13 @@ async fn forward_request( .call() .await { - Ok((StreamingCallbackHttpResponse { body, token, chunk_tree },)) => { - let decoded_chunk_tree = decode_hash_tree("chunk_tree", chunk_tree, &logger); + Ok((StreamingCallbackHttpResponse { + body, + token, + chunk_tree, + },)) => { + let decoded_chunk_tree = + decode_hash_tree("chunk_tree", chunk_tree, &logger); let chunk_headers_data = HeadersData { certificate: headers_data.certificate.clone(), tree: headers_data.tree.clone(), @@ -472,7 +474,9 @@ async fn forward_request( logger.clone(), ); - if body_valid.is_err() || sender.send_data(Bytes::from(body)).await.is_err() { + if body_valid.is_err() + || sender.send_data(Bytes::from(body)).await.is_err() + { sender.abort(); break; } @@ -542,7 +546,7 @@ fn validate( is_streaming: bool, logger: slog::Logger, ) -> Result<(), String> { - let decoded_body = decode_body(response_body, headers_data.encoding.clone()); + let body_sha = decode_body(response_body, headers_data.encoding.clone()); let body_valid = match (headers_data.certificate.clone(), headers_data.tree.clone()) { (Some(Ok(certificate)), Some(Ok(tree))) => match validate_body( Certificates { @@ -551,17 +555,19 @@ fn validate( chunk_tree: headers_data.chunk_tree.clone(), chunk_index: headers_data.chunk_index.clone(), }, - &canister_id, - &agent, - &uri, - &decoded_body, + canister_id, + agent, + uri, + &body_sha, logger.clone(), ) { - Ok(valid) => if valid { - Ok(()) - } else { - Err("Body does not pass verification".to_string()) - }, + Ok(valid) => { + if valid { + Ok(()) + } else { + Err("Body does not pass verification".to_string()) + } + } Err(e) => Err(format!("Certificate validation failed: {}", e)), }, (Some(_), _) | (_, Some(_)) => Err("Body does not pass verification".to_string()), @@ -571,35 +577,43 @@ fn validate( if body_valid.is_err() && !cfg!(feature = "skip_body_verification") { match (is_streaming, headers_data.chunk_tree.is_some()) { - (true, false) => {}, // backward compatibility. Headers could not contain chunk_tree witness for streaming + (true, false) => {} // backward compatibility. Headers could not contain chunk_tree witness for streaming _ => { - return Err(body_valid.unwrap_err()); - }, + return body_valid; + } } } Ok(()) } -fn decode_body(body: &[u8], encoding: Option) -> Vec { +fn decode_body(body: &[u8], encoding: Option) -> [u8; 32] { + let mut sha256 = Sha256::new(); match encoding { Some(enc) => match enc.as_str() { "gzip" => { - let decoded: &mut Vec = &mut vec![]; + let decoded: &mut Vec = &mut vec![]; let decoder = GzDecoder::new(body); - decoder.take(MAX_BYTES_SIZE_TO_DECOMPRESS).read_to_end(decoded).unwrap(); - decoded.to_vec() - }, + decoder + .take(MAX_BYTES_SIZE_TO_DECOMPRESS) + .read_to_end(decoded) + .unwrap(); + sha256.update(decoded); + } "deflate" => { let decoded: &mut Vec = &mut vec![]; let decoder = DeflateDecoder::new(body); - decoder.take(MAX_BYTES_SIZE_TO_DECOMPRESS).read_to_end(decoded).unwrap(); - decoded.to_vec() - }, - _ => body.to_vec(), + decoder + .take(MAX_BYTES_SIZE_TO_DECOMPRESS) + .read_to_end(decoded) + .unwrap(); + sha256.update(decoded); + } + _ => sha256.update(body), }, - _ => body.to_vec(), - } + _ => sha256.update(body), + }; + sha256.finalize().into() } struct Certificates { @@ -614,12 +628,13 @@ fn validate_body( canister_id: &Principal, agent: &Agent, uri: &Uri, - response_body: &[u8], + body_sha: &[u8; 32], logger: slog::Logger, ) -> anyhow::Result { let cert: Certificate = serde_cbor::from_slice(&certificates.certificate).map_err(AgentError::InvalidCborData)?; - let tree: HashTree = serde_cbor::from_slice(&certificates.tree).map_err(AgentError::InvalidCborData)?; + let tree: HashTree = + serde_cbor::from_slice(&certificates.tree).map_err(AgentError::InvalidCborData)?; if let Err(e) = agent.verify(&cert) { slog::trace!(logger, ">> certificate failed verification: {}", e); @@ -671,12 +686,9 @@ fn validate_body( }, }; - let mut sha256 = Sha256::new(); - sha256.update(response_body); - let body_sha: [u8; 32] = sha256.finalize().into(); - if let Some(tree) = certificates.chunk_tree { - let chunk_tree: HashTree = serde_cbor::from_slice(&tree).map_err(AgentError::InvalidCborData)?; + let chunk_tree: HashTree = + serde_cbor::from_slice(&tree).map_err(AgentError::InvalidCborData)?; let chunk_tree_digest = chunk_tree.digest(); @@ -700,7 +712,7 @@ fn validate_body( return Ok(false); } }; - + Ok(body_sha == chunk_sha) } else { Ok(body_sha == tree_sha) From 01f1c6aeb9266ff222ec42cc9b1238d01798ac65 Mon Sep 17 00:00:00 2001 From: 3cL1p5e7 <3cL1p5e7@gmail.com> Date: Fri, 4 Mar 2022 13:27:45 +0300 Subject: [PATCH 7/8] added body decoding + refactoring --- Cargo.toml | 1 + src/main.rs | 277 ++++++++++++++++++++++++++++++++++------------------ 2 files changed, 182 insertions(+), 96 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 3d0dab1..f335dd5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,6 +21,7 @@ anyhow = "1.0.34" base64 = "0.13" candid = { version = "0.7.11", features = ["mute_warnings"] } clap = { version = "3", features = ["cargo", "derive"] } +flate2 = "1.0.0" garcon = { version = "0.2.3", features = ["async"] } hex = "0.4.3" hyper = { version = "0.14.13", features = ["full"] } diff --git a/src/main.rs b/src/main.rs index 3e399b0..8c26222 100644 --- a/src/main.rs +++ b/src/main.rs @@ -1,5 +1,6 @@ use crate::config::dns_canister_config::DnsCanisterConfig; use clap::{crate_authors, crate_version, AppSettings, Parser}; +use flate2::read::{DeflateDecoder, GzDecoder}; use hyper::{ body, body::Bytes, @@ -24,6 +25,7 @@ use ic_utils::{ use lazy_regex::regex_captures; use sha2::{Digest, Sha256}; use slog::Drain; +use std::io::prelude::Read; use std::{ convert::Infallible, error::Error, @@ -45,6 +47,9 @@ static MAX_HTTP_REQUEST_STREAM_CALLBACK_CALL_COUNT: i32 = 1000; // The maximum length of a body we should log as tracing. static MAX_LOG_BODY_SIZE: usize = 100; +// The limit of a buffer we should decompress ~10mb. +static MAX_BYTES_SIZE_TO_DECOMPRESS: u64 = 10_000_000; + #[derive(Parser)] #[clap( version = crate_version!(), @@ -169,6 +174,88 @@ fn resolve_canister_id( None } +fn decode_hash_tree( + name: &str, + value: Option, + logger: &slog::Logger, +) -> Result, ()> { + match value { + Some(tree) => base64::decode(tree).map_err(|e| { + slog::warn!(logger, "Unable to decode {} from base64: {}", name, e); + }), + _ => Err(()), + } +} + +struct HeadersData { + certificate: Option, ()>>, + tree: Option, ()>>, + encoding: Option, +} + +fn extract_headers_data(headers: &[HeaderField], logger: &slog::Logger) -> HeadersData { + let mut headers_data = HeadersData { + certificate: None, + tree: None, + encoding: None, + }; + + for HeaderField(name, value) in headers { + if name.eq_ignore_ascii_case("IC-CERTIFICATE") { + for field in value.split(',') { + if let Some((_, name, b64_value)) = regex_captures!("^(.*)=:(.*):$", field.trim()) { + slog::trace!(logger, ">> certificate {}: {}", name, b64_value); + let bytes = decode_hash_tree(name, Some(b64_value.to_string()), logger); + if name == "certificate" { + headers_data.certificate = Some(match (headers_data.certificate, bytes) { + (None, bytes) => bytes, + (Some(Ok(certificate)), Ok(bytes)) => { + slog::warn!(logger, "duplicate certificate field: {:?}", bytes); + Ok(certificate) + } + (Some(Ok(certificate)), Err(_)) => { + slog::warn!( + logger, + "duplicate certificate field (failed to decode)" + ); + Ok(certificate) + } + (Some(Err(_)), bytes) => { + slog::warn!( + logger, + "duplicate certificate field (failed to decode)" + ); + bytes + } + }); + } else if name == "tree" { + headers_data.tree = Some(match (headers_data.tree, bytes) { + (None, bytes) => bytes, + (Some(Ok(tree)), Ok(bytes)) => { + slog::warn!(logger, "duplicate tree field: {:?}", bytes); + Ok(tree) + } + (Some(Ok(tree)), Err(_)) => { + slog::warn!(logger, "duplicate tree field (failed to decode)"); + Ok(tree) + } + (Some(Err(_)), bytes) => { + slog::warn!(logger, "duplicate tree field (failed to decode)"); + bytes + } + }); + } + } + } + } else if name.eq_ignore_ascii_case("CONTENT-ENCODING") { + let enc = value.trim().to_string(); + headers_data.encoding = Some(enc); + } + } + + headers_data +} + async fn forward_request( request: Request, agent: Arc, @@ -280,76 +367,20 @@ async fn forward_request( http_response }; - let mut certificate: Option, ()>> = None; - let mut tree: Option, ()>> = None; - let mut builder = Response::builder().status(StatusCode::from_u16(http_response.status_code)?); - for HeaderField(name, value) in http_response.headers { - if name.eq_ignore_ascii_case("IC-CERTIFICATE") { - for field in value.split(',') { - if let Some((_, name, b64_value)) = regex_captures!("^(.*)=:(.*):$", field.trim()) { - slog::trace!(logger, ">> certificate {}: {}", name, b64_value); - let bytes = base64::decode(b64_value).map_err(|e| { - slog::warn!( - logger, - "Unable to decode {} in ic-certificate from base64: {}", - name, - e - ); - }); - if name == "certificate" { - certificate = Some(match (certificate, bytes) { - (None, bytes) => bytes, - (Some(Ok(certificate)), Ok(bytes)) => { - slog::warn!(logger, "duplicate certificate field: {:?}", bytes); - Ok(certificate) - } - (Some(Ok(certificate)), Err(_)) => { - slog::warn!( - logger, - "duplicate certificate field (failed to decode)" - ); - Ok(certificate) - } - (Some(Err(_)), bytes) => { - slog::warn!( - logger, - "duplicate certificate field (failed to decode)" - ); - bytes - } - }); - } else if name == "tree" { - tree = Some(match (tree, bytes) { - (None, bytes) => bytes, - (Some(Ok(tree)), Ok(bytes)) => { - slog::warn!(logger, "duplicate tree field: {:?}", bytes); - Ok(tree) - } - (Some(Ok(tree)), Err(_)) => { - slog::warn!(logger, "duplicate tree field (failed to decode)"); - Ok(tree) - } - (Some(Err(_)), bytes) => { - slog::warn!(logger, "duplicate tree field (failed to decode)"); - bytes - } - }); - } - } - } - } - - builder = builder.header(&name, value); + for HeaderField(name, value) in &http_response.headers { + builder = builder.header(name, value); } + let headers_data = extract_headers_data(&http_response.headers, &logger); let body = if logger.is_trace_enabled() { Some(http_response.body.clone()) } else { None }; let is_streaming = http_response.streaming_strategy.is_some(); - let response = if let Some(streaming_strategy) = http_response.streaming_strategy { + let response = if is_streaming { + let streaming_strategy = http_response.streaming_strategy.unwrap(); let (mut sender, body) = body::Body::channel(); let agent = agent.as_ref().clone(); sender.send_data(Bytes::from(http_response.body)).await?; @@ -400,33 +431,18 @@ async fn forward_request( builder.body(body)? } else { - let body_valid = match (certificate, tree) { - (Some(Ok(certificate)), Some(Ok(tree))) => match validate_body( - &certificate, - &tree, - &canister_id, - &agent, - &uri, - &http_response.body, - logger.clone(), - ) { - Ok(valid) => valid, - Err(e) => { - return Ok(Response::builder() - .status(StatusCode::INTERNAL_SERVER_ERROR) - .body(format!("Certificate validation failed: {}", e).into()) - .unwrap()); - } - }, - (Some(_), _) | (_, Some(_)) => false, - // Canisters don't have to provide certified variables - (None, None) => true, - }; - - if !body_valid && !cfg!(feature = "skip_body_verification") { + let body_valid = validate( + &headers_data, + &canister_id, + &agent, + &uri, + &http_response.body, + logger.clone(), + ); + if body_valid.is_err() { return Ok(Response::builder() .status(StatusCode::INTERNAL_SERVER_ERROR) - .body("Body does not pass verification".into()) + .body(body_valid.unwrap_err().into()) .unwrap()); } builder.body(http_response.body.into())? @@ -467,18 +483,91 @@ async fn forward_request( Ok(response) } -fn validate_body( - certificate: &[u8], - tree: &[u8], +fn validate( + headers_data: &HeadersData, canister_id: &Principal, agent: &Agent, uri: &Uri, response_body: &[u8], logger: slog::Logger, +) -> Result<(), String> { + let body_sha = decode_body(response_body, headers_data.encoding.clone()); + let body_valid = match (headers_data.certificate.clone(), headers_data.tree.clone()) { + (Some(Ok(certificate)), Some(Ok(tree))) => match validate_body( + Certificates { certificate, tree }, + canister_id, + agent, + uri, + &body_sha, + logger.clone(), + ) { + Ok(valid) => { + if valid { + Ok(()) + } else { + Err("Body does not pass verification".to_string()) + } + } + Err(e) => Err(format!("Certificate validation failed: {}", e)), + }, + (Some(_), _) | (_, Some(_)) => Err("Body does not pass verification".to_string()), + // Canisters don't have to provide certified variables + (None, None) => Ok(()), + }; + + if body_valid.is_err() && !cfg!(feature = "skip_body_verification") { + return body_valid; + } + + Ok(()) +} + +fn decode_body(body: &[u8], encoding: Option) -> [u8; 32] { + let mut sha256 = Sha256::new(); + match encoding { + Some(enc) => match enc.as_str() { + "gzip" => { + let decoded: &mut Vec = &mut vec![]; + let decoder = GzDecoder::new(body); + decoder + .take(MAX_BYTES_SIZE_TO_DECOMPRESS) + .read_to_end(decoded) + .unwrap(); + sha256.update(decoded); + } + "deflate" => { + let decoded: &mut Vec = &mut vec![]; + let decoder = DeflateDecoder::new(body); + decoder + .take(MAX_BYTES_SIZE_TO_DECOMPRESS) + .read_to_end(decoded) + .unwrap(); + sha256.update(decoded); + } + _ => sha256.update(body), + }, + _ => sha256.update(body), + }; + sha256.finalize().into() +} + +struct Certificates { + certificate: Vec, + tree: Vec, +} + +fn validate_body( + certificates: Certificates, + canister_id: &Principal, + agent: &Agent, + uri: &Uri, + body_sha: &[u8; 32], + logger: slog::Logger, ) -> anyhow::Result { let cert: Certificate = - serde_cbor::from_slice(certificate).map_err(AgentError::InvalidCborData)?; - let tree: HashTree = serde_cbor::from_slice(tree).map_err(AgentError::InvalidCborData)?; + serde_cbor::from_slice(&certificates.certificate).map_err(AgentError::InvalidCborData)?; + let tree: HashTree = + serde_cbor::from_slice(&certificates.tree).map_err(AgentError::InvalidCborData)?; if let Err(e) = agent.verify(&cert) { slog::trace!(logger, ">> certificate failed verification: {}", e); @@ -530,11 +619,7 @@ fn validate_body( }, }; - let mut sha256 = Sha256::new(); - sha256.update(response_body); - let body_sha = sha256.finalize(); - - Ok(&body_sha[..] == tree_sha) + Ok(body_sha == tree_sha) } fn is_hop_header(name: &str) -> bool { From e55fca22a78f53e4475f2b0a9c9b9e067ed027e4 Mon Sep 17 00:00:00 2001 From: 3cL1p5e7 <3cL1p5e7@gmail.com> Date: Fri, 11 Mar 2022 17:07:27 +0300 Subject: [PATCH 8/8] chunk_tree to Vec --- src/main.rs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/main.rs b/src/main.rs index 9637cb2..0ff6f91 100644 --- a/src/main.rs +++ b/src/main.rs @@ -455,13 +455,11 @@ async fn forward_request( token, chunk_tree, },)) => { - let decoded_chunk_tree = - decode_hash_tree("chunk_tree", chunk_tree, &logger); let chunk_headers_data = HeadersData { certificate: headers_data.certificate.clone(), tree: headers_data.tree.clone(), encoding: headers_data.encoding.clone(), - chunk_tree: decoded_chunk_tree.ok(), + chunk_tree, chunk_index: chunk_index.clone(), }; let body_valid = validate(