Update blake2 to 0.10

Summary:
This updates blake2 to 0.10, as well as migrates all existing users.

This is a non-trivial diff and additional verification is appreciated.

Note that in 0.10, blake2 types were split into the MAC and non MAC variants. Migration for non-keying hash usage, such as in `zippydb`, has minimal migration steps.

For keyed hash usage (AKA using blake2's MAC functionality), migration is a little less clear. Since hashes can be used to lookup stable items, a change in hash output may lead to lost items. D49973500 was introduced to add snapshot tests to ensure that the hash value is consistent after an update.

The most notable change is that mononoke_types seems to run into a footgun with new_keyed -- blake2 0.9 does not pad an empty key to the correct size, while blake2 0.10 does. This due to the fact that in blake2 0.9, an empty key was interpreted to use the hash functionality of blake2 while a non-empty key was interpreted to use the MAC properties. This requires a change in tests.

Reviewed By: jsgf

Differential Revision: D49649655

fbshipit-source-id: 9a13e005ac998d068df8154de627e31b7fdf615e

eden/mononoke/mononoke_types/Cargo.toml

@@ -16,7 +16,7 @@ async-recursion = "1.0.2"
 async-trait = "0.1.71"
 bincode = "1.3.3"
 bit-set = "0.5"
-blake2 = "0.9"
+blake2 = "0.10"
 blobstore = { version = "0.1.0", path = "../blobstore" }
 borrowed = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
 bounded_traversal = { version = "0.1.0", path = "../common/bounded_traversal" }

eden/mononoke/mononoke_types/src/hash.rs

@@ -8,7 +8,6 @@
 use std::fmt;
 use std::fmt::Debug;
 use std::fmt::Display;
-use std::io::Write;
 use std::str::FromStr;
 
 use abomonation_derive::Abomonation;
@@ -17,9 +16,13 @@ use anyhow::Error;
 use anyhow::Result;
 use ascii::AsciiStr;
 use ascii::AsciiString;
+use blake2::digest::typenum::U32;
+use blake2::digest::Digest;
+use blake2::digest::FixedOutput;
+use blake2::digest::KeyInit;
 use blake2::digest::Update;
-use blake2::digest::VariableOutput;
-use blake2::VarBlake2b;
+use blake2::Blake2b;
+use blake2::Blake2bMac;
 use edenapi_types::Blake3 as EdenapiBlake3;
 use edenapi_types::CommitId as EdenapiCommitId;
 use edenapi_types::GitSha1 as EdenapiGitSha1;
@@ -147,35 +150,41 @@ impl Blake2 {
 
 /// Context for incrementally computing a `Blake2` hash.
 #[derive(Clone)]
-pub struct Context(VarBlake2b);
+pub enum Context {
+    Unkeyed(Blake2b<U32>),
+    Keyed(Blake2bMac<U32>),
+}
 
 impl Context {
     /// Construct a `Context`
     #[inline]
     pub fn new(key: &[u8]) -> Self {
-        Context(VarBlake2b::new_keyed(key, BLAKE2_HASH_LENGTH_BYTES))
+        if key.is_empty() {
+            Self::Unkeyed(Blake2b::new())
+        } else {
+            Self::Keyed(
+                Blake2bMac::new_from_slice(key).expect("key should not be bigger than block size"),
+            )
+        }
     }
 
     #[inline]
     pub fn update<T>(&mut self, data: T)
     where
         T: AsRef<[u8]>,
     {
-        self.0.update(data.as_ref())
+        match self {
+            Self::Unkeyed(b2) => Digest::update(b2, data.as_ref()),
+            Self::Keyed(b2) => b2.update(data.as_ref()),
+        }
     }
 
     #[inline]
     pub fn finish(self) -> Blake2 {
-        let mut ret = [0u8; BLAKE2_HASH_LENGTH_BYTES];
-        self.0.finalize_variable(|res| {
-            if let Err(e) = ret.as_mut().write_all(res) {
-                panic!(
-                    "{}-byte array must work with {}-byte blake2b: {:?}",
-                    BLAKE2_HASH_LENGTH_BYTES, BLAKE2_HASH_LENGTH_BYTES, e
-                );
-            }
-        });
-        Blake2(ret)
+        match self {
+            Self::Unkeyed(b2) => Blake2(b2.finalize_fixed().into()),
+            Self::Keyed(b2) => Blake2(b2.finalize_fixed().into()),
+        }
     }
 }
 

eden/mononoke/streaming_clone/Cargo.toml

@@ -9,7 +9,7 @@ license = "GPLv2+"
 
 [dependencies]
 anyhow = "=1.0.72"
-blake2 = "0.9"
+blake2 = "0.10"
 blobstore = { version = "0.1.0", path = "../blobstore" }
 borrowed = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
 clap = { version = "4.3.5", features = ["derive", "env", "string", "unicode", "wrap_help"] }

eden/mononoke/streaming_clone/src/main.rs

@@ -13,7 +13,7 @@ use std::path::PathBuf;
 use anyhow::anyhow;
 use anyhow::Error;
 use anyhow::Result;
-use blake2::Blake2b;
+use blake2::Blake2b512;
 use blake2::Digest;
 use blobstore::Blobstore;
 use blobstore::BlobstoreBytes;
@@ -423,7 +423,7 @@ async fn upload_data(
 }
 
 fn generate_key(chunk_id: u32, data: &[u8], suffix: &str) -> String {
-    let hash = Blake2b::digest(data);
+    let hash = Blake2b512::digest(data);
 
     format!("streaming_clone-chunk{:06}-{:x}-{}", chunk_id, hash, suffix,)
 }

eden/scm/lib/edenapi/ext/Cargo.toml

@@ -7,7 +7,7 @@ edition = "2021"
 
 [dependencies]
 anyhow = "=1.0.72"
-blake2 = "0.9"
+blake2 = "0.10"
 cloned = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
 crossbeam = "0.8"
 edenapi = { version = "0.1.0", path = ".." }

eden/scm/lib/edenapi/ext/src/util.rs

@@ -5,28 +5,17 @@
  * GNU General Public License version 2.
  */
 
-use std::io::Write;
-
-use blake2::digest::Update;
-use blake2::digest::VariableOutput;
-use blake2::VarBlake2b;
+use blake2::digest::FixedOutput;
+use blake2::digest::Mac;
+use blake2::Blake2bMac;
 use edenapi_types::ContentId;
 
 pub fn calc_contentid(data: &[u8]) -> ContentId {
-    let mut hash = VarBlake2b::new_keyed(b"content", ContentId::len());
+    let mut hash = Blake2bMac::new_from_slice(b"content").expect("key to be less than 32 bytes");
     hash.update(data);
-    let mut ret = [0u8; ContentId::len()];
-    hash.finalize_variable(|res| {
-        if let Err(e) = ret.as_mut().write_all(res) {
-            panic!(
-                "{}-byte array must work with {}-byte blake2b: {:?}",
-                ContentId::len(),
-                ContentId::len(),
-                e
-            );
-        }
-    });
-    ContentId::from(ret)
+    let mut ret = [0; ContentId::len()];
+    hash.finalize_into((&mut ret).into());
+    ContentId::from_byte_array(ret)
 }
 
 #[cfg(test)]

eden/scm/lib/revisionstore/Cargo.toml

@@ -9,7 +9,7 @@ edition = "2021"
 anyhow = "=1.0.72"
 async-runtime = { version = "0.1.0", path = "../async-runtime" }
 async-trait = "0.1.71"
-blake2 = "0.9"
+blake2 = "0.10"
 blake3 = { version = "1.2", features = ["rayon"] }
 byteorder = "1.3"
 clientinfo = { version = "0.1.0", path = "../clientinfo" }

eden/scm/lib/revisionstore/src/types.rs

@@ -5,8 +5,6 @@
  * GNU General Public License version 2.
  */
 
-use std::io::Write;
-
 use edenapi_types::Blake3;
 use edenapi_types::ContentId;
 use edenapi_types::Sha1;
@@ -57,25 +55,16 @@ impl ContentHash {
     }
 
     pub(crate) fn content_id(data: &Bytes) -> ContentId {
-        use blake2::digest::Update;
-        use blake2::digest::VariableOutput;
-        use blake2::VarBlake2b;
+        use blake2::digest::FixedOutput;
+        use blake2::digest::Mac;
+        use blake2::Blake2bMac;
 
-        // cribbed from pyedenapi
-        let mut hash = VarBlake2b::new_keyed(b"content", ContentId::len());
+        let mut hash =
+            Blake2bMac::new_from_slice(b"content").expect("key to be less than 32 bytes");
         hash.update(data);
-        let mut ret = [0u8; ContentId::len()];
-        hash.finalize_variable(|res| {
-            if let Err(e) = ret.as_mut().write_all(res) {
-                panic!(
-                    "{}-byte array must work with {}-byte blake2b: {:?}",
-                    ContentId::len(),
-                    ContentId::len(),
-                    e
-                );
-            }
-        });
-        ContentId::from(ret)
+        let mut ret = [0; ContentId::len()];
+        hash.finalize_into((&mut ret).into());
+        ContentId::from_byte_array(ret)
     }
 
     pub(crate) fn sha1(data: &Bytes) -> Sha1 {