From 88a2bb5711f7eea9f2d4d5334029e05720fb27c3 Mon Sep 17 00:00:00 2001 From: Luisa Vasquez Gomez Date: Thu, 12 Dec 2024 12:59:26 -0800 Subject: [PATCH] modern sync: upload hg changeset Summary: This is not a final version, this is simply to make sure all elements of the filenodes are conserved through edenapi. This is missing two thing: 1. Correct Parents expression (mentioned by Mark in D67034907) 2. Stream/ pipe through channel so we don't keep everything in memory But hey, it's a god POC, putting up for a review in case something is terribly wrong, otherwise I can address 1 and 2 in separate diff (will decide after chat about the parents issue) This also adds a condition so we don't try to add hg mutations if none are sent. With this we can now make sure all relevant data has been sent to the server and we can start optimising :) Reviewed By: clara-9 Differential Revision: D67123774 fbshipit-source-id: dd042bdb45d49e5c4987a974a6d81c75042c2633 --- eden/mononoke/edenapi_service/src/lib.rs | 2 +- eden/mononoke/modern_sync/Cargo.toml | 1 + eden/mononoke/modern_sync/src/sender.rs | 5 +- eden/mononoke/modern_sync/src/sender/dummy.rs | 12 +++++ .../modern_sync/src/sender/edenapi.rs | 53 +++++++++++++++++++ eden/mononoke/modern_sync/src/sync.rs | 5 +- eden/mononoke/mononoke_api_hg/src/repo.rs | 13 +++-- .../modern_sync/test-modern-sync.t | 15 ++++-- .../integration/modern_sync/test-sync-one.t | 9 +++- 9 files changed, 98 insertions(+), 17 deletions(-) diff --git a/eden/mononoke/edenapi_service/src/lib.rs b/eden/mononoke/edenapi_service/src/lib.rs index 15c20b4933a08..3a9e88175d548 100644 --- a/eden/mononoke/edenapi_service/src/lib.rs +++ b/eden/mononoke/edenapi_service/src/lib.rs @@ -13,7 +13,7 @@ mod errors; mod handlers; mod middleware; mod scuba; -mod utils; +pub mod utils; use std::path::Path; use std::sync::atomic::AtomicBool; diff --git a/eden/mononoke/modern_sync/Cargo.toml b/eden/mononoke/modern_sync/Cargo.toml index 7f4a57771477e..5f6249a5603d7 100644 --- a/eden/mononoke/modern_sync/Cargo.toml +++ b/eden/mononoke/modern_sync/Cargo.toml @@ -19,6 +19,7 @@ clap = { version = "4.5.20", features = ["derive", "env", "string", "unicode", " cloned = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" } commit_graph = { version = "0.1.0", path = "../repo_attributes/commit_graph/commit_graph" } context = { version = "0.1.0", path = "../server/context" } +edenapi_service = { version = "0.1.0", path = "../edenapi_service" } executor_lib = { version = "0.1.0", path = "../cmdlib/sharding" } facet = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" } fbinit = { version = "0.2.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" } diff --git a/eden/mononoke/modern_sync/src/sender.rs b/eden/mononoke/modern_sync/src/sender.rs index 90bfdd11eb06e..547d8a9598e05 100644 --- a/eden/mononoke/modern_sync/src/sender.rs +++ b/eden/mononoke/modern_sync/src/sender.rs @@ -7,6 +7,7 @@ use anyhow::Result; use async_trait::async_trait; +use mercurial_types::blobs::HgBlobChangeset; use mercurial_types::HgFileNodeId; use mercurial_types::HgManifestId; use mononoke_types::ContentId; @@ -18,9 +19,9 @@ pub mod edenapi; pub trait ModernSyncSender { async fn upload_content(&self, content_id: ContentId, _blob: FileContents) -> Result<()>; - #[allow(unused)] async fn upload_trees(&self, trees: Vec) -> Result<()>; - #[allow(unused)] async fn upload_filenodes(&self, filenodes: Vec) -> Result<()>; + + async fn upload_hg_changeset(&self, hg_css: Vec) -> Result<()>; } diff --git a/eden/mononoke/modern_sync/src/sender/dummy.rs b/eden/mononoke/modern_sync/src/sender/dummy.rs index 5e067d6e99769..9ed410f87e7f9 100644 --- a/eden/mononoke/modern_sync/src/sender/dummy.rs +++ b/eden/mononoke/modern_sync/src/sender/dummy.rs @@ -7,6 +7,7 @@ use anyhow::Result; use async_trait::async_trait; +use mercurial_types::blobs::HgBlobChangeset; use mercurial_types::HgFileNodeId; use mercurial_types::HgManifestId; use mononoke_types::ContentId; @@ -47,4 +48,15 @@ impl ModernSyncSender for DummySender { } Ok(()) } + + async fn upload_hg_changeset(&self, hg_css: Vec) -> Result<()> { + for hg_cs in hg_css { + info!( + &self.logger, + "Uploading hg changeset with id {}", + hg_cs.get_changeset_id() + ); + } + Ok(()) + } } diff --git a/eden/mononoke/modern_sync/src/sender/edenapi.rs b/eden/mononoke/modern_sync/src/sender/edenapi.rs index d25f0f1b71c66..0ee9b7ff8c4de 100644 --- a/eden/mononoke/modern_sync/src/sender/edenapi.rs +++ b/eden/mononoke/modern_sync/src/sender/edenapi.rs @@ -19,13 +19,18 @@ use edenapi::HttpClientConfig; use edenapi::SaplingRemoteApi; use edenapi_types::AnyFileContentId; use edenapi_types::AnyId; +use edenapi_types::Extra; +use edenapi_types::HgChangesetContent; use edenapi_types::HgFilenodeData; use edenapi_types::Parents; +use edenapi_types::RepoPathBuf; +use edenapi_types::UploadHgChangeset; use edenapi_types::UploadToken; use edenapi_types::UploadTreeEntry; use futures::stream; use futures::StreamExt; use futures::TryStreamExt; +use mercurial_types::blobs::HgBlobChangeset; use mercurial_types::fetch_manifest_envelope; use mercurial_types::HgFileNodeId; use mercurial_types::HgManifestId; @@ -161,6 +166,21 @@ impl ModernSyncSender for EdenapiSender { ); Ok(()) } + + async fn upload_hg_changeset(&self, hg_css: Vec) -> Result<()> { + let entries = stream::iter(hg_css) + .map(to_upload_hg_changeset) + .try_collect::>() + .await?; + + let res = self.client.upload_changesets(entries, vec![]).await?; + info!( + &self.logger, + "Upload hg changeset response: {:?}", + res.entries.try_collect::>().await? + ); + Ok(()) + } } pub async fn from_tree_to_entry( @@ -207,3 +227,36 @@ pub async fn from_id_to_filenode( file_content_upload_token: token, }) } + +pub fn to_upload_hg_changeset(hg_cs: HgBlobChangeset) -> Result { + let extra = hg_cs + .extra() + .iter() + .map(|(k, v)| Extra { + key: k.to_vec(), + value: v.to_vec(), + }) + .collect(); + + let hg_files: Result> = hg_cs + .files() + .iter() + .map(edenapi_service::utils::to_hg_path) + .collect(); + + let hg_content = HgChangesetContent { + parents: hg_cs.parents().into(), + manifestid: hg_cs.manifestid().into_nodehash().into(), + user: hg_cs.user().to_vec(), + time: hg_cs.time().timestamp_secs(), + tz: hg_cs.time().tz_offset_secs(), + extras: extra, + files: hg_files?, + message: hg_cs.message().to_vec(), + }; + + Ok(UploadHgChangeset { + node_id: hg_cs.get_changeset_id().into_nodehash().into(), + changeset_content: hg_content, + }) +} diff --git a/eden/mononoke/modern_sync/src/sync.rs b/eden/mononoke/modern_sync/src/sync.rs index 3012d95073334..ce966b2944292 100644 --- a/eden/mononoke/modern_sync/src/sync.rs +++ b/eden/mononoke/modern_sync/src/sync.rs @@ -261,11 +261,12 @@ pub async fn process_one_changeset( let hg_cs = hg_cs_id.load(ctx, repo.repo_blobstore()).await?; let hg_mf_id = hg_cs.manifestid(); - let (mf_ids, file_ids) = + let (mut mf_ids, file_ids) = sort_manifest_changes(ctx, repo.repo_blobstore(), hg_mf_id, mf_ids_p).await?; - + mf_ids.push(hg_mf_id); sender.upload_trees(mf_ids).await?; sender.upload_filenodes(file_ids).await?; + sender.upload_hg_changeset(vec![hg_cs]).await?; if log_completion { STATS::synced_commits.add_value(1, (repo.repo_identity().name().to_string(),)); diff --git a/eden/mononoke/mononoke_api_hg/src/repo.rs b/eden/mononoke/mononoke_api_hg/src/repo.rs index 3b9778eb48b43..3d0bb44cb9100 100644 --- a/eden/mononoke/mononoke_api_hg/src/repo.rs +++ b/eden/mononoke/mononoke_api_hg/src/repo.rs @@ -414,11 +414,14 @@ impl HgRepoContext { results.push(result); } log_new_commits(self.ctx(), self.repo_ctx().repo(), None, commits_to_log).await; - self.repo() - .hg_mutation_store() - .add_entries(self.ctx(), hg_changesets, mutations) - .await - .map_err(MononokeError::from)?; + + if !mutations.is_empty() { + self.repo() + .hg_mutation_store() + .add_entries(self.ctx(), hg_changesets, mutations) + .await + .map_err(MononokeError::from)?; + } Ok(results) } diff --git a/eden/mononoke/tests/integration/modern_sync/test-modern-sync.t b/eden/mononoke/tests/integration/modern_sync/test-modern-sync.t index 2ef671675374a..9228b4bbd1f53 100644 --- a/eden/mononoke/tests/integration/modern_sync/test-modern-sync.t +++ b/eden/mononoke/tests/integration/modern_sync/test-modern-sync.t @@ -70,8 +70,9 @@ Sync all bookmarks moves Uploading content with id: ContentId(Blake2(be87911855af0fc33a75f2c1cba2269dd90faa7f5c5358eb640d9d65f55fced3)) Uploading bytes: b"abc\n" Upload response: [UploadToken { data: UploadTokenData { id: AnyFileContentId(ContentId(ContentId("be87911855af0fc33a75f2c1cba2269dd90faa7f5c5358eb640d9d65f55fced3"))), bubble_id: None, metadata: Some(FileContentTokenMetadata(FileContentTokenMetadata { content_size: 4 })) }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } }] - Upload tree response: [UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("c1afe800646ee45232ab5e70c57247b78dbf3899")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("53b19c5f23977836390e5880ec30fd252a311384")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] + Upload tree response: [UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("c1afe800646ee45232ab5e70c57247b78dbf3899")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("53b19c5f23977836390e5880ec30fd252a311384")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("dbdaef03fd04c4a28dc29fb3fbe10c5ed7a090ec")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] Upload filenodes response: [UploadTokensResponse { token: UploadToken { data: UploadTokenData { id: HgFilenodeId(HgId("005d992c5dcf32993668f7cede29d296c494a5d9")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTokensResponse { token: UploadToken { data: UploadTokenData { id: HgFilenodeId(HgId("f9304d84edb8a8ee2d3ce3f9de3ea944c82eba8f")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] + Upload hg changeset response: [UploadTokensResponse { token: UploadToken { data: UploadTokenData { id: HgChangesetId(HgId("e20237022b1290d98c3f14049931a8f498c18c53")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] Found commit ChangesetId(Blake2(5b1c7130dde8e54b4285b9153d8e56d69fbf4ae685eaf9e9766cc409861995f8)) Commit info ChangesetInfo { changeset_id: ChangesetId(Blake2(5b1c7130dde8e54b4285b9153d8e56d69fbf4ae685eaf9e9766cc409861995f8)), parents: [ChangesetId(Blake2(ba1a2b3ca64cead35117cb2b707da1211cf43639ade917aee655f3875f4922c3))], author: "test", author_date: DateTime(1970-01-01T00:00:00+00:00), committer: None, committer_date: None, message: Message("E"), hg_extra: {}, git_extra_headers: None } File change Change(TrackedFileChange { inner: BasicFileChange { content_id: ContentId(Blake2(1b1e26f01a806e123b37492672d2756e1c25bb31f1e15cfda410c149c317e130)), file_type: Regular, size: 1, git_lfs: FullContent }, copy_from: None }) @@ -84,8 +85,9 @@ Sync all bookmarks moves Uploading content with id: ContentId(Blake2(96475ef07b63bf02679e9964ff65f0f96883f53d0718671bd44cce830bbf2ebd)) Uploading bytes: b"abcdefg\n" Upload response: [UploadToken { data: UploadTokenData { id: AnyFileContentId(ContentId(ContentId("96475ef07b63bf02679e9964ff65f0f96883f53d0718671bd44cce830bbf2ebd"))), bubble_id: None, metadata: Some(FileContentTokenMetadata(FileContentTokenMetadata { content_size: 8 })) }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } }] - Upload tree response: [UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("5e3e3ee682cdb8a61b7537cfc1a821b6283c8bb5")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("33ac88b3b11b11c3fd33fe71cec4c8852ba2eeef")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] + Upload tree response: [UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("5e3e3ee682cdb8a61b7537cfc1a821b6283c8bb5")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("33ac88b3b11b11c3fd33fe71cec4c8852ba2eeef")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("d6c162600b768f8478ea0557302b6027ed43105d")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] Upload filenodes response: [UploadTokensResponse { token: UploadToken { data: UploadTokenData { id: HgFilenodeId(HgId("dba92ad67dc1f3732ab73a5f51b77129275a1724")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTokensResponse { token: UploadToken { data: UploadTokenData { id: HgFilenodeId(HgId("b31c6c30a54b89020d5ac28a67917349512d75eb")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] + Upload hg changeset response: [] Found commit ChangesetId(Blake2(ba1a2b3ca64cead35117cb2b707da1211cf43639ade917aee655f3875f4922c3)) Commit info ChangesetInfo { changeset_id: ChangesetId(Blake2(ba1a2b3ca64cead35117cb2b707da1211cf43639ade917aee655f3875f4922c3)), parents: [ChangesetId(Blake2(41deea4804cd27d1f4efbec135d839338804a5dfcaf364863bd0289067644db5))], author: "test", author_date: DateTime(1970-01-01T00:00:00+00:00), committer: None, committer_date: None, message: Message("D"), hg_extra: {}, git_extra_headers: None } File change Change(TrackedFileChange { inner: BasicFileChange { content_id: ContentId(Blake2(90c8e211c758a9bbcd33e463c174f1693692677cb76c7aaf4ce41aa0a29334c0)), file_type: Regular, size: 1, git_lfs: FullContent }, copy_from: None }) @@ -98,8 +100,9 @@ Sync all bookmarks moves Uploading content with id: ContentId(Blake2(5d3bfab620332130430c7f540f9fe0b3b0079d0b9b632e0dae96a1424a7a4242)) Uploading bytes: b"abcdef\n" Upload response: [UploadToken { data: UploadTokenData { id: AnyFileContentId(ContentId(ContentId("5d3bfab620332130430c7f540f9fe0b3b0079d0b9b632e0dae96a1424a7a4242"))), bubble_id: None, metadata: Some(FileContentTokenMetadata(FileContentTokenMetadata { content_size: 7 })) }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } }] - Upload tree response: [UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("553b84eb92dd53cf5d757e536be1b42e46458017")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("fd1a9570853c1a068efbf6175c547a554015f850")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] + Upload tree response: [UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("553b84eb92dd53cf5d757e536be1b42e46458017")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("fd1a9570853c1a068efbf6175c547a554015f850")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("f0c9107712ba849f0bc585147093c0776cca8247")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] Upload filenodes response: [UploadTokensResponse { token: UploadToken { data: UploadTokenData { id: HgFilenodeId(HgId("4eec8cfdabce9565739489483b6ad93ef7657ea9")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTokensResponse { token: UploadToken { data: UploadTokenData { id: HgFilenodeId(HgId("aae2838d921bcc14ccbb9212f4175f300fd9f2f8")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] + Upload hg changeset response: [] Found commit ChangesetId(Blake2(41deea4804cd27d1f4efbec135d839338804a5dfcaf364863bd0289067644db5)) Commit info ChangesetInfo { changeset_id: ChangesetId(Blake2(41deea4804cd27d1f4efbec135d839338804a5dfcaf364863bd0289067644db5)), parents: [ChangesetId(Blake2(8a9d572a899acdef764b88671c24b94a8b0780c1591a5a9bca97184c2ef0f304))], author: "test", author_date: DateTime(1970-01-01T00:00:00+00:00), committer: None, committer_date: None, message: Message("C"), hg_extra: {}, git_extra_headers: None } File change Change(TrackedFileChange { inner: BasicFileChange { content_id: ContentId(Blake2(896ad5879a5df0403bfc93fc96507ad9c93b31b11f3d0fa05445da7918241e5d)), file_type: Regular, size: 1, git_lfs: FullContent }, copy_from: None }) @@ -112,8 +115,9 @@ Sync all bookmarks moves Uploading content with id: ContentId(Blake2(c86e7a7ee4c102efc1e5166dd95c1c73fcbff59dc3b04dc79fbbf3d1d10350ed)) Uploading bytes: b"abcde\n" Upload response: [UploadToken { data: UploadTokenData { id: AnyFileContentId(ContentId(ContentId("c86e7a7ee4c102efc1e5166dd95c1c73fcbff59dc3b04dc79fbbf3d1d10350ed"))), bubble_id: None, metadata: Some(FileContentTokenMetadata(FileContentTokenMetadata { content_size: 6 })) }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } }] - Upload tree response: [UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("144ae30be86d40d8a0617b7ec37a70e618df4840")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("3b6d87c4e93a918020513a57279573f4325109ef")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] + Upload tree response: [UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("144ae30be86d40d8a0617b7ec37a70e618df4840")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("3b6d87c4e93a918020513a57279573f4325109ef")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("b6c10150f12d2cc4c2d3e6bef65767962032e49f")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] Upload filenodes response: [UploadTokensResponse { token: UploadToken { data: UploadTokenData { id: HgFilenodeId(HgId("a2e456504a5e61f763f1a0b36a6c247c7541b2b3")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTokensResponse { token: UploadToken { data: UploadTokenData { id: HgFilenodeId(HgId("9bad1c227e9133a5bbae1652c889406d35e6dac1")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] + Upload hg changeset response: [] Found commit ChangesetId(Blake2(8a9d572a899acdef764b88671c24b94a8b0780c1591a5a9bca97184c2ef0f304)) Commit info ChangesetInfo { changeset_id: ChangesetId(Blake2(8a9d572a899acdef764b88671c24b94a8b0780c1591a5a9bca97184c2ef0f304)), parents: [ChangesetId(Blake2(53b034a90fe3002a707a7da9cdf6eac3dea460ad72f7c6969dfb88fd0e69f856))], author: "test", author_date: DateTime(1970-01-01T00:00:00+00:00), committer: None, committer_date: None, message: Message("B"), hg_extra: {}, git_extra_headers: None } File change Change(TrackedFileChange { inner: BasicFileChange { content_id: ContentId(Blake2(55662471e2a28db8257939b2f9a2d24e65b46a758bac12914a58f17dcde6905f)), file_type: Regular, size: 1, git_lfs: FullContent }, copy_from: None }) @@ -126,8 +130,9 @@ Sync all bookmarks moves Uploading content with id: ContentId(Blake2(fbc4b9b407225e86008840c4095edb4f66a62bad80529b6e120bfa7d605f9423)) Uploading bytes: b"abcd\n" Upload response: [UploadToken { data: UploadTokenData { id: AnyFileContentId(ContentId(ContentId("fbc4b9b407225e86008840c4095edb4f66a62bad80529b6e120bfa7d605f9423"))), bubble_id: None, metadata: Some(FileContentTokenMetadata(FileContentTokenMetadata { content_size: 5 })) }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } }] - Upload tree response: [UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("83af7e770afc39d483b9cd198c49fe919ef0461a")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("0652870aff7b4cb5e2172325519652378ae063e7")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] + Upload tree response: [UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("83af7e770afc39d483b9cd198c49fe919ef0461a")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("0652870aff7b4cb5e2172325519652378ae063e7")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("d7134cdd147e3dabe516366caee43b4622260069")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] Upload filenodes response: [UploadTokensResponse { token: UploadToken { data: UploadTokenData { id: HgFilenodeId(HgId("35e7525ce3a48913275d7061dd9a867ffef1e34d")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTokensResponse { token: UploadToken { data: UploadTokenData { id: HgFilenodeId(HgId("778675f9ec8d35ff2fce23a34f68edd15d783853")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] + Upload hg changeset response: [UploadTokensResponse { token: UploadToken { data: UploadTokenData { id: HgChangesetId(HgId("5a95ef0f59a992dcb5385649217862599de05565")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] $ mononoke_admin mutable-counters --repo-name orig get modern_sync Some(2) diff --git a/eden/mononoke/tests/integration/modern_sync/test-sync-one.t b/eden/mononoke/tests/integration/modern_sync/test-sync-one.t index 8b00dc90bf317..3a82da06fcf62 100644 --- a/eden/mononoke/tests/integration/modern_sync/test-sync-one.t +++ b/eden/mononoke/tests/integration/modern_sync/test-sync-one.t @@ -69,8 +69,9 @@ Uploading content with id: ContentId(Blake2(be87911855af0fc33a75f2c1cba2269dd90faa7f5c5358eb640d9d65f55fced3)) Uploading bytes: b"abc\n" Upload response: [UploadToken { data: UploadTokenData { id: AnyFileContentId(ContentId(ContentId("be87911855af0fc33a75f2c1cba2269dd90faa7f5c5358eb640d9d65f55fced3"))), bubble_id: None, metadata: Some(FileContentTokenMetadata(FileContentTokenMetadata { content_size: 4 })) }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } }] - Upload tree response: [UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("c1afe800646ee45232ab5e70c57247b78dbf3899")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("53b19c5f23977836390e5880ec30fd252a311384")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] + Upload tree response: [UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("c1afe800646ee45232ab5e70c57247b78dbf3899")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("53b19c5f23977836390e5880ec30fd252a311384")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTreeResponse { token: UploadToken { data: UploadTokenData { id: HgTreeId(HgId("dbdaef03fd04c4a28dc29fb3fbe10c5ed7a090ec")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] Upload filenodes response: [UploadTokensResponse { token: UploadToken { data: UploadTokenData { id: HgFilenodeId(HgId("005d992c5dcf32993668f7cede29d296c494a5d9")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }, UploadTokensResponse { token: UploadToken { data: UploadTokenData { id: HgFilenodeId(HgId("f9304d84edb8a8ee2d3ce3f9de3ea944c82eba8f")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] + Upload hg changeset response: [UploadTokensResponse { token: UploadToken { data: UploadTokenData { id: HgChangesetId(HgId("e20237022b1290d98c3f14049931a8f498c18c53")), bubble_id: None, metadata: None }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } } }] Compare content $ diff <(mononoke_admin filestore -R orig fetch --content-id eb56488e97bb4cf5eb17f05357b80108a4a71f6c3bab52dfcaec07161d105ec9) <(mononoke_admin filestore -R dest fetch --content-id eb56488e97bb4cf5eb17f05357b80108a4a71f6c3bab52dfcaec07161d105ec9) @@ -83,4 +84,8 @@ Compare hg manifests Compare filenodes $ diff <(mononoke_admin blobstore -R orig fetch hgfilenode.sha1.005d992c5dcf32993668f7cede29d296c494a5d9 --quiet) <(mononoke_admin blobstore -R dest fetch hgfilenode.sha1.005d992c5dcf32993668f7cede29d296c494a5d9 --quiet) - $ diff <(mononoke_admin blobstore -R orig fetch hgfilenode.sha1.f9304d84edb8a8ee2d3ce3f9de3ea944c82eba8f --quiet) <(mononoke_admin blobstore -R dest fetch hgfilenode.sha1.f9304d84edb8a8ee2d3ce3f9de3ea944c82eba8f --quiet) + $ diff <(mononoke_admin blobstore -R orig fetch hgfilenode.sha1.f9304d84edb8a8ee2d3ce3f9de3ea944c82eba8f --quiet) <(mononoke_admin blobstore -R dest fetch hgfilenode.sha1.f9304d84edb8a8ee2d3ce3f9de3ea944c82eba8f --quiet) + +Compare hg and bonsai changeset + $ diff <(mononoke_admin fetch -R orig --commit-id 53b034a90fe3002a707a7da9cdf6eac3dea460ad72f7c6969dfb88fd0e69f856) <(mononoke_admin fetch -R dest --commit-id 53b034a90fe3002a707a7da9cdf6eac3dea460ad72f7c6969dfb88fd0e69f856) + $ diff <(mononoke_admin fetch -R orig --commit-id e20237022b1290d98c3f14049931a8f498c18c53) <(mononoke_admin fetch -R dest --commit-id e20237022b1290d98c3f14049931a8f498c18c53)