Skip to content

Commit

Permalink
modern sync: Add simple file syncing
Browse files Browse the repository at this point in the history 
Summary: This uses edenapi to sync files between origin and destination repo

Reviewed By: andreacampi

Differential Revision: D66939510

fbshipit-source-id: 7bdb9e0a82b989bcdd51a936b2b20fd4c6619caa
  • Loading branch information
@lmvasquezg @facebook-github-bot
lmvasquezg authored and facebook-github-bot committed Dec 10, 2024
1 parent 787265a commit 8ccb1bf
Show file tree
Hide file tree
Showing 8 changed files with 101 additions and 33 deletions.
1 change: 1 addition & 0 deletions eden/mononoke/modern_sync/Cargo.toml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,7 @@ repo_derived_data = { version = "0.1.0", path = "../repo_attributes/repo_derived
repo_identity = { version = "0.1.0", path = "../repo_attributes/repo_identity" }
sapling-clientinfo = { version = "0.1.0", path = "../../scm/lib/clientinfo" }
sapling-edenapi = { version = "0.1.0", path = "../../scm/lib/edenapi" }
sapling-edenapi_types = { version = "0.1.0", path = "../../scm/lib/edenapi/types" }
sharding_ext = { version = "0.1.0", path = "../cmdlib/sharding_ext" }
slog = { version = "2.7", features = ["max_level_trace", "nested-values"] }
stats = { version = "0.1.0", git = "https://github.com/facebookexperimental/rust-shed.git", branch = "main" }
Expand Down
13 changes: 3 additions & 10 deletions eden/mononoke/modern_sync/src/commands/sync_one.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,7 +69,6 @@ pub async fn run(app: MononokeApp, args: CommandArgs) -> Result<()> {
let sender: Arc<dyn ModernSyncSender + Send + Sync> = if args.dry_run {
Arc::new(DummySender::new(logger.clone()))
} else {
let app_args = app.args::<ModernSyncArgs>()?;
let url = if let Some(socket) = app_args.dest_socket {
// Only for integration tests
format!("{}:{}/edenapi/", &config.url, socket)
Expand All @@ -82,15 +81,9 @@ pub async fn run(app: MononokeApp, args: CommandArgs) -> Result<()> {
.clone()
.ok_or_else(|| format_err!("TLS params not found for repo {}", repo_name))?;

Arc::new(
EdenapiSender::new(
Url::parse(&url)?,
repo_name.clone(),
logger.clone(),
tls_args,
)
.await?,
)
let dest_repo = app_args.dest_repo_name.clone().unwrap_or(repo_name.clone());

Arc::new(EdenapiSender::new(Url::parse(&url)?, dest_repo, logger.clone(), tls_args).await?)
};

crate::sync::process_one_changeset(&args.cs_id, &ctx, repo, &logger, sender, false).await?;
Expand Down
5 changes: 4 additions & 1 deletion eden/mononoke/modern_sync/src/sender.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,14 @@
* GNU General Public License version 2.
*/

use anyhow::Result;
use async_trait::async_trait;
use mononoke_types::ContentId;
use mononoke_types::FileContents;
pub mod dummy;
pub mod edenapi;

#[async_trait]
pub trait ModernSyncSender {
fn upload_content(&self, content_id: ContentId, _blob: FileContents);
async fn upload_content(&self, content_id: ContentId, _blob: FileContents) -> Result<()>;
}
8 changes: 6 additions & 2 deletions eden/mononoke/modern_sync/src/sender/dummy.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,8 @@
* GNU General Public License version 2.
*/

use anyhow::Result;
use async_trait::async_trait;
use mononoke_types::ContentId;
use mononoke_types::FileContents;
use slog::info;
Expand All @@ -23,8 +25,10 @@ impl DummySender {
}
}

#[async_trait]
impl ModernSyncSender for DummySender {
fn upload_content(&self, content_id: ContentId, _blob: FileContents) {
info!(&self.logger, "Uploading content with id: {:?}", content_id)
async fn upload_content(&self, content_id: ContentId, _blob: FileContents) -> Result<()> {
info!(&self.logger, "Uploading content with id: {:?}", content_id);
Ok(())
}
}
41 changes: 36 additions & 5 deletions eden/mononoke/modern_sync/src/sender/edenapi.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,17 @@
use std::collections::HashSet;

use anyhow::Result;
use async_trait::async_trait;
use clientinfo::ClientEntryPoint;
use clientinfo::ClientInfo;
use edenapi::Client;
use edenapi::HttpClientBuilder;
use edenapi::HttpClientConfig;
use edenapi::SaplingRemoteApi;
use edenapi_types::AnyFileContentId;
use futures::TryStreamExt;
use mononoke_app::args::TLSArgs;
use mononoke_types::FileContents;
use slog::info;
use slog::Logger;
use url::Url;
Expand All @@ -32,13 +38,14 @@ impl EdenapiSender {
logger: Logger,
tls_args: TLSArgs,
) -> Result<Self> {
let ci = ClientInfo::new_with_entry_point(ClientEntryPoint::ModernSync)?.to_json()?;
let http_config = HttpClientConfig {
cert_path: Some(tls_args.tls_certificate.into()),
key_path: Some(tls_args.tls_private_key.into()),
ca_path: Some(tls_args.tls_ca.into()),
convert_cert: false,

client_info: None,
client_info: Some(ci),
disable_tls_verification: false,
max_concurrent_requests: None,
unix_socket_domains: HashSet::new(),
Expand All @@ -60,12 +67,36 @@ impl EdenapiSender {
Ok(Self { client, logger })
}
}

#[async_trait]
impl ModernSyncSender for EdenapiSender {
fn upload_content(
async fn upload_content(
&self,
content_id: mononoke_types::ContentId,
_blob: mononoke_types::FileContents,
) {
info!(&self.logger, "Uploading content with id: {:?}", content_id)
blob: FileContents,
) -> Result<()> {
info!(&self.logger, "Uploading content with id: {:?}", content_id);

match blob {
FileContents::Bytes(bytes) => {
info!(&self.logger, "Uploading bytes: {:?}", bytes);
let response = self
.client
.process_files_upload(
vec![(AnyFileContentId::ContentId(content_id.into()), bytes.into())],
None,
None,
)
.await?;
info!(
&self.logger,
"Upload response: {:?}",
response.entries.try_collect::<Vec<_>>().await?
);
}
_ => (),
}

Ok(())
}
}
3 changes: 2 additions & 1 deletion eden/mononoke/modern_sync/src/sync.rs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -208,8 +208,9 @@ pub async fn process_one_changeset(
};

if let Some(bs) = bs {
info!(logger, "Blob {:?}", bs);
let blob = bs.load(ctx, &repo.repo_blobstore()).await?;
sender.upload_content(bs, blob);
sender.upload_content(bs, blob).await?;
}
}

Expand Down
42 changes: 36 additions & 6 deletions eden/mononoke/tests/integration/modern_sync/test-modern-sync.t
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,16 +11,16 @@
> "repos": {
> "orig": {
> "actions": {
> "read": ["$CLIENT0_ID_TYPE:$CLIENT0_ID_DATA"],
> "write": ["$CLIENT0_ID_TYPE:$CLIENT0_ID_DATA"],
> "bypass_readonly": ["$CLIENT0_ID_TYPE:$CLIENT0_ID_DATA"]
> "read": ["$CLIENT0_ID_TYPE:$CLIENT0_ID_DATA", "X509_SUBJECT_NAME:CN=localhost,O=Mononoke,C=US,ST=CA", "X509_SUBJECT_NAME:CN=client0,O=Mononoke,C=US,ST=CA"],
> "write": ["$CLIENT0_ID_TYPE:$CLIENT0_ID_DATA", "X509_SUBJECT_NAME:CN=localhost,O=Mononoke,C=US,ST=CA", "X509_SUBJECT_NAME:CN=client0,O=Mononoke,C=US,ST=CA"],
> "bypass_readonly": ["$CLIENT0_ID_TYPE:$CLIENT0_ID_DATA", "X509_SUBJECT_NAME:CN=localhost,O=Mononoke,C=US,ST=CA", "X509_SUBJECT_NAME:CN=client0,O=Mononoke,C=US,ST=CA"]
> }
> },
> "dest": {
> "actions": {
> "read": ["SERVICE_IDENTITY:server"],
> "write": ["SERVICE_IDENTITY:server"],
> "bypass_readonly": ["SERVICE_IDENTITY:server"]
> "read": ["SERVICE_IDENTITY:server", "X509_SUBJECT_NAME:CN=localhost,O=Mononoke,C=US,ST=CA", "X509_SUBJECT_NAME:CN=client0,O=Mononoke,C=US,ST=CA"],
> "write": ["SERVICE_IDENTITY:server", "X509_SUBJECT_NAME:CN=localhost,O=Mononoke,C=US,ST=CA", "X509_SUBJECT_NAME:CN=client0,O=Mononoke,C=US,ST=CA"],
> "bypass_readonly": ["SERVICE_IDENTITY:server", "X509_SUBJECT_NAME:CN=localhost,O=Mononoke,C=US,ST=CA", "X509_SUBJECT_NAME:CN=client0,O=Mononoke,C=US,ST=CA"]
> }
> }
> }
Expand Down Expand Up @@ -61,33 +61,63 @@ Sync all bookmarks moves
Found commit ChangesetId(Blake2(53b034a90fe3002a707a7da9cdf6eac3dea460ad72f7c6969dfb88fd0e69f856))
Commit info ChangesetInfo { changeset_id: ChangesetId(Blake2(53b034a90fe3002a707a7da9cdf6eac3dea460ad72f7c6969dfb88fd0e69f856)), parents: [], author: "test", author_date: DateTime(1970-01-01T00:00:00+00:00), committer: None, committer_date: None, message: Message("A"), hg_extra: {}, git_extra_headers: None }
File change Change(TrackedFileChange { inner: BasicFileChange { content_id: ContentId(Blake2(eb56488e97bb4cf5eb17f05357b80108a4a71f6c3bab52dfcaec07161d105ec9)), file_type: Regular, size: 1, git_lfs: FullContent }, copy_from: None })
Blob ContentId(Blake2(eb56488e97bb4cf5eb17f05357b80108a4a71f6c3bab52dfcaec07161d105ec9))
Uploading content with id: ContentId(Blake2(eb56488e97bb4cf5eb17f05357b80108a4a71f6c3bab52dfcaec07161d105ec9))
Uploading bytes: b"A"
Upload response: [UploadToken { data: UploadTokenData { id: AnyFileContentId(ContentId(ContentId("eb56488e97bb4cf5eb17f05357b80108a4a71f6c3bab52dfcaec07161d105ec9"))), bubble_id: None, metadata: Some(FileContentTokenMetadata(FileContentTokenMetadata { content_size: 1 })) }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } }]
File change Change(TrackedFileChange { inner: BasicFileChange { content_id: ContentId(Blake2(be87911855af0fc33a75f2c1cba2269dd90faa7f5c5358eb640d9d65f55fced3)), file_type: Regular, size: 4, git_lfs: FullContent }, copy_from: None })
Blob ContentId(Blake2(be87911855af0fc33a75f2c1cba2269dd90faa7f5c5358eb640d9d65f55fced3))
Uploading content with id: ContentId(Blake2(be87911855af0fc33a75f2c1cba2269dd90faa7f5c5358eb640d9d65f55fced3))
Uploading bytes: b"abc\n"
Upload response: [UploadToken { data: UploadTokenData { id: AnyFileContentId(ContentId(ContentId("be87911855af0fc33a75f2c1cba2269dd90faa7f5c5358eb640d9d65f55fced3"))), bubble_id: None, metadata: Some(FileContentTokenMetadata(FileContentTokenMetadata { content_size: 4 })) }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } }]
Found commit ChangesetId(Blake2(5b1c7130dde8e54b4285b9153d8e56d69fbf4ae685eaf9e9766cc409861995f8))
Commit info ChangesetInfo { changeset_id: ChangesetId(Blake2(5b1c7130dde8e54b4285b9153d8e56d69fbf4ae685eaf9e9766cc409861995f8)), parents: [ChangesetId(Blake2(ba1a2b3ca64cead35117cb2b707da1211cf43639ade917aee655f3875f4922c3))], author: "test", author_date: DateTime(1970-01-01T00:00:00+00:00), committer: None, committer_date: None, message: Message("E"), hg_extra: {}, git_extra_headers: None }
File change Change(TrackedFileChange { inner: BasicFileChange { content_id: ContentId(Blake2(1b1e26f01a806e123b37492672d2756e1c25bb31f1e15cfda410c149c317e130)), file_type: Regular, size: 1, git_lfs: FullContent }, copy_from: None })
Blob ContentId(Blake2(1b1e26f01a806e123b37492672d2756e1c25bb31f1e15cfda410c149c317e130))
Uploading content with id: ContentId(Blake2(1b1e26f01a806e123b37492672d2756e1c25bb31f1e15cfda410c149c317e130))
Uploading bytes: b"E"
Upload response: [UploadToken { data: UploadTokenData { id: AnyFileContentId(ContentId(ContentId("1b1e26f01a806e123b37492672d2756e1c25bb31f1e15cfda410c149c317e130"))), bubble_id: None, metadata: Some(FileContentTokenMetadata(FileContentTokenMetadata { content_size: 1 })) }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } }]
File change Change(TrackedFileChange { inner: BasicFileChange { content_id: ContentId(Blake2(96475ef07b63bf02679e9964ff65f0f96883f53d0718671bd44cce830bbf2ebd)), file_type: Regular, size: 8, git_lfs: FullContent }, copy_from: None })
Blob ContentId(Blake2(96475ef07b63bf02679e9964ff65f0f96883f53d0718671bd44cce830bbf2ebd))
Uploading content with id: ContentId(Blake2(96475ef07b63bf02679e9964ff65f0f96883f53d0718671bd44cce830bbf2ebd))
Uploading bytes: b"abcdefg\n"
Upload response: [UploadToken { data: UploadTokenData { id: AnyFileContentId(ContentId(ContentId("96475ef07b63bf02679e9964ff65f0f96883f53d0718671bd44cce830bbf2ebd"))), bubble_id: None, metadata: Some(FileContentTokenMetadata(FileContentTokenMetadata { content_size: 8 })) }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } }]
Found commit ChangesetId(Blake2(ba1a2b3ca64cead35117cb2b707da1211cf43639ade917aee655f3875f4922c3))
Commit info ChangesetInfo { changeset_id: ChangesetId(Blake2(ba1a2b3ca64cead35117cb2b707da1211cf43639ade917aee655f3875f4922c3)), parents: [ChangesetId(Blake2(41deea4804cd27d1f4efbec135d839338804a5dfcaf364863bd0289067644db5))], author: "test", author_date: DateTime(1970-01-01T00:00:00+00:00), committer: None, committer_date: None, message: Message("D"), hg_extra: {}, git_extra_headers: None }
File change Change(TrackedFileChange { inner: BasicFileChange { content_id: ContentId(Blake2(90c8e211c758a9bbcd33e463c174f1693692677cb76c7aaf4ce41aa0a29334c0)), file_type: Regular, size: 1, git_lfs: FullContent }, copy_from: None })
Blob ContentId(Blake2(90c8e211c758a9bbcd33e463c174f1693692677cb76c7aaf4ce41aa0a29334c0))
Uploading content with id: ContentId(Blake2(90c8e211c758a9bbcd33e463c174f1693692677cb76c7aaf4ce41aa0a29334c0))
Uploading bytes: b"D"
Upload response: [UploadToken { data: UploadTokenData { id: AnyFileContentId(ContentId(ContentId("90c8e211c758a9bbcd33e463c174f1693692677cb76c7aaf4ce41aa0a29334c0"))), bubble_id: None, metadata: Some(FileContentTokenMetadata(FileContentTokenMetadata { content_size: 1 })) }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } }]
File change Change(TrackedFileChange { inner: BasicFileChange { content_id: ContentId(Blake2(5d3bfab620332130430c7f540f9fe0b3b0079d0b9b632e0dae96a1424a7a4242)), file_type: Regular, size: 7, git_lfs: FullContent }, copy_from: None })
Blob ContentId(Blake2(5d3bfab620332130430c7f540f9fe0b3b0079d0b9b632e0dae96a1424a7a4242))
Uploading content with id: ContentId(Blake2(5d3bfab620332130430c7f540f9fe0b3b0079d0b9b632e0dae96a1424a7a4242))
Uploading bytes: b"abcdef\n"
Upload response: [UploadToken { data: UploadTokenData { id: AnyFileContentId(ContentId(ContentId("5d3bfab620332130430c7f540f9fe0b3b0079d0b9b632e0dae96a1424a7a4242"))), bubble_id: None, metadata: Some(FileContentTokenMetadata(FileContentTokenMetadata { content_size: 7 })) }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } }]
Found commit ChangesetId(Blake2(41deea4804cd27d1f4efbec135d839338804a5dfcaf364863bd0289067644db5))
Commit info ChangesetInfo { changeset_id: ChangesetId(Blake2(41deea4804cd27d1f4efbec135d839338804a5dfcaf364863bd0289067644db5)), parents: [ChangesetId(Blake2(8a9d572a899acdef764b88671c24b94a8b0780c1591a5a9bca97184c2ef0f304))], author: "test", author_date: DateTime(1970-01-01T00:00:00+00:00), committer: None, committer_date: None, message: Message("C"), hg_extra: {}, git_extra_headers: None }
File change Change(TrackedFileChange { inner: BasicFileChange { content_id: ContentId(Blake2(896ad5879a5df0403bfc93fc96507ad9c93b31b11f3d0fa05445da7918241e5d)), file_type: Regular, size: 1, git_lfs: FullContent }, copy_from: None })
Blob ContentId(Blake2(896ad5879a5df0403bfc93fc96507ad9c93b31b11f3d0fa05445da7918241e5d))
Uploading content with id: ContentId(Blake2(896ad5879a5df0403bfc93fc96507ad9c93b31b11f3d0fa05445da7918241e5d))
Uploading bytes: b"C"
Upload response: [UploadToken { data: UploadTokenData { id: AnyFileContentId(ContentId(ContentId("896ad5879a5df0403bfc93fc96507ad9c93b31b11f3d0fa05445da7918241e5d"))), bubble_id: None, metadata: Some(FileContentTokenMetadata(FileContentTokenMetadata { content_size: 1 })) }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } }]
File change Change(TrackedFileChange { inner: BasicFileChange { content_id: ContentId(Blake2(c86e7a7ee4c102efc1e5166dd95c1c73fcbff59dc3b04dc79fbbf3d1d10350ed)), file_type: Regular, size: 6, git_lfs: FullContent }, copy_from: Some((NonRootMPath("dir1/dir2/first"), ChangesetId(Blake2(8a9d572a899acdef764b88671c24b94a8b0780c1591a5a9bca97184c2ef0f304)))) })
Blob ContentId(Blake2(c86e7a7ee4c102efc1e5166dd95c1c73fcbff59dc3b04dc79fbbf3d1d10350ed))
Uploading content with id: ContentId(Blake2(c86e7a7ee4c102efc1e5166dd95c1c73fcbff59dc3b04dc79fbbf3d1d10350ed))
Uploading bytes: b"abcde\n"
Upload response: [UploadToken { data: UploadTokenData { id: AnyFileContentId(ContentId(ContentId("c86e7a7ee4c102efc1e5166dd95c1c73fcbff59dc3b04dc79fbbf3d1d10350ed"))), bubble_id: None, metadata: Some(FileContentTokenMetadata(FileContentTokenMetadata { content_size: 6 })) }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } }]
Found commit ChangesetId(Blake2(8a9d572a899acdef764b88671c24b94a8b0780c1591a5a9bca97184c2ef0f304))
Commit info ChangesetInfo { changeset_id: ChangesetId(Blake2(8a9d572a899acdef764b88671c24b94a8b0780c1591a5a9bca97184c2ef0f304)), parents: [ChangesetId(Blake2(53b034a90fe3002a707a7da9cdf6eac3dea460ad72f7c6969dfb88fd0e69f856))], author: "test", author_date: DateTime(1970-01-01T00:00:00+00:00), committer: None, committer_date: None, message: Message("B"), hg_extra: {}, git_extra_headers: None }
File change Change(TrackedFileChange { inner: BasicFileChange { content_id: ContentId(Blake2(55662471e2a28db8257939b2f9a2d24e65b46a758bac12914a58f17dcde6905f)), file_type: Regular, size: 1, git_lfs: FullContent }, copy_from: None })
Blob ContentId(Blake2(55662471e2a28db8257939b2f9a2d24e65b46a758bac12914a58f17dcde6905f))
Uploading content with id: ContentId(Blake2(55662471e2a28db8257939b2f9a2d24e65b46a758bac12914a58f17dcde6905f))
Uploading bytes: b"B"
Upload response: [UploadToken { data: UploadTokenData { id: AnyFileContentId(ContentId(ContentId("55662471e2a28db8257939b2f9a2d24e65b46a758bac12914a58f17dcde6905f"))), bubble_id: None, metadata: Some(FileContentTokenMetadata(FileContentTokenMetadata { content_size: 1 })) }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } }]
File change Change(TrackedFileChange { inner: BasicFileChange { content_id: ContentId(Blake2(fbc4b9b407225e86008840c4095edb4f66a62bad80529b6e120bfa7d605f9423)), file_type: Regular, size: 5, git_lfs: FullContent }, copy_from: None })
Blob ContentId(Blake2(fbc4b9b407225e86008840c4095edb4f66a62bad80529b6e120bfa7d605f9423))
Uploading content with id: ContentId(Blake2(fbc4b9b407225e86008840c4095edb4f66a62bad80529b6e120bfa7d605f9423))
Uploading bytes: b"abcd\n"
Upload response: [UploadToken { data: UploadTokenData { id: AnyFileContentId(ContentId(ContentId("fbc4b9b407225e86008840c4095edb4f66a62bad80529b6e120bfa7d605f9423"))), bubble_id: None, metadata: Some(FileContentTokenMetadata(FileContentTokenMetadata { content_size: 5 })) }, signature: UploadTokenSignature { signature: [102, 97, 107, 101, 116, 111, 107, 101, 110, 115, 105, 103, 110, 97, 116, 117, 114, 101] } }]

$ cat $TESTTMP/modern_sync_scuba_logs | jq | rg "start_id|dry_run|repo"
"start_id": 0,
Expand Down
Loading

0 comments on commit 8ccb1bf

Please sign in to comment.