diff --git a/.gitignore b/.gitignore index 9555db0d..d293c430 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ inlined.html target/ config.toml .vscode/ +.idea/ rustfmt.toml env/ .next/ diff --git a/portal/package.json b/portal/package.json index ebd25f02..88a4099d 100644 --- a/portal/package.json +++ b/portal/package.json @@ -6,6 +6,7 @@ "build:prod:worker": "pnpm --filter ./worker run build:prod", "build:dev:worker": "pnpm --filter ./worker run build:dev", "build:server": "pnpm --filter ./server run build", + "build": "pnpm --filter ./server run build", "serve:dev:server": "pnpm --filter ./server run dev", "serve:prod:server": "pnpm --filter ./server run start", "serve": "pnpm serve:dev:worker", diff --git a/site-builder/src/main.rs b/site-builder/src/main.rs index 51d86ce7..ed14c62d 100644 --- a/site-builder/src/main.rs +++ b/site-builder/src/main.rs @@ -393,6 +393,7 @@ async fn run() -> Result<()> { SiteIdentifier::ExistingSite(site_object), epochs.get(), WhenWalrusUpload::Always, + false, ) .await?; site_manager.update_single_resource(resource).await?; diff --git a/site-builder/src/publish.rs b/site-builder/src/publish.rs index 5bdf6af6..602768cc 100644 --- a/site-builder/src/publish.rs +++ b/site-builder/src/publish.rs @@ -29,13 +29,19 @@ use crate::{ site::{ builder::SitePtb, config::WSResources, - manager::{SiteIdentifier, SiteManager}, + manager::{SiteIdentifier, SiteIdentifier::ExistingSite, SiteManager}, resource::ResourceManager, RemoteSiteFactory, SITE_MODULE, }, summary::{SiteDataDiffSummary, Summarizable}, - util::{get_site_id_from_response, id_to_base36, path_or_defaults_if_exist, sign_and_send_ptb}, + util::{ + get_site_id_from_response, + id_to_base36, + load_wallet_context, + path_or_defaults_if_exist, + sign_and_send_ptb, + }, Config, }; @@ -64,6 +70,8 @@ pub struct PublishOptions { /// The maximum number of concurrent calls to the Walrus CLI for the computation of blob IDs. #[clap(long)] max_concurrent: Option, + /// By default, sites are deletable with site-builder delete command. By passing --permanent, the site is deleted only after `epochs` expiration. + permanent: Option, } /// The continuous editing options. @@ -151,6 +159,38 @@ impl SiteEditor { } pub async fn destroy(&self, site_id: ObjectID) -> Result<()> { + // Delete blobs on Walrus + let wallet_walrus = load_wallet_context(&self.config.general.wallet)?; + let all_dynamic_fields = RemoteSiteFactory::new( + // TODO(giac): make the backoff configurable. + &RetriableSuiClient::new_from_wallet( + &wallet_walrus, + ExponentialBackoffConfig::default(), + ) + .await?, + self.config.package, + ) + .await? + .get_existing_resources(site_id) + .await?; + + tracing::debug!( + "Retrieved blobs and deleting them: {:?}", + &all_dynamic_fields, + ); + + let mut site_manager = SiteManager::new( + self.config.clone(), + ExistingSite(site_id), + 0, + WhenWalrusUpload::Always, + false, + ) + .await?; + + site_manager.delete_from_walrus(all_dynamic_fields).await?; + + // Delete objects on SUI blockchain let mut wallet = self.config.wallet()?; let ptb = SitePtb::new(self.config.package, Identifier::new(SITE_MODULE)?)?; let mut ptb = ptb.with_call_arg(&wallet.get_object_ref(site_id).await?.into())?; @@ -239,6 +279,7 @@ impl SiteEditor { self.edit_options.site_id.clone(), self.edit_options.publish_options.epochs, self.edit_options.when_upload.clone(), + false, ) .await?; let (response, summary) = site_manager.update_site(&local_site_data).await?; diff --git a/site-builder/src/site/manager.rs b/site-builder/src/site/manager.rs index b017ae76..359f0efd 100644 --- a/site-builder/src/site/manager.rs +++ b/site-builder/src/site/manager.rs @@ -1,7 +1,11 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 -use std::{collections::BTreeSet, str::FromStr, time::Duration}; +use std::{ + collections::{BTreeSet, HashMap}, + str::FromStr, + time::Duration, +}; use anyhow::{anyhow, Result}; use sui_keys::keystore::AccountKeystore; @@ -52,6 +56,7 @@ pub struct SiteManager { pub epochs: u64, pub when_upload: WhenWalrusUpload, pub backoff_config: ExponentialBackoffConfig, + pub permanent: bool, } impl SiteManager { @@ -61,6 +66,7 @@ impl SiteManager { site_id: SiteIdentifier, epochs: u64, when_upload: WhenWalrusUpload, + permanent: bool, ) -> Result { Ok(SiteManager { walrus: config.walrus_client(), @@ -71,6 +77,7 @@ impl SiteManager { when_upload, // TODO(giac): This should be configurable. backoff_config: ExponentialBackoffConfig::default(), + permanent, }) } @@ -120,6 +127,8 @@ impl SiteManager { /// Publishes the resources to Walrus. async fn publish_to_walrus<'b>(&mut self, updates: &[&ResourceOp<'b>]) -> Result<()> { + let deletable = !self.permanent; + for update in updates.iter() { let resource = update.inner(); tracing::debug!( @@ -145,7 +154,7 @@ impl SiteManager { retry_num += 1; let result = self .walrus - .store(resource.full_path.clone(), self.epochs, false) + .store(resource.full_path.clone(), self.epochs, false, deletable) .await; match result { @@ -176,6 +185,17 @@ impl SiteManager { Ok(()) } + /// Deletes the resources from Walrus. + pub async fn delete_from_walrus<'b>(&mut self, blobs: HashMap) -> Result<()> { + for (name, blob_id) in blobs.iter() { + tracing::debug!(name, "deleting blob from Walrus"); + display::action(format!("Deleting resource from Walrus: {}", blob_id,)); + let _output = self.walrus.delete(blob_id.to_string()).await?; + display::done(); + } + Ok(()) + } + /// Executes the updates on Sui. async fn execute_sui_updates<'b>( &self, diff --git a/site-builder/src/walrus.rs b/site-builder/src/walrus.rs index 9208f40a..1f38855e 100644 --- a/site-builder/src/walrus.rs +++ b/site-builder/src/walrus.rs @@ -7,7 +7,15 @@ use std::{num::NonZeroU16, path::PathBuf}; use anyhow::{Context, Result}; use command::RpcArg; -use output::{try_from_output, BlobIdOutput, InfoOutput, NShards, ReadOutput, StoreOutput}; +use output::{ + try_from_output, + BlobIdOutput, + DestroyOutput, + InfoOutput, + NShards, + ReadOutput, + StoreOutput, +}; use tokio::process::Command as CliCommand; use self::types::BlobId; @@ -73,8 +81,19 @@ impl Walrus { /// Issues a `store` JSON command to the Walrus CLI, returning the parsed output. // NOTE: takes a mutable reference to ensure that only one store command is executed at every // time. The issue is that the inner wallet may lock coins if called in parallel. - pub async fn store(&mut self, file: PathBuf, epochs: u64, force: bool) -> Result { - create_command!(self, store, vec![file], epochs, force) + pub async fn store( + &mut self, + file: PathBuf, + epochs: u64, + force: bool, + deletable: bool, + ) -> Result { + create_command!(self, store, vec![file], epochs, force, deletable) + } + + /// Issues a `delete` JSON command to the Walrus CLI, returning the parsed output. + pub async fn delete(&mut self, object_id: String) -> Result { + create_command!(self, delete, object_id) } /// Issues a `read` JSON command to the Walrus CLI, returning the parsed output. diff --git a/site-builder/src/walrus/command.rs b/site-builder/src/walrus/command.rs index 9d66b78c..acb5e7d3 100644 --- a/site-builder/src/walrus/command.rs +++ b/site-builder/src/walrus/command.rs @@ -55,6 +55,7 @@ pub enum Command { /// duration. #[serde(default)] force: bool, + deletable: bool, }, /// Reads a blob from Walrus. Read { @@ -69,6 +70,11 @@ pub enum Command { #[serde(default)] rpc_arg: RpcArg, }, + /// Deletes a blob from Walrus. + Delete { + /// The objectID of the blob to be deleted. + object_id: String, + }, BlobId { file: PathBuf, /// The number of shards of the Walrus system. @@ -155,15 +161,28 @@ impl WalrusCmdBuilder { } /// Adds a [`Command::Store`] command to the builder. - pub fn store(self, files: Vec, epochs: u64, force: bool) -> WalrusCmdBuilder { + pub fn store( + self, + files: Vec, + epochs: u64, + force: bool, + deletable: bool, + ) -> WalrusCmdBuilder { let command = Command::Store { files, epochs, force, + deletable, }; self.with_command(command) } + /// Adds a [`Command::Delete`] command to the builder. + pub fn delete(self, object_id: String) -> WalrusCmdBuilder { + let command = Command::Delete { object_id }; + self.with_command(command) + } + /// Adds a [`Command::Read`] command to the builder. #[allow(dead_code)] pub fn read( diff --git a/site-builder/src/walrus/output.rs b/site-builder/src/walrus/output.rs index 6c3b12a6..87d9a149 100644 --- a/site-builder/src/walrus/output.rs +++ b/site-builder/src/walrus/output.rs @@ -206,6 +206,17 @@ pub(crate) struct InfoOutput { pub(crate) dev_info: String, } +/// The output of the `destroy` command. +#[derive(Debug, Clone, Deserialize)] +#[allow(unused)] +#[allow(non_snake_case)] +pub struct DestroyOutput { + /// The objectId deleted. + objectId: String, + /// The blobs deleted. + deletedBlobs: Box<[String]>, +} + /// The number of shards, which can be deserialized from the output of the `info` command. #[derive(Debug, Clone, Deserialize)] #[serde(rename_all = "camelCase")]