feat(site-builder): SEINT-228 Support for deletable blobs

.gitignore

@@ -9,6 +9,7 @@ inlined.html
 target/
 config.toml
 .vscode/
+.idea/
 rustfmt.toml
 env/
 .next/

portal/package.json

@@ -6,6 +6,7 @@
     "build:prod:worker": "pnpm --filter ./worker run build:prod",
     "build:dev:worker": "pnpm --filter ./worker run build:dev",
     "build:server": "pnpm --filter ./server run build",
+    "build": "pnpm --filter ./server run build",
     "serve:dev:server": "pnpm --filter ./server run dev",
     "serve:prod:server": "pnpm --filter ./server run start",
     "serve": "pnpm serve:dev:worker",

site-builder/src/main.rs

@@ -393,6 +393,7 @@ async fn run() -> Result<()> {
                 SiteIdentifier::ExistingSite(site_object),
                 epochs.get(),
                 WhenWalrusUpload::Always,
+                false,
             )
             .await?;
             site_manager.update_single_resource(resource).await?;

site-builder/src/publish.rs

@@ -29,13 +29,19 @@ use crate::{
     site::{
         builder::SitePtb,
         config::WSResources,
-        manager::{SiteIdentifier, SiteManager},
+        manager::{SiteIdentifier, SiteIdentifier::ExistingSite, SiteManager},
         resource::ResourceManager,
         RemoteSiteFactory,
         SITE_MODULE,
     },
     summary::{SiteDataDiffSummary, Summarizable},
-    util::{get_site_id_from_response, id_to_base36, path_or_defaults_if_exist, sign_and_send_ptb},
+    util::{
+        get_site_id_from_response,
+        id_to_base36,
+        load_wallet_context,
+        path_or_defaults_if_exist,
+        sign_and_send_ptb,
+    },
     Config,
 };
 
@@ -64,6 +70,8 @@ pub struct PublishOptions {
     /// The maximum number of concurrent calls to the Walrus CLI for the computation of blob IDs.
     #[clap(long)]
     max_concurrent: Option<NonZeroUsize>,
+    /// By default, sites are deletable with site-builder delete command. By passing --permanent, the site is deleted only after `epochs` expiration.
+    permanent: Option<bool>,
 }
 
 /// The continuous editing options.
@@ -151,6 +159,38 @@ impl SiteEditor {
     }
 
     pub async fn destroy(&self, site_id: ObjectID) -> Result<()> {
+        // Delete blobs on Walrus
+        let wallet_walrus = load_wallet_context(&self.config.general.wallet)?;
+        let all_dynamic_fields = RemoteSiteFactory::new(
+            // TODO(giac): make the backoff configurable.
+            &RetriableSuiClient::new_from_wallet(
+                &wallet_walrus,
+                ExponentialBackoffConfig::default(),
+            )
+            .await?,
+            self.config.package,
+        )
+        .await?
+        .get_existing_resources(site_id)
+        .await?;
+
+        tracing::debug!(
+            "Retrieved blobs and deleting them: {:?}",
+            &all_dynamic_fields,
+        );
+
+        let mut site_manager = SiteManager::new(
+            self.config.clone(),
+            ExistingSite(site_id),
+            0,
+            WhenWalrusUpload::Always,
+            false,
+        )
+        .await?;
+
+        site_manager.delete_from_walrus(all_dynamic_fields).await?;
+
+        // Delete objects on SUI blockchain
         let mut wallet = self.config.wallet()?;
         let ptb = SitePtb::new(self.config.package, Identifier::new(SITE_MODULE)?)?;
         let mut ptb = ptb.with_call_arg(&wallet.get_object_ref(site_id).await?.into())?;
@@ -239,6 +279,7 @@ impl SiteEditor<EditOptions> {
             self.edit_options.site_id.clone(),
             self.edit_options.publish_options.epochs,
             self.edit_options.when_upload.clone(),
+            false,
         )
         .await?;
         let (response, summary) = site_manager.update_site(&local_site_data).await?;

site-builder/src/site/manager.rs

@@ -1,7 +1,11 @@
 // Copyright (c) Mysten Labs, Inc.
 // SPDX-License-Identifier: Apache-2.0
 
-use std::{collections::BTreeSet, str::FromStr, time::Duration};
+use std::{
+    collections::{BTreeSet, HashMap},
+    str::FromStr,
+    time::Duration,
+};
 
 use anyhow::{anyhow, Result};
 use sui_keys::keystore::AccountKeystore;
@@ -52,6 +56,7 @@ pub struct SiteManager {
     pub epochs: u64,
     pub when_upload: WhenWalrusUpload,
     pub backoff_config: ExponentialBackoffConfig,
+    pub permanent: bool,
 }
 
 impl SiteManager {
@@ -61,6 +66,7 @@ impl SiteManager {
         site_id: SiteIdentifier,
         epochs: u64,
         when_upload: WhenWalrusUpload,
+        permanent: bool,
     ) -> Result<Self> {
         Ok(SiteManager {
             walrus: config.walrus_client(),
@@ -71,6 +77,7 @@ impl SiteManager {
             when_upload,
             // TODO(giac): This should be configurable.
             backoff_config: ExponentialBackoffConfig::default(),
+            permanent,
         })
     }
 
@@ -120,6 +127,8 @@ impl SiteManager {
 
     /// Publishes the resources to Walrus.
     async fn publish_to_walrus<'b>(&mut self, updates: &[&ResourceOp<'b>]) -> Result<()> {
+        let deletable = !self.permanent;
+
         for update in updates.iter() {
             let resource = update.inner();
             tracing::debug!(
@@ -145,7 +154,7 @@ impl SiteManager {
                 retry_num += 1;
                 let result = self
                     .walrus
-                    .store(resource.full_path.clone(), self.epochs, false)
+                    .store(resource.full_path.clone(), self.epochs, false, deletable)
                     .await;
 
                 match result {
@@ -176,6 +185,17 @@ impl SiteManager {
         Ok(())
     }
 
+    /// Deletes the resources from Walrus.
+    pub async fn delete_from_walrus<'b>(&mut self, blobs: HashMap<String, ObjectID>) -> Result<()> {
+        for (name, blob_id) in blobs.iter() {
+            tracing::debug!(name, "deleting blob from Walrus");
+            display::action(format!("Deleting resource from Walrus: {}", blob_id,));
+            let _output = self.walrus.delete(blob_id.to_string()).await?;
+            display::done();
+        }
+        Ok(())
+    }
+
     /// Executes the updates on Sui.
     async fn execute_sui_updates<'b>(
         &self,

site-builder/src/walrus.rs

@@ -7,7 +7,15 @@ use std::{num::NonZeroU16, path::PathBuf};
 
 use anyhow::{Context, Result};
 use command::RpcArg;
-use output::{try_from_output, BlobIdOutput, InfoOutput, NShards, ReadOutput, StoreOutput};
+use output::{
+    try_from_output,
+    BlobIdOutput,
+    DestroyOutput,
+    InfoOutput,
+    NShards,
+    ReadOutput,
+    StoreOutput,
+};
 use tokio::process::Command as CliCommand;
 
 use self::types::BlobId;
@@ -73,8 +81,19 @@ impl Walrus {
     /// Issues a `store` JSON command to the Walrus CLI, returning the parsed output.
     // NOTE: takes a mutable reference to ensure that only one store command is executed at every
     // time. The issue is that the inner wallet may lock coins if called in parallel.
-    pub async fn store(&mut self, file: PathBuf, epochs: u64, force: bool) -> Result<StoreOutput> {
-        create_command!(self, store, vec![file], epochs, force)
+    pub async fn store(
+        &mut self,
+        file: PathBuf,
+        epochs: u64,
+        force: bool,
+        deletable: bool,
+    ) -> Result<StoreOutput> {
+        create_command!(self, store, vec![file], epochs, force, deletable)
+    }
+
+    /// Issues a `delete` JSON command to the Walrus CLI, returning the parsed output.
+    pub async fn delete(&mut self, object_id: String) -> Result<DestroyOutput> {
+        create_command!(self, delete, object_id)
     }
 
     /// Issues a `read` JSON command to the Walrus CLI, returning the parsed output.

site-builder/src/walrus/command.rs

@@ -55,6 +55,7 @@ pub enum Command {
         /// duration.
         #[serde(default)]
         force: bool,
+        deletable: bool,
     },
     /// Reads a blob from Walrus.
     Read {
@@ -69,6 +70,11 @@ pub enum Command {
         #[serde(default)]
         rpc_arg: RpcArg,
     },
+    /// Deletes a blob from Walrus.
+    Delete {
+        /// The objectID of the blob to be deleted.
+        object_id: String,
+    },
     BlobId {
         file: PathBuf,
         /// The number of shards of the Walrus system.
@@ -155,15 +161,28 @@ impl WalrusCmdBuilder {
     }
 
     /// Adds a [`Command::Store`] command to the builder.
-    pub fn store(self, files: Vec<PathBuf>, epochs: u64, force: bool) -> WalrusCmdBuilder<Command> {
+    pub fn store(
+        self,
+        files: Vec<PathBuf>,
+        epochs: u64,
+        force: bool,
+        deletable: bool,
+    ) -> WalrusCmdBuilder<Command> {
         let command = Command::Store {
             files,
             epochs,
             force,
+            deletable,
         };
         self.with_command(command)
     }
 
+    /// Adds a [`Command::Delete`] command to the builder.
+    pub fn delete(self, object_id: String) -> WalrusCmdBuilder<Command> {
+        let command = Command::Delete { object_id };
+        self.with_command(command)
+    }
+
     /// Adds a [`Command::Read`] command to the builder.
     #[allow(dead_code)]
     pub fn read(

site-builder/src/walrus/output.rs

@@ -206,6 +206,17 @@ pub(crate) struct InfoOutput {
     pub(crate) dev_info: String,
 }
 
+/// The output of the `destroy` command.
+#[derive(Debug, Clone, Deserialize)]
+#[allow(unused)]
+#[allow(non_snake_case)]
+pub struct DestroyOutput {
+    /// The objectId deleted.
+    objectId: String,
+    /// The blobs deleted.
+    deletedBlobs: Box<[String]>,
+}
+
 /// The number of shards, which can be deserialized from the output of the `info` command.
 #[derive(Debug, Clone, Deserialize)]
 #[serde(rename_all = "camelCase")]