chore: updates for walrus redeployment (#357)

- [x] changes the aggregator endpoint in the portals - [x] adds `max` to the `--epochs` flag in the site builder --------- Signed-off-by: giac-mysten <[email protected]>
MystenLabs · Jan 16, 2025 · 9446c72 · 9446c72
1 parent 8dba022
commit 9446c72
Show file tree

Hide file tree

Showing 7 changed files with 156 additions and 69 deletions.
diff --git a/portal/common/lib/aggregator.ts b/portal/common/lib/aggregator.ts
@@ -7,5 +7,5 @@ import { AGGREGATOR } from "./constants";
  * Returns the URL to fetch the blob of given ID from the aggregator/cache.
  */
 export function aggregatorEndpoint(blob_id: string): URL {
-    return new URL(AGGREGATOR + "/v1/" + encodeURIComponent(blob_id));
+    return new URL(AGGREGATOR + "/v1/blobs/" + encodeURIComponent(blob_id));
 }
diff --git a/portal/common/lib/redirects.test.ts b/portal/common/lib/redirects.test.ts
@@ -2,46 +2,69 @@
 // SPDX-License-Identifier: Apache-2.0
 
 // Import necessary functions and types
-import { describe, expect, test } from 'vitest';
-import { redirectToPortalURLResponse, redirectToAggregatorUrlResponse } from './redirects';
-import { DomainDetails } from './types/index';
+import { describe, expect, test } from "vitest";
+import { redirectToPortalURLResponse, redirectToAggregatorUrlResponse } from "./redirects";
+import { DomainDetails } from "./types/index";
 
 const redirectToPortalURLTestCases: [string, DomainDetails, string][] = [
-    ['https://example.com', { subdomain: 'subname', path: '/index.html' },
-        'https://subname.example.com/index.html'],
-    ['https://walrus.site', { subdomain: 'name', path: '/index.html' },
-        'https://name.walrus.site/index.html'],
-    ['http://localhost:8080', { subdomain: 'docs', path: '/css/print.css' },
-        'http://docs.localhost:8080/css/print.css'],
-    ['https://portalname.co.uk', { subdomain: 'subsubname.subname', path: '/index.html' },
-        'https://subsubname.subname.portalname.co.uk/index.html'],
+    [
+        "https://example.com",
+        { subdomain: "subname", path: "/index.html" },
+        "https://subname.example.com/index.html",
+    ],
+    [
+        "https://walrus.site",
+        { subdomain: "name", path: "/index.html" },
+        "https://name.walrus.site/index.html",
+    ],
+    [
+        "http://localhost:8080",
+        { subdomain: "docs", path: "/css/print.css" },
+        "http://docs.localhost:8080/css/print.css",
+    ],
+    [
+        "https://portalname.co.uk",
+        { subdomain: "subsubname.subname", path: "/index.html" },
+        "https://subsubname.subname.portalname.co.uk/index.html",
+    ],
 ];
 
-describe('redirectToPortalURLResponse', () => {
+describe("redirectToPortalURLResponse", () => {
     redirectToPortalURLTestCases.forEach(([input, path, expected]) => {
-        test(`${input} with subdomain: ${path.subdomain} and path: ${path.path} -> ${expected}`,
-            () => {
+        test(`${input} with subdomain: ${path.subdomain} and path: ${path.path} -> ${expected}`, () => {
             const scope = new URL(input);
             const response = redirectToPortalURLResponse(scope, path);
             expect(response.status).toBe(302);
-            expect(response.headers.get('Location')).toBe(expected);
+            expect(response.headers.get("Location")).toBe(expected);
         });
     });
 });
 
 const redirectToAggregatorUrlTestCases: [string, string, string][] = [
-    ['https://example.com', '12345', 'https://aggregator.walrus-testnet.walrus.space/v1/12345'],
-    ['https://walrus.site', 'blob-id', 'https://aggregator.walrus-testnet.walrus.space/v1/blob-id'],
-    ['http://localhost:8080', 'abcde', 'https://aggregator.walrus-testnet.walrus.space/v1/abcde'],
+    [
+        "https://example.com",
+        "12345",
+        "https://aggregator.walrus-testnet.walrus.space/v1/blobs/12345",
+    ],
+    [
+        "https://walrus.site",
+        "blob-id",
+        "https://aggregator.walrus-testnet.walrus.space/v1/blobs/blob-id",
+    ],
+    [
+        "http://localhost:8080",
+        "abcde",
+        "https://aggregator.walrus-testnet.walrus.space/v1/blobs/abcde",
+    ],
 ];
 
-describe('redirectToAggregatorUrlResponse', () => {
+describe("redirectToAggregatorUrlResponse", () => {
     redirectToAggregatorUrlTestCases.forEach(([input, blobId, expected]) => {
         test(`${input} with blobId: ${blobId} -> ${expected}`, () => {
             const scope = new URL(input);
             const response = redirectToAggregatorUrlResponse(scope, blobId);
             expect(response.status).toBe(302);
-            expect(response.headers.get('Location')).toBe(expected);
+            expect(response.headers.get("Location")).toBe(expected);
         });
     });
 });
diff --git a/site-builder/src/main.rs b/site-builder/src/main.rs
@@ -11,15 +11,18 @@ mod summary;
 mod types;
 mod util;
 mod walrus;
-use std::{num::NonZeroU64, path::PathBuf};
+use std::{
+    num::{NonZeroU32, NonZeroUsize},
+    path::PathBuf,
+};
 
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, ensure, Result};
 use backoff::ExponentialBackoffConfig;
 use clap::{Parser, Subcommand};
 use futures::TryFutureExt;
-use publish::{ContinuousEditing, PublishOptions, SiteEditor, WhenWalrusUpload};
+use publish::{ContinuousEditing, SiteEditor, WhenWalrusUpload};
 use retry_client::RetriableSuiClient;
-use serde::Deserialize;
+use serde::{Deserialize, Serialize};
 use site::{
     config::WSResources,
     manager::{SiteIdentifier, SiteManager},
@@ -29,7 +32,7 @@ use site::{
 use sui_sdk::wallet_context::WalletContext;
 use sui_types::base_types::ObjectID;
 use util::path_or_defaults_if_exist;
-use walrus::Walrus;
+use walrus::{output::EpochCount, Walrus};
 
 use crate::{
     preprocessor::Preprocessor,
@@ -204,11 +207,44 @@ enum Commands {
         // TODO: deduplicate with the `publish_options` in the `Publish` and `Update` commands.
         ws_resources: Option<PathBuf>,
         /// The number of epochs for which to save the resources on Walrus.
-        #[clap(long)]
-        epochs: NonZeroU64,
+        ///
+        /// If set to `max`, the resources are stored for the maximum number of epochs allowed on
+        /// Walrus. Otherwise, the resources are stored for the specified number of epochs. The
+        /// number of epochs must be greater than 0.
+        #[clap(long, value_parser = EpochCountOrMax::parse_epoch_count)]
+        epochs: EpochCountOrMax,
     },
 }
 
+#[derive(Parser, Debug, Clone)]
+pub struct PublishOptions {
+    /// The directory containing the site sources.
+    pub directory: PathBuf,
+    /// The path to the Walrus sites resources file.
+    ///
+    /// This JSON configuration file defined HTTP resource headers and other utilities for your
+    /// files. By default, the file is expected to be named `ws-resources.json` and located in the
+    /// root of the site directory.
+    ///
+    /// The configuration file _will not_ be uploaded to Walrus.
+    #[clap(long)]
+    ws_resources: Option<PathBuf>,
+    /// The number of epochs for which to save the resources on Walrus.
+    ///
+    /// If set to `max`, the resources are stored for the maximum number of epochs allowed on
+    /// Walrus. Otherwise, the resources are stored for the specified number of epochs. The
+    /// number of epochs must be greater than 0.
+    #[clap(long, value_parser = EpochCountOrMax::parse_epoch_count)]
+    pub epochs: EpochCountOrMax,
+    /// Preprocess the directory before publishing.
+    /// See the `list-directory` command. Warning: Rewrites all `index.html` files.
+    #[clap(long, action)]
+    pub list_directory: bool,
+    /// The maximum number of concurrent calls to the Walrus CLI for the computation of blob IDs.
+    #[clap(long)]
+    max_concurrent: Option<NonZeroUsize>,
+}
+
 /// The configuration for the site builder.
 #[derive(Deserialize, Debug, Clone)]
 pub(crate) struct Config {
@@ -258,6 +294,53 @@ impl Config {
     }
 }
 
+/// The number of epochs to store the blobs for.
+///
+/// Can be either a non-zero number of epochs or the special value `max`, which will store the blobs
+/// for the maximum number of epochs allowed by the system object on chain.
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub enum EpochCountOrMax {
+    /// Store the blobs for the maximum number of epochs allowed.
+    #[serde(rename = "max")]
+    Max,
+    /// The number of epochs to store the blobs for.
+    #[serde(untagged)]
+    Epochs(NonZeroU32),
+}
+
+impl EpochCountOrMax {
+    fn parse_epoch_count(input: &str) -> Result<Self> {
+        if input == "max" {
+            Ok(Self::Max)
+        } else {
+            let epochs = input.parse::<u32>()?;
+            Ok(Self::Epochs(NonZeroU32::new(epochs).ok_or_else(|| {
+                anyhow!("invalid epoch count; please a number >0 or `max`")
+            })?))
+        }
+    }
+
+    /// Tries to convert the `EpochCountOrMax` into an `EpochCount` value.
+    ///
+    /// If the `EpochCountOrMax` is `Max`, the `max_epochs_ahead` is used as the maximum number of
+    /// epochs that can be stored ahead.
+    pub fn try_into_epoch_count(&self, max_epochs_ahead: EpochCount) -> anyhow::Result<EpochCount> {
+        match self {
+            EpochCountOrMax::Max => Ok(max_epochs_ahead),
+            EpochCountOrMax::Epochs(epochs) => {
+                let epochs = epochs.get();
+                ensure!(
+                    epochs <= max_epochs_ahead,
+                    "blobs can only be stored for up to {} epochs ahead; {} epochs were requested",
+                    max_epochs_ahead,
+                    epochs
+                );
+                Ok(epochs)
+            }
+        }
+    }
+}
+
 mod default {
     pub(crate) fn walrus_binary() -> Option<String> {
         Some("walrus".to_owned())
@@ -391,7 +474,7 @@ async fn run() -> Result<()> {
             let mut site_manager = SiteManager::new(
                 config,
                 SiteIdentifier::ExistingSite(site_object),
-                epochs.get(),
+                epochs,
                 WhenWalrusUpload::Always,
             )
             .await?;

diff --git a/site-builder/src/publish.rs b/site-builder/src/publish.rs
@@ -3,13 +3,11 @@
 
 use std::{
     collections::BTreeSet,
-    num::NonZeroUsize,
     path::{Path, PathBuf},
     sync::mpsc::channel,
 };
 
 use anyhow::{anyhow, Result};
-use clap::Parser;
 use notify::{RecursiveMode, Watcher};
 use sui_sdk::rpc_types::{
     SuiExecutionStatus,
@@ -37,35 +35,11 @@ use crate::{
     summary::{SiteDataDiffSummary, Summarizable},
     util::{get_site_id_from_response, id_to_base36, path_or_defaults_if_exist, sign_and_send_ptb},
     Config,
+    PublishOptions,
 };
 
 const DEFAULT_WS_RESOURCES_FILE: &str = "ws-resources.json";
 
-#[derive(Parser, Debug, Clone)]
-pub struct PublishOptions {
-    /// The directory containing the site sources.
-    pub directory: PathBuf,
-    /// The path to the Walrus sites resources file.
-    ///
-    /// This JSON configuration file defined HTTP resource headers and other utilities for your
-    /// files. By default, the file is expected to be named `ws-resources.json` and located in the
-    /// root of the site directory.
-    ///
-    /// The configuration file _will not_ be uploaded to Walrus.
-    #[clap(long)]
-    ws_resources: Option<PathBuf>,
-    /// The number of epochs for which to save the resources on Walrus.
-    #[clap(long, default_value_t = 1)]
-    pub epochs: u64,
-    /// Preprocess the directory before publishing.
-    /// See the `list-directory` command. Warning: Rewrites all `index.html` files.
-    #[clap(long, action)]
-    pub list_directory: bool,
-    /// The maximum number of concurrent calls to the Walrus CLI for the computation of blob IDs.
-    #[clap(long)]
-    max_concurrent: Option<NonZeroUsize>,
-}
-
 /// The continuous editing options.
 #[derive(Debug, Clone)]
 pub(crate) enum ContinuousEditing {
@@ -237,7 +211,7 @@ impl SiteEditor<EditOptions> {
         let mut site_manager = SiteManager::new(
             self.config.clone(),
             self.edit_options.site_id.clone(),
-            self.edit_options.publish_options.epochs,
+            self.edit_options.publish_options.epochs.clone(),
             self.edit_options.when_upload.clone(),
         )
         .await?;

diff --git a/site-builder/src/site/manager.rs b/site-builder/src/site/manager.rs
@@ -29,6 +29,7 @@ use crate::{
     util::{get_site_id_from_response, sign_and_send_ptb},
     walrus::Walrus,
     Config,
+    EpochCountOrMax,
 };
 
 const MAX_RESOURCES_PER_PTB: usize = 200;
@@ -49,7 +50,7 @@ pub struct SiteManager {
     pub walrus: Walrus,
     pub wallet: WalletContext,
     pub site_id: SiteIdentifier,
-    pub epochs: u64,
+    pub epochs: EpochCountOrMax,
     pub when_upload: WhenWalrusUpload,
     pub backoff_config: ExponentialBackoffConfig,
 }
@@ -59,7 +60,7 @@ impl SiteManager {
     pub async fn new(
         config: Config,
         site_id: SiteIdentifier,
-        epochs: u64,
+        epochs: EpochCountOrMax,
         when_upload: WhenWalrusUpload,
     ) -> Result<Self> {
         Ok(SiteManager {
@@ -145,7 +146,7 @@ impl SiteManager {
                 retry_num += 1;
                 let result = self
                     .walrus
-                    .store(resource.full_path.clone(), self.epochs, false)
+                    .store(resource.full_path.clone(), self.epochs.clone(), false)
                     .await;
 
                 match result {

diff --git a/site-builder/src/walrus.rs b/site-builder/src/walrus.rs
@@ -11,7 +11,7 @@ use output::{try_from_output, BlobIdOutput, InfoOutput, NShards, ReadOutput, Sto
 use tokio::process::Command as CliCommand;
 
 use self::types::BlobId;
-use crate::walrus::command::WalrusCmdBuilder;
+use crate::{walrus::command::WalrusCmdBuilder, EpochCountOrMax};
 
 pub mod command;
 pub mod output;
@@ -73,7 +73,12 @@ impl Walrus {
     /// Issues a `store` JSON command to the Walrus CLI, returning the parsed output.
     // NOTE: takes a mutable reference to ensure that only one store command is executed at every
     // time. The issue is that the inner wallet may lock coins if called in parallel.
-    pub async fn store(&mut self, file: PathBuf, epochs: u64, force: bool) -> Result<StoreOutput> {
+    pub async fn store(
+        &mut self,
+        file: PathBuf,
+        epochs: EpochCountOrMax,
+        force: bool,
+    ) -> Result<StoreOutput> {
         create_command!(self, store, vec![file], epochs, force)
     }
 

diff --git a/site-builder/src/walrus/command.rs b/site-builder/src/walrus/command.rs
@@ -10,6 +10,7 @@ use serde::{Deserialize, Serialize};
 use serde_with::{serde_as, DisplayFromStr};
 
 use super::types::BlobId;
+use crate::EpochCountOrMax;
 
 /// Represents a call to the JSON mode of the Walrus CLI.
 #[derive(Debug, Clone, Serialize, Deserialize)]
@@ -47,8 +48,7 @@ pub enum Command {
         /// The path to the file to be stored.
         files: Vec<PathBuf>,
         /// The number of epochs for which to store the file.
-        #[serde(default = "default::epochs")]
-        epochs: u64,
+        epochs: EpochCountOrMax,
         /// Do not check for the blob status before storing it.
         ///
         /// This will create a new blob even if the blob is already certified for a sufficient
@@ -112,10 +112,6 @@ mod default {
     pub(crate) fn gas_budget() -> u64 {
         500_000_000
     }
-
-    pub(crate) fn epochs() -> u64 {
-        1
-    }
 }
 
 /// Helper struct to build [`WalrusJsonCmd`] instances.
@@ -155,7 +151,12 @@ impl WalrusCmdBuilder {
     }
 
     /// Adds a [`Command::Store`] command to the builder.
-    pub fn store(self, files: Vec<PathBuf>, epochs: u64, force: bool) -> WalrusCmdBuilder<Command> {
+    pub fn store(
+        self,
+        files: Vec<PathBuf>,
+        epochs: EpochCountOrMax,
+        force: bool,
+    ) -> WalrusCmdBuilder<Command> {
         let command = Command::Store {
             files,
             epochs,