From 4aa53d70c3ba777142a96fdecd37b85c684ec2fb Mon Sep 17 00:00:00 2001 From: jk jensen Date: Fri, 27 Dec 2024 11:37:39 -0700 Subject: [PATCH] [suiop][pulumi] update command (#20731) ## Description Command to trigger a dependency update for all pulumi programs in the given directory. This will make codemod updates for things like PRO-194 much easier ## Test plan ``` cargo run -- p u ~/mysten/sui-operations/pulumi/apps Compiling suiop-cli v1.40.0 (/Users/jkjensen/mysten/sui/crates/suiop-cli) Finished `dev` profile [unoptimized + debuginfo] target(s) in 6.20s Running `/Users/jkjensen/mysten/sui/target/debug/suiop p u /Users/jkjensen/mysten/sui-operations/pulumi/apps` 2024-12-24T17:23:14.098064Z INFO suioplib::cli::pulumi::deps: Processing subdirectory: /Users/jkjensen/mysten/sui-operations/pulumi/apps/suifrens 2024-12-24T17:23:14.098584Z INFO suioplib::cli::pulumi::deps: Updating dependencies for python 2024-12-24T17:23:14.098591Z INFO suioplib::cli::pulumi::deps: Updating dependencies for python project at /Users/jkjensen/mysten/sui-operations/pulumi/apps/suifrens 2024-12-24T17:23:15.316311Z INFO suioplib::cli::pulumi::deps: Processing subdirectory: /Users/jkjensen/mysten/sui-operations/pulumi/apps/suifrens/__pycache__ ... 2024-12-24T17:26:08.786248Z INFO suioplib::cli::pulumi::deps: Successfully updated dependencies ``` --- ## Release notes Check each box that your changes affect. If none of the boxes relate to your changes, release notes aren't required. For each box you select, include information after the relevant heading that describes the impact of your changes that a user might notice and any actions they must take to implement updates. - [ ] Protocol: - [ ] Nodes (Validators and Full nodes): - [ ] gRPC: - [ ] JSON-RPC: - [ ] GraphQL: - [ ] CLI: - [ ] Rust SDK: --- Cargo.lock | 68 +++++++------ crates/suiop-cli/Cargo.toml | 2 +- crates/suiop-cli/src/cli/pulumi/deps.rs | 124 ++++++++++++++++++++++++ crates/suiop-cli/src/cli/pulumi/mod.rs | 25 +++++ crates/suiop-cli/src/command.rs | 5 + 5 files changed, 188 insertions(+), 36 deletions(-) create mode 100644 crates/suiop-cli/src/cli/pulumi/deps.rs diff --git a/Cargo.lock b/Cargo.lock index 70e1e71311ba5..50885c3e4d922 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -179,8 +179,8 @@ dependencies = [ "rand 0.8.5", "rcgen", "ring 0.17.8", - "rustls 0.23.12", - "rustls-webpki 0.102.6", + "rustls 0.23.20", + "rustls-webpki 0.102.8", "serde", "serde_json", "socket2 0.5.6", @@ -1690,7 +1690,7 @@ dependencies = [ "hyper 1.4.1", "hyper-util", "pin-project-lite", - "rustls 0.23.12", + "rustls 0.23.20", "rustls-pemfile 2.1.2", "rustls-pki-types", "tokio", @@ -2805,7 +2805,7 @@ dependencies = [ "quinn-proto", "rand 0.8.5", "rstest", - "rustls 0.23.12", + "rustls 0.23.20", "serde", "shared-crypto", "strum_macros 0.24.3", @@ -5961,7 +5961,7 @@ dependencies = [ "hyper 1.4.1", "hyper-util", "log", - "rustls 0.23.12", + "rustls 0.23.20", "rustls-native-certs 0.7.1", "rustls-pki-types", "tokio", @@ -6605,17 +6605,15 @@ dependencies = [ [[package]] name = "jsonpath-rust" -version = "0.5.1" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19d8fe85bd70ff715f31ce8c739194b423d79811a19602115d611a3ec85d6200" +checksum = "0c00ae348f9f8fd2d09f82a98ca381c60df9e0820d8d79fce43e649b4dc3128b" dependencies = [ - "lazy_static", - "once_cell", "pest", "pest_derive", "regex", "serde_json", - "thiserror 1.0.64", + "thiserror 2.0.9", ] [[package]] @@ -6850,9 +6848,9 @@ dependencies = [ [[package]] name = "kube" -version = "0.96.0" +version = "0.97.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efffeb3df0bd4ef3e5d65044573499c0e4889b988070b08c50b25b1329289a1f" +checksum = "e5fd2596428f922f784ca43907c449f104d69055c811135684474143736c67ae" dependencies = [ "k8s-openapi", "kube-client", @@ -6861,9 +6859,9 @@ dependencies = [ [[package]] name = "kube-client" -version = "0.96.0" +version = "0.97.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8bf471ece8ff8d24735ce78dac4d091e9fcb8d74811aeb6b75de4d1c3f5de0f1" +checksum = "d539b6493d162ae5ab691762be972b6a1c20f6d8ddafaae305c0e2111b589d99" dependencies = [ "base64 0.22.1", "bytes", @@ -6883,13 +6881,13 @@ dependencies = [ "k8s-openapi", "kube-core", "pem 3.0.4", - "rustls 0.23.12", + "rustls 0.23.20", "rustls-pemfile 2.1.2", "secrecy", "serde", "serde_json", "serde_yaml 0.9.21", - "thiserror 1.0.64", + "thiserror 2.0.9", "tokio", "tokio-util 0.7.10", "tower 0.5.1", @@ -6899,9 +6897,9 @@ dependencies = [ [[package]] name = "kube-core" -version = "0.96.0" +version = "0.97.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f42346d30bb34d1d7adc5c549b691bce7aa3a1e60254e68fab7e2d7b26fe3d77" +checksum = "98a87cc0046cf6b62cbb63ae1fbc366ee8ba29269f575289679473754ff5d7a7" dependencies = [ "chrono", "form_urlencoded", @@ -6910,7 +6908,7 @@ dependencies = [ "serde", "serde-value", "serde_json", - "thiserror 1.0.64", + "thiserror 2.0.9", ] [[package]] @@ -10408,7 +10406,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 1.1.0", - "rustls 0.23.12", + "rustls 0.23.20", "thiserror 1.0.64", "tokio", "tracing", @@ -10424,7 +10422,7 @@ dependencies = [ "rand 0.8.5", "ring 0.17.8", "rustc-hash 2.0.0", - "rustls 0.23.12", + "rustls 0.23.20", "slab", "thiserror 1.0.64", "tinyvec", @@ -10817,7 +10815,7 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls 0.23.12", + "rustls 0.23.20", "rustls-native-certs 0.7.1", "rustls-pemfile 2.1.2", "rustls-pki-types", @@ -11356,15 +11354,15 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.12" +version = "0.23.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c58f8c84392efc0a126acce10fa59ff7b3d2ac06ab451a33f2741989b806b044" +checksum = "5065c3f250cbd332cd894be57c40fa52387247659b14a2d6041d121547903b1b" dependencies = [ "log", "once_cell", "ring 0.17.8", "rustls-pki-types", - "rustls-webpki 0.102.6", + "rustls-webpki 0.102.8", "subtle", "zeroize", ] @@ -11415,9 +11413,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.7.0" +version = "1.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976295e77ce332211c0d24d92c0e83e50f5c5f046d11082cea19f3df13a3562d" +checksum = "d2bf47e6ff922db3825eb750c4e2ff784c6ff8fb9e13046ef6a1d1c5401b0b37" [[package]] name = "rustls-webpki" @@ -11431,9 +11429,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.102.6" +version = "0.102.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e6b52d4fda176fd835fdc55a835d4a89b8499cad995885a21149d5ad62f852e" +checksum = "64ca1bc8749bd4cf37b5ce386cc146580777b4e8572c7b97baf22c83f444bee9" dependencies = [ "ring 0.17.8", "rustls-pki-types", @@ -14884,7 +14882,7 @@ dependencies = [ "protobuf", "rand 0.8.5", "reqwest 0.12.5", - "rustls 0.23.12", + "rustls 0.23.20", "rustls-pemfile 2.1.2", "serde", "serde_json", @@ -15501,8 +15499,8 @@ dependencies = [ "rand 0.8.5", "rcgen", "reqwest 0.12.5", - "rustls 0.23.12", - "rustls-webpki 0.102.6", + "rustls 0.23.20", + "rustls-webpki 0.102.8", "tokio", "tokio-rustls 0.26.0", "tower-layer", @@ -15826,7 +15824,7 @@ dependencies = [ "object_store", "prometheus", "rand 0.8.5", - "rustls 0.23.12", + "rustls 0.23.20", "serde", "serde_json", "serde_yaml 0.8.26", @@ -16568,7 +16566,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04fb792ccd6bbcd4bba408eb8a292f70fc4a3589e5d793626f45190e6454b6ab" dependencies = [ "ring 0.17.8", - "rustls 0.23.12", + "rustls 0.23.20", "tokio", "tokio-postgres", "tokio-rustls 0.26.0", @@ -16613,7 +16611,7 @@ version = "0.26.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7bc40d0e5a97695bb96e27995cd3a08538541b0a846f65bba7a359f36700d4" dependencies = [ - "rustls 0.23.12", + "rustls 0.23.20", "rustls-pki-types", "tokio", ] diff --git a/crates/suiop-cli/Cargo.toml b/crates/suiop-cli/Cargo.toml index 8b34ff2d4a319..82a29a0606eea 100644 --- a/crates/suiop-cli/Cargo.toml +++ b/crates/suiop-cli/Cargo.toml @@ -57,7 +57,7 @@ thiserror.workspace = true strsim = "0.11.1" futures-timer = "3.0.3" tempfile.workspace = true -kube = { version = "0.96.0", features = ["client"] } +kube = { version = "0.97.0", features = ["client"] } k8s-openapi = { version = "0.23.0", features = ["latest"] } diff --git a/crates/suiop-cli/src/cli/pulumi/deps.rs b/crates/suiop-cli/src/cli/pulumi/deps.rs new file mode 100644 index 0000000000000..90f24c92c1a98 --- /dev/null +++ b/crates/suiop-cli/src/cli/pulumi/deps.rs @@ -0,0 +1,124 @@ +// Copyright (c) Mysten Labs, Inc. +// SPDX-License-Identifier: Apache-2.0 + +use crate::{command::CommandOptions, run_cmd}; +use anyhow::Result; +use serde_yaml::Value; +use std::fs; +use std::path::{Path, PathBuf}; +use tracing::{debug, info}; + +fn update_dependencies(path: &Path, runtime: &str) -> Result<()> { + info!( + "Updating dependencies for {} project at {}", + runtime, + path.display() + ); + + let mut cmd_opts = CommandOptions::new(false, false); + cmd_opts.current_dir = Some(path.to_path_buf()); + let output = match runtime { + "go" => run_cmd(vec!["go", "get", "-u"], Some(cmd_opts.clone())) + .and_then(|_o| run_cmd(vec!["go", "mod", "tidy"], Some(cmd_opts))), + "python" => run_cmd(vec!["poetry", "update"], Some(cmd_opts)), + "typescript" => run_cmd(vec!["pnpm", "update"], Some(cmd_opts)), + _ => unreachable!(), + }?; + debug!( + "Command output: {:?}", + String::from_utf8_lossy(&output.stdout) + ); + if !output.stderr.is_empty() { + debug!( + "Command stderr: {:?}", + String::from_utf8_lossy(&output.stderr) + ); + } + + Ok(()) +} + +fn process_directory( + dir_path: &Path, + runtime_filter: &Option, +) -> Result> { + let mut errors = Vec::new(); + + let pulumi_yaml = dir_path.join("Pulumi.yaml"); + + if pulumi_yaml.exists() { + let update_result = (|| -> Result<()> { + let contents = fs::read_to_string(&pulumi_yaml)?; + let yaml: Value = serde_yaml::from_str(&contents) + .map_err(|e| std::io::Error::new(std::io::ErrorKind::InvalidData, e))?; + + let runtime = yaml["runtime"] + .as_str() + .or_else(|| yaml["runtime"]["name"].as_str()) + .ok_or_else(|| { + std::io::Error::new( + std::io::ErrorKind::InvalidData, + "No runtime field found in Pulumi.yaml", + ) + })?; + + let runtime = runtime.to_lowercase(); + if !["typescript", "go", "python"].contains(&runtime.as_str()) { + return Ok(()); + } + + if runtime_filter.as_ref().map_or(true, |f| f == &runtime) { + info!("Updating dependencies for {}", runtime); + update_dependencies(dir_path, &runtime)?; + } + Ok(()) + })(); + + if let Err(e) = update_result { + errors.push((dir_path.to_path_buf(), e)); + } + } + + // Recurse into subdirectories + for entry in fs::read_dir(dir_path)? { + let entry = entry?; + let path = entry.path(); + let file_name = path.file_name().and_then(|n| n.to_str()).unwrap_or(""); + if path.is_dir() + && !file_name.starts_with('.') + && !file_name.contains("common") + && !file_name.contains("node_modules") + { + info!("Processing subdirectory: {}", path.display()); + match process_directory(&path, runtime_filter) { + Ok(mut sub_errors) => errors.append(&mut sub_errors), + Err(e) => errors.push((path, e)), + } + } + } + Ok(errors) +} + +pub fn update_deps_cmd(filepath: PathBuf, runtime: Option) -> Result<()> { + if !filepath.exists() || !filepath.is_dir() { + return Err(anyhow::anyhow!( + "Specified path does not exist or is not a directory", + )); + } + + let errors = process_directory(&filepath, &runtime)?; + if !errors.is_empty() { + let error_messages = errors + .into_iter() + .map(|(path, error)| format!("- {}: {}", path.display(), error)) + .collect::>() + .join("\n"); + Err(anyhow::anyhow!( + "Failed to update dependencies in the following directories:\n{}", + error_messages + )) + } else { + info!("Successfully updated dependencies"); + Ok(()) + } +} diff --git a/crates/suiop-cli/src/cli/pulumi/mod.rs b/crates/suiop-cli/src/cli/pulumi/mod.rs index 2d9df6d16f078..ccdb75d5399e6 100644 --- a/crates/suiop-cli/src/cli/pulumi/mod.rs +++ b/crates/suiop-cli/src/cli/pulumi/mod.rs @@ -1,17 +1,28 @@ // Copyright (c) Mysten Labs, Inc. // SPDX-License-Identifier: Apache-2.0 +mod deps; mod init; mod setup; +use std::path::PathBuf; + use anyhow::Result; use clap::arg; use clap::Parser; use clap::ValueEnum; +use deps::update_deps_cmd; use init::ProjectType; use setup::ensure_gcloud; use setup::ensure_pulumi_setup; +fn validate_runtime(s: &str) -> Result { + match s.to_lowercase().as_str() { + "typescript" | "go" | "python" => Ok(s.to_lowercase()), + _ => Err(String::from("Runtime must be typescript, go, or python")), + } +} + #[derive(ValueEnum, PartialEq, Clone, Debug)] pub enum PulumiProjectRuntime { #[clap(alias = "golang")] @@ -47,6 +58,17 @@ pub enum PulumiAction { #[arg(long, default_value = "go")] runtime: PulumiProjectRuntime, }, + /// update dependencies for pulumi programs in a given directory + #[command(name = "update-deps", aliases = ["u"])] + UpdateDeps { + /// Starting directory path + #[arg(required = true)] + filepath: PathBuf, + + /// Optional runtime filter (typescript, go, python) + #[arg(value_parser = validate_runtime)] + runtime: Option, + }, } pub fn pulumi_cmd(args: &PulumiArgs) -> Result<()> { @@ -63,5 +85,8 @@ pub fn pulumi_cmd(args: &PulumiArgs) -> Result<()> { } project_type.create_project(kms, project_name.clone(), runtime) } + PulumiAction::UpdateDeps { filepath, runtime } => { + update_deps_cmd(filepath.clone(), runtime.clone()) + } } } diff --git a/crates/suiop-cli/src/command.rs b/crates/suiop-cli/src/command.rs index 8247dbb9938f1..0af69815bd1e6 100644 --- a/crates/suiop-cli/src/command.rs +++ b/crates/suiop-cli/src/command.rs @@ -6,6 +6,7 @@ use anyhow::Context; use anyhow::Result; use spinners::Spinner; use spinners::Spinners; +use std::path::PathBuf; use std::process::Command; use std::process::Output; use std::process::Stdio; @@ -16,6 +17,7 @@ const SPINNER: Spinners = Spinners::Dots12; pub struct CommandOptions { shared_stdio: bool, show_spinner: bool, + pub current_dir: Option, } impl CommandOptions { @@ -23,6 +25,7 @@ impl CommandOptions { CommandOptions { shared_stdio, show_spinner, + current_dir: None, } } } @@ -32,6 +35,7 @@ impl Default for CommandOptions { CommandOptions { shared_stdio: false, show_spinner: true, + current_dir: None, } } } @@ -41,6 +45,7 @@ pub fn run_cmd(cmd_in: Vec<&str>, options: Option) -> Result 1 { cmd.args(cmd_in[1..].iter())