Allow overriding container image platform for crane-based operations (#…

…334)

* Allow manually overriding container image platform for crane-based operations

* Update `CHANGELOG.md`

* Allow manually overriding container image platform for docker image pulls

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Added
 
 - (cli) Added a `stage show-next-index` command to print the index of the next staged pallet bundle (if it exists).
+- (cli) Added a `--platform` flag (and `FORKLIFT_PLATFORM` env var) to override the auto-detected platform (e.g. `linux/amd64` or `linux/arm64`) used for downloading container images for file exports and for pre-downloading container images needed for the next `forklift stage apply`.
 
 ## 0.8.0-alpha.2 - 2024-09-22
 

cmd/forklift/dev/plt/downloads.go

@@ -53,7 +53,7 @@ func cacheDlAction(versions Versions) cli.ActionFunc {
 		}
 
 		if err := fcli.DownloadExportFiles(
-			0, plt, caches.r, caches.d, false, c.Bool("parallel"),
+			0, plt, caches.r, caches.d, c.String("platform"), false, c.Bool("parallel"),
 		); err != nil {
 			return err
 		}

cmd/forklift/dev/plt/images.go

@@ -51,7 +51,7 @@ func cacheImgAction(versions Versions) cli.ActionFunc {
 
 		fmt.Println("Downloading Docker container images specified by the development pallet...")
 		if err := fcli.DownloadImages(
-			0, plt, caches.r, c.Bool("include-disabled"), c.Bool("parallel"),
+			0, plt, caches.r, c.String("platform"), c.Bool("include-disabled"), c.Bool("parallel"),
 		); err != nil {
 			return err
 		}

cmd/forklift/dev/plt/pallets.go

@@ -261,7 +261,7 @@ func cacheAllAction(versions Versions) cli.ActionFunc {
 
 		if err = fcli.CacheAllReqs(
 			0, plt, caches.m, caches.p, caches.r, caches.d,
-			c.Bool("include-disabled"), c.Bool("parallel"),
+			c.String("platform"), c.Bool("include-disabled"), c.Bool("parallel"),
 		); err != nil {
 			return err
 		}
@@ -360,7 +360,8 @@ func stageAction(versions Versions) cli.ActionFunc {
 		}
 		if _, err = fcli.StagePallet(
 			0, plt, stageStore, caches.staging(), c.String("exports"),
-			versions.Staging, c.Bool("no-cache-img"), c.Bool("parallel"), c.Bool("ignore-tool-version"),
+			versions.Staging, c.Bool("no-cache-img"), c.String("platform"), c.Bool("parallel"),
+			c.Bool("ignore-tool-version"),
 		); err != nil {
 			return err
 		}
@@ -400,7 +401,8 @@ func applyAction(versions Versions) cli.ActionFunc {
 		}
 		index, err := fcli.StagePallet(
 			0, plt, stageStore, caches.staging(), c.String("exports"),
-			versions.Staging, false, c.Bool("parallel"), c.Bool("ignore-tool-version"),
+			versions.Staging, false, c.String("platform"), c.Bool("parallel"),
+			c.Bool("ignore-tool-version"),
 		)
 		if err != nil {
 			return errors.Wrap(err, "couldn't stage pallet to be applied immediately")
@@ -513,7 +515,8 @@ func addPltAction(versions Versions) cli.ActionFunc {
 				return err
 			}
 			if _, _, err = fcli.CacheStagingReqs(
-				0, plt, caches.m, caches.p, caches.r, caches.d, false, c.Bool("parallel"),
+				0, plt, caches.m, caches.p, caches.r, caches.d,
+				c.String("platform"), false, c.Bool("parallel"),
 			); err != nil {
 				return err
 			}

cmd/forklift/dev/plt/repositories.go

@@ -105,7 +105,8 @@ func addRepoAction(versions Versions) cli.ActionFunc {
 		}
 		if !c.Bool("no-cache-req") {
 			if _, _, err = fcli.CacheStagingReqs(
-				0, plt, caches.m, caches.p, caches.r, caches.d, false, c.Bool("parallel"),
+				0, plt, caches.m, caches.p, caches.r, caches.d,
+				c.String("platform"), false, c.Bool("parallel"),
 			); err != nil {
 				return err
 			}

cmd/forklift/main.go

@@ -14,6 +14,7 @@ import (
 	"github.com/PlanktoScope/forklift/cmd/forklift/plt"
 	"github.com/PlanktoScope/forklift/cmd/forklift/stage"
 	fcli "github.com/PlanktoScope/forklift/internal/app/forklift/cli"
+	"github.com/PlanktoScope/forklift/internal/clients/crane"
 )
 
 func main() {
@@ -22,7 +23,10 @@ func main() {
 	}
 }
 
-var defaultWorkspaceBase, _ = os.UserHomeDir()
+var (
+	defaultWorkspaceBase, _ = os.UserHomeDir()
+	defaultPlatform         = crane.DetectPlatform().String()
+)
 
 var fcliVersions fcli.StagingVersions = fcli.StagingVersions{
 	Core: fcli.Versions{
@@ -83,6 +87,12 @@ var app = &cli.App{
 				"or starting containers",
 			EnvVars: []string{"FORKLIFT_PARALLEL"},
 		},
+		&cli.StringFlag{
+			Name:    "platform",
+			Value:   defaultPlatform,
+			Usage:   "Select the os/arch or os/arch/variant platform for downloading container images",
+			EnvVars: []string{"FORKLIFT_PLATFORM"},
+		},
 	},
 	Suggest: true,
 }

cmd/forklift/plt/downloads.go

@@ -51,7 +51,7 @@ func cacheDlAction(versions Versions) cli.ActionFunc {
 		}
 
 		if err := fcli.DownloadExportFiles(
-			0, plt, caches.r, caches.d, false, c.Bool("parallel"),
+			0, plt, caches.r, caches.d, c.String("platform"), false, c.Bool("parallel"),
 		); err != nil {
 			return err
 		}

cmd/forklift/plt/images.go

@@ -49,7 +49,7 @@ func cacheImgAction(versions Versions) cli.ActionFunc {
 
 		fmt.Println("Downloading Docker container images specified by the local pallet...")
 		if err := fcli.DownloadImages(
-			0, plt, caches.r, c.Bool("include-disabled"), c.Bool("parallel"),
+			0, plt, caches.r, c.String("platform"), c.Bool("include-disabled"), c.Bool("parallel"),
 		); err != nil {
 			return err
 		}

cmd/forklift/plt/pallets.go

@@ -99,7 +99,7 @@ func cacheAllAction(versions Versions) cli.ActionFunc {
 
 		if err = fcli.CacheAllReqs(
 			0, plt, caches.m, caches.p, caches.r, caches.d,
-			c.Bool("include-disabled"), c.Bool("parallel"),
+			c.String("platform"), c.Bool("include-disabled"), c.Bool("parallel"),
 		); err != nil {
 			return err
 		}
@@ -136,7 +136,8 @@ func switchAction(versions Versions) cli.ActionFunc {
 		if err = preparePallet(
 			// Note: we don't cache staging requirements because that will be handled by the apply/stage
 			// step anyways:
-			workspace, query, true, false, c.Bool("parallel"), c.Bool("ignore-tool-version"), versions,
+			workspace, query, true, false, c.String("platform"), c.Bool("parallel"),
+			c.Bool("ignore-tool-version"), versions,
 		); err != nil {
 			return err
 		}
@@ -370,7 +371,8 @@ func isHeadInRemotes(indent int, gitRepo *git.Repo) (bool, error) {
 
 func preparePallet(
 	workspace *forklift.FSWorkspace, gitRepoQuery forklift.GitRepoQuery,
-	updateLocalMirror, cacheStagingReqs, parallel, ignoreToolVersion bool, versions Versions,
+	updateLocalMirror, cacheStagingReqs bool, platform string, parallel, ignoreToolVersion bool,
+	versions Versions,
 ) error {
 	// clone pallet
 	if err := fcli.CloneQueriedGitRepoUsingLocalMirror(
@@ -394,7 +396,7 @@ func preparePallet(
 	if cacheStagingReqs {
 		fmt.Println()
 		if _, _, err = fcli.CacheStagingReqs(
-			0, plt, caches.m, caches.p, caches.r, caches.d, false, parallel,
+			0, plt, caches.m, caches.p, caches.r, caches.d, platform, false, parallel,
 		); err != nil {
 			return err
 		}
@@ -431,7 +433,8 @@ func upgradeAction(versions Versions) cli.ActionFunc {
 		if err = preparePallet(
 			// Note: we don't cache staging requirements because that will be handled by the apply/stage
 			// step anyways:
-			workspace, query, false, false, c.Bool("parallel"), c.Bool("ignore-tool-version"), versions,
+			workspace, query, false, false, c.String("platform"), c.Bool("parallel"),
+			c.Bool("ignore-tool-version"), versions,
 		); err != nil {
 			return err
 		}
@@ -669,7 +672,7 @@ func cloneAction(versions Versions) cli.ActionFunc {
 		}
 
 		if err = preparePallet(
-			workspace, query, true, !c.Bool("no-cache-req"), c.Bool("parallel"),
+			workspace, query, true, !c.Bool("no-cache-req"), c.String("platform"), c.Bool("parallel"),
 			c.Bool("ignore-tool-version"), versions,
 		); err != nil {
 			return err
@@ -744,7 +747,8 @@ func pullAction(versions Versions) cli.ActionFunc {
 
 		if !c.Bool("no-cache-req") {
 			if _, _, err = fcli.CacheStagingReqs(
-				0, plt, caches.m, caches.p, caches.r, caches.d, false, c.Bool("parallel"),
+				0, plt, caches.m, caches.p, caches.r, caches.d,
+				c.String("platform"), false, c.Bool("parallel"),
 			); err != nil {
 				return err
 			}
@@ -865,7 +869,8 @@ func stageAction(versions Versions) cli.ActionFunc {
 		}
 		if _, err = fcli.StagePallet(
 			0, plt, stageStore, caches.staging(), c.String("exports"),
-			versions.Staging, c.Bool("no-cache-img"), c.Bool("parallel"), c.Bool("ignore-tool-version"),
+			versions.Staging, c.Bool("no-cache-img"), c.String("platform"), c.Bool("parallel"),
+			c.Bool("ignore-tool-version"),
 		); err != nil {
 			return err
 		}
@@ -900,7 +905,8 @@ func applyAction(versions Versions) cli.ActionFunc {
 		}
 		index, err := fcli.StagePallet(
 			0, plt, stageStore, caches.staging(), c.String("exports"),
-			versions.Staging, false, c.Bool("parallel"), c.Bool("ignore-tool-version"),
+			versions.Staging, false, c.String("platform"), c.Bool("parallel"),
+			c.Bool("ignore-tool-version"),
 		)
 		if err != nil {
 			return errors.Wrap(err, "couldn't stage pallet to be applied immediately")
@@ -1011,7 +1017,8 @@ func addPltAction(versions Versions) cli.ActionFunc {
 				return err
 			}
 			if _, _, err = fcli.CacheStagingReqs(
-				0, plt, caches.m, caches.p, caches.r, caches.d, false, c.Bool("parallel"),
+				0, plt, caches.m, caches.p, caches.r, caches.d,
+				c.String("platform"), false, c.Bool("parallel"),
 			); err != nil {
 				return err
 			}

cmd/forklift/plt/repositories.go

@@ -98,7 +98,8 @@ func addRepoAction(versions Versions) cli.ActionFunc {
 		}
 		if !c.Bool("no-cache-req") {
 			if _, _, err = fcli.CacheStagingReqs(
-				0, plt, caches.m, caches.p, caches.r, caches.d, false, c.Bool("parallel"),
+				0, plt, caches.m, caches.p, caches.r, caches.d,
+				c.String("platform"), false, c.Bool("parallel"),
 			); err != nil {
 				return err
 			}

cmd/forklift/stage/images.go

@@ -22,7 +22,7 @@ func cacheImgAction(versions Versions) cli.ActionFunc {
 
 		if err = fcli.DownloadImagesForStoreApply(
 			0, store, versions.Tool, versions.MinSupportedBundle,
-			c.Bool("parallel"), c.Bool("ignore-tool-version"),
+			c.String("platform"), c.Bool("parallel"), c.Bool("ignore-tool-version"),
 		); err != nil {
 			return err
 		}

cmd/forklift/stage/store.go

@@ -298,7 +298,8 @@ func setNextAction(versions Versions) cli.ActionFunc {
 
 		if err = fcli.SetNextStagedBundle(
 			0, store, newNext, c.String("exports"), versions.Tool, versions.MinSupportedBundle,
-			c.Bool("no-cache-img"), c.Bool("parallel"), c.Bool("ignore-tool-version"),
+			c.Bool("no-cache-img"), c.String("platform"), c.Bool("parallel"),
+			c.Bool("ignore-tool-version"),
 		); err != nil {
 			return err
 		}

internal/app/forklift/cli/caching-exports.go

@@ -40,7 +40,8 @@ func GetDownloadCache(wpath string, ensureCache bool) (*forklift.FSDownloadCache
 
 func DownloadExportFiles(
 	indent int, deplsLoader ResolvedDeplsLoader, pkgLoader forklift.FSPkgLoader,
-	dlCache *forklift.FSDownloadCache, includeDisabled, parallel bool,
+	dlCache *forklift.FSDownloadCache,
+	platform string, includeDisabled, parallel bool,
 ) error {
 	httpDownloads, ociDownloads, err := ListRequiredDownloads(deplsLoader, pkgLoader, includeDisabled)
 	if err != nil {
@@ -83,9 +84,9 @@ func DownloadExportFiles(
 	}
 
 	if parallel {
-		return downloadParallel(indent, newHTTP, newOCI, dlCache, http.DefaultClient)
+		return downloadParallel(indent, newHTTP, newOCI, platform, dlCache, http.DefaultClient)
 	}
-	return downloadSerial(indent, newHTTP, newOCI, dlCache, http.DefaultClient)
+	return downloadSerial(indent, newHTTP, newOCI, platform, dlCache, http.DefaultClient)
 }
 
 func ListRequiredDownloads(
@@ -140,7 +141,8 @@ func ListRequiredDownloads(
 }
 
 func downloadParallel(
-	indent int, httpURLs, ociImageNames []string, cache *forklift.FSDownloadCache, hc *http.Client,
+	indent int, httpURLs, ociImageNames []string, platform string, cache *forklift.FSDownloadCache,
+	hc *http.Client,
 ) error {
 	eg, egctx := errgroup.WithContext(context.Background())
 	for _, url := range httpURLs {
@@ -164,7 +166,7 @@ func downloadParallel(
 			if err != nil {
 				return errors.Wrapf(err, "couldn't determine path to cache download for %s", imageName)
 			}
-			if err = downloadOCIImage(egctx, imageName, outputPath); err != nil {
+			if err = downloadOCIImage(egctx, imageName, outputPath, platform); err != nil {
 				return errors.Wrapf(err, "couldn't download %s", imageName)
 			}
 			IndentedPrintf(indent, "Downloaded %s\n", imageName)
@@ -178,7 +180,8 @@ func downloadParallel(
 }
 
 func downloadSerial(
-	indent int, httpURLs, ociImageNames []string, cache *forklift.FSDownloadCache, hc *http.Client,
+	indent int, httpURLs, ociImageNames []string, platform string, cache *forklift.FSDownloadCache,
+	hc *http.Client,
 ) error {
 	for _, url := range httpURLs {
 		IndentedPrintf(indent, "Downloading file %s to cache...\n", url)
@@ -197,7 +200,7 @@ func downloadSerial(
 		if err != nil {
 			return errors.Wrapf(err, "couldn't determine path to cache download for %s", imageName)
 		}
-		if err = downloadOCIImage(context.Background(), imageName, outputPath); err != nil {
+		if err = downloadOCIImage(context.Background(), imageName, outputPath, platform); err != nil {
 			return errors.Wrapf(err, "couldn't download %s", imageName)
 		}
 		IndentedPrintf(indent, "Downloaded %s\n", imageName)
@@ -249,7 +252,7 @@ func downloadFile(ctx context.Context, url, outputPath string, hc *http.Client)
 	return nil
 }
 
-func downloadOCIImage(ctx context.Context, imageName, outputPath string) error {
+func downloadOCIImage(ctx context.Context, imageName, outputPath, platform string) error {
 	if err := forklift.EnsureExists(filepath.FromSlash(path.Dir(outputPath))); err != nil {
 		return err
 	}
@@ -265,7 +268,7 @@ func downloadOCIImage(ctx context.Context, imageName, outputPath string) error {
 		}
 	}()
 
-	if err = crane.ExportOCIImage(ctx, imageName, file); err != nil {
+	if err = crane.ExportOCIImage(ctx, imageName, file, platform); err != nil {
 		return errors.Wrapf(err, "couldn't download and export image as a tarball: %s", imageName)
 	}
 

internal/app/forklift/cli/caching.go

@@ -11,17 +11,23 @@ func CacheAllReqs(
 	indent int, pallet *forklift.FSPallet, mirrorsCache core.Pather,
 	palletCache forklift.PathedPalletCache, repoCache forklift.PathedRepoCache,
 	dlCache *forklift.FSDownloadCache,
-	includeDisabled, parallel bool,
+	platform string, includeDisabled, parallel bool,
 ) error {
 	pallet, repoCacheWithMerged, err := CacheStagingReqs(
-		indent, pallet, mirrorsCache, palletCache, repoCache, dlCache, includeDisabled, parallel,
+		indent, pallet, mirrorsCache, palletCache, repoCache, dlCache,
+		platform, includeDisabled, parallel,
 	)
 	if err != nil {
 		return err
 	}
 
-	IndentedPrintln(indent, "Downloading Docker container images to be deployed by the local pallet...")
-	if err := DownloadImages(1, pallet, repoCacheWithMerged, includeDisabled, parallel); err != nil {
+	IndentedPrintln(
+		indent,
+		"Downloading Docker container images to be deployed by the local pallet...",
+	)
+	if err := DownloadImages(
+		1, pallet, repoCacheWithMerged, platform, includeDisabled, parallel,
+	); err != nil {
 		return err
 	}
 	return nil
@@ -31,7 +37,7 @@ func CacheStagingReqs(
 	indent int, pallet *forklift.FSPallet, mirrorsCache core.Pather,
 	palletCache forklift.PathedPalletCache, repoCache forklift.PathedRepoCache,
 	dlCache *forklift.FSDownloadCache,
-	includeDisabled, parallel bool,
+	platform string, includeDisabled, parallel bool,
 ) (merged *forklift.FSPallet, repoCacheWithMerged *forklift.LayeredRepoCache, err error) {
 	IndentedPrintln(indent, "Caching everything needed to stage the pallet...")
 	indent++
@@ -65,7 +71,7 @@ func CacheStagingReqs(
 	}
 
 	if err = DownloadExportFiles(
-		indent, merged, repoCacheWithMerged, dlCache, includeDisabled, parallel,
+		indent, merged, repoCacheWithMerged, dlCache, platform, includeDisabled, parallel,
 	); err != nil {
 		return merged, repoCacheWithMerged, err
 	}