From 17ec0e61e65586d8495c5bb43e9651ab8b1fc995 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 02:54:51 +0000 Subject: [PATCH 01/62] WIP: added manifest_add command Signed-off-by: WYGIN --- cmd/cmd.go | 1 + internal/commands/commands.go | 8 ++ internal/commands/manifest.go | 26 +++++++ internal/commands/manifest_add.go | 61 +++++++++++++++ internal/commands/manifest_add_test.go | 103 +++++++++++++++++++++++++ 5 files changed, 199 insertions(+) create mode 100644 internal/commands/manifest.go create mode 100644 internal/commands/manifest_add.go create mode 100644 internal/commands/manifest_add_test.go diff --git a/cmd/cmd.go b/cmd/cmd.go index ace01f3f2d..1ad7d8a21e 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -104,6 +104,7 @@ func NewPackCommand(logger ConfigurableLogger) (*cobra.Command, error) { rootCmd.AddCommand(commands.SetDefaultRegistry(logger, cfg, cfgPath)) rootCmd.AddCommand(commands.RemoveRegistry(logger, cfg, cfgPath)) rootCmd.AddCommand(commands.YankBuildpack(logger, cfg, packClient)) + rootCmd.AddCommand(commands.NewManifestCommand(logger, packClient)) } packHome, err := config.PackHome() diff --git a/internal/commands/commands.go b/internal/commands/commands.go index 11fa645ca5..08bfd3933b 100644 --- a/internal/commands/commands.go +++ b/internal/commands/commands.go @@ -32,6 +32,14 @@ type PackClient interface { InspectExtension(client.InspectExtensionOptions) (*client.ExtensionInfo, error) PullBuildpack(context.Context, client.PullBuildpackOptions) error DownloadSBOM(name string, options client.DownloadSBOMOptions) error + CreateManifest(ctx context.Context, name string, images []string, opts client.CreateManifestOptions) (imageID string, err error) + AnnotateManifest(ctx context.Context, name string, image string, opts client.ManifestAnnotateOptions) error + ExistsManifest(ctx context.Context, image string) error + AddManifest(ctx context.Context, index string, images string, opts client.ManifestAddOptions) (imageID string, err error) + DeleteManifest(ctx context.Context, name []string) error + RemoveManifest(ctx context.Context, name string, images []string) error + PushManifest(ctx context.Context, index string, opts client.PushManifestOptions) (imageID string, err error) + InspectManifest(ctx context.Context, name string, opts client.InspectManifestOptions) error } func AddHelpFlag(cmd *cobra.Command, commandName string) { diff --git a/internal/commands/manifest.go b/internal/commands/manifest.go new file mode 100644 index 0000000000..9b2108a449 --- /dev/null +++ b/internal/commands/manifest.go @@ -0,0 +1,26 @@ +package commands + +import ( + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/logging" +) + +func NewManifestCommand(logger logging.Logger, client PackClient) *cobra.Command { + cmd := &cobra.Command{ + Use: "manifest", + Short: "Interact with image index or manifest list", + RunE: nil, + } + + cmd.AddCommand(ManifestCreate(logger, client)) + cmd.AddCommand(ManifestAdd(logger, client)) + cmd.AddCommand(ManifestAnnotate(logger, client)) + cmd.AddCommand(ManifestDelete(logger, client)) + cmd.AddCommand(ManifestInspect(logger, client)) + cmd.AddCommand(ManifestPush(logger, client)) + cmd.AddCommand(ManifestRemove(logger, client)) + + AddHelpFlag(cmd, "manifest") + return cmd +} \ No newline at end of file diff --git a/internal/commands/manifest_add.go b/internal/commands/manifest_add.go new file mode 100644 index 0000000000..7b85b0d662 --- /dev/null +++ b/internal/commands/manifest_add.go @@ -0,0 +1,61 @@ +package commands + +import ( + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/client" + "github.com/buildpacks/pack/pkg/logging" +) + +// ManifestAddFlags define flags provided to the ManifestAdd +type ManifestAddFlags struct { + ManifestAnnotateFlags + all bool +} + +// ManifestAdd modifies a manifest list (Image index) and add a new image to the list of manifests. +func ManifestAdd(logger logging.Logger, pack PackClient) *cobra.Command { + var flags ManifestAddFlags + + cmd := &cobra.Command{ + Use: "pack manifest add [OPTIONS] [flags]", + Args: cobra.MatchAll(cobra.ExactArgs(2), cobra.OnlyValidArgs), + Short: "manifest add modifies a manifest list (Image index) and add a new image to the list of manifests.", + Example: `pack manifest add cnbs/sample-package:hello-multiarch-universe \ + cnbs/sample-package:hello-universe-riscv-linux`, + Long: `manifest add modifies a manifest list (Image index) and add a new image to the list of manifests. + + When a manifest list exits locally, user can add a new image to the manifest list using this command`, + RunE: logError(logger, func(cmd *cobra.Command, args []string) error { + imageIndex := args[0] + manifests := args[1] + if err := validateManifestAddFlags(flags); err != nil { + return err + } + + imageID, err := pack.AddManifest(cmd.Context(), imageIndex, manifests, client.ManifestAddOptions{ + + All: flags.all, + }) + + if err != nil { + return err + } + + logger.Infof(imageID) + return nil + }), + } + + cmd.Flags().BoolVar(&flags.all, "all", false, "add all of the contents to the local list (applies only if is an index)") + cmd.Flags().StringVar(&flags.os, "os", "", "Set the operating system") + cmd.Flags().StringVar(&flags.arch, "arch", "", "Set the architecture") + cmd.Flags().StringVar(&flags.variant, "variant", "", "Set the architecture variant") + + AddHelpFlag(cmd, "add") + return cmd +} + +func validateManifestAddFlags(flags ManifestAddFlags) error { + return nil +} \ No newline at end of file diff --git a/internal/commands/manifest_add_test.go b/internal/commands/manifest_add_test.go new file mode 100644 index 0000000000..8e419bb4f9 --- /dev/null +++ b/internal/commands/manifest_add_test.go @@ -0,0 +1,103 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestAddCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestAddCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestAdd(logger, mockClient) + }) + + when("#ManifestAdd", func() { + when("no flags specified", func() { + + }) + when("when --all flags passed", func() { + + }) + when("when --os flags passed", func() { + + }) + when("when --arch flags passed", func() { + + }) + when("when --variant flags passed", func() { + + }) + when("when --os-version flags passed", func() { + + }) + when("when --features flags passed", func() { + + }) + when("when --os-features flags passed", func() { + + }) + when("when --annotations flags passed", func() { + + }) + when("when multiple flags passed", func() { + + }) + when("when no args passed", func() { + + }) + when("when manifest list reference is incorrect", func() { + + }) + when("when manifest reference is incorrect", func() { + + }) + when("when manifest passed in-place of manifest list on first arg", func() { + + }) + when("when manifest list is passed on second arg", func() { + + }) + when("when manifest is passed on second arg with --all option", func() { + + }) + when("when manifest list in locally available", func() { + + }) + when("when manifest is not locally available", func() { + + }) + when("when manifest is locally available passed", func() { + + }) + when("when multiple manifests passed", func() { + + }) + }) +} \ No newline at end of file From 45f39bc458ef3d10cb853d9019ece64f4e7f1b0b Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 02:57:44 +0000 Subject: [PATCH 02/62] WIP added manifest_annotate command Signed-off-by: WYGIN --- internal/commands/manifest_annotate.go | 42 ++++++++ internal/commands/manifest_annotate_test.go | 100 ++++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 internal/commands/manifest_annotate.go create mode 100644 internal/commands/manifest_annotate_test.go diff --git a/internal/commands/manifest_annotate.go b/internal/commands/manifest_annotate.go new file mode 100644 index 0000000000..268721b41f --- /dev/null +++ b/internal/commands/manifest_annotate.go @@ -0,0 +1,42 @@ +package commands + +import ( + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/logging" +) + +// ManifestAnnotateFlags define flags provided to the ManifestAnnotate +type ManifestAnnotateFlags struct { + os, arch, variant, osVersion string + features, osFeatures, annotations []string +} + +// ManifestAnnotate modifies a manifest list (Image index) and update the platform information for an image included in the manifest list. +func ManifestAnnotate(logger logging.Logger, pack PackClient) *cobra.Command { + var flags ManifestAnnotateFlags + + cmd := &cobra.Command{ + Use: "pack manifest annotate [OPTIONS] [...] [flags]", + Args: cobra.MatchAll(cobra.ExactArgs(2), cobra.OnlyValidArgs), + Short: "manifest annotate modifies a manifest list (Image index) and update the platform information for an image included in the manifest list.", + Example: `pack manifest annotate cnbs/sample-package:hello-universe-multiarch \ + cnbs/sample-package:hello-universe --arch amd64`, + Long: `manifest annotate modifies a manifest list (Image index) and update the platform information for an image included in the manifest list. + Sometimes a manifest list could reference an image that doesn't specify the architecture, The "annotate" command allows users to update those values before pushing the manifest list a registry`, + RunE: logError(logger, func(cmd *cobra.Command, args []string) error { + return nil + }), + } + + cmd.Flags().StringVar(&flags.os, "os", "", "Set the architecture") + cmd.Flags().StringVar(&flags.arch, "arch", "", "Set the architecture") + cmd.Flags().StringVar(&flags.variant, "variant", "", "Set the architecture") + cmd.Flags().StringVar(&flags.osVersion, "os-version", "", "override the os `version` of the specified image") + cmd.Flags().StringSliceVar(&flags.features, "features", nil, "override the `features` of the specified image") + cmd.Flags().StringSliceVar(&flags.osFeatures, "os-features", nil, "override the os `features` of the specified image") + cmd.Flags().StringSliceVar(&flags.annotations, "annotations", nil, "set an `annotation` for the specified image") + + AddHelpFlag(cmd, "annotate") + return cmd +} \ No newline at end of file diff --git a/internal/commands/manifest_annotate_test.go b/internal/commands/manifest_annotate_test.go new file mode 100644 index 0000000000..6320887605 --- /dev/null +++ b/internal/commands/manifest_annotate_test.go @@ -0,0 +1,100 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestAnnotateCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestAnnotateCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestAnnotateCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestAnnotate(logger, mockClient) + }) + + when("#ManifestAnnotate", func() { + when("no flags specified", func() { + + }) + when("when --os flags passed", func() { + + }) + when("when --arch flags passed", func() { + + }) + when("when --variant flags passed", func() { + + }) + when("when --os-version flags passed", func() { + + }) + when("when --features flags passed", func() { + + }) + when("when --os-features flags passed", func() { + + }) + when("when --annotations flags passed", func() { + + }) + when("when multiple flags passed", func() { + + }) + when("when no args passed", func() { + + }) + when("when manifest list reference is incorrect", func() { + + }) + when("when manifest reference is incorrect", func() { + + }) + when("when manifest passed in-place of manifest list on first arg", func() { + + }) + when("when manifest list is passed on second arg", func() { + + }) + when("when manifest is passed on second arg with --all option", func() { + + }) + when("when manifest list in locally available", func() { + + }) + when("when manifest is not locally available", func() { + + }) + when("when manifest is locally available passed", func() { + + }) + when("when multiple manifests passed", func() { + + }) + }) +} \ No newline at end of file From 5b75d95c2a51595fd9737e0d9cdcb2533a86310d Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 02:59:56 +0000 Subject: [PATCH 03/62] WIP added manifest_create command Signed-off-by: WYGIN --- internal/commands/manifest_create.go | 92 +++++++++++++++++++ internal/commands/manifest_create_test.go | 106 ++++++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 internal/commands/manifest_create.go create mode 100644 internal/commands/manifest_create_test.go diff --git a/internal/commands/manifest_create.go b/internal/commands/manifest_create.go new file mode 100644 index 0000000000..e8fe471f78 --- /dev/null +++ b/internal/commands/manifest_create.go @@ -0,0 +1,92 @@ +package commands + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/client" + "github.com/buildpacks/pack/pkg/logging" +) + +// ManifestCreateFlags define flags provided to the ManifestCreate +type ManifestCreateFlags struct { + format, registry, os, arch string + insecure, publish, all, amend bool +} + +// ManifestCreate creates an image-index/image-list for a multi-arch image +func ManifestCreate(logger logging.Logger, pack PackClient) *cobra.Command { + var flags ManifestCreateFlags + + cmd := &cobra.Command{ + Use: "pack manifest create [ ... ] [flags]", + Args: cobra.MatchAll(cobra.MinimumNArgs(2), cobra.OnlyValidArgs), + Short: "manifest create generates a manifest list for a multi-arch image", + Example: `pack manifest create cnbs/sample-package:hello-multiarch-universe \ + cnbs/sample-package:hello-universe \ + cnbs/sample-package:hello-universe-windows`, + Long: `Create a manifest list or image index for the image to support muti architecture for the image, it create a new ManifestList or ImageIndex with the given name and adds the list of Manifests to the newly created ImageIndex or ManifestList + + If the already exists in the registry: pack will save a local copy of the remote manifest list, + If the doestn't exist in a registry: pack will create a local representation of the manifest list that will only save on the remote registry if the user publish it`, + RunE: logError(logger, func(cmd *cobra.Command, args []string) error { + imageIndex := args[0] + manifests := args[1:] + cmdFlags := cmd.Flags() + + if err := validateManifestCreateFlags(flags); err != nil { + return err + } + + if cmdFlags.Changed("insecure") { + flags.insecure = !flags.insecure + } + + if cmdFlags.Changed("publish") { + flags.publish = !flags.publish + } + + id, err := pack.CreateManifest(cmd.Context(), imageIndex, manifests, client.CreateManifestOptions{ + Format: flags.format, + Registry: flags.registry, + Insecure: flags.insecure, + Publish: flags.publish, + }) + + if err != nil { + return err + } + logger.Infof("Successfully created ImageIndex/ManifestList with imageID: '%s'", id) + + return nil + }), + } + + cmdFlags := cmd.Flags() + + cmdFlags.StringVarP(&flags.format, "format", "f", "v2s2", "Format to save image index as ('OCI' or 'V2S2') (default 'v2s2')") + cmdFlags.StringVarP(&flags.registry, "registry", "r", "", "Publish to registry") + cmdFlags.StringVar(&flags.os, "os", "", "If any of the specified images is a list/index, choose the one for `os`") + if err := cmdFlags.MarkHidden("os"); err != nil { + panic(fmt.Sprintf("error marking --os as hidden: %v", err)) + } + cmdFlags.StringVar(&flags.arch, "arch", "", "If any of the specified images is a list/index, choose the one for `arch`") + if err := cmdFlags.MarkHidden("arch"); err != nil { + panic(fmt.Sprintf("error marking --arch as hidden: %v", err)) + } + cmdFlags.BoolVar(&flags.insecure, "insecure", false, "Allow publishing to insecure registry") + if err := cmdFlags.MarkHidden("insecure"); err != nil { + panic(fmt.Sprintf("error marking insecure as hidden: %v", err)) + } + cmdFlags.BoolVar(&flags.publish, "publish", false, "Publish to registry") + cmdFlags.BoolVar(&flags.all, "all", false, "Add all of the list's images if the images to add are lists/index") + cmdFlags.BoolVar(&flags.amend, "amend", false, "Modify an existing list/index if one with the desired name already exists") + + AddHelpFlag(cmd, "create") + return cmd +} + +func validateManifestCreateFlags(flags ManifestCreateFlags) error { + return nil +} \ No newline at end of file diff --git a/internal/commands/manifest_create_test.go b/internal/commands/manifest_create_test.go new file mode 100644 index 0000000000..673dff4b45 --- /dev/null +++ b/internal/commands/manifest_create_test.go @@ -0,0 +1,106 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestCreateCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestCreateCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestCreateCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestCreate(logger, mockClient) + }) + + when("#ManifestCreate", func() { + when("no flags specified", func() { + + }) + when("when --all flags passed", func() { + + }) + when("when --os flags passed", func() { + + }) + when("when --arch flags passed", func() { + + }) + when("when --registry flags passed", func() { + + }) + when("when --format flags passed", func() { + + }) + when("when --format flags not passed", func() { + + }) + when("when --insecure flags passed", func() { + + }) + when("when --publish flags passed", func() { + + }) + when("when --ammend flags passed", func() { + + }) + when("when multiple flags passed", func() { + + }) + when("when no args passed", func() { + + }) + when("when manifest list reference is incorrect", func() { + + }) + when("when manifest reference is incorrect", func() { + + }) + when("when manifest list is passed on second arg", func() { + + }) + when("when manifest is passed on second arg with --all option", func() { + + }) + when("when manifest list is locally available", func() { + + }) + when("when manifest list is not locally available", func() { + + }) + when("when manifest is not locally available", func() { + + }) + when("when manifest is locally available passed", func() { + + }) + when("when multiple manifests passed", func() { + + }) + }) +} \ No newline at end of file From 9799f011e8f3b9b624abf7eadea0b72e28c7de50 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:01:40 +0000 Subject: [PATCH 04/62] WIP added manifest_exists command Signed-off-by: WYGIN --- internal/commands/manifest_exists.go | 33 +++++++++++++ internal/commands/manifest_exists_test.go | 60 +++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 internal/commands/manifest_exists.go create mode 100644 internal/commands/manifest_exists_test.go diff --git a/internal/commands/manifest_exists.go b/internal/commands/manifest_exists.go new file mode 100644 index 0000000000..1d5c2062f7 --- /dev/null +++ b/internal/commands/manifest_exists.go @@ -0,0 +1,33 @@ +package commands + +import ( + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/logging" +) + +// ManifestDeleteFlags define flags provided to the ManifestDelete +// type ManifestDeleteFlags struct { +// } + +// ManifestExists checks if a manifest list exists in local storage +func ManifestExists(logger logging.Logger, pack PackClient) *cobra.Command { + // var flags ManifestDeleteFlags + + cmd := &cobra.Command{ + Use: "pack manifest exists [manifest-list]", + Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), + Short: "checks if a manifest list exists in local storage", + Example: `pack manifest exists cnbs/sample-package:hello-multiarch-universe`, + Long: `Checks if a manifest list exists in local storage`, + RunE: logError(logger, func(cmd *cobra.Command, args []string) error { + if err := pack.ExistsManifest(cmd.Context(), args[0]); err != nil { + return err + } + return nil + }), + } + + AddHelpFlag(cmd, "remove") + return cmd +} \ No newline at end of file diff --git a/internal/commands/manifest_exists_test.go b/internal/commands/manifest_exists_test.go new file mode 100644 index 0000000000..292b5429ba --- /dev/null +++ b/internal/commands/manifest_exists_test.go @@ -0,0 +1,60 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestExistsCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestExistsCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestExistsCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestExists(logger, mockClient) + }) + + when("#ManifestExists", func() { + when("only one arg is passed", func() { + + }) + when("when more than one arg passed", func() { + + }) + when("when passed arg is manifest list", func() { + it("if exists locally", func() { + + }) + it("if exists at registry only", func() { + + }) + }) + when("when passed arg is manifest", func() { + + }) + }) +} \ No newline at end of file From 1968518c437aaee9442d98b5ee500219e163e1cb Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:03:47 +0000 Subject: [PATCH 05/62] WIP added manifest_inspect command Signed-off-by: WYGIN --- internal/commands/manifest_inspect.go | 31 ++++++++++++ internal/commands/manifest_inspect_test.go | 57 ++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 internal/commands/manifest_inspect.go create mode 100644 internal/commands/manifest_inspect_test.go diff --git a/internal/commands/manifest_inspect.go b/internal/commands/manifest_inspect.go new file mode 100644 index 0000000000..80bf3a52f6 --- /dev/null +++ b/internal/commands/manifest_inspect.go @@ -0,0 +1,31 @@ +package commands + +import ( + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/logging" +) + +// ManifestInspectFlags define flags provided to the ManifestInspect +// type ManifestInspectFlags struct { +// } + +// ManifestInspect shows the manifest information stored in local storage +func ManifestInspect(logger logging.Logger, pack PackClient) *cobra.Command { + // var flags ManifestInspectFlags + + cmd := &cobra.Command{ + Use: "pack manifest inspect [flags]", + Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), + Short: "manifest inspect shows the manifest information stored in local storage", + Example: `pack manifest inspect cnbs/sample-builder:multiarch`, + Long: `manifest inspect shows the manifest information stored in local storage. + The inspect command will help users to view how their local manifest list looks like`, + RunE: logError(logger, func(cmd *cobra.Command, args []string) error { + return nil + }), + } + + AddHelpFlag(cmd, "inspect") + return cmd +} \ No newline at end of file diff --git a/internal/commands/manifest_inspect_test.go b/internal/commands/manifest_inspect_test.go new file mode 100644 index 0000000000..fdfeea5605 --- /dev/null +++ b/internal/commands/manifest_inspect_test.go @@ -0,0 +1,57 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestInspectCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestInspectCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestInspectCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestInspect(logger, mockClient) + }) + + when("#ManifestInspect", func() { + when("inspect manifest", func() { + + }) + when("inspect manifest list", func() { + it("when available locally", func() { + + }) + it("when available at registry", func() { + + }) + it("by passing multiple manifest lists", func() { + + }) + }) + }) +} \ No newline at end of file From d8f60393def94ef47a80dd2386bb290ee1984ac2 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:05:59 +0000 Subject: [PATCH 06/62] WIP added manifest_push command Signed-off-by: WYGIN --- internal/commands/manifest_push.go | 59 +++++++++++++++++ internal/commands/manifest_push_test.go | 88 +++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 internal/commands/manifest_push.go create mode 100644 internal/commands/manifest_push_test.go diff --git a/internal/commands/manifest_push.go b/internal/commands/manifest_push.go new file mode 100644 index 0000000000..2c2c8f3f0b --- /dev/null +++ b/internal/commands/manifest_push.go @@ -0,0 +1,59 @@ +package commands + +import ( + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/client" + "github.com/buildpacks/pack/pkg/logging" +) + +// ManifestPushFlags define flags provided to the ManifestPush +type ManifestPushFlags struct { + format string + insecure, purge, all, quite bool +} + +// ManifestPush pushes a manifest list (Image index) to a registry. +func ManifestPush(logger logging.Logger, pack PackClient) *cobra.Command { + var flags ManifestPushFlags + + cmd := &cobra.Command{ + Use: "pack manifest push [OPTIONS] [flags]", + Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), + Short: "manifest push pushes a manifest list (Image index) to a registry.", + Example: `pack manifest push cnbs/sample-package:hello-multiarch-universe`, + Long: `manifest push pushes a manifest list (Image index) to a registry. + Once a manifest list is ready to be published into the registry, the push command can be used`, + RunE: logError(logger, func(cmd *cobra.Command, args []string) error { + if err := parseFalgs(flags); err != nil { + return err + } + + imageID, err := pack.PushManifest(cmd.Context(), args[0], client.PushManifestOptions{ + Format: flags.format, + Insecure: flags.insecure, + Purge: flags.purge, + }) + + if err != nil { + return err + } + + logger.Infof(imageID) + return nil + }), + } + + cmd.Flags().StringVarP(&flags.format, "format", "f", "", "Format to save image index as ('OCI' or 'V2S2') (default 'v2s2')") + cmd.Flags().BoolVar(&flags.insecure, "insecure", false, "Allow publishing to insecure registry") + cmd.Flags().BoolVar(&flags.purge, "purge", false, "Delete the manifest list or image index from local storage if pushing succeeds") + cmd.Flags().BoolVar(&flags.all, "all", false, "Also push the images in the list") + cmd.Flags().BoolVarP(&flags.quite, "quite", "q", false, "Also push the images in the list") + + AddHelpFlag(cmd, "push") + return cmd +} + +func parseFalgs(flags ManifestPushFlags) error { + return nil +} \ No newline at end of file diff --git a/internal/commands/manifest_push_test.go b/internal/commands/manifest_push_test.go new file mode 100644 index 0000000000..92656d0e1b --- /dev/null +++ b/internal/commands/manifest_push_test.go @@ -0,0 +1,88 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestPushCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestPushCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestPushCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestPush(logger, mockClient) + }) + + when("#ManifestPush", func() { + when("no flags specified", func() { + + }) + when("when --all flags passed", func() { + + }) + when("when --insecure flags passed", func() { + + }) + when("when --purge flags passed", func() { + + }) + when("when --quite flags passed", func() { + + }) + when("when --format flags passed", func() { + + }) + when("when multiple flags passed", func() { + + }) + when("when no args passed", func() { + + }) + when("when manifest list reference is incorrect", func() { + + }) + when("when manifest passed in-place of manifest list on first arg", func() { + + }) + when("when manifest list is passed", func() { + + }) + when("when manifest list is passed with --all option", func() { + + }) + when("when manifest list is locally available", func() { + + }) + when("when manifest list is not locally available", func() { + + }) + when("local images are included in manifest list when pushing", func() { + + }) + }) +} \ No newline at end of file From 07b9150f061a46326c831ef290f96dcd17b47606 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:07:47 +0000 Subject: [PATCH 07/62] WIP added manifest_remove command Signed-off-by: WYGIN --- internal/commands/manifest_remove.go | 34 +++++++++++++ internal/commands/manifest_remove_test.go | 61 +++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 internal/commands/manifest_remove.go create mode 100644 internal/commands/manifest_remove_test.go diff --git a/internal/commands/manifest_remove.go b/internal/commands/manifest_remove.go new file mode 100644 index 0000000000..1b09cb493b --- /dev/null +++ b/internal/commands/manifest_remove.go @@ -0,0 +1,34 @@ +package commands + +import ( + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/logging" +) + +// ManifestDeleteFlags define flags provided to the ManifestDelete +// type ManifestDeleteFlags struct { +// } + +// ManifestDelete deletes one or more manifest lists from local storage +func ManifestDelete(logger logging.Logger, pack PackClient) *cobra.Command { + // var flags ManifestDeleteFlags + + cmd := &cobra.Command{ + Use: "pack manifest remove [manifest-list] [manifest-list...] [flags]", + Args: cobra.MatchAll(cobra.MinimumNArgs(1), cobra.OnlyValidArgs), + Short: "Delete one or more manifest lists from local storage", + Example: `pack manifest remove cnbs/sample-package:hello-multiarch-universe`, + Long: `Delete one or more manifest lists from local storage. + When a manifest list exits locally, users can remove existing images from a manifest list`, + RunE: logError(logger, func(cmd *cobra.Command, args []string) error { + if err := pack.DeleteManifest(cmd.Context(), args); err != nil { + return err + } + return nil + }), + } + + AddHelpFlag(cmd, "remove") + return cmd +} \ No newline at end of file diff --git a/internal/commands/manifest_remove_test.go b/internal/commands/manifest_remove_test.go new file mode 100644 index 0000000000..71b71e7bbf --- /dev/null +++ b/internal/commands/manifest_remove_test.go @@ -0,0 +1,61 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestDeleteCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestDeleteCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestDeleteCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestDelete(logger, mockClient) + }) + + when("#ManifestRemove", func() { + when("when no args passed", func() { + + }) + when("when manifest list reference is incorrect", func() { + + }) + when("when manifest passed in-place of manifest list on first arg", func() { + + }) + when("when manifest list is locally available", func() { + + }) + when("when manifest list is not locally available", func() { + + }) + when("when multiple manifests passed", func() { + + }) + }) +} \ No newline at end of file From a6355e196bd3a1e91760ec9f36490c2bcb97c25c Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:09:43 +0000 Subject: [PATCH 08/62] WIP added manifest_rm command Signed-off-by: WYGIN --- internal/commands/manifest_rm.go | 35 ++++++++++++ internal/commands/manifest_rm_test.go | 76 +++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 internal/commands/manifest_rm.go create mode 100644 internal/commands/manifest_rm_test.go diff --git a/internal/commands/manifest_rm.go b/internal/commands/manifest_rm.go new file mode 100644 index 0000000000..46ed6fe9ec --- /dev/null +++ b/internal/commands/manifest_rm.go @@ -0,0 +1,35 @@ +package commands + +import ( + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/logging" +) + +// ManifestRemoveFlags define flags provided to the ManifestRemove +// type ManifestRemoveFlags struct { +// } + +// ManifestRemove will remove the specified image manifest if it is already referenced in the index +func ManifestRemove(logger logging.Logger, pack PackClient) *cobra.Command { + // var flags ManifestRemoveFlags + + cmd := &cobra.Command{ + Use: "pack manifest rm [manifest-list] [manifest] [manifest...] [flags]", + Args: cobra.MatchAll(cobra.MinimumNArgs(2), cobra.OnlyValidArgs), + Short: "manifest rm will remove the specified image manifest if it is already referenced in the index", + Example: `pack manifest rm cnbs/sample-package:hello-multiarch-universe \ + cnbs/sample-package:hello-universe-windows`, + Long: `manifest rm will remove the specified image manifest if it is already referenced in the index. + Sometimes users can just experiment with the feature locally and they want to discard all the local information created by pack. 'rm' command just delete the local manifest list`, + RunE: logError(logger, func(cmd *cobra.Command, args []string) error { + if err := pack.RemoveManifest(cmd.Context(), args[0], args[1:]); err != nil { + return err + } + return nil + }), + } + + AddHelpFlag(cmd, "rm") + return cmd +} \ No newline at end of file diff --git a/internal/commands/manifest_rm_test.go b/internal/commands/manifest_rm_test.go new file mode 100644 index 0000000000..e66d84f41c --- /dev/null +++ b/internal/commands/manifest_rm_test.go @@ -0,0 +1,76 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestRemoveCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestRemoveCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestRemoveCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestRemove(logger, mockClient) + }) + + when("#ManifestRm", func() { + when("when no args passed", func() { + + }) + when("when manifest list reference is incorrect", func() { + + }) + when("when manifest reference is incorrect", func() { + + }) + when("when manifest passed in-place of manifest list on first arg", func() { + + }) + when("when manifest list is passed on second arg", func() { + + }) + when("when manifest is passed on second arg", func() { + + }) + when("when manifest list is locally available", func() { + + }) + when("when manifest list is not locally available", func() { + + }) + when("when manifest is locally available", func() { + + }) + when("when manifest is not locally available", func() { + + }) + when("when multiple manifests passed", func() { + + }) + }) +} \ No newline at end of file From f24b167f0edc2c766e9181e3837e8d816348c5d2 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:13:41 +0000 Subject: [PATCH 09/62] WIP added add_manifest client Signed-off-by: WYGIN --- internal/commands/manifest_test.go | 67 +++++++++++++++++++ pkg/client/add_manifest.go | 87 ++++++++++++++++++++++++ pkg/client/add_manifest_test.go | 103 +++++++++++++++++++++++++++++ 3 files changed, 257 insertions(+) create mode 100644 internal/commands/manifest_test.go create mode 100644 pkg/client/add_manifest.go create mode 100644 pkg/client/add_manifest_test.go diff --git a/internal/commands/manifest_test.go b/internal/commands/manifest_test.go new file mode 100644 index 0000000000..6313b2fdf7 --- /dev/null +++ b/internal/commands/manifest_test.go @@ -0,0 +1,67 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.NewManifestCommand(logger, mockClient) + }) + + when("#ManifestAdd", func() { + when("no flags specified", func() { + + }) + when("add is passed as flag", func() { + + }) + when("create is passed as flag", func() { + + }) + when("annotate is passed as flag", func() { + + }) + when("remove is passed as flag", func() { + + }) + when("inspect is passed as flag", func() { + + }) + when("rm is passed as flag", func() { + + }) + when("push is passed as flag", func() { + + }) + }) +} \ No newline at end of file diff --git a/pkg/client/add_manifest.go b/pkg/client/add_manifest.go new file mode 100644 index 0000000000..ab52ba87eb --- /dev/null +++ b/pkg/client/add_manifest.go @@ -0,0 +1,87 @@ +package client + +import ( + "context" + "fmt" + "strings" +) + +type ManifestAddOptions struct { + ManifestAnnotateOptions + All bool +} + +// AddManifest implements commands.PackClient. +func (c *Client) AddManifest(ctx context.Context, index string, image string, opts ManifestAddOptions) (indexID string, err error) { + ref, err := c.runtime.ParseReference(image) + if err != nil { + return + } + + imgIndex, err := c.runtime.LookupImageIndex(index) + if err != nil { + return + } + + digest, err := imgIndex.Add(ctx, ref, opts.All) + if err != nil { + if ref, _, err = c.imageFactory.FindImage(image); err != nil { + return indexID, fmt.Errorf("Error while trying to find image on local storage: %v", err) + } + digest, err = imgIndex.Add(ctx, ref, opts.All) + if err != nil { + return indexID, fmt.Errorf("Error while trying to add on manifest list: %v", err) + } + } + + if opts.OS != "" { + if _, err := imgIndex.Index.SetOS(digest, opts.OS); err != nil { + return indexID, err + } + } + + if opts.OSArch != "" { + if _, err := imgIndex.Index.SetArchitecture(digest, opts.OSArch); err != nil { + return indexID, err + } + } + + if opts.OSVariant != "" { + if _, err := imgIndex.Index.SetVariant(digest, opts.OSVariant); err != nil { + return indexID, err + } + } + + if opts.OSVersion != "" { + if _, err := imgIndex.Index.SetOSVersion(digest, opts.OSVersion); err != nil { + return indexID, err + } + } + + if len(opts.Features) != 0 { + if _, err := imgIndex.Index.SetFeatures(digest, opts.Features); err != nil { + return indexID, err + } + } + + if len(opts.Annotations) != 0 { + annotations := make(map[string]string) + for _, annotationSpec := range opts.Annotations { + spec := strings.SplitN(annotationSpec, "=", 2) + if len(spec) != 2 { + return indexID, fmt.Errorf("no value given for annotation %q", spec[0]) + } + annotations[spec[0]] = spec[1] + } + if err := imgIndex.Index.SetAnnotations(&digest, annotations); err != nil { + return err + } + } + + indexID, err = imgIndex.Index.Save(index, nil, "") + if err == nil { + fmt.Printf("%s: %s\n", indexID, digest.String()) + } + + return indexID, err +} \ No newline at end of file diff --git a/pkg/client/add_manifest_test.go b/pkg/client/add_manifest_test.go new file mode 100644 index 0000000000..7aa3101c36 --- /dev/null +++ b/pkg/client/add_manifest_test.go @@ -0,0 +1,103 @@ +package client_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestAddCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestAddCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestAdd(logger, mockClient) + }) + + when("#AddManifest", func() { + when("no flags specified", func() { + + }) + when("when --all flags passed", func() { + + }) + when("when --os flags passed", func() { + + }) + when("when --arch flags passed", func() { + + }) + when("when --variant flags passed", func() { + + }) + when("when --os-version flags passed", func() { + + }) + when("when --features flags passed", func() { + + }) + when("when --os-features flags passed", func() { + + }) + when("when --annotations flags passed", func() { + + }) + when("when multiple flags passed", func() { + + }) + when("when no args passed", func() { + + }) + when("when manifest list reference is incorrect", func() { + + }) + when("when manifest reference is incorrect", func() { + + }) + when("when manifest passed in-place of manifest list on first arg", func() { + + }) + when("when manifest list is passed on second arg", func() { + + }) + when("when manifest is passed on second arg with --all option", func() { + + }) + when("when manifest list in locally available", func() { + + }) + when("when manifest is not locally available", func() { + + }) + when("when manifest is locally available passed", func() { + + }) + when("when multiple manifests passed", func() { + + }) + }) +} \ No newline at end of file From 15ed584a061e4ccda38cd31f60fc3f795e5f398f Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:15:33 +0000 Subject: [PATCH 10/62] WIP added annotate_manifest client Signed-off-by: WYGIN --- pkg/client/annotate_manifest.go | 83 ++++++++++++++++++++++++++++ pkg/client/annotate_manifest_test.go | 1 + 2 files changed, 84 insertions(+) create mode 100644 pkg/client/annotate_manifest.go create mode 100644 pkg/client/annotate_manifest_test.go diff --git a/pkg/client/annotate_manifest.go b/pkg/client/annotate_manifest.go new file mode 100644 index 0000000000..a279509a60 --- /dev/null +++ b/pkg/client/annotate_manifest.go @@ -0,0 +1,83 @@ +package client + +import ( + "context" + "fmt" + "strings" +) + +type ManifestAnnotateOptions struct { + OS, OSVersion, OSArch, OSVariant string + OSFeatures, Annotations, Features map[string]string +} + +// AnnotateManifest implements commands.PackClient. +func (c *Client) AnnotateManifest(ctx context.Context, name string, image string, opts ManifestAnnotateOptions) error { + manifestList, err := c.runtime.LookupImageIndex(name) + if err != nil { + return err + } + + digest, err := c.runtime.ParseDigest(image) + if err != nil { + ref, _, err := c.imageFactory.FindImage(image) + if err != nil { + return err + } + digest, err = c.runtime.ParseDigest(ref.Name()) + if err != nil { + return err + } + } + + if opts.OS != "" { + if err := manifestList.Index.SetOS(digest, opts.OS); err != nil { + return err + } + } + if opts.OSVersion != "" { + if err := manifestList.Index.SetOSVersion(digest, opts.OSVersion); err != nil { + return err + } + } + if len(opts.OSFeatures) != 0 { + if err := manifestList.Index.SetOSFeatures(digest, opts.OSFeatures); err != nil { + return err + } + } + if opts.OSArch != "" { + if err := manifestList.Index.SetArchitecture(digest, opts.OSArch); err != nil { + return err + } + } + if opts.OSVariant != "" { + if err := manifestList.Index.SetVariant(digest, opts.OSVariant); err != nil { + return err + } + } + if len(opts.Features) != 0 { + if err := manifestList.Index.SetFeatures(digest, opts.Features); err != nil { + return err + } + } + if len(opts.Annotations) != 0 { + annotations := make(map[string]string) + for _, annotationSpec := range opts.Annotations { + spec := strings.SplitN(annotationSpec, "=", 2) + if len(spec) != 2 { + return fmt.Errorf("no value given for annotation %q", spec[0]) + } + annotations[spec[0]] = spec[1] + } + if err := manifestList.Index.SetAnnotations(&digest, annotations); err != nil { + return err + } + } + + updatedListID, err := manifestList.Index.Save(name, nil, "") + if err == nil { + fmt.Printf("%s: %s\n", updatedListID, digest.String()) + } + + return nil +} \ No newline at end of file diff --git a/pkg/client/annotate_manifest_test.go b/pkg/client/annotate_manifest_test.go new file mode 100644 index 0000000000..e169c0b61a --- /dev/null +++ b/pkg/client/annotate_manifest_test.go @@ -0,0 +1 @@ +package client_test From 8cf016e8a09b9d14ac61671467b0fdffc60bc02d Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:21:34 +0000 Subject: [PATCH 11/62] WIP added IndexFactory to Client Signed-off-by: WYGIN --- pkg/client/client.go | 73 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/pkg/client/client.go b/pkg/client/client.go index b3862cc4cc..b058d84182 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -20,10 +20,12 @@ import ( "path/filepath" "github.com/buildpacks/imgutil" + "github.com/buildpacks/imgutil/layout" "github.com/buildpacks/imgutil/local" "github.com/buildpacks/imgutil/remote" dockerClient "github.com/docker/docker/client" "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" "github.com/pkg/errors" "github.com/buildpacks/pack" @@ -70,6 +72,15 @@ type ImageFactory interface { // NewImage initializes an image object with required settings so that it // can be written either locally or to a registry. NewImage(repoName string, local bool, imageOS string) (imgutil.Image, error) + // FindImage locates the locally-stored image which corresponds to a given name. + FindImage(name string) (ref name.Reference, image imgutil.Image, err error) +} + +// IndexFactory is an interface representing the ability to create a ImageIndex/ManifestList. +type IndexFactory interface { + NewIndex(reponame string, opts imgutil.IndexOptions) (imgutil.Index, error) + // Fetch ManifestList from Registry with the given name + FetchIndex(name string) (imgutil.Index, error) } //go:generate mockgen -package testmocks -destination ../testmocks/mock_buildpack_downloader.go github.com/buildpacks/pack/pkg/client BuildpackDownloader @@ -90,6 +101,7 @@ type Client struct { keychain authn.Keychain imageFactory ImageFactory imageFetcher ImageFetcher + indexFactory IndexFactory downloader BlobDownloader lifecycleExecutor LifecycleExecutor buildpackDownloader BuildpackDownloader @@ -117,6 +129,13 @@ func WithImageFactory(f ImageFactory) Option { } } +// WithIndexFactory supply your own index factory +func WithIndexFactory(f IndexFactory) Option { + return func(c *Client) { + c.indexFactory = f + } +} + // WithFetcher supply your own Fetcher. // A Fetcher retrieves both local and remote images to make them available. func WithFetcher(f ImageFetcher) Option { @@ -225,6 +244,12 @@ func NewClient(opts ...Option) (*Client, error) { } } + if client.indexFactory == nil { + client.indexFactory = &indexFactory{ + keychain: client.keychain, + } + } + if client.buildpackDownloader == nil { client.buildpackDownloader = buildpack.NewDownloader( client.logger, @@ -273,3 +298,51 @@ func (f *imageFactory) NewImage(repoName string, daemon bool, imageOS string) (i return remote.NewImage(repoName, f.keychain, remote.WithDefaultPlatform(platform)) } + +func (f *imageFactory) LoadImage(repoName string, platform imgutil.Platform) (img imgutil.Image, err error) { + img, err = local.NewImage(repoName, f.dockerClient, local.WithDefaultPlatform(platform), local.WithPreviousImage(repoName)) + if err == nil { + return + } + + img, err = layout.NewImage(repoName, layout.WithPreviousImage(repoName), layout.WithDefaultPlatform(platform)) + if err == nil { + return + } + return nil, errors.Errorf("Image: '%s' not found", repoName) +} + +func (f *imageFactory) FindImage(repoName string) (ref name.Reference, img imgutil.Image, err error) { + img, err = local.NewImage(repoName, f.dockerClient, local.WithPreviousImage(repoName)) + if err == nil { + return + } + + img, err = layout.NewImage(repoName, layout.WithPreviousImage(repoName)) + if err == nil { + return + } + return ref, img, errors.Errorf("Image: '%s' not found", repoName) +} + +type indexFactory struct { + keychain authn.Keychain +} + +func (f *indexFactory) NewIndex(name string, opts imgutil.IndexOptions) (index imgutil.Index, err error) { + + index, err = remote.NewIndex(name, f.keychain, opts) + if err != nil { + if opts.MediaType == imgutil.MediaTypes.OCI { + return layout.NewIndex(name, opts) + } else { + return local.NewIndex(name, opts) + } + } + + return index, err +} + +func (f *indexFactory) FetchIndex(repoName string) (index imgutil.Index, err error) { + return remote.NewIndex(repoName, f.keychain, imgutil.IndexOptions{}) +} From 0ed954ff5a81fbcd97cdb47b90684090006326d8 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:23:46 +0000 Subject: [PATCH 12/62] WIP added create_manifest client Signed-off-by: WYGIN --- pkg/client/create_manifest.go | 70 ++++++++++++++++++++++++++++++ pkg/client/create_manifest_test.go | 1 + 2 files changed, 71 insertions(+) create mode 100644 pkg/client/create_manifest.go create mode 100644 pkg/client/create_manifest_test.go diff --git a/pkg/client/create_manifest.go b/pkg/client/create_manifest.go new file mode 100644 index 0000000000..557df033eb --- /dev/null +++ b/pkg/client/create_manifest.go @@ -0,0 +1,70 @@ +package client + +import ( + "context" + "errors" + "fmt" + + "github.com/buildpacks/imgutil" + + packErrors "github.com/buildpacks/pack/pkg/errors" +) + +type CreateManifestOptions struct { + Format, Registry string + Insecure, Publish, amend, all bool +} + +// CreateManifest implements commands.PackClient. +func (c *Client) CreateManifest(ctx context.Context, name string, images []string, opts CreateManifestOptions) (imageID string, err error) { + index, err := c.indexFactory.NewIndex(name, parseOptsToIndexOptions(opts)) + if err != nil { + return + } + index.Create() + if err != nil { + return + } + if imageID, err = index.Save(name, c.runtime.ImageType(opts.Format)); err != nil { + if errors.Is(err, packErrors.ErrDuplicateName) && opts.amend { + _, err := c.runtime.LookupImageIndex(name) + if err != nil { + fmt.Printf("no list named %q found: %v", name, err) + } + + if index == nil { + return imageID, fmt.Errorf("--amend specified but no matching manifest list found with name %q", name) + } + } else { + return + } + } + + for _, img := range images { + ref, err := c.runtime.ParseReference(img) + if err != nil { + return imageID, err + } + if localRef, _, err := c.imageFactory.FindImage(img); err == nil { + ref = localRef + } + if _, err = index.Add(ctx, ref, opts.all); err != nil { + return imageID, err + } + } + + imageID, err = index.Save(name, c.runtime.ImageType(opts.Format)) + if err == nil { + fmt.Printf("%s\n", imageID) + } + + if opts.Publish { + + } + + return imageID, err +} + +func parseOptsToIndexOptions(opts CreateManifestOptions) (idxOpts imgutil.IndexOptions) { + return idxOpts +} \ No newline at end of file diff --git a/pkg/client/create_manifest_test.go b/pkg/client/create_manifest_test.go new file mode 100644 index 0000000000..e169c0b61a --- /dev/null +++ b/pkg/client/create_manifest_test.go @@ -0,0 +1 @@ +package client_test From dfd4bc400bc02fde28e577e424c76e7d58d66393 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:25:26 +0000 Subject: [PATCH 13/62] WIP added delete_manifest client Signed-off-by: WYGIN --- pkg/client/delete_manifest.go | 12 ++++++++++++ pkg/client/delete_manifest_test.go | 1 + 2 files changed, 13 insertions(+) create mode 100644 pkg/client/delete_manifest.go create mode 100644 pkg/client/delete_manifest_test.go diff --git a/pkg/client/delete_manifest.go b/pkg/client/delete_manifest.go new file mode 100644 index 0000000000..ac72dd6f46 --- /dev/null +++ b/pkg/client/delete_manifest.go @@ -0,0 +1,12 @@ +package client + +import ( + "context" + // "fmt" + // "strings" +) + +// DeleteManifest implements commands.PackClient. +func (c *Client) DeleteManifest(ctx context.Context, names []string) error { + return c.runtime.RemoveManifests(ctx, names) +} \ No newline at end of file diff --git a/pkg/client/delete_manifest_test.go b/pkg/client/delete_manifest_test.go new file mode 100644 index 0000000000..e169c0b61a --- /dev/null +++ b/pkg/client/delete_manifest_test.go @@ -0,0 +1 @@ +package client_test From 94ba3e37cd75d6efc0da3c49a6bf2378d63b3587 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:27:01 +0000 Subject: [PATCH 14/62] WIP added exists_manifest client Signed-off-by: WYGIN --- pkg/client/exists_manifest.go | 17 +++++++++++++++++ pkg/client/exists_manifest_test.go | 1 + 2 files changed, 18 insertions(+) create mode 100644 pkg/client/exists_manifest.go create mode 100644 pkg/client/exists_manifest_test.go diff --git a/pkg/client/exists_manifest.go b/pkg/client/exists_manifest.go new file mode 100644 index 0000000000..f9530040a1 --- /dev/null +++ b/pkg/client/exists_manifest.go @@ -0,0 +1,17 @@ +package client + +import ( + "context" + + // "github.com/buildpacks/imgutil" + "github.com/pkg/errors" +) + +func (c *Client) ExistsManifest(ctx context.Context, image string) error { + + if _, err := c.runtime.LookupImageIndex(image); err != nil { + return errors.Errorf("image '%s' is not found", image) + } + + return nil +} \ No newline at end of file diff --git a/pkg/client/exists_manifest_test.go b/pkg/client/exists_manifest_test.go new file mode 100644 index 0000000000..e169c0b61a --- /dev/null +++ b/pkg/client/exists_manifest_test.go @@ -0,0 +1 @@ +package client_test From 4ec02e721c24c037eb25f85b366d25da6a1ed1cf Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:29:04 +0000 Subject: [PATCH 15/62] WIP added inspect_manifest client Signed-off-by: WYGIN --- pkg/client/inspect_manifest.go | 62 +++++++++++++++++++++++++++++ pkg/client/inspect_manifest_test.go | 1 + 2 files changed, 63 insertions(+) create mode 100644 pkg/client/inspect_manifest.go create mode 100644 pkg/client/inspect_manifest_test.go diff --git a/pkg/client/inspect_manifest.go b/pkg/client/inspect_manifest.go new file mode 100644 index 0000000000..bb73d24c2a --- /dev/null +++ b/pkg/client/inspect_manifest.go @@ -0,0 +1,62 @@ +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + + "github.com/pkg/errors" + + packErrors "github.com/buildpacks/pack/pkg/errors" +) + +type InspectManifestOptions struct { +} + +// InspectManifest implements commands.PackClient. +func (c *Client) InspectManifest(ctx context.Context, name string, opts InspectManifestOptions) error { + printManifest := func(manifest []byte) error { + var b bytes.Buffer + err := json.Indent(&b, manifest, "", " ") + if err != nil { + return fmt.Errorf("rendering manifest for display: %w", err) + } + + fmt.Printf("%s\n", b.String()) + return nil + } + + // Before doing a remote lookup, attempt to resolve the manifest list + // locally. + manifestList, err := c.runtime.LookupImageIndex(name) + if err == nil { + schema2List, err := manifestList.Index.Inspect() + if err != nil { + rawSchema2List, err := json.Marshal(schema2List) + if err != nil { + return err + } + + return printManifest(rawSchema2List) + } + if !errors.Is(err, packErrors.ErrIndexUnknown) && !errors.Is(err, packErrors.ErrNotAddManifestList) { + return err + } + + _, err = c.runtime.ParseReference(name) + if err != nil { + fmt.Printf("error parsing reference to image %q: %v", name, err) + } + + index, err := c.indexFactory.FetchIndex(name) + + if err != nil { + return err + } + + return printManifest(index) + } + + return fmt.Errorf("unable to locate manifest list locally or at registry") +} \ No newline at end of file diff --git a/pkg/client/inspect_manifest_test.go b/pkg/client/inspect_manifest_test.go new file mode 100644 index 0000000000..e169c0b61a --- /dev/null +++ b/pkg/client/inspect_manifest_test.go @@ -0,0 +1 @@ +package client_test From f2161a917a0d7695b6d25b1d726b91c54d0bb019 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:30:41 +0000 Subject: [PATCH 16/62] WIP added push_manifest client Signed-off-by: WYGIN --- pkg/client/push_manifest.go | 32 ++++++++++++++++++++++++++++++++ pkg/client/push_manifest_test.go | 1 + 2 files changed, 33 insertions(+) create mode 100644 pkg/client/push_manifest.go create mode 100644 pkg/client/push_manifest_test.go diff --git a/pkg/client/push_manifest.go b/pkg/client/push_manifest.go new file mode 100644 index 0000000000..e4ecbb4cd0 --- /dev/null +++ b/pkg/client/push_manifest.go @@ -0,0 +1,32 @@ +package client + +import ( + "context" + + runtime "github.com/buildpacks/pack/internal/runtime" +) + +type PushManifestOptions struct { + Format string + Insecure, Purge bool +} + +// PushManifest implements commands.PackClient. +func (c *Client) PushManifest(ctx context.Context, index string, opts PushManifestOptions) (imageID string, err error) { + manifestList, err := c.runtime.LookupImageIndex(index) + if err != nil { + return + } + + _, err = manifestList.Push(ctx, parseFalgsForImgUtil(opts)) + + if err == nil && opts.Purge { + c.runtime.RemoveManifests(ctx, []string{index}) + } + + return imageID, err +} + +func parseFalgsForImgUtil(opts PushManifestOptions) (idxOptions runtime.PushOptions) { + return idxOptions +} \ No newline at end of file diff --git a/pkg/client/push_manifest_test.go b/pkg/client/push_manifest_test.go new file mode 100644 index 0000000000..e169c0b61a --- /dev/null +++ b/pkg/client/push_manifest_test.go @@ -0,0 +1 @@ +package client_test From 20802085c00f802a69590ce5774cf5f99dc2e405 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:32:30 +0000 Subject: [PATCH 17/62] WIP added remove_manifest client Signed-off-by: WYGIN --- pkg/client/remove_manifest.go | 27 +++++++++++++++++++++++++++ pkg/client/remove_manifest_test.go | 1 + 2 files changed, 28 insertions(+) create mode 100644 pkg/client/remove_manifest.go create mode 100644 pkg/client/remove_manifest_test.go diff --git a/pkg/client/remove_manifest.go b/pkg/client/remove_manifest.go new file mode 100644 index 0000000000..f6319c5dc5 --- /dev/null +++ b/pkg/client/remove_manifest.go @@ -0,0 +1,27 @@ +package client + +import ( + "context" + "fmt" +) + +// RemoveManifest implements commands.PackClient. +func (c *Client) RemoveManifest(ctx context.Context, name string, images []string) error { + imgIndex, err := c.runtime.LookupImageIndex(name) + if err != nil { + return err + } + + for _, image := range images { + _, err := c.runtime.ParseReference(image) + if err != nil { + fmt.Errorf(`Invalid instance "%s": %v`, image, err) + } + if err := imgIndex.Remove(image); err != nil { + return err + } + fmt.Printf("Successfully removed %s from %s", image, name) + } + + return nil +} \ No newline at end of file diff --git a/pkg/client/remove_manifest_test.go b/pkg/client/remove_manifest_test.go new file mode 100644 index 0000000000..e169c0b61a --- /dev/null +++ b/pkg/client/remove_manifest_test.go @@ -0,0 +1 @@ +package client_test From 5aa4b9f12a7146538635d8e91c6f4bd458baf93e Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:33:42 +0000 Subject: [PATCH 18/62] WIP added errors Signed-off-by: WYGIN --- pkg/errors/errors.go | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 pkg/errors/errors.go diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go new file mode 100644 index 0000000000..2d6943cf86 --- /dev/null +++ b/pkg/errors/errors.go @@ -0,0 +1,9 @@ +package errors + +import "errors" + +var ( + ErrDuplicateName = errors.New("Image/ImageIndex with the given name exists") + ErrIndexUnknown = errors.New("cannot find Image Index with the given name") + ErrNotAddManifestList = errors.New("error while adding ImageIndex to the list") +) \ No newline at end of file From d10a6175cdec8cfdaeb37923ed515ca8ccdfb76e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 22:50:55 +0000 Subject: [PATCH 19/62] build(deps): bump github.com/buildpacks/lifecycle from 0.17.1 to 0.17.2 Bumps [github.com/buildpacks/lifecycle](https://github.com/buildpacks/lifecycle) from 0.17.1 to 0.17.2. - [Release notes](https://github.com/buildpacks/lifecycle/releases) - [Changelog](https://github.com/buildpacks/lifecycle/blob/main/RELEASE.md) - [Commits](https://github.com/buildpacks/lifecycle/compare/v0.17.1...v0.17.2) --- updated-dependencies: - dependency-name: github.com/buildpacks/lifecycle dependency-type: direct:production update-type: version-update:semver-patch ... Signed-off-by: dependabot[bot] Signed-off-by: Juan Bustamante --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d4a8ecda69..833398d0e1 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/Microsoft/go-winio v0.6.1 github.com/apex/log v1.9.0 github.com/buildpacks/imgutil v0.0.0-20230626185301-726f02e4225c - github.com/buildpacks/lifecycle v0.17.1 + github.com/buildpacks/lifecycle v0.17.2 github.com/docker/cli v24.0.6+incompatible github.com/docker/docker v24.0.6+incompatible github.com/docker/go-connections v0.4.0 diff --git a/go.sum b/go.sum index 14aae3e627..9c312db205 100644 --- a/go.sum +++ b/go.sum @@ -97,8 +97,8 @@ github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230522190001- github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/buildpacks/imgutil v0.0.0-20230626185301-726f02e4225c h1:HlRuSz+JGAzudNtNCfHIzXe0AEuHX6Vx8uZgmjvX02o= github.com/buildpacks/imgutil v0.0.0-20230626185301-726f02e4225c/go.mod h1:mBG5M3GJW5nknCEOOqtmMHyPYnSpw/5GEiciuYU/COw= -github.com/buildpacks/lifecycle v0.17.1 h1:sCNj83TH1YE8Z3+CKHoFx/HK+llCVF1RlQUbj3xdNBQ= -github.com/buildpacks/lifecycle v0.17.1/go.mod h1:WFzcNp1WG4bwgHuXtKxMg4tdU3AguL44ZlP3knANeVs= +github.com/buildpacks/lifecycle v0.17.2 h1:CfJYWHIC5v996idgjDamYHBTk+G+c1Qt7Yk80MlbWpw= +github.com/buildpacks/lifecycle v0.17.2/go.mod h1:h8MrqltqMM+HQnn2F2JOQaKWmeybZ54qvlNV3pAiAqw= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589 h1:krfRl01rzPzxSxyLyrChD+U+MzsBXbm0OwYYB67uF+4= github.com/chrismellard/docker-credential-acr-env v0.0.0-20230304212654-82a0ddb27589/go.mod h1:OuDyvmLnMCwa2ep4Jkm6nyA0ocJuZlGyk2gGseVzERM= From 32e5d1d292f700bbfae92284eacd5509907be38e Mon Sep 17 00:00:00 2001 From: Juan Bustamante Date: Thu, 26 Oct 2023 16:52:01 -0500 Subject: [PATCH 20/62] Updating default lifecycle version to 0.17.2 Signed-off-by: Juan Bustamante --- acceptance/testdata/pack_fixtures/report_output.txt | 2 +- internal/builder/lifecycle.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/acceptance/testdata/pack_fixtures/report_output.txt b/acceptance/testdata/pack_fixtures/report_output.txt index 58e3baa5d7..beaae617bd 100644 --- a/acceptance/testdata/pack_fixtures/report_output.txt +++ b/acceptance/testdata/pack_fixtures/report_output.txt @@ -2,7 +2,7 @@ Pack: Version: {{ .Version }} OS/Arch: {{ .OS }}/{{ .Arch }} -Default Lifecycle Version: 0.17.1 +Default Lifecycle Version: 0.17.2 Supported Platform APIs: 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9, 0.10, 0.11, 0.12 diff --git a/internal/builder/lifecycle.go b/internal/builder/lifecycle.go index 3184279021..6c3193d47b 100644 --- a/internal/builder/lifecycle.go +++ b/internal/builder/lifecycle.go @@ -14,7 +14,7 @@ import ( // A snapshot of the latest tested lifecycle version values const ( - DefaultLifecycleVersion = "0.17.1" + DefaultLifecycleVersion = "0.17.2" DefaultBuildpackAPIVersion = "0.2" ) From d5d98bc843285f0fe295e5c5b903cfec96712feb Mon Sep 17 00:00:00 2001 From: Ed Morley <501702+edmorley@users.noreply.github.com> Date: Fri, 27 Oct 2023 09:20:05 +0100 Subject: [PATCH 21/62] Group minor/patch version Go Dependabot updates into one PR Go minor/patch dependencies will now be grouped, using the new Dependabot grouping feature: https://github.blog/changelog/2023-08-17-grouped-version-updates-by-semantic-version-level-for-dependabot/ Major updates, as well as security updates will still be opened as separate PRs. I've not grouped GitHub Actions update PRs, since the volume is typically much lower for those. In addition, the schedule has been changed from daily to weekly. This reduces project maintenance toil (no more having to manually create combined update PRs), plus makes it less painful for contributors to subscribe to repository notifications (currently there is a lot of noise from Dependabot PRs being opened/auto-rebased etc). Signed-off-by: Ed Morley <501702+edmorley@users.noreply.github.com> --- .github/dependabot.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 393b765af7..f1b8489baa 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -4,8 +4,13 @@ updates: - package-ecosystem: "gomod" directory: "/" schedule: - # Check for updates to GitHub Actions every weekday - interval: "daily" + interval: "weekly" + groups: + # Group all minor/patch go dependencies into a single PR. + go-dependencies: + update-types: + - "minor" + - "patch" labels: - "dependencies" - "go" @@ -15,8 +20,7 @@ updates: - package-ecosystem: "github-actions" directory: "/" schedule: - # Check for updates to GitHub Actions every weekday - interval: "daily" + interval: "weekly" labels: - "dependencies" - "github_actions" From bf88725565dfd4692180e1ab67d239295e13ef14 Mon Sep 17 00:00:00 2001 From: Natalie Arellano Date: Fri, 27 Oct 2023 12:41:30 -0400 Subject: [PATCH 22/62] Add buildpacksio/pack:-base images to delivery Signed-off-by: Natalie Arellano --- .github/workflows/delivery-docker.yml | 26 +++++++++---------- .../workflows/delivery/docker/project.toml | 7 ----- Dockerfile | 6 ++--- 3 files changed, 16 insertions(+), 23 deletions(-) delete mode 100644 .github/workflows/delivery/docker/project.toml diff --git a/.github/workflows/delivery-docker.yml b/.github/workflows/delivery-docker.yml index 5db2c7e4b2..10675f99db 100644 --- a/.github/workflows/delivery-docker.yml +++ b/.github/workflows/delivery-docker.yml @@ -16,12 +16,21 @@ on: default: false env: - BUILDER: "paketobuildpacks/builder-jammy-tiny" IMG_NAME: 'pack' USERNAME: 'buildpacksio' jobs: deliver-docker: + strategy: + matrix: + config: [tiny, base] + include: + - config: tiny + base_image: gcr.io/distroless/static + suffix: + - config: base + base_image: ubuntu:jammy + suffix: -base runs-on: ubuntu-latest steps: - name: Determine version @@ -42,16 +51,6 @@ jobs: uses: actions/checkout@v4 with: ref: v${{ steps.version.outputs.result }} - # This has to come after the first checkout, so it isn't clobbered - - name: Checkout delivery configuration - uses: actions/checkout@v4 - with: - path: ./head - - name: Setup Working Dir - shell: bash - run: | - rm project.toml || true - cp head/.github/workflows/delivery/docker/project.toml project.toml - name: Determine App Name run: 'echo "IMG_NAME=${{ env.USERNAME }}/${{ env.IMG_NAME }}" >> $GITHUB_ENV' - name: Login to Dockerhub @@ -65,12 +64,13 @@ jobs: - name: Buildx Build/Publish run: | docker buildx build . \ - --tag ${{ env.IMG_NAME }}:${{ steps.version.outputs.result }} \ + --tag ${{ env.IMG_NAME }}:${{ steps.version.outputs.result }}${{ matrix.suffix }} \ --platform linux/amd64,linux/arm64 \ --build-arg pack_version=${{ steps.version.outputs.result }} \ + --build-arg base_image=${{ matrix.base_image }} \ --provenance=false \ --push - name: Tag Image as Latest - if: ${{ github.event.release != '' || github.event.inputs.tag_latest }} + if: ${{ (github.event.release != '' || github.event.inputs.tag_latest) && matrix.config != 'base' }} run: | crane copy ${{ env.IMG_NAME }}:${{ steps.version.outputs.result }} ${{ env.IMG_NAME }}:latest diff --git a/.github/workflows/delivery/docker/project.toml b/.github/workflows/delivery/docker/project.toml deleted file mode 100644 index da9c5c982c..0000000000 --- a/.github/workflows/delivery/docker/project.toml +++ /dev/null @@ -1,7 +0,0 @@ -[project] -version = "1.0.2" -source-url = "https://github.com/buildpacks/pack" - -[[build.env]] -name = "BP_GO_TARGETS" -value = "./cmd/pack" diff --git a/Dockerfile b/Dockerfile index 70c75ed45e..c81b2f0c04 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,5 @@ +ARG base_image=gcr.io/distroless/static + FROM golang:1.20 as builder ARG pack_version ENV PACK_VERSION=$pack_version @@ -5,8 +7,6 @@ WORKDIR /app COPY . . RUN make build -FROM scratch +FROM ${base_image} COPY --from=builder /app/out/pack /usr/local/bin/pack -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ -COPY --from=builder /tmp /tmp ENTRYPOINT [ "/usr/local/bin/pack" ] From 31654b819074c24b820709fafc1a9490ea62ad95 Mon Sep 17 00:00:00 2001 From: Natalie Arellano Date: Mon, 30 Oct 2023 09:19:54 -0400 Subject: [PATCH 23/62] Add floating :base tag Signed-off-by: Natalie Arellano --- .github/workflows/delivery-docker.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/delivery-docker.yml b/.github/workflows/delivery-docker.yml index 10675f99db..c956195175 100644 --- a/.github/workflows/delivery-docker.yml +++ b/.github/workflows/delivery-docker.yml @@ -70,6 +70,10 @@ jobs: --build-arg base_image=${{ matrix.base_image }} \ --provenance=false \ --push + - name: Tag Image as Base + if: ${{ (github.event.release != '' || github.event.inputs.tag_latest) && matrix.config == 'base' }} + run: | + crane copy ${{ env.IMG_NAME }}:${{ steps.version.outputs.result }} ${{ env.IMG_NAME }}:base - name: Tag Image as Latest if: ${{ (github.event.release != '' || github.event.inputs.tag_latest) && matrix.config != 'base' }} run: | From f70fdfe33edf9384eb529ca039b70d002e7cc4d7 Mon Sep 17 00:00:00 2001 From: Natalie Arellano Date: Thu, 12 Oct 2023 10:01:06 -0400 Subject: [PATCH 24/62] Ensure the run image os/arch always matches: - the builder for `pack build` - the previous image for `pack rebase` Signed-off-by: Natalie Arellano --- pkg/client/build.go | 36 ++++++++++++++++++++++-------------- pkg/client/build_test.go | 9 +++++---- pkg/client/rebase.go | 17 ++++++++++++++++- pkg/client/rebase_test.go | 2 ++ 4 files changed, 45 insertions(+), 19 deletions(-) diff --git a/pkg/client/build.go b/pkg/client/build.go index 058dd89c54..30757a7f57 100644 --- a/pkg/client/build.go +++ b/pkg/client/build.go @@ -318,6 +318,16 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error { return errors.Wrapf(err, "failed to fetch builder image '%s'", builderRef.Name()) } + builderOS, err := rawBuilderImage.OS() + if err != nil { + return errors.Wrapf(err, "getting builder OS") + } + + builderArch, err := rawBuilderImage.Architecture() + if err != nil { + return errors.Wrapf(err, "getting builder architecture") + } + bldr, err := c.getBuilder(rawBuilderImage) if err != nil { return errors.Wrapf(err, "invalid builder %s", style.Symbol(opts.Builder)) @@ -325,7 +335,11 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error { runImageName := c.resolveRunImage(opts.RunImage, imgRegistry, builderRef.Context().RegistryStr(), bldr.DefaultRunImage(), opts.AdditionalMirrors, opts.Publish) - fetchOptions := image.FetchOptions{Daemon: !opts.Publish, PullPolicy: opts.PullPolicy} + fetchOptions := image.FetchOptions{ + Daemon: !opts.Publish, + PullPolicy: opts.PullPolicy, + Platform: fmt.Sprintf("%s/%s", builderOS, builderArch), + } if opts.Layout() { targetRunImagePath, err := layout.ParseRefToPath(runImageName) if err != nil { @@ -361,11 +375,6 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error { return err } - imgOS, err := rawBuilderImage.OS() - if err != nil { - return errors.Wrapf(err, "getting builder OS") - } - // Default mode: if the TrustBuilder option is not set, trust the suggested builders. if opts.TrustBuilder == nil { opts.TrustBuilder = IsSuggestedBuilderFunc @@ -396,15 +405,14 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error { lifecycleImageName = fmt.Sprintf("%s:%s", internalConfig.DefaultLifecycleImageRepo, lifecycleVersion.String()) } - imgArch, err := rawBuilderImage.Architecture() - if err != nil { - return errors.Wrapf(err, "getting builder architecture") - } - lifecycleImage, err := c.imageFetcher.Fetch( ctx, lifecycleImageName, - image.FetchOptions{Daemon: true, PullPolicy: opts.PullPolicy, Platform: fmt.Sprintf("%s/%s", imgOS, imgArch)}, + image.FetchOptions{ + Daemon: true, + PullPolicy: opts.PullPolicy, + Platform: fmt.Sprintf("%s/%s", builderOS, builderArch), + }, ) if err != nil { return fmt.Errorf("fetching lifecycle image: %w", err) @@ -455,7 +463,7 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error { if !c.experimental { return fmt.Errorf("experimental features must be enabled when builder contains image extensions") } - if imgOS == "windows" { + if builderOS == "windows" { return fmt.Errorf("builder contains image extensions which are not supported for Windows builds") } if !(opts.PullPolicy == image.PullAlways) { @@ -467,7 +475,7 @@ func (c *Client) Build(ctx context.Context, opts BuildOptions) error { opts.ContainerConfig.Volumes = appendLayoutVolumes(opts.ContainerConfig.Volumes, pathsConfig) } - processedVolumes, warnings, err := processVolumes(imgOS, opts.ContainerConfig.Volumes) + processedVolumes, warnings, err := processVolumes(builderOS, opts.ContainerConfig.Volumes) if err != nil { return err } diff --git a/pkg/client/build_test.go b/pkg/client/build_test.go index 06b215104b..2f43619c53 100644 --- a/pkg/client/build_test.go +++ b/pkg/client/build_test.go @@ -2085,11 +2085,12 @@ api = "0.2" })) h.AssertEq(t, fakeLifecycle.Opts.Publish, true) - args := fakeImageFetcher.FetchCalls["default/run"] - h.AssertEq(t, args.Daemon, false) - - args = fakeImageFetcher.FetchCalls[defaultBuilderName] + args := fakeImageFetcher.FetchCalls[defaultBuilderName] h.AssertEq(t, args.Daemon, true) + + args = fakeImageFetcher.FetchCalls["default/run"] + h.AssertEq(t, args.Daemon, false) + h.AssertEq(t, args.Platform, "linux/amd64") }) when("builder is untrusted", func() { diff --git a/pkg/client/rebase.go b/pkg/client/rebase.go index 168c727573..276c56026e 100644 --- a/pkg/client/rebase.go +++ b/pkg/client/rebase.go @@ -2,6 +2,7 @@ package client import ( "context" + "fmt" "os" "path/filepath" @@ -60,6 +61,16 @@ func (c *Client) Rebase(ctx context.Context, opts RebaseOptions) error { return err } + appOS, err := appImage.OS() + if err != nil { + return errors.Wrapf(err, "getting app OS") + } + + appArch, err := appImage.Architecture() + if err != nil { + return errors.Wrapf(err, "getting app architecture") + } + var md files.LayersMetadataCompat if ok, err := dist.GetLabel(appImage, platform.LifecycleMetadataLabel, &md); err != nil { return err @@ -90,7 +101,11 @@ func (c *Client) Rebase(ctx context.Context, opts RebaseOptions) error { return errors.New("run image must be specified") } - baseImage, err := c.imageFetcher.Fetch(ctx, runImageName, image.FetchOptions{Daemon: !opts.Publish, PullPolicy: opts.PullPolicy}) + baseImage, err := c.imageFetcher.Fetch(ctx, runImageName, image.FetchOptions{ + Daemon: !opts.Publish, + PullPolicy: opts.PullPolicy, + Platform: fmt.Sprintf("%s/%s", appOS, appArch), + }) if err != nil { return err } diff --git a/pkg/client/rebase_test.go b/pkg/client/rebase_test.go index 86eff76de9..101fb683a9 100644 --- a/pkg/client/rebase_test.go +++ b/pkg/client/rebase_test.go @@ -258,6 +258,8 @@ func testRebase(t *testing.T, when spec.G, it spec.S) { h.AssertEq(t, fakeAppImage.Base(), "some/run") lbl, _ := fakeAppImage.Label("io.buildpacks.lifecycle.metadata") h.AssertContains(t, lbl, `"runImage":{"topLayer":"remote-top-layer-sha","reference":"remote-digest"`) + args := fakeImageFetcher.FetchCalls["some/run"] + h.AssertEq(t, args.Platform, "linux/amd64") }) }) }) From 0133424e3537e89e589eeb20f10b343f72d17488 Mon Sep 17 00:00:00 2001 From: Natalie Arellano Date: Thu, 12 Oct 2023 11:30:07 -0400 Subject: [PATCH 25/62] When downloading buildpacks or extensions for `pack build` or `pack builder create`, ensure the os/arch matches the builder Signed-off-by: Natalie Arellano --- pkg/buildpack/downloader.go | 15 +++++++++++++-- pkg/buildpack/downloader_test.go | 24 +++++++++++++++--------- pkg/client/build.go | 13 ++++++++++--- pkg/client/build_test.go | 2 ++ pkg/client/create_builder.go | 12 +++++++++--- pkg/client/create_builder_test.go | 14 ++++++++++++-- 6 files changed, 61 insertions(+), 19 deletions(-) diff --git a/pkg/buildpack/downloader.go b/pkg/buildpack/downloader.go index 41a5a61cdb..454afda0cb 100644 --- a/pkg/buildpack/downloader.go +++ b/pkg/buildpack/downloader.go @@ -67,6 +67,9 @@ type DownloadOptions struct { // The OS of the builder image ImageOS string + // The OS/Architecture to download + Platform string + // Deprecated: the older alternative to buildpack URI ImageName string @@ -102,7 +105,11 @@ func (c *buildpackDownloader) Download(ctx context.Context, moduleURI string, op case PackageLocator: imageName := ParsePackageLocator(moduleURI) c.logger.Debugf("Downloading %s from image: %s", kind, style.Symbol(imageName)) - mainBP, depBPs, err = extractPackaged(ctx, kind, imageName, c.imageFetcher, image.FetchOptions{Daemon: opts.Daemon, PullPolicy: opts.PullPolicy}) + mainBP, depBPs, err = extractPackaged(ctx, kind, imageName, c.imageFetcher, image.FetchOptions{ + Daemon: opts.Daemon, + PullPolicy: opts.PullPolicy, + Platform: opts.Platform, + }) if err != nil { return nil, nil, errors.Wrapf(err, "extracting from registry %s", style.Symbol(moduleURI)) } @@ -113,7 +120,11 @@ func (c *buildpackDownloader) Download(ctx context.Context, moduleURI string, op return nil, nil, errors.Wrapf(err, "locating in registry: %s", style.Symbol(moduleURI)) } - mainBP, depBPs, err = extractPackaged(ctx, kind, address, c.imageFetcher, image.FetchOptions{Daemon: opts.Daemon, PullPolicy: opts.PullPolicy}) + mainBP, depBPs, err = extractPackaged(ctx, kind, address, c.imageFetcher, image.FetchOptions{ + Daemon: opts.Daemon, + PullPolicy: opts.PullPolicy, + Platform: opts.Platform, + }) if err != nil { return nil, nil, errors.Wrapf(err, "extracting from registry %s", style.Symbol(moduleURI)) } diff --git a/pkg/buildpack/downloader_test.go b/pkg/buildpack/downloader_test.go index 16c30b6e10..2aefdb571d 100644 --- a/pkg/buildpack/downloader_test.go +++ b/pkg/buildpack/downloader_test.go @@ -127,8 +127,12 @@ func testBuildpackDownloader(t *testing.T, when spec.G, it spec.S) { downloadOptions = buildpack.DownloadOptions{ImageOS: "linux"} ) - shouldFetchPackageImageWith := func(demon bool, pull image.PullPolicy) { - mockImageFetcher.EXPECT().Fetch(gomock.Any(), packageImage.Name(), image.FetchOptions{Daemon: demon, PullPolicy: pull}).Return(packageImage, nil) + shouldFetchPackageImageWith := func(demon bool, pull image.PullPolicy, platform string) { + mockImageFetcher.EXPECT().Fetch(gomock.Any(), packageImage.Name(), image.FetchOptions{ + Daemon: demon, + PullPolicy: pull, + Platform: platform, + }).Return(packageImage, nil) } when("package image lives in cnb registry", func() { @@ -141,11 +145,12 @@ func testBuildpackDownloader(t *testing.T, when spec.G, it spec.S) { downloadOptions = buildpack.DownloadOptions{ RegistryName: "some-registry", ImageOS: "linux", + Platform: "linux/amd64", Daemon: true, PullPolicy: image.PullAlways, } - shouldFetchPackageImageWith(true, image.PullAlways) + shouldFetchPackageImageWith(true, image.PullAlways, "linux/amd64") mainBP, _, err := buildpackDownloader.Download(context.TODO(), "urn:cnb:registry:example/foo@1.1.0", downloadOptions) h.AssertNil(t, err) h.AssertEq(t, mainBP.Descriptor().Info().ID, "example/foo") @@ -161,7 +166,7 @@ func testBuildpackDownloader(t *testing.T, when spec.G, it spec.S) { PullPolicy: image.PullAlways, } - shouldFetchPackageImageWith(true, image.PullAlways) + shouldFetchPackageImageWith(true, image.PullAlways, "") mainBP, _, err := buildpackDownloader.Download(context.TODO(), "example/foo@1.1.0", downloadOptions) h.AssertNil(t, err) h.AssertEq(t, mainBP.Descriptor().Info().ID, "example/foo") @@ -185,10 +190,11 @@ func testBuildpackDownloader(t *testing.T, when spec.G, it spec.S) { Daemon: true, PullPolicy: image.PullAlways, ImageOS: "linux", + Platform: "linux/amd64", ImageName: "some/package:tag", } - shouldFetchPackageImageWith(true, image.PullAlways) + shouldFetchPackageImageWith(true, image.PullAlways, "linux/amd64") mainBP, _, err := buildpackDownloader.Download(context.TODO(), "", downloadOptions) h.AssertNil(t, err) h.AssertEq(t, mainBP.Descriptor().Info().ID, "example/foo") @@ -204,7 +210,7 @@ func testBuildpackDownloader(t *testing.T, when spec.G, it spec.S) { PullPolicy: image.PullAlways, } - shouldFetchPackageImageWith(true, image.PullAlways) + shouldFetchPackageImageWith(true, image.PullAlways, "") mainBP, _, err := buildpackDownloader.Download(context.TODO(), "", downloadOptions) h.AssertNil(t, err) h.AssertEq(t, mainBP.Descriptor().Info().ID, "example/foo") @@ -220,7 +226,7 @@ func testBuildpackDownloader(t *testing.T, when spec.G, it spec.S) { PullPolicy: image.PullAlways, } - shouldFetchPackageImageWith(false, image.PullAlways) + shouldFetchPackageImageWith(false, image.PullAlways, "") mainBP, _, err := buildpackDownloader.Download(context.TODO(), "", downloadOptions) h.AssertNil(t, err) h.AssertEq(t, mainBP.Descriptor().Info().ID, "example/foo") @@ -234,7 +240,7 @@ func testBuildpackDownloader(t *testing.T, when spec.G, it spec.S) { Daemon: false, PullPolicy: image.PullAlways, } - shouldFetchPackageImageWith(false, image.PullAlways) + shouldFetchPackageImageWith(false, image.PullAlways, "") mainBP, _, err := buildpackDownloader.Download(context.TODO(), packageImage.Name(), downloadOptions) h.AssertNil(t, err) h.AssertEq(t, mainBP.Descriptor().Info().ID, "example/foo") @@ -250,7 +256,7 @@ func testBuildpackDownloader(t *testing.T, when spec.G, it spec.S) { PullPolicy: image.PullNever, } - shouldFetchPackageImageWith(false, image.PullNever) + shouldFetchPackageImageWith(false, image.PullNever, "") mainBP, _, err := buildpackDownloader.Download(context.TODO(), "", downloadOptions) h.AssertNil(t, err) h.AssertEq(t, mainBP.Descriptor().Info().ID, "example/foo") diff --git a/pkg/client/build.go b/pkg/client/build.go index 30757a7f57..4371e311b0 100644 --- a/pkg/client/build.go +++ b/pkg/client/build.go @@ -1032,13 +1032,19 @@ func (c *Client) fetchBuildpack(ctx context.Context, bp string, relativeBaseDir Version: version, } default: - imageOS, err := builderImage.OS() + builderOS, err := builderImage.OS() if err != nil { - return nil, nil, errors.Wrapf(err, "getting OS from %s", style.Symbol(builderImage.Name())) + return nil, nil, errors.Wrapf(err, "getting builder OS") + } + + builderArch, err := builderImage.Architecture() + if err != nil { + return nil, nil, errors.Wrapf(err, "getting builder architecture") } downloadOptions := buildpack.DownloadOptions{ RegistryName: registry, - ImageOS: imageOS, + ImageOS: builderOS, + Platform: fmt.Sprintf("%s/%s", builderOS, builderArch), RelativeBaseDir: relativeBaseDir, Daemon: !publish, PullPolicy: pullPolicy, @@ -1076,6 +1082,7 @@ func (c *Client) fetchBuildpackDependencies(ctx context.Context, bp string, pack mainBP, deps, err := c.buildpackDownloader.Download(ctx, dep.URI, buildpack.DownloadOptions{ RegistryName: downloadOptions.RegistryName, ImageOS: downloadOptions.ImageOS, + Platform: downloadOptions.Platform, Daemon: downloadOptions.Daemon, PullPolicy: downloadOptions.PullPolicy, RelativeBaseDir: filepath.Join(bp, packageCfg.Buildpack.URI), diff --git a/pkg/client/build_test.go b/pkg/client/build_test.go index 2f43619c53..ed5693f7c9 100644 --- a/pkg/client/build_test.go +++ b/pkg/client/build_test.go @@ -1240,6 +1240,8 @@ api = "0.2" Version: "child.buildpack.version", }, }) + args := fakeImageFetcher.FetchCalls[fakePackage.Name()] + h.AssertEq(t, args.Platform, "linux/amd64") }) it("fails when no metadata label on package", func() { diff --git a/pkg/client/create_builder.go b/pkg/client/create_builder.go index 28cd2fa7c5..dc03d73d6b 100644 --- a/pkg/client/create_builder.go +++ b/pkg/client/create_builder.go @@ -256,14 +256,20 @@ func (c *Client) addExtensionsToBuilder(ctx context.Context, opts CreateBuilderO func (c *Client) addConfig(ctx context.Context, kind string, config pubbldr.ModuleConfig, opts CreateBuilderOptions, bldr *builder.Builder) error { c.logger.Debugf("Looking up %s %s", kind, style.Symbol(config.DisplayString())) - imageOS, err := bldr.Image().OS() + builderOS, err := bldr.Image().OS() if err != nil { - return errors.Wrapf(err, "getting OS from %s", style.Symbol(bldr.Image().Name())) + return errors.Wrapf(err, "getting builder OS") } + builderArch, err := bldr.Image().Architecture() + if err != nil { + return errors.Wrapf(err, "getting builder architecture") + } + mainBP, depBPs, err := c.buildpackDownloader.Download(ctx, config.URI, buildpack.DownloadOptions{ Daemon: !opts.Publish, ImageName: config.ImageName, - ImageOS: imageOS, + ImageOS: builderOS, + Platform: fmt.Sprintf("%s/%s", builderOS, builderArch), ModuleKind: kind, PullPolicy: opts.PullPolicy, RegistryName: opts.Registry, diff --git a/pkg/client/create_builder_test.go b/pkg/client/create_builder_test.go index 3b4cfc41ef..ea0c95a939 100644 --- a/pkg/client/create_builder_test.go +++ b/pkg/client/create_builder_test.go @@ -843,12 +843,22 @@ func testCreateBuilder(t *testing.T, when spec.G, it spec.S) { buildpackBlob := blob.NewBlob(filepath.Join("testdata", "buildpack-api-0.4")) bp, err := buildpack.FromBuildpackRootBlob(buildpackBlob, archive.DefaultTarWriterFactory()) h.AssertNil(t, err) - mockBuildpackDownloader.EXPECT().Download(gomock.Any(), "https://example.fake/bp-one-with-api-4.tgz", gomock.Any()).Return(bp, bpDependencies, nil) + mockBuildpackDownloader.EXPECT().Download(gomock.Any(), "https://example.fake/bp-one-with-api-4.tgz", gomock.Any()).DoAndReturn( + func(ctx context.Context, buildpackURI string, opts buildpack.DownloadOptions) (buildpack.BuildModule, []buildpack.BuildModule, error) { + // test options + h.AssertEq(t, opts.Platform, "linux/amd64") + return bp, bpDependencies, nil + }) extensionBlob := blob.NewBlob(filepath.Join("testdata", "extension-api-0.9")) extension, err := buildpack.FromExtensionRootBlob(extensionBlob, archive.DefaultTarWriterFactory()) h.AssertNil(t, err) - mockBuildpackDownloader.EXPECT().Download(gomock.Any(), "https://example.fake/ext-one-with-api-9.tgz", gomock.Any()).Return(extension, nil, nil) + mockBuildpackDownloader.EXPECT().Download(gomock.Any(), "https://example.fake/ext-one-with-api-9.tgz", gomock.Any()).DoAndReturn( + func(ctx context.Context, buildpackURI string, opts buildpack.DownloadOptions) (buildpack.BuildModule, []buildpack.BuildModule, error) { + // test options + h.AssertEq(t, opts.Platform, "linux/amd64") + return extension, nil, nil + }) successfullyCreateDeterministicBuilder() From 0848fec4c074e4b615b3232f8ccb288585de258e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 31 Oct 2023 01:44:30 +0000 Subject: [PATCH 26/62] build(deps): bump the go-dependencies group with 6 updates Bumps the go-dependencies group with 6 updates: | Package | From | To | | --- | --- | --- | | [github.com/docker/cli](https://github.com/docker/cli) | `24.0.6+incompatible` | `24.0.7+incompatible` | | [github.com/docker/docker](https://github.com/docker/docker) | `24.0.6+incompatible` | `24.0.7+incompatible` | | [github.com/go-git/go-git/v5](https://github.com/go-git/go-git) | `5.9.0` | `5.10.0` | | [github.com/google/go-cmp](https://github.com/google/go-cmp) | `0.5.9` | `0.6.0` | | [github.com/onsi/gomega](https://github.com/onsi/gomega) | `1.28.0` | `1.29.0` | | [golang.org/x/oauth2](https://github.com/golang/oauth2) | `0.12.0` | `0.13.0` | Updates `github.com/docker/cli` from 24.0.6+incompatible to 24.0.7+incompatible - [Commits](https://github.com/docker/cli/compare/v24.0.6...v24.0.7) Updates `github.com/docker/docker` from 24.0.6+incompatible to 24.0.7+incompatible - [Release notes](https://github.com/docker/docker/releases) - [Commits](https://github.com/docker/docker/compare/v24.0.6...v24.0.7) Updates `github.com/go-git/go-git/v5` from 5.9.0 to 5.10.0 - [Release notes](https://github.com/go-git/go-git/releases) - [Commits](https://github.com/go-git/go-git/compare/v5.9.0...v5.10.0) Updates `github.com/google/go-cmp` from 0.5.9 to 0.6.0 - [Release notes](https://github.com/google/go-cmp/releases) - [Commits](https://github.com/google/go-cmp/compare/v0.5.9...v0.6.0) Updates `github.com/onsi/gomega` from 1.28.0 to 1.29.0 - [Release notes](https://github.com/onsi/gomega/releases) - [Changelog](https://github.com/onsi/gomega/blob/master/CHANGELOG.md) - [Commits](https://github.com/onsi/gomega/compare/v1.28.0...v1.29.0) Updates `golang.org/x/oauth2` from 0.12.0 to 0.13.0 - [Commits](https://github.com/golang/oauth2/compare/v0.12.0...v0.13.0) --- updated-dependencies: - dependency-name: github.com/docker/cli dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go-dependencies - dependency-name: github.com/docker/docker dependency-type: direct:production update-type: version-update:semver-patch dependency-group: go-dependencies - dependency-name: github.com/go-git/go-git/v5 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-dependencies - dependency-name: github.com/google/go-cmp dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-dependencies - dependency-name: github.com/onsi/gomega dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-dependencies - dependency-name: golang.org/x/oauth2 dependency-type: direct:production update-type: version-update:semver-minor dependency-group: go-dependencies ... Signed-off-by: dependabot[bot] --- go.mod | 12 ++++++------ go.sum | 28 ++++++++++++++-------------- 2 files changed, 20 insertions(+), 20 deletions(-) diff --git a/go.mod b/go.mod index 833398d0e1..95378c7be6 100644 --- a/go.mod +++ b/go.mod @@ -7,21 +7,21 @@ require ( github.com/apex/log v1.9.0 github.com/buildpacks/imgutil v0.0.0-20230626185301-726f02e4225c github.com/buildpacks/lifecycle v0.17.2 - github.com/docker/cli v24.0.6+incompatible - github.com/docker/docker v24.0.6+incompatible + github.com/docker/cli v24.0.7+incompatible + github.com/docker/docker v24.0.7+incompatible github.com/docker/go-connections v0.4.0 github.com/dustin/go-humanize v1.0.1 github.com/gdamore/tcell/v2 v2.6.0 github.com/ghodss/yaml v1.0.0 - github.com/go-git/go-git/v5 v5.9.0 + github.com/go-git/go-git/v5 v5.10.0 github.com/golang/mock v1.6.0 - github.com/google/go-cmp v0.5.9 + github.com/google/go-cmp v0.6.0 github.com/google/go-containerregistry v0.16.1 github.com/google/go-github/v30 v30.1.0 github.com/hectane/go-acl v0.0.0-20190604041725-da78bae5fc95 github.com/heroku/color v0.0.6 github.com/mitchellh/ioprogress v0.0.0-20180201004757-6a23b12fa88e - github.com/onsi/gomega v1.28.0 + github.com/onsi/gomega v1.29.0 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.1.0-rc5 github.com/pelletier/go-toml v1.9.5 @@ -32,7 +32,7 @@ require ( github.com/spf13/cobra v1.7.0 golang.org/x/crypto v0.14.0 golang.org/x/mod v0.13.0 - golang.org/x/oauth2 v0.12.0 + golang.org/x/oauth2 v0.13.0 golang.org/x/sync v0.4.0 golang.org/x/term v0.13.0 golang.org/x/text v0.13.0 diff --git a/go.sum b/go.sum index 9c312db205..677cd5be10 100644 --- a/go.sum +++ b/go.sum @@ -120,12 +120,12 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dimchansky/utfbom v1.1.1 h1:vV6w1AhK4VMnhBno/TPVCoK9U/LP0PkLCS9tbxHdi/U= github.com/dimchansky/utfbom v1.1.1/go.mod h1:SxdoEBH5qIqFocHMyGOXVAybYJdr71b1Q/j0mACtrfE= -github.com/docker/cli v24.0.6+incompatible h1:fF+XCQCgJjjQNIMjzaSmiKJSCcfcXb3TWTcc7GAneOY= -github.com/docker/cli v24.0.6+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= +github.com/docker/cli v24.0.7+incompatible h1:wa/nIwYFW7BVTGa7SWPVyyXU9lgORqUb1xfI36MSkFg= +github.com/docker/cli v24.0.7+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.6+incompatible h1:hceabKCtUgDqPu+qm0NgsaXf28Ljf4/pWFL7xjWWDgE= -github.com/docker/docker v24.0.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= +github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= @@ -151,9 +151,9 @@ github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66D github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= github.com/go-git/go-billy/v5 v5.5.0 h1:yEY4yhzCDuMGSv83oGxiBotRzhwhNr8VZyphhiu+mTU= github.com/go-git/go-billy/v5 v5.5.0/go.mod h1:hmexnoNsr2SJU1Ju67OaNz5ASJY3+sHgFRpCtpDCKow= -github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20230305113008-0c11038e723f h1:Pz0DHeFij3XFhoBRGUDPzSJ+w2UcK5/0JvF8DRI58r8= -github.com/go-git/go-git/v5 v5.9.0 h1:cD9SFA7sHVRdJ7AYck1ZaAa/yeuBvGPxwXDL8cxrObY= -github.com/go-git/go-git/v5 v5.9.0/go.mod h1:RKIqga24sWdMGZF+1Ekv9kylsDz6LzdTSI2s/OsZWE0= +github.com/go-git/go-git-fixtures/v4 v4.3.2-0.20231010084843-55a94097c399 h1:eMje31YglSBqCdIqdhKBW8lokaMrL3uTkpGYlE2OOT4= +github.com/go-git/go-git/v5 v5.10.0 h1:F0x3xXrAWmhwtzoCokU4IMPcBdncG+HAAqi9FcOOjbQ= +github.com/go-git/go-git/v5 v5.10.0/go.mod h1:1FOZ/pQnqw24ghP2n7cunVl0ON55BsjPYvhWHvZGhoo= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= @@ -176,8 +176,8 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-containerregistry v0.16.1 h1:rUEt426sR6nyrL3gt+18ibRcvYpKYdpsa5ZW7MA08dQ= github.com/google/go-containerregistry v0.16.1/go.mod h1:u0qB2l7mvtWVR5kNcbFIhFY1hLbf8eeGapA+vbFDCtQ= github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= @@ -250,10 +250,10 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/onsi/ginkgo v1.6.0 h1:Ix8l273rp3QzYgXSR+c8d1fTG7UPgYkOSELPhiY/YGw= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo/v2 v2.12.0 h1:UIVDowFPwpg6yMUpPjGkYvf06K3RAiJXUhCxEwQVHRI= +github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.28.0 h1:i2rg/p9n/UqIDAMFUJ6qIUUMcsqOuUHgbpbu235Vr1c= -github.com/onsi/gomega v1.28.0/go.mod h1:A1H2JE76sI14WIP57LMKj7FVfCHx3g3BcZVjJG8bjX8= +github.com/onsi/gomega v1.29.0 h1:KIA/t2t5UBzoirT4H9tsML45GEbo3ouUnBHsCfD2tVg= +github.com/onsi/gomega v1.29.0/go.mod h1:9sxs+SwGrKI0+PWe4Fxa9tFQQBG5xSsSbMXOI8PPpoQ= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc5 h1:Ygwkfw9bpDvs+c9E34SdgGOj41dX/cbdlwvlWt0pnFI= @@ -365,8 +365,8 @@ golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 71821d8d349b6a2a96a2a7c27cd651fabc8db96c Mon Sep 17 00:00:00 2001 From: Natalie Arellano Date: Fri, 22 Sep 2023 11:48:49 -0400 Subject: [PATCH 27/62] Fix misleading log message when publishing a buildpack package When using --publish it should say "saved to registry" not "saved to docker daemon" Signed-off-by: Natalie Arellano --- internal/commands/buildpack_package.go | 4 ++-- internal/commands/extension_package.go | 5 +++-- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/internal/commands/buildpack_package.go b/internal/commands/buildpack_package.go index 6cac0ad74d..dff2bf3981 100644 --- a/internal/commands/buildpack_package.go +++ b/internal/commands/buildpack_package.go @@ -118,14 +118,14 @@ func BuildpackPackage(logger logging.Logger, cfg config.Config, packager Buildpa } action := "created" + location := "docker daemon" if flags.Publish { action = "published" + location = "registry" } - location := "docker daemon" if flags.Format == client.FormatFile { location = "file" } - logger.Infof("Successfully %s package %s and saved to %s", action, style.Symbol(name), location) return nil }), diff --git a/internal/commands/extension_package.go b/internal/commands/extension_package.go index 69370c4604..f8d729b2c3 100644 --- a/internal/commands/extension_package.go +++ b/internal/commands/extension_package.go @@ -84,15 +84,16 @@ func ExtensionPackage(logger logging.Logger, cfg config.Config, packager Extensi }); err != nil { return err } + action := "created" + location := "docker daemon" if flags.Publish { action = "published" + location = "registry" } - location := "docker daemon" if flags.Format == client.FormatFile { location = "file" } - logger.Infof("Successfully %s package %s and saved to %s", action, style.Symbol(name), location) return nil }), From 4624639fe54dd3f520a6a64f04f3e0d6035d8fda Mon Sep 17 00:00:00 2001 From: WYGIN Date: Wed, 4 Oct 2023 16:15:54 +0530 Subject: [PATCH 28/62] WIP added code to generate env files in buildConfigEnv dir Signed-off-by: WYGIN --- internal/commands/build.go | 62 ++++++++++++++++++++++++++++++++++++++ pkg/project/types/types.go | 18 +++++++++-- 2 files changed, 78 insertions(+), 2 deletions(-) diff --git a/internal/commands/build.go b/internal/commands/build.go index 3984da8503..4f4bc9a66a 100644 --- a/internal/commands/build.go +++ b/internal/commands/build.go @@ -147,6 +147,9 @@ func Build(logger logging.Logger, cfg config.Config, packClient PackClient) *cob if err != nil { return errors.Wrapf(err, "parsing creation time %s", flags.DateTime) } + if err := generateBuildConfigEnvFiles(descriptor.Build.Env); err != nil { + return err + } if err := packClient.Build(cmd.Context(), client.BuildOptions{ AppPath: flags.AppPath, Builder: builder, @@ -367,3 +370,62 @@ func parseProjectToml(appPath, descriptorPath string) (projectTypes.Descriptor, descriptor, err := project.ReadProjectDescriptor(actualPath) return descriptor, actualPath, err } + +func generateBuildConfigEnvFiles(envList []projectTypes.EnvVar) error { + dir, err := createBuildConfigEnvDir() + if err != nil { + return err + } + for _, env := range envList { + f, err := os.Create(filepath.Join(dir, env.Name+getActionType(env.Action))) + if err != nil { + return err + } + f.WriteString(env.Value) + if e := f.Close(); e != nil { + return e + } + } + return nil +} + +func cnbBuildConfigDir() string { + if v := os.Getenv("CNB_BUILD_CONFIG_DIR"); v == "" { + return "/cnb/build-config" + } else { + return v + } +} + +func createBuildConfigEnvDir() (dir string, err error) { + dir = filepath.Join(cnbBuildConfigDir(), "env") + _, err = os.Stat(dir) + if os.IsNotExist(err) { + err := os.MkdirAll(dir, os.ModePerm) + if err != nil { + return dir, err + } + return dir, nil + } + return dir, err +} + +func getActionType(action projectTypes.ActionType) string { + const delim = "." + switch action { + case projectTypes.NONE: + return "" + case projectTypes.DEFAULT: + return delim + string(projectTypes.DEFAULT) + case projectTypes.OVERRIDE: + return delim + string(projectTypes.OVERRIDE) + case projectTypes.APPEND: + return delim + string(projectTypes.APPEND) + case projectTypes.PREPEND: + return delim + string(projectTypes.PREPEND) + case projectTypes.DELIMIT: + return delim + string(projectTypes.DELIMIT) + default: + return delim + string(projectTypes.DEFAULT) + } +} diff --git a/pkg/project/types/types.go b/pkg/project/types/types.go index 3371d8d388..fdbb3bd3a5 100644 --- a/pkg/project/types/types.go +++ b/pkg/project/types/types.go @@ -17,9 +17,23 @@ type Buildpack struct { Script Script `toml:"script"` } +type ActionType string + +var ActionTypes []ActionType = []ActionType{NONE, DEFAULT, OVERRIDE, APPEND, PREPEND, DELIMIT} + +const ( + NONE ActionType = "" + DEFAULT ActionType = "default" + OVERRIDE ActionType = "override" + APPEND ActionType = "append" + PREPEND ActionType = "prepend" + DELIMIT ActionType = "delim" +) + type EnvVar struct { - Name string `toml:"name"` - Value string `toml:"value"` + Name string `toml:"name"` + Value string `toml:"value"` + Action ActionType `toml:"action"` } type Build struct { From 98542316f1c15021bc3f40ce6e0764202dd9f196 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Wed, 4 Oct 2023 15:45:11 +0000 Subject: [PATCH 29/62] WIP added required abstract test cases Signed-off-by: WYGIN --- builder/config_reader.go | 22 +++++- internal/commands/build.go | 62 ----------------- internal/commands/builder_create.go | 70 +++++++++++++++++++ internal/commands/builder_create_test.go | 87 +++++++++++++++++++++++- pkg/project/types/types.go | 18 +---- 5 files changed, 178 insertions(+), 81 deletions(-) diff --git a/builder/config_reader.go b/builder/config_reader.go index d719c5df8c..35ccd7ac92 100644 --- a/builder/config_reader.go +++ b/builder/config_reader.go @@ -70,7 +70,27 @@ type RunImageConfig struct { // BuildConfig build image configuration type BuildConfig struct { - Image string `toml:"image"` + Image string `toml:"image"` + Env []BuildConfigEnv `toml:"env"` +} + +type ActionType string + +var ActionTypes []ActionType = []ActionType{NONE, DEFAULT, OVERRIDE, APPEND, PREPEND, DELIMIT} + +const ( + NONE ActionType = "" + DEFAULT ActionType = "default" + OVERRIDE ActionType = "override" + APPEND ActionType = "append" + PREPEND ActionType = "prepend" + DELIMIT ActionType = "delim" +) + +type BuildConfigEnv struct { + Name string `toml:"name"` + Value string `toml:"value"` + Action ActionType `toml:"action"` } // ReadConfig reads a builder configuration from the file path provided and returns the diff --git a/internal/commands/build.go b/internal/commands/build.go index 4f4bc9a66a..3984da8503 100644 --- a/internal/commands/build.go +++ b/internal/commands/build.go @@ -147,9 +147,6 @@ func Build(logger logging.Logger, cfg config.Config, packClient PackClient) *cob if err != nil { return errors.Wrapf(err, "parsing creation time %s", flags.DateTime) } - if err := generateBuildConfigEnvFiles(descriptor.Build.Env); err != nil { - return err - } if err := packClient.Build(cmd.Context(), client.BuildOptions{ AppPath: flags.AppPath, Builder: builder, @@ -370,62 +367,3 @@ func parseProjectToml(appPath, descriptorPath string) (projectTypes.Descriptor, descriptor, err := project.ReadProjectDescriptor(actualPath) return descriptor, actualPath, err } - -func generateBuildConfigEnvFiles(envList []projectTypes.EnvVar) error { - dir, err := createBuildConfigEnvDir() - if err != nil { - return err - } - for _, env := range envList { - f, err := os.Create(filepath.Join(dir, env.Name+getActionType(env.Action))) - if err != nil { - return err - } - f.WriteString(env.Value) - if e := f.Close(); e != nil { - return e - } - } - return nil -} - -func cnbBuildConfigDir() string { - if v := os.Getenv("CNB_BUILD_CONFIG_DIR"); v == "" { - return "/cnb/build-config" - } else { - return v - } -} - -func createBuildConfigEnvDir() (dir string, err error) { - dir = filepath.Join(cnbBuildConfigDir(), "env") - _, err = os.Stat(dir) - if os.IsNotExist(err) { - err := os.MkdirAll(dir, os.ModePerm) - if err != nil { - return dir, err - } - return dir, nil - } - return dir, err -} - -func getActionType(action projectTypes.ActionType) string { - const delim = "." - switch action { - case projectTypes.NONE: - return "" - case projectTypes.DEFAULT: - return delim + string(projectTypes.DEFAULT) - case projectTypes.OVERRIDE: - return delim + string(projectTypes.OVERRIDE) - case projectTypes.APPEND: - return delim + string(projectTypes.APPEND) - case projectTypes.PREPEND: - return delim + string(projectTypes.PREPEND) - case projectTypes.DELIMIT: - return delim + string(projectTypes.DELIMIT) - default: - return delim + string(projectTypes.DEFAULT) - } -} diff --git a/internal/commands/builder_create.go b/internal/commands/builder_create.go index 1b0d709895..cd8cb098a9 100644 --- a/internal/commands/builder_create.go +++ b/internal/commands/builder_create.go @@ -2,6 +2,7 @@ package commands import ( "fmt" + "os" "path/filepath" "strings" @@ -75,6 +76,10 @@ Creating a custom builder allows you to control what buildpacks are used and wha return errors.Wrap(err, "getting absolute path for config") } + if err := generateBuildConfigEnvFiles(builderConfig.Build.Env); err != nil { + return err + } + imageName := args[0] if err := pack.CreateBuilder(cmd.Context(), client.CreateBuilderOptions{ RelativeBaseDir: relativeBaseDir, @@ -137,3 +142,68 @@ func validateCreateFlags(flags *BuilderCreateFlags, cfg config.Config) error { return nil } + +func generateBuildConfigEnvFiles(envList []builder.BuildConfigEnv) error { + dir, err := createBuildConfigEnvDir() + if err != nil { + return err + } + for _, env := range envList { + var path string + if a := getActionType(env.Action); a == "" || len(a) == 0 { + path = env.Name + } else { + path = env.Name + getActionType(env.Action) + } + f, err := os.Create(filepath.Join(dir, path)) + if err != nil { + return err + } + f.WriteString(env.Value) + if e := f.Close(); e != nil { + return e + } + } + return nil +} + +func cnbBuildConfigDir() string { + if v := os.Getenv("CNB_BUILD_CONFIG_DIR"); v == "" || len(v) == 0 { + return "/cnb/build-config" + } else { + return v + } +} + +func createBuildConfigEnvDir() (dir string, err error) { + dir = filepath.Join(cnbBuildConfigDir(), "env") + _, err = os.Stat(dir) + if os.IsNotExist(err) { + err := os.MkdirAll(dir, os.ModePerm) + if err != nil { + return dir, err + } + return dir, nil + } + return dir, err +} + +func getActionType(action builder.ActionType) string { + const delim = "." + switch action { + case builder.NONE: + return "" + case builder.DEFAULT: + return delim + string(builder.DEFAULT) + case builder.OVERRIDE: + return delim + string(builder.OVERRIDE) + case builder.APPEND: + return delim + string(builder.APPEND) + case builder.PREPEND: + return delim + string(builder.PREPEND) + case builder.DELIMIT: + return delim + string(builder.DELIMIT) + default: + return delim + string(builder.DEFAULT) + } +} diff --git a/internal/commands/builder_create_test.go b/internal/commands/builder_create_test.go index 12c89465f4..a70b5cd59b 100644 --- a/internal/commands/builder_create_test.go +++ b/internal/commands/builder_create_test.go @@ -157,15 +157,98 @@ func testCreateCommand(t *testing.T, when spec.G, it spec.S) { }) }) - when("uses --builder-config", func() { + when("uses --config", func() { it.Before(func() { h.AssertNil(t, os.WriteFile(builderConfigPath, []byte(validConfig), 0666)) }) + when("buildConfigEnv files generated", func() { + const ActionNONE = validConfig + ` + [[build.env]] + name = "actionNone" + value = "actionNoneValue" + action = "" + ` + const ActionDEFAULT = validConfig + ` + [[build.env]] + name = "actionDefault" + value = "actionDefaultValue" + action = "default" + ` + const ActionOVERRIDE = validConfig + ` + [[build.env]] + name = "actionOverride" + value = "actionOverrideValue" + action = "override" + ` + const ActionAPPEND = validConfig + ` + [[build.env]] + name = "actionAppend" + value = "actionAppendValue" + action = "append" + ` + const ActionPREPEND = validConfig + ` + [[build.env]] + name = "actionPrepend" + value = "actionPrependValue" + action = "prepend" + ` + const ActionDELIMIT = validConfig + ` + [[build.env]] + name = "actionDelimit" + value = "actionDelimitValue" + action = "delim" + ` + const ActionUNKNOWN = validConfig + ` + [[build.env]] + name = "actionUnknown" + value = "actionUnknownValue" + action = "unknown" + ` + const ActionMULTIPLE = validConfig + ` + [[build.env]] + name = "actionAppend" + value = "actionAppendValue" + action = "append" + [[build.env]] + name = "actionPrepend" + value = "actionPrependValue" + action = "prepend" + [[build.env]] + name = "actionDelimit" + value = "actionDelimitValue" + action = "delim" + ` + it("should create content as expected when ActionType `NONE`", func() { + + }) + it("should create content as expected when ActionType `DEFAULT`", func() { + + }) + it("should create content as expected when ActionType `OVERRIDE`", func() { + + }) + it("should create content as expected when ActionType `APPEND`", func() { + + }) + it("should create content as expected when ActionType `PREPEND`", func() { + + }) + it("should create content as expected when ActionType `DELIMIT`", func() { + + }) + it("should create content as expected when unknown ActionType passed", func() { + + }) + it("should create content as expected when multiple ActionTypes passed", func() { + + }) + }) + it("errors with a descriptive message", func() { command.SetArgs([]string{ "some/builder", - "--builder-config", builderConfigPath, + "--config", builderConfigPath, }) h.AssertError(t, command.Execute(), "unknown flag: --builder-config") }) diff --git a/pkg/project/types/types.go b/pkg/project/types/types.go index fdbb3bd3a5..3371d8d388 100644 --- a/pkg/project/types/types.go +++ b/pkg/project/types/types.go @@ -17,23 +17,9 @@ type Buildpack struct { Script Script `toml:"script"` } -type ActionType string - -var ActionTypes []ActionType = []ActionType{NONE, DEFAULT, OVERRIDE, APPEND, PREPEND, DELIMIT} - -const ( - NONE ActionType = "" - DEFAULT ActionType = "default" - OVERRIDE ActionType = "override" - APPEND ActionType = "append" - PREPEND ActionType = "prepend" - DELIMIT ActionType = "delim" -) - type EnvVar struct { - Name string `toml:"name"` - Value string `toml:"value"` - Action ActionType `toml:"action"` + Name string `toml:"name"` + Value string `toml:"value"` } type Build struct { From 161d06ae32a26b523909c846da06042d8f22a627 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 5 Oct 2023 10:00:34 +0000 Subject: [PATCH 30/62] WIP added tests that still nedd to be fixed Signed-off-by: WYGIN --- builder/config_reader.go | 2 +- internal/commands/builder_create.go | 74 +++-- internal/commands/builder_create_test.go | 366 ++++++++++++++++++----- 3 files changed, 344 insertions(+), 98 deletions(-) diff --git a/builder/config_reader.go b/builder/config_reader.go index 35ccd7ac92..7c17ac331a 100644 --- a/builder/config_reader.go +++ b/builder/config_reader.go @@ -90,7 +90,7 @@ const ( type BuildConfigEnv struct { Name string `toml:"name"` Value string `toml:"value"` - Action ActionType `toml:"action"` + Action ActionType `toml:"action,omitempty"` } // ReadConfig reads a builder configuration from the file path provided and returns the diff --git a/internal/commands/builder_create.go b/internal/commands/builder_create.go index cd8cb098a9..7cdf1faa75 100644 --- a/internal/commands/builder_create.go +++ b/internal/commands/builder_create.go @@ -76,7 +76,14 @@ Creating a custom builder allows you to control what buildpacks are used and wha return errors.Wrap(err, "getting absolute path for config") } - if err := generateBuildConfigEnvFiles(builderConfig.Build.Env); err != nil { + envMap, warnings, err := parseBuildConfigEnv(builderConfig.Build.Env, flags.BuilderTomlPath) + for _, v := range warnings { + logger.Warn(v) + } + if err != nil { + return err + } + if err := generateBuildConfigEnvFiles(envMap); err != nil { return err } @@ -143,23 +150,17 @@ func validateCreateFlags(flags *BuilderCreateFlags, cfg config.Config) error { return nil } -func generateBuildConfigEnvFiles(envList []builder.BuildConfigEnv) error { +func generateBuildConfigEnvFiles(envMap map[string]string) error { dir, err := createBuildConfigEnvDir() if err != nil { return err } - for _, env := range envList { - var path string - if a := getActionType(env.Action); a == "" || len(a) == 0 { - path = env.Name - } else { - path = env.Name + getActionType(env.Action) - } - f, err := os.Create(filepath.Join(dir, path)) + for k, v := range envMap { + f, err := os.Create(filepath.Join(dir, k)) if err != nil { return err } - f.WriteString(env.Value) + f.WriteString(v) if e := f.Close(); e != nil { return e } @@ -167,7 +168,7 @@ func generateBuildConfigEnvFiles(envList []builder.BuildConfigEnv) error { return nil } -func cnbBuildConfigDir() string { +func CnbBuildConfigDir() string { if v := os.Getenv("CNB_BUILD_CONFIG_DIR"); v == "" || len(v) == 0 { return "/cnb/build-config" } else { @@ -176,7 +177,7 @@ func cnbBuildConfigDir() string { } func createBuildConfigEnvDir() (dir string, err error) { - dir = filepath.Join(cnbBuildConfigDir(), "env") + dir = filepath.Join(CnbBuildConfigDir(), "env") _, err = os.Stat(dir) if os.IsNotExist(err) { err := os.MkdirAll(dir, os.ModePerm) @@ -188,22 +189,53 @@ func createBuildConfigEnvDir() (dir string, err error) { return dir, err } -func getActionType(action builder.ActionType) string { +func getActionType(action builder.ActionType) (actionString string, err error) { const delim = "." switch action { case builder.NONE: - return "" + return "", nil case builder.DEFAULT: - return delim + string(builder.DEFAULT) + return delim + string(builder.DEFAULT), nil case builder.OVERRIDE: - return delim + string(builder.OVERRIDE) + return delim + string(builder.OVERRIDE), nil case builder.APPEND: - return delim + string(builder.APPEND) + return delim + string(builder.APPEND), nil case builder.PREPEND: - return delim + string(builder.PREPEND) + return delim + string(builder.PREPEND), nil case builder.DELIMIT: - return delim + string(builder.DELIMIT) + return delim + string(builder.DELIMIT), nil default: - return delim + string(builder.DEFAULT) + return actionString, errors.Errorf("unknown action type %s", style.Symbol(string(action))) + } +} +func GetBuildConfigEnvFileName(env builder.BuildConfigEnv) (path string, err error) { + if a, err := getActionType(env.Action); err != nil { + return path, err + } else if a == "" || len(a) == 0 { + path = strings.ToUpper(env.Name) + } else { + path = strings.ToUpper(env.Name) + a + } + return path, err +} + +func parseBuildConfigEnv(env []builder.BuildConfigEnv, path string) (envMap map[string]string, warnings []string, err error) { + envMap = map[string]string{} + for _, v := range env { + if name := v.Name; name == "" || len(name) == 0 { + return nil, nil, errors.Wrapf(errors.Errorf("env name should not be empty"), "parse contents of '%s'", path) + } + if val := v.Value; val == "" || len(val) == 0 { + warnings = append(warnings, fmt.Sprintf("empty value for key/name %s", style.Symbol(v.Name))) + } + val, err := GetBuildConfigEnvFileName(v) + if err != nil { + return envMap, warnings, err + } + if _, e := envMap[val]; e { + return nil, nil, errors.Wrapf(errors.Errorf("env with name: %s and action: %s is already defined", style.Symbol(v.Name), style.Symbol(string(v.Action))), "parse contents of '%s'", path) + } + envMap[val] = v.Value } + return envMap, warnings, err } diff --git a/internal/commands/builder_create_test.go b/internal/commands/builder_create_test.go index a70b5cd59b..fcf3cc59cd 100644 --- a/internal/commands/builder_create_test.go +++ b/internal/commands/builder_create_test.go @@ -47,6 +47,80 @@ const validConfigWithExtensions = ` ` +const ActionNONE = validConfig + ` +[[build.env]] +name = "actionNone" +value = "actionNoneValue" +action = "" +` +const ActionDEFAULT = validConfig + ` +[[build.env]] +name = "actionDefault" +value = "actionDefaultValue" +action = "default" +` +const ActionOVERRIDE = validConfig + ` +[[build.env]] +name = "actionOverride" +value = "actionOverrideValue" +action = "override" +` +const ActionAPPEND = validConfig + ` +[[build.env]] +name = "actionAppend" +value = "actionAppendValue" +action = "append" +` +const ActionPREPEND = validConfig + ` +[[build.env]] +name = "actionPrepend" +value = "actionPrependValue" +action = "prepend" +` +const ActionDELIMIT = validConfig + ` +[[build.env]] +name = "actionDelimit" +value = ":" +action = "delim" +` +const ActionUNKNOWN = validConfig + ` +[[build.env]] +name = "actionUnknown" +value = "actionUnknownValue" +action = "unknown" +` +const ActionMULTIPLE = validConfig + ` +[[build.env]] +name = "MY_VAR" +value = "actionAppendValue" +action = "append" +[[build.env]] +name = "MY_VAR" +value = "actionDefaultValue" +action = "default" +[[build.env]] +name = "MY_VAR" +value = "actionPrependValue" +action = "prepend" +[[build.env]] +name = "MY_VAR" +value = ":" +action = "delim" +` + +const ActionWarning = validConfig + ` +[[build.env]] +name = "actionWarning" +value = "" +` + +const ActionError = validConfig + ` +[[build.env]] +name = "" +value = "some-value" +action = "default" +` + func TestCreateCommand(t *testing.T) { color.Disable(true) defer color.Disable(false) @@ -157,100 +231,195 @@ func testCreateCommand(t *testing.T, when spec.G, it spec.S) { }) }) - when("uses --config", func() { + when("uses --builder-config", func() { it.Before(func() { h.AssertNil(t, os.WriteFile(builderConfigPath, []byte(validConfig), 0666)) }) - when("buildConfigEnv files generated", func() { - const ActionNONE = validConfig + ` - [[build.env]] - name = "actionNone" - value = "actionNoneValue" - action = "" - ` - const ActionDEFAULT = validConfig + ` - [[build.env]] - name = "actionDefault" - value = "actionDefaultValue" - action = "default" - ` - const ActionOVERRIDE = validConfig + ` - [[build.env]] - name = "actionOverride" - value = "actionOverrideValue" - action = "override" - ` - const ActionAPPEND = validConfig + ` - [[build.env]] - name = "actionAppend" - value = "actionAppendValue" - action = "append" - ` - const ActionPREPEND = validConfig + ` - [[build.env]] - name = "actionPrepend" - value = "actionPrependValue" - action = "prepend" - ` - const ActionDELIMIT = validConfig + ` - [[build.env]] - name = "actionDelimit" - value = "actionDelimitValue" - action = "delim" - ` - const ActionUNKNOWN = validConfig + ` - [[build.env]] - name = "actionUnknown" - value = "actionUnknownValue" - action = "unknown" - ` - const ActionMULTIPLE = validConfig + ` - [[build.env]] - name = "actionAppend" - value = "actionAppendValue" - action = "append" - [[build.env]] - name = "actionPrepend" - value = "actionPrependValue" - action = "prepend" - [[build.env]] - name = "actionDelimit" - value = "actionDelimitValue" - action = "delim" - ` - it("should create content as expected when ActionType `NONE`", func() { - + it("errors with a descriptive message", func() { + command.SetArgs([]string{ + "some/builder", + "--builder-config", builderConfigPath, }) - it("should create content as expected when ActionType `DEFAULT`", func() { + h.AssertError(t, command.Execute(), "unknown flag: --builder-config") + }) + }) + when("buildConfigEnv files generated", func() { + var fileIndex = 0 + buildConfigEnvDir := commands.CnbBuildConfigDir() + it.Before(func() { + h.AssertNil(t, os.WriteFile(builderConfigPath, []byte(getBuildConfigEnvFileContent(fileIndex)), 0666)) + }) + it.After(func() { + err := os.RemoveAll(buildConfigEnvDir) + h.AssertNil(t, err) + fileIndex++ + }) + it("should create content as expected when ActionType `NONE`", func() { + command.SetArgs([]string{ + "some/builder", + "--config", builderConfigPath, }) - it("should create content as expected when ActionType `OVERRIDE`", func() { - + h.AssertNil(t, command.Execute()) + name := actionTypesMap[fileIndex][0][0] + file, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) + h.AssertNil(t, err) + h.AssertEq(t, name, file.Name()) + content, err := os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) + h.AssertNil(t, err) + h.AssertEq(t, actionTypesMap[fileIndex][0][1], string(content)) + }) + it("should create content as expected when ActionType `DEFAULT`", func() { + command.SetArgs([]string{ + "some/builder", + "--config", builderConfigPath, }) - it("should create content as expected when ActionType `APPEND`", func() { - + h.AssertNil(t, command.Execute()) + name := actionTypesMap[fileIndex][0][0] + file, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) + h.AssertNil(t, err) + h.AssertEq(t, name, file.Name()) + content, err := os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) + h.AssertNil(t, err) + h.AssertEq(t, actionTypesMap[fileIndex][0][1], string(content)) + }) + it("should create content as expected when ActionType `OVERRIDE`", func() { + command.SetArgs([]string{ + "some/builder", + "--config", builderConfigPath, }) - it("should create content as expected when ActionType `PREPEND`", func() { - + h.AssertNil(t, command.Execute()) + name := actionTypesMap[fileIndex][0][0] + file, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) + h.AssertNil(t, err) + h.AssertEq(t, name, file.Name()) + content, err := os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) + h.AssertNil(t, err) + h.AssertEq(t, actionTypesMap[fileIndex][0][1], string(content)) + }) + it("should create content as expected when ActionType `APPEND`", func() { + command.SetArgs([]string{ + "some/builder", + "--config", builderConfigPath, }) - it("should create content as expected when ActionType `DELIMIT`", func() { - + h.AssertNil(t, command.Execute()) + name := actionTypesMap[fileIndex][0][0] + file, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) + h.AssertNil(t, err) + h.AssertEq(t, name, file.Name()) + content, err := os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) + h.AssertNil(t, err) + h.AssertEq(t, actionTypesMap[fileIndex][0][1], string(content)) + }) + it("should create content as expected when ActionType `PREPEND`", func() { + command.SetArgs([]string{ + "some/builder", + "--config", builderConfigPath, }) - it("should create content as expected when unknown ActionType passed", func() { - + h.AssertNil(t, command.Execute()) + name := actionTypesMap[fileIndex][0][0] + file, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) + h.AssertNil(t, err) + h.AssertEq(t, name, file.Name()) + content, err := os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) + h.AssertNil(t, err) + h.AssertEq(t, actionTypesMap[fileIndex][0][1], string(content)) + }) + it("should create content as expected when ActionType `DELIMIT`", func() { + command.SetArgs([]string{ + "some/builder", + "--config", builderConfigPath, }) - it("should create content as expected when multiple ActionTypes passed", func() { - + h.AssertNil(t, command.Execute()) + name := actionTypesMap[fileIndex][0][0] + file, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) + h.AssertNil(t, err) + h.AssertEq(t, name, file.Name()) + content, err := os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) + h.AssertNil(t, err) + h.AssertEq(t, actionTypesMap[fileIndex][0][1], string(content)) + }) + it("should return an error when unknown ActionType passed", func() { + command.SetArgs([]string{ + "some/builder", + "--config", builderConfigPath, + }) + var bufErr bytes.Buffer + command.SetErr(&bufErr) + h.AssertNil(t, command.Execute()) + h.AssertNotEq(t, bufErr.String(), "") + name := actionTypesMap[fileIndex][0][0] + _, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) + h.AssertNotNil(t, err) + }) + it("should create content as expected when multiple ActionTypes passed", func() { + command.SetArgs([]string{ + "some/builder", + "--config", builderConfigPath, }) + h.AssertNil(t, command.Execute()) + name := actionTypesMap[fileIndex][0][0] + file, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) + h.AssertNil(t, err) + h.AssertEq(t, name, file.Name()) + content, err := os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) + h.AssertNil(t, err) + h.AssertEq(t, actionTypesMap[fileIndex][0][1], string(content)) + + name = actionTypesMap[fileIndex][1][0] + file, err = os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) + h.AssertNil(t, err) + h.AssertEq(t, name, file.Name()) + content, err = os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) + h.AssertNil(t, err) + h.AssertEq(t, actionTypesMap[fileIndex][1][1], string(content)) + + name = actionTypesMap[fileIndex][2][0] + file, err = os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) + h.AssertNil(t, err) + h.AssertEq(t, name, file.Name()) + content, err = os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) + h.AssertNil(t, err) + h.AssertEq(t, actionTypesMap[fileIndex][2][1], string(content)) + + name = actionTypesMap[fileIndex][3][0] + file, err = os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) + h.AssertNil(t, err) + h.AssertEq(t, name, file.Name()) + content, err = os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) + h.AssertNil(t, err) + h.AssertEq(t, actionTypesMap[fileIndex][3][1], string(content)) }) - - it("errors with a descriptive message", func() { + it("should show warnings when env value is empty", func() { command.SetArgs([]string{ "some/builder", "--config", builderConfigPath, }) - h.AssertError(t, command.Execute(), "unknown flag: --builder-config") + var bufOut bytes.Buffer + command.SetOut(&bufOut) + h.AssertNil(t, command.Execute()) + h.AssertNotEq(t, bufOut.String(), "") + name := actionTypesMap[fileIndex][0][0] + file, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) + h.AssertNil(t, err) + h.AssertEq(t, name, file.Name()) + content, err := os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) + h.AssertNil(t, err) + h.AssertEq(t, actionTypesMap[fileIndex][0][1], string(content)) + }) + it("should return an error when env.Name is empty", func() { + command.SetArgs([]string{ + "some/builder", + "--config", builderConfigPath, + }) + var bufErr bytes.Buffer + command.SetErr(&bufErr) + h.AssertNil(t, command.Execute()) + h.AssertNotEq(t, bufErr.String(), "") + name := actionTypesMap[fileIndex][0][0] + _, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) + h.AssertNotNil(t, err) }) }) @@ -296,3 +465,48 @@ func testCreateCommand(t *testing.T, when spec.G, it spec.S) { }) }) } + +func getBuildConfigEnvFileContent(index int) string { + switch index { + case 0: + return ActionNONE + case 1: + return ActionDEFAULT + case 2: + return ActionOVERRIDE + case 3: + return ActionAPPEND + case 4: + return ActionPREPEND + case 5: + return ActionDELIMIT + case 6: + return ActionUNKNOWN + case 7: + return ActionMULTIPLE + case 8: + return ActionWarning + case 9: + return ActionError + default: + return "" + } +} + +var actionTypesMap = map[int]map[int]map[int]string{ + 0: {0: {0: "ACTIONNONE", 1: "actionNoneValue"}}, + 1: {0: {0: "ACTIONDEFAULT.default", 1: "actionDefaultValue"}}, + 2: {0: {0: "ACTIONOVERRIDE.override", 1: "actionOverrideValue"}}, + 3: {0: {0: "ACTIONAPPEND.append", 1: "actionAppendValue"}}, + 4: {0: {0: "ACTIONPREPEND.prepend", 1: "actionPrependValue"}}, + 5: {0: {0: "ACTIONDELIMIT.delim", 1: ":"}}, + 6: {0: {0: "ACTIONUNKNOWN.unknown", 1: "actionUnknownValue"}}, + 7: { + 0: {0: "MY_VAR.append", 1: "actionAppendValue"}, + 1: {0: "MY_VAR.default", 1: "actionDefaultValue"}, + 2: {0: "MY_VAR.prepend", 1: "actionPrependValue"}, + 3: {0: "MY_VAR.delim", 1: ":"}, + }, + 8: {0: {0: "actionWarning", 1: ""}}, + 9: {0: {0: ".default", 1: "some-value"}}, +} From 7183dcfde13c7bf990c9e032f3e6bd56c22566e2 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 5 Oct 2023 15:47:15 +0000 Subject: [PATCH 31/62] WIP change from action to suffix and delim in builder.toml Signed-off-by: WYGIN --- builder/config_reader.go | 22 ++++++------ internal/builder/builder.go | 5 +-- internal/commands/builder_create.go | 43 +++++++++++++--------- internal/commands/builder_create_test.go | 45 ++++++++++++------------ 4 files changed, 62 insertions(+), 53 deletions(-) diff --git a/builder/config_reader.go b/builder/config_reader.go index 7c17ac331a..9cd6117a2c 100644 --- a/builder/config_reader.go +++ b/builder/config_reader.go @@ -74,23 +74,23 @@ type BuildConfig struct { Env []BuildConfigEnv `toml:"env"` } -type ActionType string +type Suffix string -var ActionTypes []ActionType = []ActionType{NONE, DEFAULT, OVERRIDE, APPEND, PREPEND, DELIMIT} +var SuffixSlice []Suffix = []Suffix{NONE, DEFAULT, OVERRIDE, APPEND, PREPEND} const ( - NONE ActionType = "" - DEFAULT ActionType = "default" - OVERRIDE ActionType = "override" - APPEND ActionType = "append" - PREPEND ActionType = "prepend" - DELIMIT ActionType = "delim" + NONE Suffix = "" + DEFAULT Suffix = "default" + OVERRIDE Suffix = "override" + APPEND Suffix = "append" + PREPEND Suffix = "prepend" ) type BuildConfigEnv struct { - Name string `toml:"name"` - Value string `toml:"value"` - Action ActionType `toml:"action,omitempty"` + Name string `toml:"name"` + Value string `toml:"value"` + Suffix Suffix `toml:"suffix,omitempty"` + Delim string `toml:"delim,omitempty"` } // ReadConfig reads a builder configuration from the file path provided and returns the diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 51b5a4a00e..8228ab779b 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -35,8 +35,9 @@ import ( const ( packName = "Pack CLI" - cnbDir = "/cnb" - buildpacksDir = "/cnb/buildpacks" + cnbDir = "/cnb" + buildConfigDir = "/cnb/build-config" + buildpacksDir = "/cnb/buildpacks" orderPath = "/cnb/order.toml" stackPath = "/cnb/stack.toml" diff --git a/internal/commands/builder_create.go b/internal/commands/builder_create.go index 7cdf1faa75..b606b08738 100644 --- a/internal/commands/builder_create.go +++ b/internal/commands/builder_create.go @@ -186,12 +186,12 @@ func createBuildConfigEnvDir() (dir string, err error) { } return dir, nil } - return dir, err + return dir, nil } -func getActionType(action builder.ActionType) (actionString string, err error) { +func getActionType(suffix builder.Suffix) (suffixString string, err error) { const delim = "." - switch action { + switch suffix { case builder.NONE: return "", nil case builder.DEFAULT: @@ -202,21 +202,24 @@ func getActionType(action builder.ActionType) (actionString string, err error) { return delim + string(builder.APPEND), nil case builder.PREPEND: return delim + string(builder.PREPEND), nil - case builder.DELIMIT: - return delim + string(builder.DELIMIT), nil default: - return actionString, errors.Errorf("unknown action type %s", style.Symbol(string(action))) + return suffixString, errors.Errorf("unknown action type %s", style.Symbol(string(suffix))) } } -func GetBuildConfigEnvFileName(env builder.BuildConfigEnv) (path string, err error) { - if a, err := getActionType(env.Action); err != nil { - return path, err - } else if a == "" || len(a) == 0 { - path = strings.ToUpper(env.Name) +func GetBuildConfigEnvFileName(env builder.BuildConfigEnv) (suffixName, delimName string, err error) { + suffix, err := getActionType(env.Suffix) + if err != nil { + return suffixName, delimName, err + } + if suffix == "" || len(suffix) == 0 { + suffixName = env.Name } else { - path = strings.ToUpper(env.Name) + a + suffixName = env.Name + suffix + } + if delim := env.Delim; delim != "" || len(delim) != 0 { + delimName = env.Name + ".delim" } - return path, err + return suffixName, delimName, err } func parseBuildConfigEnv(env []builder.BuildConfigEnv, path string) (envMap map[string]string, warnings []string, err error) { @@ -228,14 +231,20 @@ func parseBuildConfigEnv(env []builder.BuildConfigEnv, path string) (envMap map[ if val := v.Value; val == "" || len(val) == 0 { warnings = append(warnings, fmt.Sprintf("empty value for key/name %s", style.Symbol(v.Name))) } - val, err := GetBuildConfigEnvFileName(v) + suffixName, delimName, err := GetBuildConfigEnvFileName(v) if err != nil { return envMap, warnings, err } - if _, e := envMap[val]; e { - return nil, nil, errors.Wrapf(errors.Errorf("env with name: %s and action: %s is already defined", style.Symbol(v.Name), style.Symbol(string(v.Action))), "parse contents of '%s'", path) + if val, e := envMap[suffixName]; e { + warnings = append(warnings, fmt.Sprintf(errors.Errorf("overriding env with name: %s and suffix: %s from %s to %s", style.Symbol(v.Name), style.Symbol(string(v.Suffix)), style.Symbol(val), style.Symbol(v.Value)).Error(), "parse contents of '%s'", path)) + } + if val, e := envMap[delimName]; e { + warnings = append(warnings, fmt.Sprintf(errors.Errorf("overriding env with name: %s and delim: %s from %s to %s", style.Symbol(v.Name), style.Symbol(v.Delim), style.Symbol(val), style.Symbol(v.Value)).Error(), "parse contents of '%s'", path)) + } + if delim := v.Delim; (delim != "" || len(delim) != 0) && (delimName != "" || len(delimName) != 0) { + envMap[delimName] = delim } - envMap[val] = v.Value + envMap[suffixName] = v.Value } return envMap, warnings, err } diff --git a/internal/commands/builder_create_test.go b/internal/commands/builder_create_test.go index fcf3cc59cd..b36b67611c 100644 --- a/internal/commands/builder_create_test.go +++ b/internal/commands/builder_create_test.go @@ -51,61 +51,58 @@ const ActionNONE = validConfig + ` [[build.env]] name = "actionNone" value = "actionNoneValue" -action = "" ` const ActionDEFAULT = validConfig + ` [[build.env]] name = "actionDefault" value = "actionDefaultValue" -action = "default" +suffix = "default" ` const ActionOVERRIDE = validConfig + ` [[build.env]] name = "actionOverride" value = "actionOverrideValue" -action = "override" +suffix = "override" ` const ActionAPPEND = validConfig + ` [[build.env]] name = "actionAppend" value = "actionAppendValue" -action = "append" +suffix = "append" ` const ActionPREPEND = validConfig + ` [[build.env]] name = "actionPrepend" value = "actionPrependValue" -action = "prepend" +suffix = "prepend" ` const ActionDELIMIT = validConfig + ` [[build.env]] name = "actionDelimit" -value = ":" -action = "delim" +delim = ":" ` const ActionUNKNOWN = validConfig + ` [[build.env]] name = "actionUnknown" value = "actionUnknownValue" -action = "unknown" +suffix = "unknown" ` const ActionMULTIPLE = validConfig + ` [[build.env]] name = "MY_VAR" value = "actionAppendValue" -action = "append" +suffix = "append" +delim = ":" [[build.env]] name = "MY_VAR" value = "actionDefaultValue" -action = "default" +suffix = "default" +delim = ":" [[build.env]] name = "MY_VAR" value = "actionPrependValue" -action = "prepend" -[[build.env]] -name = "MY_VAR" -value = ":" -action = "delim" +suffix = "prepend" +delim = ":" ` const ActionWarning = validConfig + ` @@ -118,7 +115,7 @@ const ActionError = validConfig + ` [[build.env]] name = "" value = "some-value" -action = "default" +suffix = "default" ` func TestCreateCommand(t *testing.T) { @@ -249,6 +246,8 @@ func testCreateCommand(t *testing.T, when spec.G, it spec.S) { var fileIndex = 0 buildConfigEnvDir := commands.CnbBuildConfigDir() it.Before(func() { + err := os.MkdirAll(buildConfigEnvDir, os.ModePerm) + h.AssertNil(t, err) h.AssertNil(t, os.WriteFile(builderConfigPath, []byte(getBuildConfigEnvFileContent(fileIndex)), 0666)) }) it.After(func() { @@ -494,13 +493,13 @@ func getBuildConfigEnvFileContent(index int) string { } var actionTypesMap = map[int]map[int]map[int]string{ - 0: {0: {0: "ACTIONNONE", 1: "actionNoneValue"}}, - 1: {0: {0: "ACTIONDEFAULT.default", 1: "actionDefaultValue"}}, - 2: {0: {0: "ACTIONOVERRIDE.override", 1: "actionOverrideValue"}}, - 3: {0: {0: "ACTIONAPPEND.append", 1: "actionAppendValue"}}, - 4: {0: {0: "ACTIONPREPEND.prepend", 1: "actionPrependValue"}}, - 5: {0: {0: "ACTIONDELIMIT.delim", 1: ":"}}, - 6: {0: {0: "ACTIONUNKNOWN.unknown", 1: "actionUnknownValue"}}, + 0: {0: {0: "actionNone", 1: "actionNoneValue"}}, + 1: {0: {0: "actionDefault.default", 1: "actionDefaultValue"}}, + 2: {0: {0: "actionOverride.override", 1: "actionOverrideValue"}}, + 3: {0: {0: "actionAppend.append", 1: "actionAppendValue"}}, + 4: {0: {0: "actionPrepend.prepend", 1: "actionPrependValue"}}, + 5: {0: {0: "actionDelim.delim", 1: ":"}}, + 6: {0: {0: "actionUnknown.unknown", 1: "actionUnknownValue"}}, 7: { 0: {0: "MY_VAR.append", 1: "actionAppendValue"}, 1: {0: "MY_VAR.default", 1: "actionDefaultValue"}, From 148003724e0b765f744e8efedd4afeb3e04c56b4 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Sun, 8 Oct 2023 08:31:18 +0000 Subject: [PATCH 32/62] WIP added logic to add buildConfigEnvs to layer of image Signed-off-by: WYGIN --- internal/builder/builder.go | 64 +++- internal/builder/builder_test.go | 43 +++ internal/commands/builder_create.go | 81 ++--- internal/commands/builder_create_test.go | 438 +++++++++-------------- pkg/client/create_builder.go | 13 + 5 files changed, 328 insertions(+), 311 deletions(-) diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 8228ab779b..47d392c82d 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -32,12 +32,13 @@ import ( lifecycleplatform "github.com/buildpacks/lifecycle/platform" ) +var buildConfigDir = cnbBuildConfigDir() + const ( packName = "Pack CLI" - cnbDir = "/cnb" - buildConfigDir = "/cnb/build-config" - buildpacksDir = "/cnb/buildpacks" + cnbDir = "/cnb" + buildpacksDir = "/cnb/buildpacks" orderPath = "/cnb/order.toml" stackPath = "/cnb/stack.toml" @@ -68,6 +69,7 @@ const ( // Builder represents a pack builder, used to build images type Builder struct { baseImageName string + buildConfigEnv map[string]string image imgutil.Image layerWriterFactory archive.TarWriterFactory lifecycle Lifecycle @@ -147,6 +149,7 @@ func constructBuilder(img imgutil.Image, newName string, errOnMissingLabel bool, metadata: metadata, lifecycleDescriptor: constructLifecycleDescriptor(metadata), env: map[string]string{}, + buildConfigEnv: map[string]string{}, validateMixins: true, additionalBuildpacks: *buildpack.NewModuleManager(opts.flatten, opts.depth), additionalExtensions: *buildpack.NewModuleManager(opts.flatten, opts.depth), @@ -350,6 +353,11 @@ func (b *Builder) SetEnv(env map[string]string) { b.env = env } +// SetBuildConfigEnv sets an environment variable to a value that will take action on platform environment variables basedon filename suffix +func (b *Builder) SetBuildConfigEnv(env map[string]string) { + b.buildConfigEnv = env +} + // SetOrder sets the order of the builder func (b *Builder) SetOrder(order dist.Order) { b.order = order @@ -526,6 +534,19 @@ func (b *Builder) Save(logger logging.Logger, creatorMetadata CreatorMetadata) e return errors.Wrap(err, "adding run.tar layer") } + if len(b.buildConfigEnv) > 0 { + logger.Debugf("Provided Build Config Environment Variables\n %s", style.Map(b.env, " ", "\n")) + } + + buildConfigEnvTar, err := b.buildConfigEnvLayer(tmpDir, b.buildConfigEnv) + if err != nil { + return errors.Wrap(err, "retrieving build-config-env layer") + } + + if err := b.image.AddLayer(buildConfigEnvTar); err != nil { + return errors.Wrap(err, "adding build-config-env layer") + } + if len(b.env) > 0 { logger.Debugf("Provided Environment Variables\n %s", style.Map(b.env, " ", "\n")) } @@ -904,7 +925,7 @@ func (b *Builder) defaultDirsLayer(dest string) (string, error) { } // can't use filepath.Join(), to ensure Windows doesn't transform it to Windows join - for _, path := range []string{cnbDir, dist.BuildpacksDir, dist.ExtensionsDir, platformDir, platformDir + "/env"} { + for _, path := range []string{cnbDir, dist.BuildpacksDir, dist.ExtensionsDir, platformDir, platformDir + "/env", buildConfigDir, buildConfigDir + "/env"} { if err := lw.WriteHeader(b.rootOwnedDir(path, ts)); err != nil { return "", errors.Wrapf(err, "creating %s dir in layer", style.Symbol(path)) } @@ -1103,6 +1124,33 @@ func (b *Builder) envLayer(dest string, env map[string]string) (string, error) { return fh.Name(), nil } +func (b *Builder) buildConfigEnvLayer(dest string, env map[string]string) (string, error) { + fh, err := os.Create(filepath.Join(dest, "build-config-env.tar")) + if err != nil { + return "", err + } + defer fh.Close() + + lw := b.layerWriterFactory.NewWriter(fh) + defer lw.Close() + + for k, v := range env { + if err := lw.WriteHeader(&tar.Header{ + Name: path.Join(buildConfigDir, "env", k), + Size: int64(len(v)), + Mode: 0644, + ModTime: archive.NormalizedDateTime, + }); err != nil { + return "", err + } + if _, err := lw.Write([]byte(v)); err != nil { + return "", err + } + } + + return fh.Name(), nil +} + func (b *Builder) whiteoutLayer(tmpDir string, i int, bpInfo dist.ModuleInfo) (string, error) { bpWhiteoutsTmpDir := filepath.Join(tmpDir, strconv.Itoa(i)+"_whiteouts") if err := os.MkdirAll(bpWhiteoutsTmpDir, os.ModePerm); err != nil { @@ -1258,3 +1306,11 @@ func (e errModuleTar) Info() dist.ModuleInfo { func (e errModuleTar) Path() string { return "" } + +func cnbBuildConfigDir() string { + if env, ok := os.LookupEnv("CNB_BUILD_CONFIG_DIR"); !ok { + return "/cnb/build-config" + } else { + return env + } +} diff --git a/internal/builder/builder_test.go b/internal/builder/builder_test.go index 13a958c72e..2745e9e97b 100644 --- a/internal/builder/builder_test.go +++ b/internal/builder/builder_test.go @@ -384,6 +384,20 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) { ) }) + it("creates the build-config dir", func() { + h.AssertNil(t, subject.Save(logger, builder.CreatorMetadata{})) + h.AssertEq(t, baseImage.IsSaved(), true) + + layerTar, err := baseImage.FindLayerWithPath("/cnb/build-config") + h.AssertNil(t, err) + h.AssertOnTarEntry(t, layerTar, "/cnb/build-config", + h.IsDirectory(), + h.HasOwnerAndGroup(0, 0), + h.HasFileMode(0755), + h.HasModTime(archive.NormalizedDateTime), + ) + }) + it("creates the buildpacks dir", func() { h.AssertNil(t, subject.Save(logger, builder.CreatorMetadata{})) h.AssertEq(t, baseImage.IsSaved(), true) @@ -1607,6 +1621,35 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) { }) }) + when("#SetBuildConfigEnv", func() { + it.Before(func() { + subject.SetBuildConfigEnv(map[string]string{ + "SOME_KEY": "some-val", + "OTHER_KEY.append": "other-val", + "OTHER_KEY.delim": ":", + }) + h.AssertNil(t, subject.Save(logger, builder.CreatorMetadata{})) + h.AssertEq(t, baseImage.IsSaved(), true) + }) + + it("adds the env vars as files to the image", func() { + layerTar, err := baseImage.FindLayerWithPath("/cnb/build-config/env/SOME_KEY") + h.AssertNil(t, err) + h.AssertOnTarEntry(t, layerTar, "/cnb/build-config/env/SOME_KEY", + h.ContentEquals(`some-val`), + h.HasModTime(archive.NormalizedDateTime), + ) + h.AssertOnTarEntry(t, layerTar, "/cnb/build-config/env/OTHER_KEY.append", + h.ContentEquals(`other-val`), + h.HasModTime(archive.NormalizedDateTime), + ) + h.AssertOnTarEntry(t, layerTar, "/cnb/build-config/env/OTHER_KEY.delim", + h.ContentEquals(`:`), + h.HasModTime(archive.NormalizedDateTime), + ) + }) + }) + when("#SetEnv", func() { it.Before(func() { subject.SetEnv(map[string]string{ diff --git a/internal/commands/builder_create.go b/internal/commands/builder_create.go index b606b08738..a98cb43521 100644 --- a/internal/commands/builder_create.go +++ b/internal/commands/builder_create.go @@ -2,7 +2,6 @@ package commands import ( "fmt" - "os" "path/filepath" "strings" @@ -76,20 +75,18 @@ Creating a custom builder allows you to control what buildpacks are used and wha return errors.Wrap(err, "getting absolute path for config") } - envMap, warnings, err := parseBuildConfigEnv(builderConfig.Build.Env, flags.BuilderTomlPath) + envMap, warnings, err := ParseBuildConfigEnv(builderConfig.Build.Env, flags.BuilderTomlPath) for _, v := range warnings { logger.Warn(v) } if err != nil { return err } - if err := generateBuildConfigEnvFiles(envMap); err != nil { - return err - } imageName := args[0] if err := pack.CreateBuilder(cmd.Context(), client.CreateBuilderOptions{ RelativeBaseDir: relativeBaseDir, + BuildConfigEnv: envMap, BuilderName: imageName, Config: builderConfig, Publish: flags.Publish, @@ -150,45 +147,6 @@ func validateCreateFlags(flags *BuilderCreateFlags, cfg config.Config) error { return nil } -func generateBuildConfigEnvFiles(envMap map[string]string) error { - dir, err := createBuildConfigEnvDir() - if err != nil { - return err - } - for k, v := range envMap { - f, err := os.Create(filepath.Join(dir, k)) - if err != nil { - return err - } - f.WriteString(v) - if e := f.Close(); e != nil { - return e - } - } - return nil -} - -func CnbBuildConfigDir() string { - if v := os.Getenv("CNB_BUILD_CONFIG_DIR"); v == "" || len(v) == 0 { - return "/cnb/build-config" - } else { - return v - } -} - -func createBuildConfigEnvDir() (dir string, err error) { - dir = filepath.Join(CnbBuildConfigDir(), "env") - _, err = os.Stat(dir) - if os.IsNotExist(err) { - err := os.MkdirAll(dir, os.ModePerm) - if err != nil { - return dir, err - } - return dir, nil - } - return dir, nil -} - func getActionType(suffix builder.Suffix) (suffixString string, err error) { const delim = "." switch suffix { @@ -206,7 +164,21 @@ func getActionType(suffix builder.Suffix) (suffixString string, err error) { return suffixString, errors.Errorf("unknown action type %s", style.Symbol(string(suffix))) } } -func GetBuildConfigEnvFileName(env builder.BuildConfigEnv) (suffixName, delimName string, err error) { + +func getFilePrefixSuffix(filename string) (prefix, suffix string, err error) { + val := strings.Split(filename, ".") + if len(val) <= 1 { + return val[0], suffix, errors.Errorf("Suffix might be null") + } + if len(val) == 2 { + suffix = val[1] + } else { + strings.Join(val[1:], ".") + } + return val[0], suffix, err +} + +func getBuildConfigEnvFileName(env builder.BuildConfigEnv) (suffixName, delimName string, err error) { suffix, err := getActionType(env.Suffix) if err != nil { return suffixName, delimName, err @@ -222,8 +194,9 @@ func GetBuildConfigEnvFileName(env builder.BuildConfigEnv) (suffixName, delimNam return suffixName, delimName, err } -func parseBuildConfigEnv(env []builder.BuildConfigEnv, path string) (envMap map[string]string, warnings []string, err error) { +func ParseBuildConfigEnv(env []builder.BuildConfigEnv, path string) (envMap map[string]string, warnings []string, err error) { envMap = map[string]string{} + var appendOrPrependWithoutDelim = 0 for _, v := range env { if name := v.Name; name == "" || len(name) == 0 { return nil, nil, errors.Wrapf(errors.Errorf("env name should not be empty"), "parse contents of '%s'", path) @@ -231,7 +204,7 @@ func parseBuildConfigEnv(env []builder.BuildConfigEnv, path string) (envMap map[ if val := v.Value; val == "" || len(val) == 0 { warnings = append(warnings, fmt.Sprintf("empty value for key/name %s", style.Symbol(v.Name))) } - suffixName, delimName, err := GetBuildConfigEnvFileName(v) + suffixName, delimName, err := getBuildConfigEnvFileName(v) if err != nil { return envMap, warnings, err } @@ -246,5 +219,19 @@ func parseBuildConfigEnv(env []builder.BuildConfigEnv, path string) (envMap map[ } envMap[suffixName] = v.Value } + + for k := range envMap { + name, suffix, err := getFilePrefixSuffix(k) + if err != nil { + continue + } + if _, ok := envMap[name+".delim"]; (suffix == "append" || suffix == "prepend") && !ok { + warnings = append(warnings, fmt.Sprintf(errors.Errorf("env with name/key %s with suffix %s must to have a %s value", style.Symbol(name), style.Symbol(suffix), style.Symbol("delim")).Error(), "parse contents of '%s'", path)) + appendOrPrependWithoutDelim++ + } + } + if appendOrPrependWithoutDelim > 0 { + return envMap, warnings, errors.Errorf("error parsing [[build.env]] in file '%s'", path) + } return envMap, warnings, err } diff --git a/internal/commands/builder_create_test.go b/internal/commands/builder_create_test.go index b36b67611c..f5f9684390 100644 --- a/internal/commands/builder_create_test.go +++ b/internal/commands/builder_create_test.go @@ -13,6 +13,7 @@ import ( "github.com/sclevine/spec/report" "github.com/spf13/cobra" + "github.com/buildpacks/pack/builder" "github.com/buildpacks/pack/internal/commands" "github.com/buildpacks/pack/internal/commands/testmocks" "github.com/buildpacks/pack/internal/config" @@ -47,76 +48,109 @@ const validConfigWithExtensions = ` ` -const ActionNONE = validConfig + ` -[[build.env]] -name = "actionNone" -value = "actionNoneValue" -` -const ActionDEFAULT = validConfig + ` -[[build.env]] -name = "actionDefault" -value = "actionDefaultValue" -suffix = "default" -` -const ActionOVERRIDE = validConfig + ` -[[build.env]] -name = "actionOverride" -value = "actionOverrideValue" -suffix = "override" -` -const ActionAPPEND = validConfig + ` -[[build.env]] -name = "actionAppend" -value = "actionAppendValue" -suffix = "append" -` -const ActionPREPEND = validConfig + ` -[[build.env]] -name = "actionPrepend" -value = "actionPrependValue" -suffix = "prepend" -` -const ActionDELIMIT = validConfig + ` -[[build.env]] -name = "actionDelimit" -delim = ":" -` -const ActionUNKNOWN = validConfig + ` -[[build.env]] -name = "actionUnknown" -value = "actionUnknownValue" -suffix = "unknown" -` -const ActionMULTIPLE = validConfig + ` -[[build.env]] -name = "MY_VAR" -value = "actionAppendValue" -suffix = "append" -delim = ":" -[[build.env]] -name = "MY_VAR" -value = "actionDefaultValue" -suffix = "default" -delim = ":" -[[build.env]] -name = "MY_VAR" -value = "actionPrependValue" -suffix = "prepend" -delim = ":" -` +var BuildConfigEnvSuffixNone = builder.BuildConfigEnv{ + Name: "suffixNone", + Value: "suffixNoneValue", +} -const ActionWarning = validConfig + ` -[[build.env]] -name = "actionWarning" -value = "" -` +var BuildConfigEnvSuffixNoneWithEmptySuffix = builder.BuildConfigEnv{ + Name: "suffixNoneWithEmptySuffix", + Value: "suffixNoneWithEmptySuffixValue", + Suffix: "", +} -const ActionError = validConfig + ` -[[build.env]] -name = "" -value = "some-value" -suffix = "default" -` +var BuildConfigEnvSuffixDefault = builder.BuildConfigEnv{ + Name: "suffixDefault", + Value: "suffixDefaultValue", + Suffix: "default", +} + +var BuildConfigEnvSuffixOverride = builder.BuildConfigEnv{ + Name: "suffixOverride", + Value: "suffixOverrideValue", + Suffix: "override", +} + +var BuildConfigEnvSuffixAppend = builder.BuildConfigEnv{ + Name: "suffixAppend", + Value: "suffixAppendValue", + Suffix: "append", + Delim: ":", +} + +var BuildConfigEnvSuffixPrepend = builder.BuildConfigEnv{ + Name: "suffixPrepend", + Value: "suffixPrependValue", + Suffix: "prepend", + Delim: ":", +} + +var BuildConfigEnvDelimWithoutSuffix = builder.BuildConfigEnv{ + Name: "delimWithoutSuffix", + Delim: ":", +} + +var BuildConfigEnvSuffixUnknown = builder.BuildConfigEnv{ + Name: "suffixUnknown", + Value: "suffixUnknownValue", + Suffix: "unknown", +} + +var BuildConfigEnvSuffixMultiple = []builder.BuildConfigEnv{ + { + Name: "MY_VAR", + Value: "suffixAppendValueValue", + Suffix: "append", + Delim: ";", + }, + { + Name: "MY_VAR", + Value: "suffixDefaultValue", + Suffix: "default", + Delim: "%", + }, + { + Name: "MY_VAR", + Value: "suffixPrependValue", + Suffix: "prepend", + Delim: ":", + }, +} + +var BuildConfigEnvEmptyValue = builder.BuildConfigEnv{ + Name: "warning", + Value: "", +} + +var BuildConfigEnvEmptyName = builder.BuildConfigEnv{ + Name: "", + Value: "suffixUnknownValue", + Suffix: "default", +} + +var BuildConfigEnvSuffixPrependWithoutDelim = builder.BuildConfigEnv{ + Name: "suffixPrepend", + Value: "suffixPrependValue", + Suffix: "prepend", +} + +var BuildConfigEnvDelimWithoutSuffixAppendOrPrepend = builder.BuildConfigEnv{ + Name: "delimWithoutActionAppendOrPrepend", + Value: "some-value", + Delim: ":", +} + +var BuildConfigEnvDelimWithSameSuffixAndName = []builder.BuildConfigEnv{ + { + Name: "MY_VAR", + Value: "some-value", + Suffix: "", + }, + { + Name: "MY_VAR", + Value: "some-value", + }, +} func TestCreateCommand(t *testing.T) { color.Disable(true) @@ -242,183 +276,112 @@ func testCreateCommand(t *testing.T, when spec.G, it spec.S) { }) }) - when("buildConfigEnv files generated", func() { - var fileIndex = 0 - buildConfigEnvDir := commands.CnbBuildConfigDir() - it.Before(func() { - err := os.MkdirAll(buildConfigEnvDir, os.ModePerm) + when("#ParseBuildpackConfigEnv", func() { + it("should create envMap as expected when suffix is omitted", func() { + envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixNone}, "") + h.AssertEq(t, envMap, map[string]string{ + BuildConfigEnvSuffixNone.Name: BuildConfigEnvSuffixNone.Value, + }) + h.AssertEq(t, len(warnings), 0) h.AssertNil(t, err) - h.AssertNil(t, os.WriteFile(builderConfigPath, []byte(getBuildConfigEnvFileContent(fileIndex)), 0666)) }) - it.After(func() { - err := os.RemoveAll(buildConfigEnvDir) + it("should create envMap as expected when suffix is empty string", func() { + envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixNoneWithEmptySuffix}, "") + h.AssertEq(t, envMap, map[string]string{ + BuildConfigEnvSuffixNoneWithEmptySuffix.Name: BuildConfigEnvSuffixNoneWithEmptySuffix.Value, + }) + h.AssertEq(t, len(warnings), 0) h.AssertNil(t, err) - fileIndex++ }) - it("should create content as expected when ActionType `NONE`", func() { - command.SetArgs([]string{ - "some/builder", - "--config", builderConfigPath, + it("should create envMap as expected when suffix is `default`", func() { + envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixDefault}, "") + h.AssertEq(t, envMap, map[string]string{ + BuildConfigEnvSuffixDefault.Name + "." + string(BuildConfigEnvSuffixDefault.Suffix): BuildConfigEnvSuffixDefault.Value, }) - h.AssertNil(t, command.Execute()) - name := actionTypesMap[fileIndex][0][0] - file, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) - h.AssertNil(t, err) - h.AssertEq(t, name, file.Name()) - content, err := os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) + h.AssertEq(t, len(warnings), 0) h.AssertNil(t, err) - h.AssertEq(t, actionTypesMap[fileIndex][0][1], string(content)) }) - it("should create content as expected when ActionType `DEFAULT`", func() { - command.SetArgs([]string{ - "some/builder", - "--config", builderConfigPath, + it("should create envMap as expected when suffix is `override`", func() { + envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixOverride}, "") + h.AssertEq(t, envMap, map[string]string{ + BuildConfigEnvSuffixOverride.Name + "." + string(BuildConfigEnvSuffixOverride.Suffix): BuildConfigEnvSuffixOverride.Value, }) - h.AssertNil(t, command.Execute()) - name := actionTypesMap[fileIndex][0][0] - file, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) - h.AssertNil(t, err) - h.AssertEq(t, name, file.Name()) - content, err := os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) + h.AssertEq(t, len(warnings), 0) h.AssertNil(t, err) - h.AssertEq(t, actionTypesMap[fileIndex][0][1], string(content)) }) - it("should create content as expected when ActionType `OVERRIDE`", func() { - command.SetArgs([]string{ - "some/builder", - "--config", builderConfigPath, + it("should create envMap as expected when suffix is `append`", func() { + envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixAppend}, "") + h.AssertEq(t, envMap, map[string]string{ + BuildConfigEnvSuffixAppend.Name + "." + string(BuildConfigEnvSuffixAppend.Suffix): BuildConfigEnvSuffixAppend.Value, + BuildConfigEnvSuffixAppend.Name + ".delim": BuildConfigEnvSuffixAppend.Delim, }) - h.AssertNil(t, command.Execute()) - name := actionTypesMap[fileIndex][0][0] - file, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) - h.AssertNil(t, err) - h.AssertEq(t, name, file.Name()) - content, err := os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) + h.AssertEq(t, len(warnings), 0) h.AssertNil(t, err) - h.AssertEq(t, actionTypesMap[fileIndex][0][1], string(content)) }) - it("should create content as expected when ActionType `APPEND`", func() { - command.SetArgs([]string{ - "some/builder", - "--config", builderConfigPath, + it("should create envMap as expected when suffix is `prepend`", func() { + envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixPrepend}, "") + h.AssertEq(t, envMap, map[string]string{ + BuildConfigEnvSuffixPrepend.Name + "." + string(BuildConfigEnvSuffixPrepend.Suffix): BuildConfigEnvSuffixPrepend.Value, + BuildConfigEnvSuffixPrepend.Name + ".delim": BuildConfigEnvSuffixPrepend.Delim, }) - h.AssertNil(t, command.Execute()) - name := actionTypesMap[fileIndex][0][0] - file, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) - h.AssertNil(t, err) - h.AssertEq(t, name, file.Name()) - content, err := os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) + h.AssertEq(t, len(warnings), 0) h.AssertNil(t, err) - h.AssertEq(t, actionTypesMap[fileIndex][0][1], string(content)) }) - it("should create content as expected when ActionType `PREPEND`", func() { - command.SetArgs([]string{ - "some/builder", - "--config", builderConfigPath, + it("should create envMap as expected when delim is specified", func() { + envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvDelimWithoutSuffix}, "") + h.AssertEq(t, envMap, map[string]string{ + BuildConfigEnvDelimWithoutSuffix.Name: BuildConfigEnvDelimWithoutSuffix.Value, + BuildConfigEnvDelimWithoutSuffix.Name + ".delim": BuildConfigEnvDelimWithoutSuffix.Delim, }) - h.AssertNil(t, command.Execute()) - name := actionTypesMap[fileIndex][0][0] - file, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) + h.AssertNotEq(t, len(warnings), 0) h.AssertNil(t, err) - h.AssertEq(t, name, file.Name()) - content, err := os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) - h.AssertNil(t, err) - h.AssertEq(t, actionTypesMap[fileIndex][0][1], string(content)) }) - it("should create content as expected when ActionType `DELIMIT`", func() { - command.SetArgs([]string{ - "some/builder", - "--config", builderConfigPath, + it("should create envMap with a warning when `value` is empty", func() { + envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvEmptyValue}, "") + h.AssertEq(t, envMap, map[string]string{ + BuildConfigEnvEmptyValue.Name: BuildConfigEnvEmptyValue.Value, }) - h.AssertNil(t, command.Execute()) - name := actionTypesMap[fileIndex][0][0] - file, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) + h.AssertNotEq(t, len(warnings), 0) h.AssertNil(t, err) - h.AssertEq(t, name, file.Name()) - content, err := os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) - h.AssertNil(t, err) - h.AssertEq(t, actionTypesMap[fileIndex][0][1], string(content)) }) - it("should return an error when unknown ActionType passed", func() { - command.SetArgs([]string{ - "some/builder", - "--config", builderConfigPath, - }) - var bufErr bytes.Buffer - command.SetErr(&bufErr) - h.AssertNil(t, command.Execute()) - h.AssertNotEq(t, bufErr.String(), "") - name := actionTypesMap[fileIndex][0][0] - _, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) + it("should return an error when `name` is empty", func() { + envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvEmptyName}, "") + h.AssertEq(t, envMap, map[string]string(nil)) + h.AssertEq(t, len(warnings), 0) h.AssertNotNil(t, err) }) - it("should create content as expected when multiple ActionTypes passed", func() { - command.SetArgs([]string{ - "some/builder", - "--config", builderConfigPath, + it("should return warnings when `apprend` or `prepend` is used without `delim`", func() { + envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixPrependWithoutDelim}, "") + h.AssertEq(t, envMap, map[string]string{ + BuildConfigEnvSuffixPrependWithoutDelim.Name + "." + string(BuildConfigEnvSuffixPrependWithoutDelim.Suffix): BuildConfigEnvSuffixPrependWithoutDelim.Value, }) - h.AssertNil(t, command.Execute()) - name := actionTypesMap[fileIndex][0][0] - file, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) - h.AssertNil(t, err) - h.AssertEq(t, name, file.Name()) - content, err := os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) - h.AssertNil(t, err) - h.AssertEq(t, actionTypesMap[fileIndex][0][1], string(content)) - - name = actionTypesMap[fileIndex][1][0] - file, err = os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) - h.AssertNil(t, err) - h.AssertEq(t, name, file.Name()) - content, err = os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) - h.AssertNil(t, err) - h.AssertEq(t, actionTypesMap[fileIndex][1][1], string(content)) - - name = actionTypesMap[fileIndex][2][0] - file, err = os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) - h.AssertNil(t, err) - h.AssertEq(t, name, file.Name()) - content, err = os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) - h.AssertNil(t, err) - h.AssertEq(t, actionTypesMap[fileIndex][2][1], string(content)) - - name = actionTypesMap[fileIndex][3][0] - file, err = os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) - h.AssertNil(t, err) - h.AssertEq(t, name, file.Name()) - content, err = os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) - h.AssertNil(t, err) - h.AssertEq(t, actionTypesMap[fileIndex][3][1], string(content)) + h.AssertNotEq(t, len(warnings), 0) + h.AssertNotNil(t, err) }) - it("should show warnings when env value is empty", func() { - command.SetArgs([]string{ - "some/builder", - "--config", builderConfigPath, + it("should return an error when unknown `suffix` is used", func() { + envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixUnknown}, "") + h.AssertEq(t, envMap, map[string]string{}) + h.AssertEq(t, len(warnings), 0) + h.AssertNotNil(t, err) + }) + it("should override with the last specified delim when `[[build.env]]` has multiple delims with same `name` with a `append` or `prepend` suffix", func() { + envMap, warnings, err := commands.ParseBuildConfigEnv(BuildConfigEnvSuffixMultiple, "") + h.AssertEq(t, envMap, map[string]string{ + BuildConfigEnvSuffixMultiple[0].Name + "." + string(BuildConfigEnvSuffixMultiple[0].Suffix): BuildConfigEnvSuffixMultiple[0].Value, + BuildConfigEnvSuffixMultiple[1].Name + "." + string(BuildConfigEnvSuffixMultiple[1].Suffix): BuildConfigEnvSuffixMultiple[1].Value, + BuildConfigEnvSuffixMultiple[2].Name + "." + string(BuildConfigEnvSuffixMultiple[2].Suffix): BuildConfigEnvSuffixMultiple[2].Value, + BuildConfigEnvSuffixMultiple[2].Name + ".delim": BuildConfigEnvSuffixMultiple[2].Delim, }) - var bufOut bytes.Buffer - command.SetOut(&bufOut) - h.AssertNil(t, command.Execute()) - h.AssertNotEq(t, bufOut.String(), "") - name := actionTypesMap[fileIndex][0][0] - file, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) + h.AssertNotEq(t, len(warnings), 0) h.AssertNil(t, err) - h.AssertEq(t, name, file.Name()) - content, err := os.ReadFile(filepath.Join(buildConfigEnvDir, file.Name())) - h.AssertNil(t, err) - h.AssertEq(t, actionTypesMap[fileIndex][0][1], string(content)) }) - it("should return an error when env.Name is empty", func() { - command.SetArgs([]string{ - "some/builder", - "--config", builderConfigPath, + it("should override `value` with the last read value when a `[[build.env]]` has same `name` with same `suffix`", func() { + envMap, warnings, err := commands.ParseBuildConfigEnv(BuildConfigEnvDelimWithSameSuffixAndName, "") + h.AssertEq(t, envMap, map[string]string{ + BuildConfigEnvDelimWithSameSuffixAndName[1].Name: BuildConfigEnvDelimWithSameSuffixAndName[1].Value, }) - var bufErr bytes.Buffer - command.SetErr(&bufErr) - h.AssertNil(t, command.Execute()) - h.AssertNotEq(t, bufErr.String(), "") - name := actionTypesMap[fileIndex][0][0] - _, err := os.Stat(filepath.Clean(filepath.Join(buildConfigEnvDir, name))) - h.AssertNotNil(t, err) + h.AssertNotEq(t, len(warnings), 0) + h.AssertNil(t, err) }) }) @@ -464,48 +427,3 @@ func testCreateCommand(t *testing.T, when spec.G, it spec.S) { }) }) } - -func getBuildConfigEnvFileContent(index int) string { - switch index { - case 0: - return ActionNONE - case 1: - return ActionDEFAULT - case 2: - return ActionOVERRIDE - case 3: - return ActionAPPEND - case 4: - return ActionPREPEND - case 5: - return ActionDELIMIT - case 6: - return ActionUNKNOWN - case 7: - return ActionMULTIPLE - case 8: - return ActionWarning - case 9: - return ActionError - default: - return "" - } -} - -var actionTypesMap = map[int]map[int]map[int]string{ - 0: {0: {0: "actionNone", 1: "actionNoneValue"}}, - 1: {0: {0: "actionDefault.default", 1: "actionDefaultValue"}}, - 2: {0: {0: "actionOverride.override", 1: "actionOverrideValue"}}, - 3: {0: {0: "actionAppend.append", 1: "actionAppendValue"}}, - 4: {0: {0: "actionPrepend.prepend", 1: "actionPrependValue"}}, - 5: {0: {0: "actionDelim.delim", 1: ":"}}, - 6: {0: {0: "actionUnknown.unknown", 1: "actionUnknownValue"}}, - 7: { - 0: {0: "MY_VAR.append", 1: "actionAppendValue"}, - 1: {0: "MY_VAR.default", 1: "actionDefaultValue"}, - 2: {0: "MY_VAR.prepend", 1: "actionPrependValue"}, - 3: {0: "MY_VAR.delim", 1: ":"}, - }, - 8: {0: {0: "actionWarning", 1: ""}}, - 9: {0: {0: ".default", 1: "some-value"}}, -} diff --git a/pkg/client/create_builder.go b/pkg/client/create_builder.go index dc03d73d6b..f94c827501 100644 --- a/pkg/client/create_builder.go +++ b/pkg/client/create_builder.go @@ -29,6 +29,9 @@ type CreateBuilderOptions struct { // Name of the builder. BuilderName string + // BuildConfigEnv for Builder + BuildConfigEnv map[string]string + // Configuration that defines the functionality a builder provides. Config pubbldr.Config @@ -79,6 +82,11 @@ func (c *Client) CreateBuilder(ctx context.Context, opts CreateBuilderOptions) e bldr.SetStack(opts.Config.Stack) } bldr.SetRunImage(opts.Config.Run) + if opts.BuildConfigEnv == nil || len(opts.BuildConfigEnv) == 0 { + bldr.SetBuildConfigEnv(make(map[string]string)) + } else { + bldr.SetBuildConfigEnv(opts.BuildConfigEnv) + } return bldr.Save(c.logger, builder.CreatorMetadata{Version: c.version}) } @@ -191,6 +199,11 @@ func (c *Client) createBaseBuilder(ctx context.Context, opts CreateBuilderOption } bldr.SetLifecycle(lifecycle) + if opts.BuildConfigEnv == nil || len(opts.BuildConfigEnv) == 0 { + bldr.SetBuildConfigEnv(make(map[string]string)) + } else { + bldr.SetBuildConfigEnv(opts.BuildConfigEnv) + } return bldr, nil } From b8409a9f1f1561d23646f9bf0d8e69c1b05598bf Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 12 Oct 2023 06:11:46 +0000 Subject: [PATCH 33/62] build-config-env layer oly added when it is not empty Signed-off-by: WYGIN --- internal/builder/builder.go | 15 +++++++-------- pkg/client/create_builder.go | 6 +----- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 47d392c82d..7c4e8d9a9a 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -536,15 +536,14 @@ func (b *Builder) Save(logger logging.Logger, creatorMetadata CreatorMetadata) e if len(b.buildConfigEnv) > 0 { logger.Debugf("Provided Build Config Environment Variables\n %s", style.Map(b.env, " ", "\n")) - } - - buildConfigEnvTar, err := b.buildConfigEnvLayer(tmpDir, b.buildConfigEnv) - if err != nil { - return errors.Wrap(err, "retrieving build-config-env layer") - } + buildConfigEnvTar, err := b.buildConfigEnvLayer(tmpDir, b.buildConfigEnv) + if err != nil { + return errors.Wrap(err, "retrieving build-config-env layer") + } - if err := b.image.AddLayer(buildConfigEnvTar); err != nil { - return errors.Wrap(err, "adding build-config-env layer") + if err := b.image.AddLayer(buildConfigEnvTar); err != nil { + return errors.Wrap(err, "adding build-config-env layer") + } } if len(b.env) > 0 { diff --git a/pkg/client/create_builder.go b/pkg/client/create_builder.go index f94c827501..a773b0b7ee 100644 --- a/pkg/client/create_builder.go +++ b/pkg/client/create_builder.go @@ -82,11 +82,7 @@ func (c *Client) CreateBuilder(ctx context.Context, opts CreateBuilderOptions) e bldr.SetStack(opts.Config.Stack) } bldr.SetRunImage(opts.Config.Run) - if opts.BuildConfigEnv == nil || len(opts.BuildConfigEnv) == 0 { - bldr.SetBuildConfigEnv(make(map[string]string)) - } else { - bldr.SetBuildConfigEnv(opts.BuildConfigEnv) - } + bldr.SetBuildConfigEnv(opts.BuildConfigEnv) return bldr.Save(c.logger, builder.CreatorMetadata{Version: c.version}) } From fc47fd8dbe910fdd8eaf699f79c52bf2053abf4d Mon Sep 17 00:00:00 2001 From: WYGIN Date: Wed, 25 Oct 2023 15:35:28 +0000 Subject: [PATCH 34/62] added requested changes and fix bugs Signed-off-by: WYGIN --- builder/config_reader.go | 93 +++++++++++++++++++++++- internal/builder/builder.go | 10 +-- internal/builder/builder_test.go | 37 +++++++++- internal/commands/builder_create.go | 91 +---------------------- internal/commands/builder_create_test.go | 26 +++---- pkg/client/create_builder.go | 6 +- 6 files changed, 146 insertions(+), 117 deletions(-) diff --git a/builder/config_reader.go b/builder/config_reader.go index 9cd6117a2c..f0195f9e4c 100644 --- a/builder/config_reader.go +++ b/builder/config_reader.go @@ -4,6 +4,7 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/BurntSushi/toml" "github.com/pkg/errors" @@ -76,8 +77,6 @@ type BuildConfig struct { type Suffix string -var SuffixSlice []Suffix = []Suffix{NONE, DEFAULT, OVERRIDE, APPEND, PREPEND} - const ( NONE Suffix = "" DEFAULT Suffix = "default" @@ -182,3 +181,93 @@ func parseConfig(file *os.File) (Config, error) { return builderConfig, nil } + +func ParseBuildConfigEnv(env []BuildConfigEnv, path string) (envMap map[string]string, warnings []string, err error) { + envMap = map[string]string{} + var appendOrPrependWithoutDelim = 0 + for _, v := range env { + if name := v.Name; name == "" || len(name) == 0 { + return nil, nil, errors.Wrapf(errors.Errorf("env name should not be empty"), "parse contents of '%s'", path) + } + if val := v.Value; val == "" || len(val) == 0 { + warnings = append(warnings, fmt.Sprintf("empty value for key/name %s", style.Symbol(v.Name))) + } + suffixName, delimName, err := getBuildConfigEnvFileName(v) + if err != nil { + return envMap, warnings, err + } + if val, e := envMap[suffixName]; e { + warnings = append(warnings, fmt.Sprintf(errors.Errorf("overriding env with name: %s and suffix: %s from %s to %s", style.Symbol(v.Name), style.Symbol(string(v.Suffix)), style.Symbol(val), style.Symbol(v.Value)).Error(), "parse contents of '%s'", path)) + } + if val, e := envMap[delimName]; e { + warnings = append(warnings, fmt.Sprintf(errors.Errorf("overriding env with name: %s and delim: %s from %s to %s", style.Symbol(v.Name), style.Symbol(v.Delim), style.Symbol(val), style.Symbol(v.Value)).Error(), "parse contents of '%s'", path)) + } + if delim := v.Delim; (delim != "" || len(delim) != 0) && (delimName != "" || len(delimName) != 0) { + envMap[delimName] = delim + } + envMap[suffixName] = v.Value + } + + for k := range envMap { + name, suffix, err := getFilePrefixSuffix(k) + if err != nil { + continue + } + if _, ok := envMap[name+".delim"]; (suffix == "append" || suffix == "prepend") && !ok { + warnings = append(warnings, fmt.Sprintf(errors.Errorf("env with name/key %s with suffix %s must to have a %s value", style.Symbol(name), style.Symbol(suffix), style.Symbol("delim")).Error(), "parse contents of '%s'", path)) + appendOrPrependWithoutDelim++ + } + } + if appendOrPrependWithoutDelim > 0 { + return envMap, warnings, errors.Errorf("error parsing [[build.env]] in file '%s'", path) + } + return envMap, warnings, err +} + +func getBuildConfigEnvFileName(env BuildConfigEnv) (suffixName, delimName string, err error) { + suffix, err := getActionType(env.Suffix) + if err != nil { + return suffixName, delimName, err + } + if suffix == "" || len(suffix) == 0 { + suffixName = env.Name + } else { + suffixName = env.Name + suffix + } + if delim := env.Delim; delim != "" || len(delim) != 0 { + delimName = env.Name + ".delim" + } + return suffixName, delimName, err +} + +func getActionType(suffix Suffix) (suffixString string, err error) { + const delim = "." + switch suffix { + case NONE: + return "", nil + case DEFAULT: + return delim + string(DEFAULT), nil + case OVERRIDE: + return delim + string(OVERRIDE), nil + case APPEND: + return delim + string(APPEND), nil + case PREPEND: + return delim + string(PREPEND), nil + default: + return suffixString, errors.Errorf("unknown action type %s", style.Symbol(string(suffix))) + } +} + +func getFilePrefixSuffix(filename string) (prefix, suffix string, err error) { + val := strings.Split(filename, ".") + if len(val) <= 1 { + return val[0], suffix, errors.Errorf("Suffix might be null") + } + if len(val) == 2 { + suffix = val[1] + } else { + suffix = strings.Join(val[1:], ".") + } + return val[0], suffix, err +} + diff --git a/internal/builder/builder.go b/internal/builder/builder.go index 7c4e8d9a9a..fa7dd6e9d3 100644 --- a/internal/builder/builder.go +++ b/internal/builder/builder.go @@ -1129,13 +1129,11 @@ func (b *Builder) buildConfigEnvLayer(dest string, env map[string]string) (strin return "", err } defer fh.Close() - lw := b.layerWriterFactory.NewWriter(fh) defer lw.Close() - for k, v := range env { if err := lw.WriteHeader(&tar.Header{ - Name: path.Join(buildConfigDir, "env", k), + Name: path.Join(cnbBuildConfigDir(), "env", k), Size: int64(len(v)), Mode: 0644, ModTime: archive.NormalizedDateTime, @@ -1307,9 +1305,9 @@ func (e errModuleTar) Path() string { } func cnbBuildConfigDir() string { - if env, ok := os.LookupEnv("CNB_BUILD_CONFIG_DIR"); !ok { - return "/cnb/build-config" - } else { + if env, ok := os.LookupEnv("CNB_BUILD_CONFIG_DIR"); ok { return env } + + return "/cnb/build-config" } diff --git a/internal/builder/builder_test.go b/internal/builder/builder_test.go index 2745e9e97b..8f0e18350c 100644 --- a/internal/builder/builder_test.go +++ b/internal/builder/builder_test.go @@ -397,7 +397,6 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) { h.HasModTime(archive.NormalizedDateTime), ) }) - it("creates the buildpacks dir", func() { h.AssertNil(t, subject.Save(logger, builder.CreatorMetadata{})) h.AssertEq(t, baseImage.IsSaved(), true) @@ -1621,8 +1620,44 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) { }) }) + when("when CNB_BUILD_CONFIG_DIR is defined", func () { + var buildConfigEnvName = "CNB_BUILD_CONFIG_DIR" + var buildConfigEnvValue = "/cnb/dup-build-config-dir" + it.Before(func() { + os.Setenv(buildConfigEnvName, buildConfigEnvValue) + subject.SetBuildConfigEnv(map[string]string{ + "SOME_KEY": "some-val", + "OTHER_KEY.append": "other-val", + "OTHER_KEY.delim": ":", + }) + h.AssertNil(t, subject.Save(logger, builder.CreatorMetadata{})) + h.AssertEq(t, baseImage.IsSaved(), true) + }) + it.After(func() { + os.Unsetenv(buildConfigEnvName) + }) + + it("adds the env vars as files to the image", func() { + layerTar, err := baseImage.FindLayerWithPath(buildConfigEnvValue + "/env/SOME_KEY") + h.AssertNil(t, err) + h.AssertOnTarEntry(t, layerTar, buildConfigEnvValue + "/env/SOME_KEY", + h.ContentEquals(`some-val`), + h.HasModTime(archive.NormalizedDateTime), + ) + h.AssertOnTarEntry(t, layerTar, buildConfigEnvValue + "/env/OTHER_KEY.append", + h.ContentEquals(`other-val`), + h.HasModTime(archive.NormalizedDateTime), + ) + h.AssertOnTarEntry(t, layerTar, buildConfigEnvValue + "/env/OTHER_KEY.delim", + h.ContentEquals(`:`), + h.HasModTime(archive.NormalizedDateTime), + ) + }) + }) + when("#SetBuildConfigEnv", func() { it.Before(func() { + os.Unsetenv("CNB_BUILD_CONFIG_DIR") subject.SetBuildConfigEnv(map[string]string{ "SOME_KEY": "some-val", "OTHER_KEY.append": "other-val", diff --git a/internal/commands/builder_create.go b/internal/commands/builder_create.go index a98cb43521..c3c1ccb90b 100644 --- a/internal/commands/builder_create.go +++ b/internal/commands/builder_create.go @@ -75,7 +75,7 @@ Creating a custom builder allows you to control what buildpacks are used and wha return errors.Wrap(err, "getting absolute path for config") } - envMap, warnings, err := ParseBuildConfigEnv(builderConfig.Build.Env, flags.BuilderTomlPath) + envMap, warnings, err := builder.ParseBuildConfigEnv(builderConfig.Build.Env, flags.BuilderTomlPath) for _, v := range warnings { logger.Warn(v) } @@ -146,92 +146,3 @@ func validateCreateFlags(flags *BuilderCreateFlags, cfg config.Config) error { return nil } - -func getActionType(suffix builder.Suffix) (suffixString string, err error) { - const delim = "." - switch suffix { - case builder.NONE: - return "", nil - case builder.DEFAULT: - return delim + string(builder.DEFAULT), nil - case builder.OVERRIDE: - return delim + string(builder.OVERRIDE), nil - case builder.APPEND: - return delim + string(builder.APPEND), nil - case builder.PREPEND: - return delim + string(builder.PREPEND), nil - default: - return suffixString, errors.Errorf("unknown action type %s", style.Symbol(string(suffix))) - } -} - -func getFilePrefixSuffix(filename string) (prefix, suffix string, err error) { - val := strings.Split(filename, ".") - if len(val) <= 1 { - return val[0], suffix, errors.Errorf("Suffix might be null") - } - if len(val) == 2 { - suffix = val[1] - } else { - strings.Join(val[1:], ".") - } - return val[0], suffix, err -} - -func getBuildConfigEnvFileName(env builder.BuildConfigEnv) (suffixName, delimName string, err error) { - suffix, err := getActionType(env.Suffix) - if err != nil { - return suffixName, delimName, err - } - if suffix == "" || len(suffix) == 0 { - suffixName = env.Name - } else { - suffixName = env.Name + suffix - } - if delim := env.Delim; delim != "" || len(delim) != 0 { - delimName = env.Name + ".delim" - } - return suffixName, delimName, err -} - -func ParseBuildConfigEnv(env []builder.BuildConfigEnv, path string) (envMap map[string]string, warnings []string, err error) { - envMap = map[string]string{} - var appendOrPrependWithoutDelim = 0 - for _, v := range env { - if name := v.Name; name == "" || len(name) == 0 { - return nil, nil, errors.Wrapf(errors.Errorf("env name should not be empty"), "parse contents of '%s'", path) - } - if val := v.Value; val == "" || len(val) == 0 { - warnings = append(warnings, fmt.Sprintf("empty value for key/name %s", style.Symbol(v.Name))) - } - suffixName, delimName, err := getBuildConfigEnvFileName(v) - if err != nil { - return envMap, warnings, err - } - if val, e := envMap[suffixName]; e { - warnings = append(warnings, fmt.Sprintf(errors.Errorf("overriding env with name: %s and suffix: %s from %s to %s", style.Symbol(v.Name), style.Symbol(string(v.Suffix)), style.Symbol(val), style.Symbol(v.Value)).Error(), "parse contents of '%s'", path)) - } - if val, e := envMap[delimName]; e { - warnings = append(warnings, fmt.Sprintf(errors.Errorf("overriding env with name: %s and delim: %s from %s to %s", style.Symbol(v.Name), style.Symbol(v.Delim), style.Symbol(val), style.Symbol(v.Value)).Error(), "parse contents of '%s'", path)) - } - if delim := v.Delim; (delim != "" || len(delim) != 0) && (delimName != "" || len(delimName) != 0) { - envMap[delimName] = delim - } - envMap[suffixName] = v.Value - } - - for k := range envMap { - name, suffix, err := getFilePrefixSuffix(k) - if err != nil { - continue - } - if _, ok := envMap[name+".delim"]; (suffix == "append" || suffix == "prepend") && !ok { - warnings = append(warnings, fmt.Sprintf(errors.Errorf("env with name/key %s with suffix %s must to have a %s value", style.Symbol(name), style.Symbol(suffix), style.Symbol("delim")).Error(), "parse contents of '%s'", path)) - appendOrPrependWithoutDelim++ - } - } - if appendOrPrependWithoutDelim > 0 { - return envMap, warnings, errors.Errorf("error parsing [[build.env]] in file '%s'", path) - } - return envMap, warnings, err -} diff --git a/internal/commands/builder_create_test.go b/internal/commands/builder_create_test.go index f5f9684390..4ddf115b32 100644 --- a/internal/commands/builder_create_test.go +++ b/internal/commands/builder_create_test.go @@ -278,7 +278,7 @@ func testCreateCommand(t *testing.T, when spec.G, it spec.S) { when("#ParseBuildpackConfigEnv", func() { it("should create envMap as expected when suffix is omitted", func() { - envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixNone}, "") + envMap, warnings, err := builder.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixNone}, "") h.AssertEq(t, envMap, map[string]string{ BuildConfigEnvSuffixNone.Name: BuildConfigEnvSuffixNone.Value, }) @@ -286,7 +286,7 @@ func testCreateCommand(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, err) }) it("should create envMap as expected when suffix is empty string", func() { - envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixNoneWithEmptySuffix}, "") + envMap, warnings, err := builder.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixNoneWithEmptySuffix}, "") h.AssertEq(t, envMap, map[string]string{ BuildConfigEnvSuffixNoneWithEmptySuffix.Name: BuildConfigEnvSuffixNoneWithEmptySuffix.Value, }) @@ -294,7 +294,7 @@ func testCreateCommand(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, err) }) it("should create envMap as expected when suffix is `default`", func() { - envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixDefault}, "") + envMap, warnings, err := builder.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixDefault}, "") h.AssertEq(t, envMap, map[string]string{ BuildConfigEnvSuffixDefault.Name + "." + string(BuildConfigEnvSuffixDefault.Suffix): BuildConfigEnvSuffixDefault.Value, }) @@ -302,7 +302,7 @@ func testCreateCommand(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, err) }) it("should create envMap as expected when suffix is `override`", func() { - envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixOverride}, "") + envMap, warnings, err := builder.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixOverride}, "") h.AssertEq(t, envMap, map[string]string{ BuildConfigEnvSuffixOverride.Name + "." + string(BuildConfigEnvSuffixOverride.Suffix): BuildConfigEnvSuffixOverride.Value, }) @@ -310,7 +310,7 @@ func testCreateCommand(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, err) }) it("should create envMap as expected when suffix is `append`", func() { - envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixAppend}, "") + envMap, warnings, err := builder.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixAppend}, "") h.AssertEq(t, envMap, map[string]string{ BuildConfigEnvSuffixAppend.Name + "." + string(BuildConfigEnvSuffixAppend.Suffix): BuildConfigEnvSuffixAppend.Value, BuildConfigEnvSuffixAppend.Name + ".delim": BuildConfigEnvSuffixAppend.Delim, @@ -319,7 +319,7 @@ func testCreateCommand(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, err) }) it("should create envMap as expected when suffix is `prepend`", func() { - envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixPrepend}, "") + envMap, warnings, err := builder.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixPrepend}, "") h.AssertEq(t, envMap, map[string]string{ BuildConfigEnvSuffixPrepend.Name + "." + string(BuildConfigEnvSuffixPrepend.Suffix): BuildConfigEnvSuffixPrepend.Value, BuildConfigEnvSuffixPrepend.Name + ".delim": BuildConfigEnvSuffixPrepend.Delim, @@ -328,7 +328,7 @@ func testCreateCommand(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, err) }) it("should create envMap as expected when delim is specified", func() { - envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvDelimWithoutSuffix}, "") + envMap, warnings, err := builder.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvDelimWithoutSuffix}, "") h.AssertEq(t, envMap, map[string]string{ BuildConfigEnvDelimWithoutSuffix.Name: BuildConfigEnvDelimWithoutSuffix.Value, BuildConfigEnvDelimWithoutSuffix.Name + ".delim": BuildConfigEnvDelimWithoutSuffix.Delim, @@ -337,7 +337,7 @@ func testCreateCommand(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, err) }) it("should create envMap with a warning when `value` is empty", func() { - envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvEmptyValue}, "") + envMap, warnings, err := builder.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvEmptyValue}, "") h.AssertEq(t, envMap, map[string]string{ BuildConfigEnvEmptyValue.Name: BuildConfigEnvEmptyValue.Value, }) @@ -345,13 +345,13 @@ func testCreateCommand(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, err) }) it("should return an error when `name` is empty", func() { - envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvEmptyName}, "") + envMap, warnings, err := builder.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvEmptyName}, "") h.AssertEq(t, envMap, map[string]string(nil)) h.AssertEq(t, len(warnings), 0) h.AssertNotNil(t, err) }) it("should return warnings when `apprend` or `prepend` is used without `delim`", func() { - envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixPrependWithoutDelim}, "") + envMap, warnings, err := builder.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixPrependWithoutDelim}, "") h.AssertEq(t, envMap, map[string]string{ BuildConfigEnvSuffixPrependWithoutDelim.Name + "." + string(BuildConfigEnvSuffixPrependWithoutDelim.Suffix): BuildConfigEnvSuffixPrependWithoutDelim.Value, }) @@ -359,13 +359,13 @@ func testCreateCommand(t *testing.T, when spec.G, it spec.S) { h.AssertNotNil(t, err) }) it("should return an error when unknown `suffix` is used", func() { - envMap, warnings, err := commands.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixUnknown}, "") + envMap, warnings, err := builder.ParseBuildConfigEnv([]builder.BuildConfigEnv{BuildConfigEnvSuffixUnknown}, "") h.AssertEq(t, envMap, map[string]string{}) h.AssertEq(t, len(warnings), 0) h.AssertNotNil(t, err) }) it("should override with the last specified delim when `[[build.env]]` has multiple delims with same `name` with a `append` or `prepend` suffix", func() { - envMap, warnings, err := commands.ParseBuildConfigEnv(BuildConfigEnvSuffixMultiple, "") + envMap, warnings, err := builder.ParseBuildConfigEnv(BuildConfigEnvSuffixMultiple, "") h.AssertEq(t, envMap, map[string]string{ BuildConfigEnvSuffixMultiple[0].Name + "." + string(BuildConfigEnvSuffixMultiple[0].Suffix): BuildConfigEnvSuffixMultiple[0].Value, BuildConfigEnvSuffixMultiple[1].Name + "." + string(BuildConfigEnvSuffixMultiple[1].Suffix): BuildConfigEnvSuffixMultiple[1].Value, @@ -376,7 +376,7 @@ func testCreateCommand(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, err) }) it("should override `value` with the last read value when a `[[build.env]]` has same `name` with same `suffix`", func() { - envMap, warnings, err := commands.ParseBuildConfigEnv(BuildConfigEnvDelimWithSameSuffixAndName, "") + envMap, warnings, err := builder.ParseBuildConfigEnv(BuildConfigEnvDelimWithSameSuffixAndName, "") h.AssertEq(t, envMap, map[string]string{ BuildConfigEnvDelimWithSameSuffixAndName[1].Name: BuildConfigEnvDelimWithSameSuffixAndName[1].Value, }) diff --git a/pkg/client/create_builder.go b/pkg/client/create_builder.go index a773b0b7ee..b697313b33 100644 --- a/pkg/client/create_builder.go +++ b/pkg/client/create_builder.go @@ -195,11 +195,7 @@ func (c *Client) createBaseBuilder(ctx context.Context, opts CreateBuilderOption } bldr.SetLifecycle(lifecycle) - if opts.BuildConfigEnv == nil || len(opts.BuildConfigEnv) == 0 { - bldr.SetBuildConfigEnv(make(map[string]string)) - } else { - bldr.SetBuildConfigEnv(opts.BuildConfigEnv) - } + bldr.SetBuildConfigEnv(opts.BuildConfigEnv) return bldr, nil } From 4feb437661db4199ab502458f29fb624c52025b7 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Wed, 25 Oct 2023 15:56:16 +0000 Subject: [PATCH 35/62] fix: code format Signed-off-by: WYGIN --- builder/config_reader.go | 1 - internal/builder/builder_test.go | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/builder/config_reader.go b/builder/config_reader.go index f0195f9e4c..008984bff3 100644 --- a/builder/config_reader.go +++ b/builder/config_reader.go @@ -270,4 +270,3 @@ func getFilePrefixSuffix(filename string) (prefix, suffix string, err error) { } return val[0], suffix, err } - diff --git a/internal/builder/builder_test.go b/internal/builder/builder_test.go index 8f0e18350c..37124533da 100644 --- a/internal/builder/builder_test.go +++ b/internal/builder/builder_test.go @@ -1620,7 +1620,7 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) { }) }) - when("when CNB_BUILD_CONFIG_DIR is defined", func () { + when("when CNB_BUILD_CONFIG_DIR is defined", func() { var buildConfigEnvName = "CNB_BUILD_CONFIG_DIR" var buildConfigEnvValue = "/cnb/dup-build-config-dir" it.Before(func() { @@ -1640,15 +1640,15 @@ func testBuilder(t *testing.T, when spec.G, it spec.S) { it("adds the env vars as files to the image", func() { layerTar, err := baseImage.FindLayerWithPath(buildConfigEnvValue + "/env/SOME_KEY") h.AssertNil(t, err) - h.AssertOnTarEntry(t, layerTar, buildConfigEnvValue + "/env/SOME_KEY", + h.AssertOnTarEntry(t, layerTar, buildConfigEnvValue+"/env/SOME_KEY", h.ContentEquals(`some-val`), h.HasModTime(archive.NormalizedDateTime), ) - h.AssertOnTarEntry(t, layerTar, buildConfigEnvValue + "/env/OTHER_KEY.append", + h.AssertOnTarEntry(t, layerTar, buildConfigEnvValue+"/env/OTHER_KEY.append", h.ContentEquals(`other-val`), h.HasModTime(archive.NormalizedDateTime), ) - h.AssertOnTarEntry(t, layerTar, buildConfigEnvValue + "/env/OTHER_KEY.delim", + h.AssertOnTarEntry(t, layerTar, buildConfigEnvValue+"/env/OTHER_KEY.delim", h.ContentEquals(`:`), h.HasModTime(archive.NormalizedDateTime), ) From e41d99e4d0b36d9469cf730effc2234eff385d23 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 26 Oct 2023 10:34:16 +0000 Subject: [PATCH 36/62] improved code coverage Signed-off-by: WYGIN --- builder/config_reader.go | 2 +- builder/config_reader_test.go | 107 ++++++++++++++++++++++++++++++++++ testhelpers/testhelpers.go | 27 +++++++++ 3 files changed, 135 insertions(+), 1 deletion(-) diff --git a/builder/config_reader.go b/builder/config_reader.go index 008984bff3..6ebd688d55 100644 --- a/builder/config_reader.go +++ b/builder/config_reader.go @@ -229,7 +229,7 @@ func getBuildConfigEnvFileName(env BuildConfigEnv) (suffixName, delimName string if err != nil { return suffixName, delimName, err } - if suffix == "" || len(suffix) == 0 { + if suffix == "" { suffixName = env.Name } else { suffixName = env.Name + suffix diff --git a/builder/config_reader_test.go b/builder/config_reader_test.go index 93a1bf851d..6c5512f897 100644 --- a/builder/config_reader_test.go +++ b/builder/config_reader_test.go @@ -229,4 +229,111 @@ uri = "noop-buildpack.tgz" h.AssertError(t, builder.ValidateConfig(config), "build.image is required") }) }) + when("#ParseBuildConfigEnv()", func() { + it("should return an error when name is not defined", func() { + _, _, err := builder.ParseBuildConfigEnv([]builder.BuildConfigEnv{ + { + Name: "", + Value: "vaiue", + }, + }, "") + h.AssertNotNil(t, err) + }) + it("should warn when the value is nil or empty string", func() { + env, warn, err := builder.ParseBuildConfigEnv([]builder.BuildConfigEnv{ + { + Name: "key", + Value: "", + Suffix: "override", + }, + }, "") + + h.AssertNotNil(t, warn) + h.AssertNil(t, err) + h.AssertMapContains[string, string](t, env, h.NewKeyValue[string, string]("key.override", "")) + }) + it("should return an error when unknown suffix is specified", func() { + _, _, err := builder.ParseBuildConfigEnv([]builder.BuildConfigEnv{ + { + Name: "key", + Value: "", + Suffix: "invalid", + }, + }, "") + + h.AssertNotNil(t, err) + }) + it("should override and show a warning when suffix or delim is defined multiple times", func() { + env, warn, err := builder.ParseBuildConfigEnv([]builder.BuildConfigEnv{ + { + Name: "key1", + Value: "value1", + Suffix: "append", + Delim: "%", + }, + { + Name: "key1", + Value: "value2", + Suffix: "append", + Delim: ",", + }, + { + Name: "key1", + Value: "value3", + Suffix: "default", + Delim: ";", + }, + { + Name: "key1", + Value: "value4", + Suffix: "prepend", + Delim: ":", + }, + }, "") + + h.AssertNotNil(t, warn) + h.AssertNil(t, err) + h.AssertMapContains[string, string]( + t, + env, + h.NewKeyValue[string, string]("key1.append", "value2"), + h.NewKeyValue[string, string]("key1.default", "value3"), + h.NewKeyValue[string, string]("key1.prepend", "value4"), + h.NewKeyValue[string, string]("key1.delim", ":"), + ) + h.AssertMapNotContains[string, string]( + t, + env, + h.NewKeyValue[string, string]("key1.append", "value1"), + h.NewKeyValue[string, string]("key1.delim", "%"), + h.NewKeyValue[string, string]("key1.delim", ","), + h.NewKeyValue[string, string]("key1.delim", ";"), + ) + }) + it("should return an error when `suffix` is defined as `append` or `prepend` without a `delim`", func() { + _, warn, err := builder.ParseBuildConfigEnv([]builder.BuildConfigEnv{ + { + Name: "key", + Value: "value", + Suffix: "append", + }, + }, "") + + h.AssertNotNil(t, warn) + h.AssertNotNil(t, err) + }) + it("when suffix is NONE or omitted should default to `override`", func() { + env, warn, err := builder.ParseBuildConfigEnv([]builder.BuildConfigEnv{ + { + Name: "key", + Value: "value", + Suffix: "", + }, + }, "") + + h.AssertNotNil(t, warn) + h.AssertNil(t, err) + h.AssertMapContains[string, string](t, env, h.NewKeyValue[string, string]("key", "value")) + }) + }) } diff --git a/testhelpers/testhelpers.go b/testhelpers/testhelpers.go index 7868691a3c..3eeb4b6431 100644 --- a/testhelpers/testhelpers.go +++ b/testhelpers/testhelpers.go @@ -179,6 +179,33 @@ func AssertNotContains(t *testing.T, actual, expected string) { } } +type KeyValue[k comparable, v any] struct { + key k + value v +} + +func NewKeyValue[k comparable, v any](key k, value v) KeyValue[k, v] { + return KeyValue[k, v]{key: key, value: value} +} + +func AssertMapContains[key comparable, value any](t *testing.T, actual map[key]value, expected ...KeyValue[key, value]) { + t.Helper() + for _, i := range expected { + if v, ok := actual[i.key]; !ok || !reflect.DeepEqual(v, i.value) { + t.Fatalf("Expected %s to contain elements %s", reflect.ValueOf(actual), reflect.ValueOf(expected)) + } + } +} + +func AssertMapNotContains[key comparable, value any](t *testing.T, actual map[key]value, expected ...KeyValue[key, value]) { + t.Helper() + for _, i := range expected { + if v, ok := actual[i.key]; ok && reflect.DeepEqual(v, i.value) { + t.Fatalf("Expected %s to not contain elements %s", reflect.ValueOf(actual), reflect.ValueOf(expected)) + } + } +} + func AssertSliceContains(t *testing.T, slice []string, expected ...string) { t.Helper() _, missing, _ := stringset.Compare(slice, expected) From e3cc4d9b982f9ae7b3483ebde7b9bea4fb745b13 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 1 Nov 2023 14:39:31 +0000 Subject: [PATCH 37/62] build(deps): bump buildpacks/github-actions from 5.4.0 to 5.5.0 Bumps [buildpacks/github-actions](https://github.com/buildpacks/github-actions) from 5.4.0 to 5.5.0. - [Release notes](https://github.com/buildpacks/github-actions/releases) - [Commits](https://github.com/buildpacks/github-actions/compare/v5.4.0...v5.5.0) --- updated-dependencies: - dependency-name: buildpacks/github-actions dependency-type: direct:production update-type: version-update:semver-minor ... Signed-off-by: dependabot[bot] --- .github/workflows/delivery-docker.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/delivery-docker.yml b/.github/workflows/delivery-docker.yml index c956195175..9c4232732b 100644 --- a/.github/workflows/delivery-docker.yml +++ b/.github/workflows/delivery-docker.yml @@ -60,7 +60,7 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - uses: docker/setup-qemu-action@v3 - uses: docker/setup-buildx-action@v3 - - uses: buildpacks/github-actions/setup-tools@v5.4.0 + - uses: buildpacks/github-actions/setup-tools@v5.5.0 - name: Buildx Build/Publish run: | docker buildx build . \ From 8e69f1bb19611568b2473429558221c187946050 Mon Sep 17 00:00:00 2001 From: Jerico Pena Date: Wed, 1 Nov 2023 12:27:50 -0400 Subject: [PATCH 38/62] Add ServerVersion to docker client interface which is needed by imgutil Signed-off-by: Jerico Pena --- pkg/client/docker.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/client/docker.go b/pkg/client/docker.go index f637066e11..dfdf2c6139 100644 --- a/pkg/client/docker.go +++ b/pkg/client/docker.go @@ -21,6 +21,7 @@ type DockerClient interface { ImageRemove(ctx context.Context, image string, options types.ImageRemoveOptions) ([]types.ImageDeleteResponseItem, error) ImagePull(ctx context.Context, ref string, options types.ImagePullOptions) (io.ReadCloser, error) Info(ctx context.Context) (types.Info, error) + ServerVersion(ctx context.Context) (types.Version, error) VolumeRemove(ctx context.Context, volumeID string, force bool) error ContainerCreate(ctx context.Context, config *containertypes.Config, hostConfig *containertypes.HostConfig, networkingConfig *networktypes.NetworkingConfig, platform *specs.Platform, containerName string) (containertypes.CreateResponse, error) CopyFromContainer(ctx context.Context, container, srcPath string) (io.ReadCloser, types.ContainerPathStat, error) From a8335fb71c555637fdad871f768cb7d788081eff Mon Sep 17 00:00:00 2001 From: Jerico Pena Date: Wed, 1 Nov 2023 23:32:25 -0400 Subject: [PATCH 39/62] Use imgutil fork to verify platform changes Signed-off-by: Jerico Pena --- go.mod | 2 ++ 1 file changed, 2 insertions(+) diff --git a/go.mod b/go.mod index 95378c7be6..8046101adf 100644 --- a/go.mod +++ b/go.mod @@ -125,3 +125,5 @@ require ( ) go 1.20 + +replace github.com/buildpacks/imgutil => github.com/jericop/imgutil v1.0.0 \ No newline at end of file From f9ec20eb7e196f75ae788f34877d0a56bfadf9d1 Mon Sep 17 00:00:00 2001 From: Jerico Pena Date: Thu, 2 Nov 2023 09:57:26 -0400 Subject: [PATCH 40/62] Bump imgutil dependency for default platform change Signed-off-by: Jerico Pena --- go.mod | 4 +--- go.sum | 2 ++ 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8046101adf..95ac60f314 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ require ( github.com/Masterminds/semver v1.5.0 github.com/Microsoft/go-winio v0.6.1 github.com/apex/log v1.9.0 - github.com/buildpacks/imgutil v0.0.0-20230626185301-726f02e4225c + github.com/buildpacks/imgutil v0.0.0-20231102131059-84d632186b59 github.com/buildpacks/lifecycle v0.17.2 github.com/docker/cli v24.0.7+incompatible github.com/docker/docker v24.0.7+incompatible @@ -125,5 +125,3 @@ require ( ) go 1.20 - -replace github.com/buildpacks/imgutil => github.com/jericop/imgutil v1.0.0 \ No newline at end of file diff --git a/go.sum b/go.sum index 677cd5be10..9e80695228 100644 --- a/go.sum +++ b/go.sum @@ -97,6 +97,8 @@ github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20230522190001- github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/buildpacks/imgutil v0.0.0-20230626185301-726f02e4225c h1:HlRuSz+JGAzudNtNCfHIzXe0AEuHX6Vx8uZgmjvX02o= github.com/buildpacks/imgutil v0.0.0-20230626185301-726f02e4225c/go.mod h1:mBG5M3GJW5nknCEOOqtmMHyPYnSpw/5GEiciuYU/COw= +github.com/buildpacks/imgutil v0.0.0-20231102131059-84d632186b59 h1:5g+dMdOO6Ufx/bDa5gUH/Aw9FN6wt7T3HzKSkHVvue4= +github.com/buildpacks/imgutil v0.0.0-20231102131059-84d632186b59/go.mod h1:PsazEB9yz+NG/cgm0Z1oQ0Xq6rD/U7eNMt5Su41afYY= github.com/buildpacks/lifecycle v0.17.2 h1:CfJYWHIC5v996idgjDamYHBTk+G+c1Qt7Yk80MlbWpw= github.com/buildpacks/lifecycle v0.17.2/go.mod h1:h8MrqltqMM+HQnn2F2JOQaKWmeybZ54qvlNV3pAiAqw= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= From b71462648b464951052daa6c9152ce6d889af72c Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 02:54:51 +0000 Subject: [PATCH 41/62] WIP: added manifest_add command Signed-off-by: WYGIN --- cmd/cmd.go | 1 + internal/commands/commands.go | 8 ++ internal/commands/manifest.go | 26 +++++++ internal/commands/manifest_add.go | 61 +++++++++++++++ internal/commands/manifest_add_test.go | 103 +++++++++++++++++++++++++ 5 files changed, 199 insertions(+) create mode 100644 internal/commands/manifest.go create mode 100644 internal/commands/manifest_add.go create mode 100644 internal/commands/manifest_add_test.go diff --git a/cmd/cmd.go b/cmd/cmd.go index ace01f3f2d..1ad7d8a21e 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -104,6 +104,7 @@ func NewPackCommand(logger ConfigurableLogger) (*cobra.Command, error) { rootCmd.AddCommand(commands.SetDefaultRegistry(logger, cfg, cfgPath)) rootCmd.AddCommand(commands.RemoveRegistry(logger, cfg, cfgPath)) rootCmd.AddCommand(commands.YankBuildpack(logger, cfg, packClient)) + rootCmd.AddCommand(commands.NewManifestCommand(logger, packClient)) } packHome, err := config.PackHome() diff --git a/internal/commands/commands.go b/internal/commands/commands.go index 11fa645ca5..08bfd3933b 100644 --- a/internal/commands/commands.go +++ b/internal/commands/commands.go @@ -32,6 +32,14 @@ type PackClient interface { InspectExtension(client.InspectExtensionOptions) (*client.ExtensionInfo, error) PullBuildpack(context.Context, client.PullBuildpackOptions) error DownloadSBOM(name string, options client.DownloadSBOMOptions) error + CreateManifest(ctx context.Context, name string, images []string, opts client.CreateManifestOptions) (imageID string, err error) + AnnotateManifest(ctx context.Context, name string, image string, opts client.ManifestAnnotateOptions) error + ExistsManifest(ctx context.Context, image string) error + AddManifest(ctx context.Context, index string, images string, opts client.ManifestAddOptions) (imageID string, err error) + DeleteManifest(ctx context.Context, name []string) error + RemoveManifest(ctx context.Context, name string, images []string) error + PushManifest(ctx context.Context, index string, opts client.PushManifestOptions) (imageID string, err error) + InspectManifest(ctx context.Context, name string, opts client.InspectManifestOptions) error } func AddHelpFlag(cmd *cobra.Command, commandName string) { diff --git a/internal/commands/manifest.go b/internal/commands/manifest.go new file mode 100644 index 0000000000..9b2108a449 --- /dev/null +++ b/internal/commands/manifest.go @@ -0,0 +1,26 @@ +package commands + +import ( + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/logging" +) + +func NewManifestCommand(logger logging.Logger, client PackClient) *cobra.Command { + cmd := &cobra.Command{ + Use: "manifest", + Short: "Interact with image index or manifest list", + RunE: nil, + } + + cmd.AddCommand(ManifestCreate(logger, client)) + cmd.AddCommand(ManifestAdd(logger, client)) + cmd.AddCommand(ManifestAnnotate(logger, client)) + cmd.AddCommand(ManifestDelete(logger, client)) + cmd.AddCommand(ManifestInspect(logger, client)) + cmd.AddCommand(ManifestPush(logger, client)) + cmd.AddCommand(ManifestRemove(logger, client)) + + AddHelpFlag(cmd, "manifest") + return cmd +} \ No newline at end of file diff --git a/internal/commands/manifest_add.go b/internal/commands/manifest_add.go new file mode 100644 index 0000000000..7b85b0d662 --- /dev/null +++ b/internal/commands/manifest_add.go @@ -0,0 +1,61 @@ +package commands + +import ( + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/client" + "github.com/buildpacks/pack/pkg/logging" +) + +// ManifestAddFlags define flags provided to the ManifestAdd +type ManifestAddFlags struct { + ManifestAnnotateFlags + all bool +} + +// ManifestAdd modifies a manifest list (Image index) and add a new image to the list of manifests. +func ManifestAdd(logger logging.Logger, pack PackClient) *cobra.Command { + var flags ManifestAddFlags + + cmd := &cobra.Command{ + Use: "pack manifest add [OPTIONS] [flags]", + Args: cobra.MatchAll(cobra.ExactArgs(2), cobra.OnlyValidArgs), + Short: "manifest add modifies a manifest list (Image index) and add a new image to the list of manifests.", + Example: `pack manifest add cnbs/sample-package:hello-multiarch-universe \ + cnbs/sample-package:hello-universe-riscv-linux`, + Long: `manifest add modifies a manifest list (Image index) and add a new image to the list of manifests. + + When a manifest list exits locally, user can add a new image to the manifest list using this command`, + RunE: logError(logger, func(cmd *cobra.Command, args []string) error { + imageIndex := args[0] + manifests := args[1] + if err := validateManifestAddFlags(flags); err != nil { + return err + } + + imageID, err := pack.AddManifest(cmd.Context(), imageIndex, manifests, client.ManifestAddOptions{ + + All: flags.all, + }) + + if err != nil { + return err + } + + logger.Infof(imageID) + return nil + }), + } + + cmd.Flags().BoolVar(&flags.all, "all", false, "add all of the contents to the local list (applies only if is an index)") + cmd.Flags().StringVar(&flags.os, "os", "", "Set the operating system") + cmd.Flags().StringVar(&flags.arch, "arch", "", "Set the architecture") + cmd.Flags().StringVar(&flags.variant, "variant", "", "Set the architecture variant") + + AddHelpFlag(cmd, "add") + return cmd +} + +func validateManifestAddFlags(flags ManifestAddFlags) error { + return nil +} \ No newline at end of file diff --git a/internal/commands/manifest_add_test.go b/internal/commands/manifest_add_test.go new file mode 100644 index 0000000000..8e419bb4f9 --- /dev/null +++ b/internal/commands/manifest_add_test.go @@ -0,0 +1,103 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestAddCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestAddCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestAdd(logger, mockClient) + }) + + when("#ManifestAdd", func() { + when("no flags specified", func() { + + }) + when("when --all flags passed", func() { + + }) + when("when --os flags passed", func() { + + }) + when("when --arch flags passed", func() { + + }) + when("when --variant flags passed", func() { + + }) + when("when --os-version flags passed", func() { + + }) + when("when --features flags passed", func() { + + }) + when("when --os-features flags passed", func() { + + }) + when("when --annotations flags passed", func() { + + }) + when("when multiple flags passed", func() { + + }) + when("when no args passed", func() { + + }) + when("when manifest list reference is incorrect", func() { + + }) + when("when manifest reference is incorrect", func() { + + }) + when("when manifest passed in-place of manifest list on first arg", func() { + + }) + when("when manifest list is passed on second arg", func() { + + }) + when("when manifest is passed on second arg with --all option", func() { + + }) + when("when manifest list in locally available", func() { + + }) + when("when manifest is not locally available", func() { + + }) + when("when manifest is locally available passed", func() { + + }) + when("when multiple manifests passed", func() { + + }) + }) +} \ No newline at end of file From 8b109bf962bdfb268dde255075cb5e000091494d Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 02:57:44 +0000 Subject: [PATCH 42/62] WIP added manifest_annotate command Signed-off-by: WYGIN --- internal/commands/manifest_annotate.go | 42 ++++++++ internal/commands/manifest_annotate_test.go | 100 ++++++++++++++++++++ 2 files changed, 142 insertions(+) create mode 100644 internal/commands/manifest_annotate.go create mode 100644 internal/commands/manifest_annotate_test.go diff --git a/internal/commands/manifest_annotate.go b/internal/commands/manifest_annotate.go new file mode 100644 index 0000000000..268721b41f --- /dev/null +++ b/internal/commands/manifest_annotate.go @@ -0,0 +1,42 @@ +package commands + +import ( + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/logging" +) + +// ManifestAnnotateFlags define flags provided to the ManifestAnnotate +type ManifestAnnotateFlags struct { + os, arch, variant, osVersion string + features, osFeatures, annotations []string +} + +// ManifestAnnotate modifies a manifest list (Image index) and update the platform information for an image included in the manifest list. +func ManifestAnnotate(logger logging.Logger, pack PackClient) *cobra.Command { + var flags ManifestAnnotateFlags + + cmd := &cobra.Command{ + Use: "pack manifest annotate [OPTIONS] [...] [flags]", + Args: cobra.MatchAll(cobra.ExactArgs(2), cobra.OnlyValidArgs), + Short: "manifest annotate modifies a manifest list (Image index) and update the platform information for an image included in the manifest list.", + Example: `pack manifest annotate cnbs/sample-package:hello-universe-multiarch \ + cnbs/sample-package:hello-universe --arch amd64`, + Long: `manifest annotate modifies a manifest list (Image index) and update the platform information for an image included in the manifest list. + Sometimes a manifest list could reference an image that doesn't specify the architecture, The "annotate" command allows users to update those values before pushing the manifest list a registry`, + RunE: logError(logger, func(cmd *cobra.Command, args []string) error { + return nil + }), + } + + cmd.Flags().StringVar(&flags.os, "os", "", "Set the architecture") + cmd.Flags().StringVar(&flags.arch, "arch", "", "Set the architecture") + cmd.Flags().StringVar(&flags.variant, "variant", "", "Set the architecture") + cmd.Flags().StringVar(&flags.osVersion, "os-version", "", "override the os `version` of the specified image") + cmd.Flags().StringSliceVar(&flags.features, "features", nil, "override the `features` of the specified image") + cmd.Flags().StringSliceVar(&flags.osFeatures, "os-features", nil, "override the os `features` of the specified image") + cmd.Flags().StringSliceVar(&flags.annotations, "annotations", nil, "set an `annotation` for the specified image") + + AddHelpFlag(cmd, "annotate") + return cmd +} \ No newline at end of file diff --git a/internal/commands/manifest_annotate_test.go b/internal/commands/manifest_annotate_test.go new file mode 100644 index 0000000000..6320887605 --- /dev/null +++ b/internal/commands/manifest_annotate_test.go @@ -0,0 +1,100 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestAnnotateCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestAnnotateCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestAnnotateCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestAnnotate(logger, mockClient) + }) + + when("#ManifestAnnotate", func() { + when("no flags specified", func() { + + }) + when("when --os flags passed", func() { + + }) + when("when --arch flags passed", func() { + + }) + when("when --variant flags passed", func() { + + }) + when("when --os-version flags passed", func() { + + }) + when("when --features flags passed", func() { + + }) + when("when --os-features flags passed", func() { + + }) + when("when --annotations flags passed", func() { + + }) + when("when multiple flags passed", func() { + + }) + when("when no args passed", func() { + + }) + when("when manifest list reference is incorrect", func() { + + }) + when("when manifest reference is incorrect", func() { + + }) + when("when manifest passed in-place of manifest list on first arg", func() { + + }) + when("when manifest list is passed on second arg", func() { + + }) + when("when manifest is passed on second arg with --all option", func() { + + }) + when("when manifest list in locally available", func() { + + }) + when("when manifest is not locally available", func() { + + }) + when("when manifest is locally available passed", func() { + + }) + when("when multiple manifests passed", func() { + + }) + }) +} \ No newline at end of file From a76513dd831a11450a8f0f87eca98840fa5c6a47 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 02:59:56 +0000 Subject: [PATCH 43/62] WIP added manifest_create command Signed-off-by: WYGIN --- internal/commands/manifest_create.go | 92 +++++++++++++++++++ internal/commands/manifest_create_test.go | 106 ++++++++++++++++++++++ 2 files changed, 198 insertions(+) create mode 100644 internal/commands/manifest_create.go create mode 100644 internal/commands/manifest_create_test.go diff --git a/internal/commands/manifest_create.go b/internal/commands/manifest_create.go new file mode 100644 index 0000000000..e8fe471f78 --- /dev/null +++ b/internal/commands/manifest_create.go @@ -0,0 +1,92 @@ +package commands + +import ( + "fmt" + + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/client" + "github.com/buildpacks/pack/pkg/logging" +) + +// ManifestCreateFlags define flags provided to the ManifestCreate +type ManifestCreateFlags struct { + format, registry, os, arch string + insecure, publish, all, amend bool +} + +// ManifestCreate creates an image-index/image-list for a multi-arch image +func ManifestCreate(logger logging.Logger, pack PackClient) *cobra.Command { + var flags ManifestCreateFlags + + cmd := &cobra.Command{ + Use: "pack manifest create [ ... ] [flags]", + Args: cobra.MatchAll(cobra.MinimumNArgs(2), cobra.OnlyValidArgs), + Short: "manifest create generates a manifest list for a multi-arch image", + Example: `pack manifest create cnbs/sample-package:hello-multiarch-universe \ + cnbs/sample-package:hello-universe \ + cnbs/sample-package:hello-universe-windows`, + Long: `Create a manifest list or image index for the image to support muti architecture for the image, it create a new ManifestList or ImageIndex with the given name and adds the list of Manifests to the newly created ImageIndex or ManifestList + + If the already exists in the registry: pack will save a local copy of the remote manifest list, + If the doestn't exist in a registry: pack will create a local representation of the manifest list that will only save on the remote registry if the user publish it`, + RunE: logError(logger, func(cmd *cobra.Command, args []string) error { + imageIndex := args[0] + manifests := args[1:] + cmdFlags := cmd.Flags() + + if err := validateManifestCreateFlags(flags); err != nil { + return err + } + + if cmdFlags.Changed("insecure") { + flags.insecure = !flags.insecure + } + + if cmdFlags.Changed("publish") { + flags.publish = !flags.publish + } + + id, err := pack.CreateManifest(cmd.Context(), imageIndex, manifests, client.CreateManifestOptions{ + Format: flags.format, + Registry: flags.registry, + Insecure: flags.insecure, + Publish: flags.publish, + }) + + if err != nil { + return err + } + logger.Infof("Successfully created ImageIndex/ManifestList with imageID: '%s'", id) + + return nil + }), + } + + cmdFlags := cmd.Flags() + + cmdFlags.StringVarP(&flags.format, "format", "f", "v2s2", "Format to save image index as ('OCI' or 'V2S2') (default 'v2s2')") + cmdFlags.StringVarP(&flags.registry, "registry", "r", "", "Publish to registry") + cmdFlags.StringVar(&flags.os, "os", "", "If any of the specified images is a list/index, choose the one for `os`") + if err := cmdFlags.MarkHidden("os"); err != nil { + panic(fmt.Sprintf("error marking --os as hidden: %v", err)) + } + cmdFlags.StringVar(&flags.arch, "arch", "", "If any of the specified images is a list/index, choose the one for `arch`") + if err := cmdFlags.MarkHidden("arch"); err != nil { + panic(fmt.Sprintf("error marking --arch as hidden: %v", err)) + } + cmdFlags.BoolVar(&flags.insecure, "insecure", false, "Allow publishing to insecure registry") + if err := cmdFlags.MarkHidden("insecure"); err != nil { + panic(fmt.Sprintf("error marking insecure as hidden: %v", err)) + } + cmdFlags.BoolVar(&flags.publish, "publish", false, "Publish to registry") + cmdFlags.BoolVar(&flags.all, "all", false, "Add all of the list's images if the images to add are lists/index") + cmdFlags.BoolVar(&flags.amend, "amend", false, "Modify an existing list/index if one with the desired name already exists") + + AddHelpFlag(cmd, "create") + return cmd +} + +func validateManifestCreateFlags(flags ManifestCreateFlags) error { + return nil +} \ No newline at end of file diff --git a/internal/commands/manifest_create_test.go b/internal/commands/manifest_create_test.go new file mode 100644 index 0000000000..673dff4b45 --- /dev/null +++ b/internal/commands/manifest_create_test.go @@ -0,0 +1,106 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestCreateCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestCreateCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestCreateCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestCreate(logger, mockClient) + }) + + when("#ManifestCreate", func() { + when("no flags specified", func() { + + }) + when("when --all flags passed", func() { + + }) + when("when --os flags passed", func() { + + }) + when("when --arch flags passed", func() { + + }) + when("when --registry flags passed", func() { + + }) + when("when --format flags passed", func() { + + }) + when("when --format flags not passed", func() { + + }) + when("when --insecure flags passed", func() { + + }) + when("when --publish flags passed", func() { + + }) + when("when --ammend flags passed", func() { + + }) + when("when multiple flags passed", func() { + + }) + when("when no args passed", func() { + + }) + when("when manifest list reference is incorrect", func() { + + }) + when("when manifest reference is incorrect", func() { + + }) + when("when manifest list is passed on second arg", func() { + + }) + when("when manifest is passed on second arg with --all option", func() { + + }) + when("when manifest list is locally available", func() { + + }) + when("when manifest list is not locally available", func() { + + }) + when("when manifest is not locally available", func() { + + }) + when("when manifest is locally available passed", func() { + + }) + when("when multiple manifests passed", func() { + + }) + }) +} \ No newline at end of file From 128f17892b13cee32992616c29ac9711489dba1f Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:01:40 +0000 Subject: [PATCH 44/62] WIP added manifest_exists command Signed-off-by: WYGIN --- internal/commands/manifest_exists.go | 33 +++++++++++++ internal/commands/manifest_exists_test.go | 60 +++++++++++++++++++++++ 2 files changed, 93 insertions(+) create mode 100644 internal/commands/manifest_exists.go create mode 100644 internal/commands/manifest_exists_test.go diff --git a/internal/commands/manifest_exists.go b/internal/commands/manifest_exists.go new file mode 100644 index 0000000000..1d5c2062f7 --- /dev/null +++ b/internal/commands/manifest_exists.go @@ -0,0 +1,33 @@ +package commands + +import ( + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/logging" +) + +// ManifestDeleteFlags define flags provided to the ManifestDelete +// type ManifestDeleteFlags struct { +// } + +// ManifestExists checks if a manifest list exists in local storage +func ManifestExists(logger logging.Logger, pack PackClient) *cobra.Command { + // var flags ManifestDeleteFlags + + cmd := &cobra.Command{ + Use: "pack manifest exists [manifest-list]", + Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), + Short: "checks if a manifest list exists in local storage", + Example: `pack manifest exists cnbs/sample-package:hello-multiarch-universe`, + Long: `Checks if a manifest list exists in local storage`, + RunE: logError(logger, func(cmd *cobra.Command, args []string) error { + if err := pack.ExistsManifest(cmd.Context(), args[0]); err != nil { + return err + } + return nil + }), + } + + AddHelpFlag(cmd, "remove") + return cmd +} \ No newline at end of file diff --git a/internal/commands/manifest_exists_test.go b/internal/commands/manifest_exists_test.go new file mode 100644 index 0000000000..292b5429ba --- /dev/null +++ b/internal/commands/manifest_exists_test.go @@ -0,0 +1,60 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestExistsCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestExistsCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestExistsCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestExists(logger, mockClient) + }) + + when("#ManifestExists", func() { + when("only one arg is passed", func() { + + }) + when("when more than one arg passed", func() { + + }) + when("when passed arg is manifest list", func() { + it("if exists locally", func() { + + }) + it("if exists at registry only", func() { + + }) + }) + when("when passed arg is manifest", func() { + + }) + }) +} \ No newline at end of file From d238c644fcfa2f4e627119410b48b4e3b70523e8 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:03:47 +0000 Subject: [PATCH 45/62] WIP added manifest_inspect command Signed-off-by: WYGIN --- internal/commands/manifest_inspect.go | 31 ++++++++++++ internal/commands/manifest_inspect_test.go | 57 ++++++++++++++++++++++ 2 files changed, 88 insertions(+) create mode 100644 internal/commands/manifest_inspect.go create mode 100644 internal/commands/manifest_inspect_test.go diff --git a/internal/commands/manifest_inspect.go b/internal/commands/manifest_inspect.go new file mode 100644 index 0000000000..80bf3a52f6 --- /dev/null +++ b/internal/commands/manifest_inspect.go @@ -0,0 +1,31 @@ +package commands + +import ( + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/logging" +) + +// ManifestInspectFlags define flags provided to the ManifestInspect +// type ManifestInspectFlags struct { +// } + +// ManifestInspect shows the manifest information stored in local storage +func ManifestInspect(logger logging.Logger, pack PackClient) *cobra.Command { + // var flags ManifestInspectFlags + + cmd := &cobra.Command{ + Use: "pack manifest inspect [flags]", + Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), + Short: "manifest inspect shows the manifest information stored in local storage", + Example: `pack manifest inspect cnbs/sample-builder:multiarch`, + Long: `manifest inspect shows the manifest information stored in local storage. + The inspect command will help users to view how their local manifest list looks like`, + RunE: logError(logger, func(cmd *cobra.Command, args []string) error { + return nil + }), + } + + AddHelpFlag(cmd, "inspect") + return cmd +} \ No newline at end of file diff --git a/internal/commands/manifest_inspect_test.go b/internal/commands/manifest_inspect_test.go new file mode 100644 index 0000000000..fdfeea5605 --- /dev/null +++ b/internal/commands/manifest_inspect_test.go @@ -0,0 +1,57 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestInspectCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestInspectCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestInspectCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestInspect(logger, mockClient) + }) + + when("#ManifestInspect", func() { + when("inspect manifest", func() { + + }) + when("inspect manifest list", func() { + it("when available locally", func() { + + }) + it("when available at registry", func() { + + }) + it("by passing multiple manifest lists", func() { + + }) + }) + }) +} \ No newline at end of file From 9006ad4a91e11e43ea90a7364a84204403202e7b Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:05:59 +0000 Subject: [PATCH 46/62] WIP added manifest_push command Signed-off-by: WYGIN --- internal/commands/manifest_push.go | 59 +++++++++++++++++ internal/commands/manifest_push_test.go | 88 +++++++++++++++++++++++++ 2 files changed, 147 insertions(+) create mode 100644 internal/commands/manifest_push.go create mode 100644 internal/commands/manifest_push_test.go diff --git a/internal/commands/manifest_push.go b/internal/commands/manifest_push.go new file mode 100644 index 0000000000..2c2c8f3f0b --- /dev/null +++ b/internal/commands/manifest_push.go @@ -0,0 +1,59 @@ +package commands + +import ( + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/client" + "github.com/buildpacks/pack/pkg/logging" +) + +// ManifestPushFlags define flags provided to the ManifestPush +type ManifestPushFlags struct { + format string + insecure, purge, all, quite bool +} + +// ManifestPush pushes a manifest list (Image index) to a registry. +func ManifestPush(logger logging.Logger, pack PackClient) *cobra.Command { + var flags ManifestPushFlags + + cmd := &cobra.Command{ + Use: "pack manifest push [OPTIONS] [flags]", + Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs), + Short: "manifest push pushes a manifest list (Image index) to a registry.", + Example: `pack manifest push cnbs/sample-package:hello-multiarch-universe`, + Long: `manifest push pushes a manifest list (Image index) to a registry. + Once a manifest list is ready to be published into the registry, the push command can be used`, + RunE: logError(logger, func(cmd *cobra.Command, args []string) error { + if err := parseFalgs(flags); err != nil { + return err + } + + imageID, err := pack.PushManifest(cmd.Context(), args[0], client.PushManifestOptions{ + Format: flags.format, + Insecure: flags.insecure, + Purge: flags.purge, + }) + + if err != nil { + return err + } + + logger.Infof(imageID) + return nil + }), + } + + cmd.Flags().StringVarP(&flags.format, "format", "f", "", "Format to save image index as ('OCI' or 'V2S2') (default 'v2s2')") + cmd.Flags().BoolVar(&flags.insecure, "insecure", false, "Allow publishing to insecure registry") + cmd.Flags().BoolVar(&flags.purge, "purge", false, "Delete the manifest list or image index from local storage if pushing succeeds") + cmd.Flags().BoolVar(&flags.all, "all", false, "Also push the images in the list") + cmd.Flags().BoolVarP(&flags.quite, "quite", "q", false, "Also push the images in the list") + + AddHelpFlag(cmd, "push") + return cmd +} + +func parseFalgs(flags ManifestPushFlags) error { + return nil +} \ No newline at end of file diff --git a/internal/commands/manifest_push_test.go b/internal/commands/manifest_push_test.go new file mode 100644 index 0000000000..92656d0e1b --- /dev/null +++ b/internal/commands/manifest_push_test.go @@ -0,0 +1,88 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestPushCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestPushCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestPushCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestPush(logger, mockClient) + }) + + when("#ManifestPush", func() { + when("no flags specified", func() { + + }) + when("when --all flags passed", func() { + + }) + when("when --insecure flags passed", func() { + + }) + when("when --purge flags passed", func() { + + }) + when("when --quite flags passed", func() { + + }) + when("when --format flags passed", func() { + + }) + when("when multiple flags passed", func() { + + }) + when("when no args passed", func() { + + }) + when("when manifest list reference is incorrect", func() { + + }) + when("when manifest passed in-place of manifest list on first arg", func() { + + }) + when("when manifest list is passed", func() { + + }) + when("when manifest list is passed with --all option", func() { + + }) + when("when manifest list is locally available", func() { + + }) + when("when manifest list is not locally available", func() { + + }) + when("local images are included in manifest list when pushing", func() { + + }) + }) +} \ No newline at end of file From 11d416d062008fc12e3416f99e20ff7b34795e15 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:07:47 +0000 Subject: [PATCH 47/62] WIP added manifest_remove command Signed-off-by: WYGIN --- internal/commands/manifest_remove.go | 34 +++++++++++++ internal/commands/manifest_remove_test.go | 61 +++++++++++++++++++++++ 2 files changed, 95 insertions(+) create mode 100644 internal/commands/manifest_remove.go create mode 100644 internal/commands/manifest_remove_test.go diff --git a/internal/commands/manifest_remove.go b/internal/commands/manifest_remove.go new file mode 100644 index 0000000000..1b09cb493b --- /dev/null +++ b/internal/commands/manifest_remove.go @@ -0,0 +1,34 @@ +package commands + +import ( + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/logging" +) + +// ManifestDeleteFlags define flags provided to the ManifestDelete +// type ManifestDeleteFlags struct { +// } + +// ManifestDelete deletes one or more manifest lists from local storage +func ManifestDelete(logger logging.Logger, pack PackClient) *cobra.Command { + // var flags ManifestDeleteFlags + + cmd := &cobra.Command{ + Use: "pack manifest remove [manifest-list] [manifest-list...] [flags]", + Args: cobra.MatchAll(cobra.MinimumNArgs(1), cobra.OnlyValidArgs), + Short: "Delete one or more manifest lists from local storage", + Example: `pack manifest remove cnbs/sample-package:hello-multiarch-universe`, + Long: `Delete one or more manifest lists from local storage. + When a manifest list exits locally, users can remove existing images from a manifest list`, + RunE: logError(logger, func(cmd *cobra.Command, args []string) error { + if err := pack.DeleteManifest(cmd.Context(), args); err != nil { + return err + } + return nil + }), + } + + AddHelpFlag(cmd, "remove") + return cmd +} \ No newline at end of file diff --git a/internal/commands/manifest_remove_test.go b/internal/commands/manifest_remove_test.go new file mode 100644 index 0000000000..71b71e7bbf --- /dev/null +++ b/internal/commands/manifest_remove_test.go @@ -0,0 +1,61 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestDeleteCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestDeleteCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestDeleteCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestDelete(logger, mockClient) + }) + + when("#ManifestRemove", func() { + when("when no args passed", func() { + + }) + when("when manifest list reference is incorrect", func() { + + }) + when("when manifest passed in-place of manifest list on first arg", func() { + + }) + when("when manifest list is locally available", func() { + + }) + when("when manifest list is not locally available", func() { + + }) + when("when multiple manifests passed", func() { + + }) + }) +} \ No newline at end of file From 749b6fe1827c27e9b3d6efe36de5fd6372f4bf18 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:09:43 +0000 Subject: [PATCH 48/62] WIP added manifest_rm command Signed-off-by: WYGIN --- internal/commands/manifest_rm.go | 35 ++++++++++++ internal/commands/manifest_rm_test.go | 76 +++++++++++++++++++++++++++ 2 files changed, 111 insertions(+) create mode 100644 internal/commands/manifest_rm.go create mode 100644 internal/commands/manifest_rm_test.go diff --git a/internal/commands/manifest_rm.go b/internal/commands/manifest_rm.go new file mode 100644 index 0000000000..46ed6fe9ec --- /dev/null +++ b/internal/commands/manifest_rm.go @@ -0,0 +1,35 @@ +package commands + +import ( + "github.com/spf13/cobra" + + "github.com/buildpacks/pack/pkg/logging" +) + +// ManifestRemoveFlags define flags provided to the ManifestRemove +// type ManifestRemoveFlags struct { +// } + +// ManifestRemove will remove the specified image manifest if it is already referenced in the index +func ManifestRemove(logger logging.Logger, pack PackClient) *cobra.Command { + // var flags ManifestRemoveFlags + + cmd := &cobra.Command{ + Use: "pack manifest rm [manifest-list] [manifest] [manifest...] [flags]", + Args: cobra.MatchAll(cobra.MinimumNArgs(2), cobra.OnlyValidArgs), + Short: "manifest rm will remove the specified image manifest if it is already referenced in the index", + Example: `pack manifest rm cnbs/sample-package:hello-multiarch-universe \ + cnbs/sample-package:hello-universe-windows`, + Long: `manifest rm will remove the specified image manifest if it is already referenced in the index. + Sometimes users can just experiment with the feature locally and they want to discard all the local information created by pack. 'rm' command just delete the local manifest list`, + RunE: logError(logger, func(cmd *cobra.Command, args []string) error { + if err := pack.RemoveManifest(cmd.Context(), args[0], args[1:]); err != nil { + return err + } + return nil + }), + } + + AddHelpFlag(cmd, "rm") + return cmd +} \ No newline at end of file diff --git a/internal/commands/manifest_rm_test.go b/internal/commands/manifest_rm_test.go new file mode 100644 index 0000000000..e66d84f41c --- /dev/null +++ b/internal/commands/manifest_rm_test.go @@ -0,0 +1,76 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestRemoveCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestRemoveCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestRemoveCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestRemove(logger, mockClient) + }) + + when("#ManifestRm", func() { + when("when no args passed", func() { + + }) + when("when manifest list reference is incorrect", func() { + + }) + when("when manifest reference is incorrect", func() { + + }) + when("when manifest passed in-place of manifest list on first arg", func() { + + }) + when("when manifest list is passed on second arg", func() { + + }) + when("when manifest is passed on second arg", func() { + + }) + when("when manifest list is locally available", func() { + + }) + when("when manifest list is not locally available", func() { + + }) + when("when manifest is locally available", func() { + + }) + when("when manifest is not locally available", func() { + + }) + when("when multiple manifests passed", func() { + + }) + }) +} \ No newline at end of file From 8b25d63ea60486887d28d116cf5904a62e9e7be8 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:13:41 +0000 Subject: [PATCH 49/62] WIP added add_manifest client Signed-off-by: WYGIN --- internal/commands/manifest_test.go | 67 +++++++++++++++++++ pkg/client/add_manifest.go | 87 ++++++++++++++++++++++++ pkg/client/add_manifest_test.go | 103 +++++++++++++++++++++++++++++ 3 files changed, 257 insertions(+) create mode 100644 internal/commands/manifest_test.go create mode 100644 pkg/client/add_manifest.go create mode 100644 pkg/client/add_manifest_test.go diff --git a/internal/commands/manifest_test.go b/internal/commands/manifest_test.go new file mode 100644 index 0000000000..6313b2fdf7 --- /dev/null +++ b/internal/commands/manifest_test.go @@ -0,0 +1,67 @@ +package commands_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.NewManifestCommand(logger, mockClient) + }) + + when("#ManifestAdd", func() { + when("no flags specified", func() { + + }) + when("add is passed as flag", func() { + + }) + when("create is passed as flag", func() { + + }) + when("annotate is passed as flag", func() { + + }) + when("remove is passed as flag", func() { + + }) + when("inspect is passed as flag", func() { + + }) + when("rm is passed as flag", func() { + + }) + when("push is passed as flag", func() { + + }) + }) +} \ No newline at end of file diff --git a/pkg/client/add_manifest.go b/pkg/client/add_manifest.go new file mode 100644 index 0000000000..ab52ba87eb --- /dev/null +++ b/pkg/client/add_manifest.go @@ -0,0 +1,87 @@ +package client + +import ( + "context" + "fmt" + "strings" +) + +type ManifestAddOptions struct { + ManifestAnnotateOptions + All bool +} + +// AddManifest implements commands.PackClient. +func (c *Client) AddManifest(ctx context.Context, index string, image string, opts ManifestAddOptions) (indexID string, err error) { + ref, err := c.runtime.ParseReference(image) + if err != nil { + return + } + + imgIndex, err := c.runtime.LookupImageIndex(index) + if err != nil { + return + } + + digest, err := imgIndex.Add(ctx, ref, opts.All) + if err != nil { + if ref, _, err = c.imageFactory.FindImage(image); err != nil { + return indexID, fmt.Errorf("Error while trying to find image on local storage: %v", err) + } + digest, err = imgIndex.Add(ctx, ref, opts.All) + if err != nil { + return indexID, fmt.Errorf("Error while trying to add on manifest list: %v", err) + } + } + + if opts.OS != "" { + if _, err := imgIndex.Index.SetOS(digest, opts.OS); err != nil { + return indexID, err + } + } + + if opts.OSArch != "" { + if _, err := imgIndex.Index.SetArchitecture(digest, opts.OSArch); err != nil { + return indexID, err + } + } + + if opts.OSVariant != "" { + if _, err := imgIndex.Index.SetVariant(digest, opts.OSVariant); err != nil { + return indexID, err + } + } + + if opts.OSVersion != "" { + if _, err := imgIndex.Index.SetOSVersion(digest, opts.OSVersion); err != nil { + return indexID, err + } + } + + if len(opts.Features) != 0 { + if _, err := imgIndex.Index.SetFeatures(digest, opts.Features); err != nil { + return indexID, err + } + } + + if len(opts.Annotations) != 0 { + annotations := make(map[string]string) + for _, annotationSpec := range opts.Annotations { + spec := strings.SplitN(annotationSpec, "=", 2) + if len(spec) != 2 { + return indexID, fmt.Errorf("no value given for annotation %q", spec[0]) + } + annotations[spec[0]] = spec[1] + } + if err := imgIndex.Index.SetAnnotations(&digest, annotations); err != nil { + return err + } + } + + indexID, err = imgIndex.Index.Save(index, nil, "") + if err == nil { + fmt.Printf("%s: %s\n", indexID, digest.String()) + } + + return indexID, err +} \ No newline at end of file diff --git a/pkg/client/add_manifest_test.go b/pkg/client/add_manifest_test.go new file mode 100644 index 0000000000..7aa3101c36 --- /dev/null +++ b/pkg/client/add_manifest_test.go @@ -0,0 +1,103 @@ +package client_test + +import ( + "bytes" + "testing" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" + "github.com/golang/mock/gomock" + "github.com/heroku/color" + "github.com/sclevine/spec" + "github.com/sclevine/spec/report" + "github.com/spf13/cobra" +) + +func TestManifestAddCommand(t *testing.T) { + color.Disable(true) + defer color.Disable(false) + + spec.Run(t, "Commands", testManifestAddCommand, spec.Random(), spec.Report(report.Terminal{})) +} + +func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { + var ( + command *cobra.Command + logger *logging.LogWithWriters + outBuf bytes.Buffer + mockController *gomock.Controller + mockClient *testmocks.MockPackClient + ) + + it.Before(func() { + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + mockController = gomock.NewController(t) + mockClient = testmocks.NewMockPackClient(mockController) + + command = commands.ManifestAdd(logger, mockClient) + }) + + when("#AddManifest", func() { + when("no flags specified", func() { + + }) + when("when --all flags passed", func() { + + }) + when("when --os flags passed", func() { + + }) + when("when --arch flags passed", func() { + + }) + when("when --variant flags passed", func() { + + }) + when("when --os-version flags passed", func() { + + }) + when("when --features flags passed", func() { + + }) + when("when --os-features flags passed", func() { + + }) + when("when --annotations flags passed", func() { + + }) + when("when multiple flags passed", func() { + + }) + when("when no args passed", func() { + + }) + when("when manifest list reference is incorrect", func() { + + }) + when("when manifest reference is incorrect", func() { + + }) + when("when manifest passed in-place of manifest list on first arg", func() { + + }) + when("when manifest list is passed on second arg", func() { + + }) + when("when manifest is passed on second arg with --all option", func() { + + }) + when("when manifest list in locally available", func() { + + }) + when("when manifest is not locally available", func() { + + }) + when("when manifest is locally available passed", func() { + + }) + when("when multiple manifests passed", func() { + + }) + }) +} \ No newline at end of file From 6b8f798dfc3ed98e00f7871aab60ec1bba1c0621 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:15:33 +0000 Subject: [PATCH 50/62] WIP added annotate_manifest client Signed-off-by: WYGIN --- pkg/client/annotate_manifest.go | 83 ++++++++++++++++++++++++++++ pkg/client/annotate_manifest_test.go | 1 + 2 files changed, 84 insertions(+) create mode 100644 pkg/client/annotate_manifest.go create mode 100644 pkg/client/annotate_manifest_test.go diff --git a/pkg/client/annotate_manifest.go b/pkg/client/annotate_manifest.go new file mode 100644 index 0000000000..a279509a60 --- /dev/null +++ b/pkg/client/annotate_manifest.go @@ -0,0 +1,83 @@ +package client + +import ( + "context" + "fmt" + "strings" +) + +type ManifestAnnotateOptions struct { + OS, OSVersion, OSArch, OSVariant string + OSFeatures, Annotations, Features map[string]string +} + +// AnnotateManifest implements commands.PackClient. +func (c *Client) AnnotateManifest(ctx context.Context, name string, image string, opts ManifestAnnotateOptions) error { + manifestList, err := c.runtime.LookupImageIndex(name) + if err != nil { + return err + } + + digest, err := c.runtime.ParseDigest(image) + if err != nil { + ref, _, err := c.imageFactory.FindImage(image) + if err != nil { + return err + } + digest, err = c.runtime.ParseDigest(ref.Name()) + if err != nil { + return err + } + } + + if opts.OS != "" { + if err := manifestList.Index.SetOS(digest, opts.OS); err != nil { + return err + } + } + if opts.OSVersion != "" { + if err := manifestList.Index.SetOSVersion(digest, opts.OSVersion); err != nil { + return err + } + } + if len(opts.OSFeatures) != 0 { + if err := manifestList.Index.SetOSFeatures(digest, opts.OSFeatures); err != nil { + return err + } + } + if opts.OSArch != "" { + if err := manifestList.Index.SetArchitecture(digest, opts.OSArch); err != nil { + return err + } + } + if opts.OSVariant != "" { + if err := manifestList.Index.SetVariant(digest, opts.OSVariant); err != nil { + return err + } + } + if len(opts.Features) != 0 { + if err := manifestList.Index.SetFeatures(digest, opts.Features); err != nil { + return err + } + } + if len(opts.Annotations) != 0 { + annotations := make(map[string]string) + for _, annotationSpec := range opts.Annotations { + spec := strings.SplitN(annotationSpec, "=", 2) + if len(spec) != 2 { + return fmt.Errorf("no value given for annotation %q", spec[0]) + } + annotations[spec[0]] = spec[1] + } + if err := manifestList.Index.SetAnnotations(&digest, annotations); err != nil { + return err + } + } + + updatedListID, err := manifestList.Index.Save(name, nil, "") + if err == nil { + fmt.Printf("%s: %s\n", updatedListID, digest.String()) + } + + return nil +} \ No newline at end of file diff --git a/pkg/client/annotate_manifest_test.go b/pkg/client/annotate_manifest_test.go new file mode 100644 index 0000000000..e169c0b61a --- /dev/null +++ b/pkg/client/annotate_manifest_test.go @@ -0,0 +1 @@ +package client_test From 4aa93621d0492dc5971b4fa70f5577b99b614710 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:21:34 +0000 Subject: [PATCH 51/62] WIP added IndexFactory to Client Signed-off-by: WYGIN --- pkg/client/client.go | 73 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/pkg/client/client.go b/pkg/client/client.go index b3862cc4cc..b058d84182 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -20,10 +20,12 @@ import ( "path/filepath" "github.com/buildpacks/imgutil" + "github.com/buildpacks/imgutil/layout" "github.com/buildpacks/imgutil/local" "github.com/buildpacks/imgutil/remote" dockerClient "github.com/docker/docker/client" "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" "github.com/pkg/errors" "github.com/buildpacks/pack" @@ -70,6 +72,15 @@ type ImageFactory interface { // NewImage initializes an image object with required settings so that it // can be written either locally or to a registry. NewImage(repoName string, local bool, imageOS string) (imgutil.Image, error) + // FindImage locates the locally-stored image which corresponds to a given name. + FindImage(name string) (ref name.Reference, image imgutil.Image, err error) +} + +// IndexFactory is an interface representing the ability to create a ImageIndex/ManifestList. +type IndexFactory interface { + NewIndex(reponame string, opts imgutil.IndexOptions) (imgutil.Index, error) + // Fetch ManifestList from Registry with the given name + FetchIndex(name string) (imgutil.Index, error) } //go:generate mockgen -package testmocks -destination ../testmocks/mock_buildpack_downloader.go github.com/buildpacks/pack/pkg/client BuildpackDownloader @@ -90,6 +101,7 @@ type Client struct { keychain authn.Keychain imageFactory ImageFactory imageFetcher ImageFetcher + indexFactory IndexFactory downloader BlobDownloader lifecycleExecutor LifecycleExecutor buildpackDownloader BuildpackDownloader @@ -117,6 +129,13 @@ func WithImageFactory(f ImageFactory) Option { } } +// WithIndexFactory supply your own index factory +func WithIndexFactory(f IndexFactory) Option { + return func(c *Client) { + c.indexFactory = f + } +} + // WithFetcher supply your own Fetcher. // A Fetcher retrieves both local and remote images to make them available. func WithFetcher(f ImageFetcher) Option { @@ -225,6 +244,12 @@ func NewClient(opts ...Option) (*Client, error) { } } + if client.indexFactory == nil { + client.indexFactory = &indexFactory{ + keychain: client.keychain, + } + } + if client.buildpackDownloader == nil { client.buildpackDownloader = buildpack.NewDownloader( client.logger, @@ -273,3 +298,51 @@ func (f *imageFactory) NewImage(repoName string, daemon bool, imageOS string) (i return remote.NewImage(repoName, f.keychain, remote.WithDefaultPlatform(platform)) } + +func (f *imageFactory) LoadImage(repoName string, platform imgutil.Platform) (img imgutil.Image, err error) { + img, err = local.NewImage(repoName, f.dockerClient, local.WithDefaultPlatform(platform), local.WithPreviousImage(repoName)) + if err == nil { + return + } + + img, err = layout.NewImage(repoName, layout.WithPreviousImage(repoName), layout.WithDefaultPlatform(platform)) + if err == nil { + return + } + return nil, errors.Errorf("Image: '%s' not found", repoName) +} + +func (f *imageFactory) FindImage(repoName string) (ref name.Reference, img imgutil.Image, err error) { + img, err = local.NewImage(repoName, f.dockerClient, local.WithPreviousImage(repoName)) + if err == nil { + return + } + + img, err = layout.NewImage(repoName, layout.WithPreviousImage(repoName)) + if err == nil { + return + } + return ref, img, errors.Errorf("Image: '%s' not found", repoName) +} + +type indexFactory struct { + keychain authn.Keychain +} + +func (f *indexFactory) NewIndex(name string, opts imgutil.IndexOptions) (index imgutil.Index, err error) { + + index, err = remote.NewIndex(name, f.keychain, opts) + if err != nil { + if opts.MediaType == imgutil.MediaTypes.OCI { + return layout.NewIndex(name, opts) + } else { + return local.NewIndex(name, opts) + } + } + + return index, err +} + +func (f *indexFactory) FetchIndex(repoName string) (index imgutil.Index, err error) { + return remote.NewIndex(repoName, f.keychain, imgutil.IndexOptions{}) +} From 89763fd278d25250a8034185a6ecbb90af1ab3a8 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:23:46 +0000 Subject: [PATCH 52/62] WIP added create_manifest client Signed-off-by: WYGIN --- pkg/client/create_manifest.go | 70 ++++++++++++++++++++++++++++++ pkg/client/create_manifest_test.go | 1 + 2 files changed, 71 insertions(+) create mode 100644 pkg/client/create_manifest.go create mode 100644 pkg/client/create_manifest_test.go diff --git a/pkg/client/create_manifest.go b/pkg/client/create_manifest.go new file mode 100644 index 0000000000..557df033eb --- /dev/null +++ b/pkg/client/create_manifest.go @@ -0,0 +1,70 @@ +package client + +import ( + "context" + "errors" + "fmt" + + "github.com/buildpacks/imgutil" + + packErrors "github.com/buildpacks/pack/pkg/errors" +) + +type CreateManifestOptions struct { + Format, Registry string + Insecure, Publish, amend, all bool +} + +// CreateManifest implements commands.PackClient. +func (c *Client) CreateManifest(ctx context.Context, name string, images []string, opts CreateManifestOptions) (imageID string, err error) { + index, err := c.indexFactory.NewIndex(name, parseOptsToIndexOptions(opts)) + if err != nil { + return + } + index.Create() + if err != nil { + return + } + if imageID, err = index.Save(name, c.runtime.ImageType(opts.Format)); err != nil { + if errors.Is(err, packErrors.ErrDuplicateName) && opts.amend { + _, err := c.runtime.LookupImageIndex(name) + if err != nil { + fmt.Printf("no list named %q found: %v", name, err) + } + + if index == nil { + return imageID, fmt.Errorf("--amend specified but no matching manifest list found with name %q", name) + } + } else { + return + } + } + + for _, img := range images { + ref, err := c.runtime.ParseReference(img) + if err != nil { + return imageID, err + } + if localRef, _, err := c.imageFactory.FindImage(img); err == nil { + ref = localRef + } + if _, err = index.Add(ctx, ref, opts.all); err != nil { + return imageID, err + } + } + + imageID, err = index.Save(name, c.runtime.ImageType(opts.Format)) + if err == nil { + fmt.Printf("%s\n", imageID) + } + + if opts.Publish { + + } + + return imageID, err +} + +func parseOptsToIndexOptions(opts CreateManifestOptions) (idxOpts imgutil.IndexOptions) { + return idxOpts +} \ No newline at end of file diff --git a/pkg/client/create_manifest_test.go b/pkg/client/create_manifest_test.go new file mode 100644 index 0000000000..e169c0b61a --- /dev/null +++ b/pkg/client/create_manifest_test.go @@ -0,0 +1 @@ +package client_test From a5b1faaa8afeed670c0ddd75ca409032ca326c8a Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:25:26 +0000 Subject: [PATCH 53/62] WIP added delete_manifest client Signed-off-by: WYGIN --- pkg/client/delete_manifest.go | 12 ++++++++++++ pkg/client/delete_manifest_test.go | 1 + 2 files changed, 13 insertions(+) create mode 100644 pkg/client/delete_manifest.go create mode 100644 pkg/client/delete_manifest_test.go diff --git a/pkg/client/delete_manifest.go b/pkg/client/delete_manifest.go new file mode 100644 index 0000000000..ac72dd6f46 --- /dev/null +++ b/pkg/client/delete_manifest.go @@ -0,0 +1,12 @@ +package client + +import ( + "context" + // "fmt" + // "strings" +) + +// DeleteManifest implements commands.PackClient. +func (c *Client) DeleteManifest(ctx context.Context, names []string) error { + return c.runtime.RemoveManifests(ctx, names) +} \ No newline at end of file diff --git a/pkg/client/delete_manifest_test.go b/pkg/client/delete_manifest_test.go new file mode 100644 index 0000000000..e169c0b61a --- /dev/null +++ b/pkg/client/delete_manifest_test.go @@ -0,0 +1 @@ +package client_test From 6090d118bfa52b760b4ca8c5227231b81d4f0f72 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:27:01 +0000 Subject: [PATCH 54/62] WIP added exists_manifest client Signed-off-by: WYGIN --- pkg/client/exists_manifest.go | 17 +++++++++++++++++ pkg/client/exists_manifest_test.go | 1 + 2 files changed, 18 insertions(+) create mode 100644 pkg/client/exists_manifest.go create mode 100644 pkg/client/exists_manifest_test.go diff --git a/pkg/client/exists_manifest.go b/pkg/client/exists_manifest.go new file mode 100644 index 0000000000..f9530040a1 --- /dev/null +++ b/pkg/client/exists_manifest.go @@ -0,0 +1,17 @@ +package client + +import ( + "context" + + // "github.com/buildpacks/imgutil" + "github.com/pkg/errors" +) + +func (c *Client) ExistsManifest(ctx context.Context, image string) error { + + if _, err := c.runtime.LookupImageIndex(image); err != nil { + return errors.Errorf("image '%s' is not found", image) + } + + return nil +} \ No newline at end of file diff --git a/pkg/client/exists_manifest_test.go b/pkg/client/exists_manifest_test.go new file mode 100644 index 0000000000..e169c0b61a --- /dev/null +++ b/pkg/client/exists_manifest_test.go @@ -0,0 +1 @@ +package client_test From b8f31861fb78dbff2428241c720d714d68844f93 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:29:04 +0000 Subject: [PATCH 55/62] WIP added inspect_manifest client Signed-off-by: WYGIN --- pkg/client/inspect_manifest.go | 62 +++++++++++++++++++++++++++++ pkg/client/inspect_manifest_test.go | 1 + 2 files changed, 63 insertions(+) create mode 100644 pkg/client/inspect_manifest.go create mode 100644 pkg/client/inspect_manifest_test.go diff --git a/pkg/client/inspect_manifest.go b/pkg/client/inspect_manifest.go new file mode 100644 index 0000000000..bb73d24c2a --- /dev/null +++ b/pkg/client/inspect_manifest.go @@ -0,0 +1,62 @@ +package client + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + + "github.com/pkg/errors" + + packErrors "github.com/buildpacks/pack/pkg/errors" +) + +type InspectManifestOptions struct { +} + +// InspectManifest implements commands.PackClient. +func (c *Client) InspectManifest(ctx context.Context, name string, opts InspectManifestOptions) error { + printManifest := func(manifest []byte) error { + var b bytes.Buffer + err := json.Indent(&b, manifest, "", " ") + if err != nil { + return fmt.Errorf("rendering manifest for display: %w", err) + } + + fmt.Printf("%s\n", b.String()) + return nil + } + + // Before doing a remote lookup, attempt to resolve the manifest list + // locally. + manifestList, err := c.runtime.LookupImageIndex(name) + if err == nil { + schema2List, err := manifestList.Index.Inspect() + if err != nil { + rawSchema2List, err := json.Marshal(schema2List) + if err != nil { + return err + } + + return printManifest(rawSchema2List) + } + if !errors.Is(err, packErrors.ErrIndexUnknown) && !errors.Is(err, packErrors.ErrNotAddManifestList) { + return err + } + + _, err = c.runtime.ParseReference(name) + if err != nil { + fmt.Printf("error parsing reference to image %q: %v", name, err) + } + + index, err := c.indexFactory.FetchIndex(name) + + if err != nil { + return err + } + + return printManifest(index) + } + + return fmt.Errorf("unable to locate manifest list locally or at registry") +} \ No newline at end of file diff --git a/pkg/client/inspect_manifest_test.go b/pkg/client/inspect_manifest_test.go new file mode 100644 index 0000000000..e169c0b61a --- /dev/null +++ b/pkg/client/inspect_manifest_test.go @@ -0,0 +1 @@ +package client_test From b9dd8a479f0b95449f19916e7a999f0d5e53ef45 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:30:41 +0000 Subject: [PATCH 56/62] WIP added push_manifest client Signed-off-by: WYGIN --- pkg/client/push_manifest.go | 32 ++++++++++++++++++++++++++++++++ pkg/client/push_manifest_test.go | 1 + 2 files changed, 33 insertions(+) create mode 100644 pkg/client/push_manifest.go create mode 100644 pkg/client/push_manifest_test.go diff --git a/pkg/client/push_manifest.go b/pkg/client/push_manifest.go new file mode 100644 index 0000000000..e4ecbb4cd0 --- /dev/null +++ b/pkg/client/push_manifest.go @@ -0,0 +1,32 @@ +package client + +import ( + "context" + + runtime "github.com/buildpacks/pack/internal/runtime" +) + +type PushManifestOptions struct { + Format string + Insecure, Purge bool +} + +// PushManifest implements commands.PackClient. +func (c *Client) PushManifest(ctx context.Context, index string, opts PushManifestOptions) (imageID string, err error) { + manifestList, err := c.runtime.LookupImageIndex(index) + if err != nil { + return + } + + _, err = manifestList.Push(ctx, parseFalgsForImgUtil(opts)) + + if err == nil && opts.Purge { + c.runtime.RemoveManifests(ctx, []string{index}) + } + + return imageID, err +} + +func parseFalgsForImgUtil(opts PushManifestOptions) (idxOptions runtime.PushOptions) { + return idxOptions +} \ No newline at end of file diff --git a/pkg/client/push_manifest_test.go b/pkg/client/push_manifest_test.go new file mode 100644 index 0000000000..e169c0b61a --- /dev/null +++ b/pkg/client/push_manifest_test.go @@ -0,0 +1 @@ +package client_test From 54c3c6c2c75003f2730265a9ef60649402cea355 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:32:30 +0000 Subject: [PATCH 57/62] WIP added remove_manifest client Signed-off-by: WYGIN --- pkg/client/remove_manifest.go | 27 +++++++++++++++++++++++++++ pkg/client/remove_manifest_test.go | 1 + 2 files changed, 28 insertions(+) create mode 100644 pkg/client/remove_manifest.go create mode 100644 pkg/client/remove_manifest_test.go diff --git a/pkg/client/remove_manifest.go b/pkg/client/remove_manifest.go new file mode 100644 index 0000000000..f6319c5dc5 --- /dev/null +++ b/pkg/client/remove_manifest.go @@ -0,0 +1,27 @@ +package client + +import ( + "context" + "fmt" +) + +// RemoveManifest implements commands.PackClient. +func (c *Client) RemoveManifest(ctx context.Context, name string, images []string) error { + imgIndex, err := c.runtime.LookupImageIndex(name) + if err != nil { + return err + } + + for _, image := range images { + _, err := c.runtime.ParseReference(image) + if err != nil { + fmt.Errorf(`Invalid instance "%s": %v`, image, err) + } + if err := imgIndex.Remove(image); err != nil { + return err + } + fmt.Printf("Successfully removed %s from %s", image, name) + } + + return nil +} \ No newline at end of file diff --git a/pkg/client/remove_manifest_test.go b/pkg/client/remove_manifest_test.go new file mode 100644 index 0000000000..e169c0b61a --- /dev/null +++ b/pkg/client/remove_manifest_test.go @@ -0,0 +1 @@ +package client_test From 553f9273f9878581905f619c33203f7836c242cd Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 03:33:42 +0000 Subject: [PATCH 58/62] WIP added errors Signed-off-by: WYGIN Date: Thu Nov 9 03:33:42 2023 +0000 --- pkg/errors/errors.go | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 pkg/errors/errors.go diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go new file mode 100644 index 0000000000..2d6943cf86 --- /dev/null +++ b/pkg/errors/errors.go @@ -0,0 +1,9 @@ +package errors + +import "errors" + +var ( + ErrDuplicateName = errors.New("Image/ImageIndex with the given name exists") + ErrIndexUnknown = errors.New("cannot find Image Index with the given name") + ErrNotAddManifestList = errors.New("error while adding ImageIndex to the list") +) \ No newline at end of file From 4bbb4834c73c0a5d6fbb2689121bd2f2ec315745 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 09:31:42 +0000 Subject: [PATCH 59/62] WIP generate mock code with 'make generate' Signed-off-by: WYGIN --- internal/commands/manifest.go | 2 +- internal/commands/manifest_add.go | 2 +- internal/commands/manifest_add_test.go | 9 +- internal/commands/manifest_annotate.go | 2 +- internal/commands/manifest_annotate_test.go | 9 +- internal/commands/manifest_create.go | 2 +- internal/commands/manifest_create_test.go | 9 +- internal/commands/manifest_exists.go | 2 +- internal/commands/manifest_exists_test.go | 9 +- internal/commands/manifest_inspect.go | 2 +- internal/commands/manifest_inspect_test.go | 9 +- internal/commands/manifest_push.go | 2 +- internal/commands/manifest_push_test.go | 9 +- internal/commands/manifest_remove.go | 2 +- internal/commands/manifest_remove_test.go | 9 +- internal/commands/manifest_rm.go | 2 +- internal/commands/manifest_rm_test.go | 9 +- internal/commands/manifest_test.go | 9 +- .../commands/testmocks/mock_pack_client.go | 115 ++++++++++++++++++ pkg/client/add_manifest.go | 35 +++--- pkg/client/add_manifest_test.go | 9 +- pkg/client/annotate_manifest.go | 28 +++-- pkg/client/client.go | 2 + pkg/client/create_manifest.go | 11 +- pkg/client/delete_manifest.go | 15 ++- pkg/client/exists_manifest.go | 6 +- pkg/client/inspect_manifest.go | 7 +- pkg/client/push_manifest.go | 10 +- pkg/client/remove_manifest.go | 8 +- pkg/errors/errors.go | 2 +- pkg/testmocks/mock_docker_client.go | 8 +- 31 files changed, 245 insertions(+), 110 deletions(-) diff --git a/internal/commands/manifest.go b/internal/commands/manifest.go index 9b2108a449..d7f671148a 100644 --- a/internal/commands/manifest.go +++ b/internal/commands/manifest.go @@ -23,4 +23,4 @@ func NewManifestCommand(logger logging.Logger, client PackClient) *cobra.Command AddHelpFlag(cmd, "manifest") return cmd -} \ No newline at end of file +} diff --git a/internal/commands/manifest_add.go b/internal/commands/manifest_add.go index 7b85b0d662..05f56a4481 100644 --- a/internal/commands/manifest_add.go +++ b/internal/commands/manifest_add.go @@ -58,4 +58,4 @@ func ManifestAdd(logger logging.Logger, pack PackClient) *cobra.Command { func validateManifestAddFlags(flags ManifestAddFlags) error { return nil -} \ No newline at end of file +} diff --git a/internal/commands/manifest_add_test.go b/internal/commands/manifest_add_test.go index 8e419bb4f9..b04b9fe8d9 100644 --- a/internal/commands/manifest_add_test.go +++ b/internal/commands/manifest_add_test.go @@ -4,14 +4,15 @@ import ( "bytes" "testing" - "github.com/buildpacks/pack/internal/commands" - "github.com/buildpacks/pack/internal/commands/testmocks" - "github.com/buildpacks/pack/pkg/logging" "github.com/golang/mock/gomock" "github.com/heroku/color" "github.com/sclevine/spec" "github.com/sclevine/spec/report" "github.com/spf13/cobra" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" ) func TestManifestAddCommand(t *testing.T) { @@ -100,4 +101,4 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { }) }) -} \ No newline at end of file +} diff --git a/internal/commands/manifest_annotate.go b/internal/commands/manifest_annotate.go index 268721b41f..81f0b25561 100644 --- a/internal/commands/manifest_annotate.go +++ b/internal/commands/manifest_annotate.go @@ -39,4 +39,4 @@ func ManifestAnnotate(logger logging.Logger, pack PackClient) *cobra.Command { AddHelpFlag(cmd, "annotate") return cmd -} \ No newline at end of file +} diff --git a/internal/commands/manifest_annotate_test.go b/internal/commands/manifest_annotate_test.go index 6320887605..29d53f20d4 100644 --- a/internal/commands/manifest_annotate_test.go +++ b/internal/commands/manifest_annotate_test.go @@ -4,14 +4,15 @@ import ( "bytes" "testing" - "github.com/buildpacks/pack/internal/commands" - "github.com/buildpacks/pack/internal/commands/testmocks" - "github.com/buildpacks/pack/pkg/logging" "github.com/golang/mock/gomock" "github.com/heroku/color" "github.com/sclevine/spec" "github.com/sclevine/spec/report" "github.com/spf13/cobra" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" ) func TestManifestAnnotateCommand(t *testing.T) { @@ -97,4 +98,4 @@ func testManifestAnnotateCommand(t *testing.T, when spec.G, it spec.S) { }) }) -} \ No newline at end of file +} diff --git a/internal/commands/manifest_create.go b/internal/commands/manifest_create.go index e8fe471f78..b42c779618 100644 --- a/internal/commands/manifest_create.go +++ b/internal/commands/manifest_create.go @@ -89,4 +89,4 @@ func ManifestCreate(logger logging.Logger, pack PackClient) *cobra.Command { func validateManifestCreateFlags(flags ManifestCreateFlags) error { return nil -} \ No newline at end of file +} diff --git a/internal/commands/manifest_create_test.go b/internal/commands/manifest_create_test.go index 673dff4b45..1d8b6a955c 100644 --- a/internal/commands/manifest_create_test.go +++ b/internal/commands/manifest_create_test.go @@ -4,14 +4,15 @@ import ( "bytes" "testing" - "github.com/buildpacks/pack/internal/commands" - "github.com/buildpacks/pack/internal/commands/testmocks" - "github.com/buildpacks/pack/pkg/logging" "github.com/golang/mock/gomock" "github.com/heroku/color" "github.com/sclevine/spec" "github.com/sclevine/spec/report" "github.com/spf13/cobra" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" ) func TestManifestCreateCommand(t *testing.T) { @@ -103,4 +104,4 @@ func testManifestCreateCommand(t *testing.T, when spec.G, it spec.S) { }) }) -} \ No newline at end of file +} diff --git a/internal/commands/manifest_exists.go b/internal/commands/manifest_exists.go index 1d5c2062f7..5cf9b9462e 100644 --- a/internal/commands/manifest_exists.go +++ b/internal/commands/manifest_exists.go @@ -30,4 +30,4 @@ func ManifestExists(logger logging.Logger, pack PackClient) *cobra.Command { AddHelpFlag(cmd, "remove") return cmd -} \ No newline at end of file +} diff --git a/internal/commands/manifest_exists_test.go b/internal/commands/manifest_exists_test.go index 292b5429ba..8d69067c0f 100644 --- a/internal/commands/manifest_exists_test.go +++ b/internal/commands/manifest_exists_test.go @@ -4,14 +4,15 @@ import ( "bytes" "testing" - "github.com/buildpacks/pack/internal/commands" - "github.com/buildpacks/pack/internal/commands/testmocks" - "github.com/buildpacks/pack/pkg/logging" "github.com/golang/mock/gomock" "github.com/heroku/color" "github.com/sclevine/spec" "github.com/sclevine/spec/report" "github.com/spf13/cobra" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" ) func TestManifestExistsCommand(t *testing.T) { @@ -57,4 +58,4 @@ func testManifestExistsCommand(t *testing.T, when spec.G, it spec.S) { }) }) -} \ No newline at end of file +} diff --git a/internal/commands/manifest_inspect.go b/internal/commands/manifest_inspect.go index 80bf3a52f6..8e71b4a7b3 100644 --- a/internal/commands/manifest_inspect.go +++ b/internal/commands/manifest_inspect.go @@ -28,4 +28,4 @@ func ManifestInspect(logger logging.Logger, pack PackClient) *cobra.Command { AddHelpFlag(cmd, "inspect") return cmd -} \ No newline at end of file +} diff --git a/internal/commands/manifest_inspect_test.go b/internal/commands/manifest_inspect_test.go index fdfeea5605..2b7328aca7 100644 --- a/internal/commands/manifest_inspect_test.go +++ b/internal/commands/manifest_inspect_test.go @@ -4,14 +4,15 @@ import ( "bytes" "testing" - "github.com/buildpacks/pack/internal/commands" - "github.com/buildpacks/pack/internal/commands/testmocks" - "github.com/buildpacks/pack/pkg/logging" "github.com/golang/mock/gomock" "github.com/heroku/color" "github.com/sclevine/spec" "github.com/sclevine/spec/report" "github.com/spf13/cobra" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" ) func TestManifestInspectCommand(t *testing.T) { @@ -54,4 +55,4 @@ func testManifestInspectCommand(t *testing.T, when spec.G, it spec.S) { }) }) }) -} \ No newline at end of file +} diff --git a/internal/commands/manifest_push.go b/internal/commands/manifest_push.go index 2c2c8f3f0b..bd00f60092 100644 --- a/internal/commands/manifest_push.go +++ b/internal/commands/manifest_push.go @@ -56,4 +56,4 @@ func ManifestPush(logger logging.Logger, pack PackClient) *cobra.Command { func parseFalgs(flags ManifestPushFlags) error { return nil -} \ No newline at end of file +} diff --git a/internal/commands/manifest_push_test.go b/internal/commands/manifest_push_test.go index 92656d0e1b..5e5dd388c8 100644 --- a/internal/commands/manifest_push_test.go +++ b/internal/commands/manifest_push_test.go @@ -4,14 +4,15 @@ import ( "bytes" "testing" - "github.com/buildpacks/pack/internal/commands" - "github.com/buildpacks/pack/internal/commands/testmocks" - "github.com/buildpacks/pack/pkg/logging" "github.com/golang/mock/gomock" "github.com/heroku/color" "github.com/sclevine/spec" "github.com/sclevine/spec/report" "github.com/spf13/cobra" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" ) func TestManifestPushCommand(t *testing.T) { @@ -85,4 +86,4 @@ func testManifestPushCommand(t *testing.T, when spec.G, it spec.S) { }) }) -} \ No newline at end of file +} diff --git a/internal/commands/manifest_remove.go b/internal/commands/manifest_remove.go index 1b09cb493b..e4998b72d9 100644 --- a/internal/commands/manifest_remove.go +++ b/internal/commands/manifest_remove.go @@ -31,4 +31,4 @@ func ManifestDelete(logger logging.Logger, pack PackClient) *cobra.Command { AddHelpFlag(cmd, "remove") return cmd -} \ No newline at end of file +} diff --git a/internal/commands/manifest_remove_test.go b/internal/commands/manifest_remove_test.go index 71b71e7bbf..340dd46616 100644 --- a/internal/commands/manifest_remove_test.go +++ b/internal/commands/manifest_remove_test.go @@ -4,14 +4,15 @@ import ( "bytes" "testing" - "github.com/buildpacks/pack/internal/commands" - "github.com/buildpacks/pack/internal/commands/testmocks" - "github.com/buildpacks/pack/pkg/logging" "github.com/golang/mock/gomock" "github.com/heroku/color" "github.com/sclevine/spec" "github.com/sclevine/spec/report" "github.com/spf13/cobra" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" ) func TestManifestDeleteCommand(t *testing.T) { @@ -58,4 +59,4 @@ func testManifestDeleteCommand(t *testing.T, when spec.G, it spec.S) { }) }) -} \ No newline at end of file +} diff --git a/internal/commands/manifest_rm.go b/internal/commands/manifest_rm.go index 46ed6fe9ec..99a0af17bc 100644 --- a/internal/commands/manifest_rm.go +++ b/internal/commands/manifest_rm.go @@ -32,4 +32,4 @@ func ManifestRemove(logger logging.Logger, pack PackClient) *cobra.Command { AddHelpFlag(cmd, "rm") return cmd -} \ No newline at end of file +} diff --git a/internal/commands/manifest_rm_test.go b/internal/commands/manifest_rm_test.go index e66d84f41c..0049ec2e1d 100644 --- a/internal/commands/manifest_rm_test.go +++ b/internal/commands/manifest_rm_test.go @@ -4,14 +4,15 @@ import ( "bytes" "testing" - "github.com/buildpacks/pack/internal/commands" - "github.com/buildpacks/pack/internal/commands/testmocks" - "github.com/buildpacks/pack/pkg/logging" "github.com/golang/mock/gomock" "github.com/heroku/color" "github.com/sclevine/spec" "github.com/sclevine/spec/report" "github.com/spf13/cobra" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" ) func TestManifestRemoveCommand(t *testing.T) { @@ -73,4 +74,4 @@ func testManifestRemoveCommand(t *testing.T, when spec.G, it spec.S) { }) }) -} \ No newline at end of file +} diff --git a/internal/commands/manifest_test.go b/internal/commands/manifest_test.go index 6313b2fdf7..ebdb2d4de6 100644 --- a/internal/commands/manifest_test.go +++ b/internal/commands/manifest_test.go @@ -4,14 +4,15 @@ import ( "bytes" "testing" - "github.com/buildpacks/pack/internal/commands" - "github.com/buildpacks/pack/internal/commands/testmocks" - "github.com/buildpacks/pack/pkg/logging" "github.com/golang/mock/gomock" "github.com/heroku/color" "github.com/sclevine/spec" "github.com/sclevine/spec/report" "github.com/spf13/cobra" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" ) func TestManifestCommand(t *testing.T) { @@ -64,4 +65,4 @@ func testManifestCommand(t *testing.T, when spec.G, it spec.S) { }) }) -} \ No newline at end of file +} diff --git a/internal/commands/testmocks/mock_pack_client.go b/internal/commands/testmocks/mock_pack_client.go index f49b92def9..957a80a13d 100644 --- a/internal/commands/testmocks/mock_pack_client.go +++ b/internal/commands/testmocks/mock_pack_client.go @@ -36,6 +36,35 @@ func (m *MockPackClient) EXPECT() *MockPackClientMockRecorder { return m.recorder } +// AddManifest mocks base method. +func (m *MockPackClient) AddManifest(arg0 context.Context, arg1, arg2 string, arg3 client.ManifestAddOptions) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddManifest", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AddManifest indicates an expected call of AddManifest. +func (mr *MockPackClientMockRecorder) AddManifest(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddManifest", reflect.TypeOf((*MockPackClient)(nil).AddManifest), arg0, arg1, arg2, arg3) +} + +// AnnotateManifest mocks base method. +func (m *MockPackClient) AnnotateManifest(arg0 context.Context, arg1, arg2 string, arg3 client.ManifestAnnotateOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AnnotateManifest", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// AnnotateManifest indicates an expected call of AnnotateManifest. +func (mr *MockPackClientMockRecorder) AnnotateManifest(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AnnotateManifest", reflect.TypeOf((*MockPackClient)(nil).AnnotateManifest), arg0, arg1, arg2, arg3) +} + // Build mocks base method. func (m *MockPackClient) Build(arg0 context.Context, arg1 client.BuildOptions) error { m.ctrl.T.Helper() @@ -64,6 +93,35 @@ func (mr *MockPackClientMockRecorder) CreateBuilder(arg0, arg1 interface{}) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateBuilder", reflect.TypeOf((*MockPackClient)(nil).CreateBuilder), arg0, arg1) } +// CreateManifest mocks base method. +func (m *MockPackClient) CreateManifest(arg0 context.Context, arg1 string, arg2 []string, arg3 client.CreateManifestOptions) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateManifest", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// CreateManifest indicates an expected call of CreateManifest. +func (mr *MockPackClientMockRecorder) CreateManifest(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateManifest", reflect.TypeOf((*MockPackClient)(nil).CreateManifest), arg0, arg1, arg2, arg3) +} + +// DeleteManifest mocks base method. +func (m *MockPackClient) DeleteManifest(arg0 context.Context, arg1 []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteManifest", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteManifest indicates an expected call of DeleteManifest. +func (mr *MockPackClientMockRecorder) DeleteManifest(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteManifest", reflect.TypeOf((*MockPackClient)(nil).DeleteManifest), arg0, arg1) +} + // DownloadSBOM mocks base method. func (m *MockPackClient) DownloadSBOM(arg0 string, arg1 client.DownloadSBOMOptions) error { m.ctrl.T.Helper() @@ -78,6 +136,20 @@ func (mr *MockPackClientMockRecorder) DownloadSBOM(arg0, arg1 interface{}) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DownloadSBOM", reflect.TypeOf((*MockPackClient)(nil).DownloadSBOM), arg0, arg1) } +// ExistsManifest mocks base method. +func (m *MockPackClient) ExistsManifest(arg0 context.Context, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ExistsManifest", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// ExistsManifest indicates an expected call of ExistsManifest. +func (mr *MockPackClientMockRecorder) ExistsManifest(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ExistsManifest", reflect.TypeOf((*MockPackClient)(nil).ExistsManifest), arg0, arg1) +} + // InspectBuilder mocks base method. func (m *MockPackClient) InspectBuilder(arg0 string, arg1 bool, arg2 ...client.BuilderInspectionModifier) (*client.BuilderInfo, error) { m.ctrl.T.Helper() @@ -143,6 +215,20 @@ func (mr *MockPackClientMockRecorder) InspectImage(arg0, arg1 interface{}) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InspectImage", reflect.TypeOf((*MockPackClient)(nil).InspectImage), arg0, arg1) } +// InspectManifest mocks base method. +func (m *MockPackClient) InspectManifest(arg0 context.Context, arg1 string, arg2 client.InspectManifestOptions) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InspectManifest", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// InspectManifest indicates an expected call of InspectManifest. +func (mr *MockPackClientMockRecorder) InspectManifest(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InspectManifest", reflect.TypeOf((*MockPackClient)(nil).InspectManifest), arg0, arg1, arg2) +} + // NewBuildpack mocks base method. func (m *MockPackClient) NewBuildpack(arg0 context.Context, arg1 client.NewBuildpackOptions) error { m.ctrl.T.Helper() @@ -199,6 +285,21 @@ func (mr *MockPackClientMockRecorder) PullBuildpack(arg0, arg1 interface{}) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PullBuildpack", reflect.TypeOf((*MockPackClient)(nil).PullBuildpack), arg0, arg1) } +// PushManifest mocks base method. +func (m *MockPackClient) PushManifest(arg0 context.Context, arg1 string, arg2 client.PushManifestOptions) (string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "PushManifest", arg0, arg1, arg2) + ret0, _ := ret[0].(string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// PushManifest indicates an expected call of PushManifest. +func (mr *MockPackClientMockRecorder) PushManifest(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "PushManifest", reflect.TypeOf((*MockPackClient)(nil).PushManifest), arg0, arg1, arg2) +} + // Rebase mocks base method. func (m *MockPackClient) Rebase(arg0 context.Context, arg1 client.RebaseOptions) error { m.ctrl.T.Helper() @@ -227,6 +328,20 @@ func (mr *MockPackClientMockRecorder) RegisterBuildpack(arg0, arg1 interface{}) return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterBuildpack", reflect.TypeOf((*MockPackClient)(nil).RegisterBuildpack), arg0, arg1) } +// RemoveManifest mocks base method. +func (m *MockPackClient) RemoveManifest(arg0 context.Context, arg1 string, arg2 []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RemoveManifest", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// RemoveManifest indicates an expected call of RemoveManifest. +func (mr *MockPackClientMockRecorder) RemoveManifest(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RemoveManifest", reflect.TypeOf((*MockPackClient)(nil).RemoveManifest), arg0, arg1, arg2) +} + // YankBuildpack mocks base method. func (m *MockPackClient) YankBuildpack(arg0 client.YankBuildpackOptions) error { m.ctrl.T.Helper() diff --git a/pkg/client/add_manifest.go b/pkg/client/add_manifest.go index ab52ba87eb..db8be14157 100644 --- a/pkg/client/add_manifest.go +++ b/pkg/client/add_manifest.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "strings" + + "github.com/google/go-containerregistry/pkg/name" ) type ManifestAddOptions struct { @@ -13,53 +15,52 @@ type ManifestAddOptions struct { // AddManifest implements commands.PackClient. func (c *Client) AddManifest(ctx context.Context, index string, image string, opts ManifestAddOptions) (indexID string, err error) { - ref, err := c.runtime.ParseReference(image) + _, err = name.ParseReference(index) if err != nil { return } - imgIndex, err := c.runtime.LookupImageIndex(index) + ref, err := name.ParseReference(image) if err != nil { return } + imgIndex, err := c.indexFactory.FindIndex(index) + if err != nil { + return indexID, fmt.Errorf("Error while trying to find image on local storage: %v", image) + } + digest, err := imgIndex.Add(ctx, ref, opts.All) if err != nil { - if ref, _, err = c.imageFactory.FindImage(image); err != nil { - return indexID, fmt.Errorf("Error while trying to find image on local storage: %v", err) - } - digest, err = imgIndex.Add(ctx, ref, opts.All) - if err != nil { - return indexID, fmt.Errorf("Error while trying to add on manifest list: %v", err) - } + return indexID, fmt.Errorf("Error while trying to add on manifest list: %v", err) } if opts.OS != "" { - if _, err := imgIndex.Index.SetOS(digest, opts.OS); err != nil { + if _, err := imgIndex.SetOS(digest, opts.OS); err != nil { return indexID, err } } if opts.OSArch != "" { - if _, err := imgIndex.Index.SetArchitecture(digest, opts.OSArch); err != nil { + if _, err := imgIndex.SetArchitecture(digest, opts.OSArch); err != nil { return indexID, err } } if opts.OSVariant != "" { - if _, err := imgIndex.Index.SetVariant(digest, opts.OSVariant); err != nil { + if _, err := imgIndex.SetVariant(digest, opts.OSVariant); err != nil { return indexID, err } } if opts.OSVersion != "" { - if _, err := imgIndex.Index.SetOSVersion(digest, opts.OSVersion); err != nil { + if _, err := imgIndex.SetOSVersion(digest, opts.OSVersion); err != nil { return indexID, err } } if len(opts.Features) != 0 { - if _, err := imgIndex.Index.SetFeatures(digest, opts.Features); err != nil { + if _, err := imgIndex.SetFeatures(digest, opts.Features); err != nil { return indexID, err } } @@ -73,15 +74,15 @@ func (c *Client) AddManifest(ctx context.Context, index string, image string, op } annotations[spec[0]] = spec[1] } - if err := imgIndex.Index.SetAnnotations(&digest, annotations); err != nil { + if err := imgIndex.SetAnnotations(&digest, annotations); err != nil { return err } } - indexID, err = imgIndex.Index.Save(index, nil, "") + indexID, err = imgIndex.Save(index, nil, "") if err == nil { fmt.Printf("%s: %s\n", indexID, digest.String()) } return indexID, err -} \ No newline at end of file +} diff --git a/pkg/client/add_manifest_test.go b/pkg/client/add_manifest_test.go index 7aa3101c36..ff42a2698e 100644 --- a/pkg/client/add_manifest_test.go +++ b/pkg/client/add_manifest_test.go @@ -4,14 +4,15 @@ import ( "bytes" "testing" - "github.com/buildpacks/pack/internal/commands" - "github.com/buildpacks/pack/internal/commands/testmocks" - "github.com/buildpacks/pack/pkg/logging" "github.com/golang/mock/gomock" "github.com/heroku/color" "github.com/sclevine/spec" "github.com/sclevine/spec/report" "github.com/spf13/cobra" + + "github.com/buildpacks/pack/internal/commands" + "github.com/buildpacks/pack/internal/commands/testmocks" + "github.com/buildpacks/pack/pkg/logging" ) func TestManifestAddCommand(t *testing.T) { @@ -100,4 +101,4 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { }) }) -} \ No newline at end of file +} diff --git a/pkg/client/annotate_manifest.go b/pkg/client/annotate_manifest.go index a279509a60..8811205d96 100644 --- a/pkg/client/annotate_manifest.go +++ b/pkg/client/annotate_manifest.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "strings" + + ggcrName "github.com/google/go-containerregistry/pkg/name" ) type ManifestAnnotateOptions struct { @@ -13,50 +15,50 @@ type ManifestAnnotateOptions struct { // AnnotateManifest implements commands.PackClient. func (c *Client) AnnotateManifest(ctx context.Context, name string, image string, opts ManifestAnnotateOptions) error { - manifestList, err := c.runtime.LookupImageIndex(name) + manifestList, err := c.indexFactory.FindIndex(name) if err != nil { return err } - digest, err := c.runtime.ParseDigest(image) + digest, err := ggcrName.NewDigest(image) if err != nil { ref, _, err := c.imageFactory.FindImage(image) if err != nil { - return err + return fmt.Errorf("Error while trying to find image on local storage: %v", err.Error()) } - digest, err = c.runtime.ParseDigest(ref.Name()) + digest, err = ggcrName.NewDigest(ref.Name()) if err != nil { return err } } if opts.OS != "" { - if err := manifestList.Index.SetOS(digest, opts.OS); err != nil { + if err := manifestList.SetOS(digest, opts.OS); err != nil { return err } } if opts.OSVersion != "" { - if err := manifestList.Index.SetOSVersion(digest, opts.OSVersion); err != nil { + if err := manifestList.SetOSVersion(digest, opts.OSVersion); err != nil { return err } } if len(opts.OSFeatures) != 0 { - if err := manifestList.Index.SetOSFeatures(digest, opts.OSFeatures); err != nil { + if err := manifestList.SetOSFeatures(digest, opts.OSFeatures); err != nil { return err } } if opts.OSArch != "" { - if err := manifestList.Index.SetArchitecture(digest, opts.OSArch); err != nil { + if err := manifestList.SetArchitecture(digest, opts.OSArch); err != nil { return err } } if opts.OSVariant != "" { - if err := manifestList.Index.SetVariant(digest, opts.OSVariant); err != nil { + if err := manifestList.SetVariant(digest, opts.OSVariant); err != nil { return err } } if len(opts.Features) != 0 { - if err := manifestList.Index.SetFeatures(digest, opts.Features); err != nil { + if err := manifestList.SetFeatures(digest, opts.Features); err != nil { return err } } @@ -69,15 +71,15 @@ func (c *Client) AnnotateManifest(ctx context.Context, name string, image string } annotations[spec[0]] = spec[1] } - if err := manifestList.Index.SetAnnotations(&digest, annotations); err != nil { + if err := manifestList.SetAnnotations(&digest, annotations); err != nil { return err } } - updatedListID, err := manifestList.Index.Save(name, nil, "") + updatedListID, err := manifestList.Save(name, nil, "") if err == nil { fmt.Printf("%s: %s\n", updatedListID, digest.String()) } return nil -} \ No newline at end of file +} diff --git a/pkg/client/client.go b/pkg/client/client.go index b058d84182..3eabe4e019 100644 --- a/pkg/client/client.go +++ b/pkg/client/client.go @@ -25,6 +25,7 @@ import ( "github.com/buildpacks/imgutil/remote" dockerClient "github.com/docker/docker/client" "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" "github.com/pkg/errors" @@ -78,6 +79,7 @@ type ImageFactory interface { // IndexFactory is an interface representing the ability to create a ImageIndex/ManifestList. type IndexFactory interface { + // load ManifestList from local storage with the given name NewIndex(reponame string, opts imgutil.IndexOptions) (imgutil.Index, error) // Fetch ManifestList from Registry with the given name FetchIndex(name string) (imgutil.Index, error) diff --git a/pkg/client/create_manifest.go b/pkg/client/create_manifest.go index 557df033eb..2722caa16e 100644 --- a/pkg/client/create_manifest.go +++ b/pkg/client/create_manifest.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/buildpacks/imgutil" + ggcrName "github.com/google/go-containerregistry/pkg/name" packErrors "github.com/buildpacks/pack/pkg/errors" ) @@ -25,9 +26,9 @@ func (c *Client) CreateManifest(ctx context.Context, name string, images []strin if err != nil { return } - if imageID, err = index.Save(name, c.runtime.ImageType(opts.Format)); err != nil { + if imageID, err = index.Save(name, imgutil.Index.ImageType(opts.Format)); err != nil { if errors.Is(err, packErrors.ErrDuplicateName) && opts.amend { - _, err := c.runtime.LookupImageIndex(name) + _, err := c.indexFactory.FindIndex(name) if err != nil { fmt.Printf("no list named %q found: %v", name, err) } @@ -41,7 +42,7 @@ func (c *Client) CreateManifest(ctx context.Context, name string, images []strin } for _, img := range images { - ref, err := c.runtime.ParseReference(img) + ref, err := ggcrName.ParseReference(img) if err != nil { return imageID, err } @@ -53,7 +54,7 @@ func (c *Client) CreateManifest(ctx context.Context, name string, images []strin } } - imageID, err = index.Save(name, c.runtime.ImageType(opts.Format)) + imageID, err = index.Save(name, imgutil.Index.ImageType(opts.Format)) if err == nil { fmt.Printf("%s\n", imageID) } @@ -67,4 +68,4 @@ func (c *Client) CreateManifest(ctx context.Context, name string, images []strin func parseOptsToIndexOptions(opts CreateManifestOptions) (idxOpts imgutil.IndexOptions) { return idxOpts -} \ No newline at end of file +} diff --git a/pkg/client/delete_manifest.go b/pkg/client/delete_manifest.go index ac72dd6f46..887eef58ad 100644 --- a/pkg/client/delete_manifest.go +++ b/pkg/client/delete_manifest.go @@ -2,11 +2,18 @@ package client import ( "context" - // "fmt" - // "strings" ) // DeleteManifest implements commands.PackClient. func (c *Client) DeleteManifest(ctx context.Context, names []string) error { - return c.runtime.RemoveManifests(ctx, names) -} \ No newline at end of file + for _, name := range names { + imgIndex, err := c.indexFactory.FindIndex(name) + if err != nil { + return err + } + + imgIndex.Delete() + } + + return nil +} diff --git a/pkg/client/exists_manifest.go b/pkg/client/exists_manifest.go index f9530040a1..caa1b7c0ee 100644 --- a/pkg/client/exists_manifest.go +++ b/pkg/client/exists_manifest.go @@ -3,15 +3,13 @@ package client import ( "context" - // "github.com/buildpacks/imgutil" "github.com/pkg/errors" ) func (c *Client) ExistsManifest(ctx context.Context, image string) error { - - if _, err := c.runtime.LookupImageIndex(image); err != nil { + if _, err := c.indexFactory.FindIndex(image); err != nil { return errors.Errorf("image '%s' is not found", image) } return nil -} \ No newline at end of file +} diff --git a/pkg/client/inspect_manifest.go b/pkg/client/inspect_manifest.go index bb73d24c2a..cf3d4331cf 100644 --- a/pkg/client/inspect_manifest.go +++ b/pkg/client/inspect_manifest.go @@ -6,6 +6,7 @@ import ( "encoding/json" "fmt" + ggcrName "github.com/google/go-containerregistry/pkg/name" "github.com/pkg/errors" packErrors "github.com/buildpacks/pack/pkg/errors" @@ -29,7 +30,7 @@ func (c *Client) InspectManifest(ctx context.Context, name string, opts InspectM // Before doing a remote lookup, attempt to resolve the manifest list // locally. - manifestList, err := c.runtime.LookupImageIndex(name) + manifestList, err := c.indexFactory.FindIndex(name) if err == nil { schema2List, err := manifestList.Index.Inspect() if err != nil { @@ -44,7 +45,7 @@ func (c *Client) InspectManifest(ctx context.Context, name string, opts InspectM return err } - _, err = c.runtime.ParseReference(name) + _, err = ggcrName.ParseReference(name) if err != nil { fmt.Printf("error parsing reference to image %q: %v", name, err) } @@ -59,4 +60,4 @@ func (c *Client) InspectManifest(ctx context.Context, name string, opts InspectM } return fmt.Errorf("unable to locate manifest list locally or at registry") -} \ No newline at end of file +} diff --git a/pkg/client/push_manifest.go b/pkg/client/push_manifest.go index e4ecbb4cd0..1ae7fa3e59 100644 --- a/pkg/client/push_manifest.go +++ b/pkg/client/push_manifest.go @@ -2,8 +2,6 @@ package client import ( "context" - - runtime "github.com/buildpacks/pack/internal/runtime" ) type PushManifestOptions struct { @@ -13,20 +11,18 @@ type PushManifestOptions struct { // PushManifest implements commands.PackClient. func (c *Client) PushManifest(ctx context.Context, index string, opts PushManifestOptions) (imageID string, err error) { - manifestList, err := c.runtime.LookupImageIndex(index) + manifestList, err := c.indexFactory.FindIndex(index) if err != nil { return } _, err = manifestList.Push(ctx, parseFalgsForImgUtil(opts)) - if err == nil && opts.Purge { - c.runtime.RemoveManifests(ctx, []string{index}) - } + manifestList.Delete() return imageID, err } func parseFalgsForImgUtil(opts PushManifestOptions) (idxOptions runtime.PushOptions) { return idxOptions -} \ No newline at end of file +} diff --git a/pkg/client/remove_manifest.go b/pkg/client/remove_manifest.go index f6319c5dc5..110314d9a6 100644 --- a/pkg/client/remove_manifest.go +++ b/pkg/client/remove_manifest.go @@ -3,17 +3,19 @@ package client import ( "context" "fmt" + + gccrName "github.com/google/go-containerregistry/pkg/name" ) // RemoveManifest implements commands.PackClient. func (c *Client) RemoveManifest(ctx context.Context, name string, images []string) error { - imgIndex, err := c.runtime.LookupImageIndex(name) + imgIndex, err := c.indexFactory.FindIndex(name) if err != nil { return err } for _, image := range images { - _, err := c.runtime.ParseReference(image) + _, err := gccrName.ParseReference(image) if err != nil { fmt.Errorf(`Invalid instance "%s": %v`, image, err) } @@ -24,4 +26,4 @@ func (c *Client) RemoveManifest(ctx context.Context, name string, images []strin } return nil -} \ No newline at end of file +} diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go index 2d6943cf86..669d3e8ab7 100644 --- a/pkg/errors/errors.go +++ b/pkg/errors/errors.go @@ -6,4 +6,4 @@ var ( ErrDuplicateName = errors.New("Image/ImageIndex with the given name exists") ErrIndexUnknown = errors.New("cannot find Image Index with the given name") ErrNotAddManifestList = errors.New("error while adding ImageIndex to the list") -) \ No newline at end of file +) diff --git a/pkg/testmocks/mock_docker_client.go b/pkg/testmocks/mock_docker_client.go index 18b51c18f1..5ba29fec53 100644 --- a/pkg/testmocks/mock_docker_client.go +++ b/pkg/testmocks/mock_docker_client.go @@ -224,10 +224,10 @@ func (mr *MockCommonAPIClientMockRecorder) ContainerCreate(arg0, arg1, arg2, arg } // ContainerDiff mocks base method. -func (m *MockCommonAPIClient) ContainerDiff(arg0 context.Context, arg1 string) ([]container.ContainerChangeResponseItem, error) { +func (m *MockCommonAPIClient) ContainerDiff(arg0 context.Context, arg1 string) ([]container.FilesystemChange, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "ContainerDiff", arg0, arg1) - ret0, _ := ret[0].([]container.ContainerChangeResponseItem) + ret0, _ := ret[0].([]container.FilesystemChange) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -1338,7 +1338,7 @@ func (mr *MockCommonAPIClientMockRecorder) PluginUpgrade(arg0, arg1, arg2 interf } // RegistryLogin mocks base method. -func (m *MockCommonAPIClient) RegistryLogin(arg0 context.Context, arg1 types.AuthConfig) (registry.AuthenticateOKBody, error) { +func (m *MockCommonAPIClient) RegistryLogin(arg0 context.Context, arg1 registry.AuthConfig) (registry.AuthenticateOKBody, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "RegistryLogin", arg0, arg1) ret0, _ := ret[0].(registry.AuthenticateOKBody) @@ -1725,7 +1725,7 @@ func (mr *MockCommonAPIClientMockRecorder) VolumeInspectWithRaw(arg0, arg1 inter } // VolumeList mocks base method. -func (m *MockCommonAPIClient) VolumeList(arg0 context.Context, arg1 filters.Args) (volume.ListResponse, error) { +func (m *MockCommonAPIClient) VolumeList(arg0 context.Context, arg1 volume.ListOptions) (volume.ListResponse, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "VolumeList", arg0, arg1) ret0, _ := ret[0].(volume.ListResponse) From 8f81b6c157dd17e19b8ed6a4fb7b31db54ca2e13 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Thu, 9 Nov 2023 16:12:41 +0000 Subject: [PATCH 60/62] WIP added tests for add_manifest Signed-off-by: WYGIN --- pkg/client/add_manifest_test.go | 302 +++++++++++++++++++++++++++----- 1 file changed, 260 insertions(+), 42 deletions(-) diff --git a/pkg/client/add_manifest_test.go b/pkg/client/add_manifest_test.go index ff42a2698e..374f95a646 100644 --- a/pkg/client/add_manifest_test.go +++ b/pkg/client/add_manifest_test.go @@ -2,9 +2,13 @@ package client_test import ( "bytes" + "encoding/json" + "os" "testing" "github.com/golang/mock/gomock" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/heroku/color" "github.com/sclevine/spec" "github.com/sclevine/spec/report" @@ -13,6 +17,7 @@ import ( "github.com/buildpacks/pack/internal/commands" "github.com/buildpacks/pack/internal/commands/testmocks" "github.com/buildpacks/pack/pkg/logging" + h "github.com/buildpacks/pack/testhelpers" ) func TestManifestAddCommand(t *testing.T) { @@ -29,9 +34,11 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { outBuf bytes.Buffer mockController *gomock.Controller mockClient *testmocks.MockPackClient + tmpDir string ) it.Before(func() { + outBuf = bytes.Buffer{} logger = logging.NewLogWithWriters(&outBuf, &outBuf) mockController = gomock.NewController(t) mockClient = testmocks.NewMockPackClient(mockController) @@ -40,65 +47,276 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { }) when("#AddManifest", func() { + it.Before(func() { + tmpDir, err := os.MkdirTemp("", "manifest-test") + h.AssertNil(t, err) + os.Setenv("XDG_RUNTIME_DIR", tmpDir) + logger = logging.NewLogWithWriters(&outBuf, &outBuf) + }) when("no flags specified", func() { - + it("should be able to add manifest list", func() { + command.SetArgs([]string{"node:latest"}) + err := command.Execute() + h.AssertNil(t, err) + os.ReadFile() + }) + it("should be able to add manifest", func() { + command.SetArgs([]string{"node@sha256:a59381eeb372ade238dcde65dce1fb6ad48c4eda288bf4e3e50b94176ee67d60"}) + err := command.Execute() + h.AssertNil(t, err) + }) }) when("when --all flags passed", func() { - + it("should print a warning if no imageIndex is passed", func() { + command.SetArgs([]string{"node@sha256:a59381eeb372ade238dcde65dce1fb6ad48c4eda288bf4e3e50b94176ee67d60", "--all"}) + err := command.Execute() + h.AssertNil(t, err) + h.AssertEq(t, outBuf.String(), "some warning") + }) + it("should add all images in ImageIndex if imageIndex is passed", func() { + var manifestList fakeBadIndexStruct + var node = "node:18.18.2-slim" + command.SetArgs([]string{node, "--all"}) + var hashes = []string{ + "sha256:ef52e84aa85baadfcdfe6f40162c368c08d4def29751ed1429abe1908316b198", + "sha256:a02f6e55993fdb04a45017f1a9bd1876dc0a3fe89a1d88e53393752f80859e22", + "sha256:c1e67d1a099e50d37d6aef7ee2917496a99aff6876b046613ed822bf8c72d371", + "sha256:d1beb3473334d238c62c94ed644cfac5c7df27920579a28d15d9ed85f25e87d5", + "sha256:581d640ed5c99190e7afcf9ffb05b030a0094a3775b35bbe05328e58877dd63a", + } + err := command.Execute() + h.AssertNil(t, err) + h.AssertEq(t, outBuf.String(), "") + ref, err := name.ParseReference(node) + h.AssertNil(t, err) + file, err := os.ReadFile(tmpDir+string(os.PathSeparator)+ref.Name()+string(os.PathSeparator)+ref.Name()) + h.AssertNil(t, err) + json.Unmarshal(file, &manifestList) + for _, hash := range hashes { + hashRef, err := v1.NewHash(hash) + h.AssertNil(t, err) + img, err := manifestList.Index.Image(hashRef) + h.AssertNil(t, err) + h.AssertNotNil(t, img) + } + }) }) when("when --os flags passed", func() { - + it("should return an error when ImageIndex is not passed", func() { + var node = "node@sha256:ef52e84aa85baadfcdfe6f40162c368c08d4def29751ed1429abe1908316b198" + command.SetArgs([]string{node, "--os", "linux"}) + err := command.Execute() + h.AssertNotNil(t, err) + }) + it("should add image with given os to local list if it exists in imageIndex", func() { + var manifestList fakeBadIndexStruct + var node = "traefik:v3.0.0-beta4-windowsservercore-1809" + command.SetArgs([]string{node, "--os", "windows"}) + err := command.Execute() + h.AssertNil(t, err) + ref, err := name.ParseReference(node) + h.AssertNil(t, err) + file, err := os.ReadFile(tmpDir+string(os.PathSeparator)+ref.Name()+string(os.PathSeparator)+ref.Name()) + h.AssertNil(t, err) + json.Unmarshal(file, &manifestList) + var hashRef = "sha256:6bbe10dec34e310f581af3d2f4b1bca020b4b4d097063e77a5301e74af5a0196" + img, err := manifestList.Index.Image(hashRef) + h.AssertNil(t, err) + h.AssertNotNil(t, img) + }) + it("should return an error when given os doesn't exists in the ImageIndex", func() { + var manifestList fakeBadIndexStruct + var node = "traefik:v3.0.0-beta4-windowsservercore-1809" + command.SetArgs([]string{node, "--os", "linux"}) + err := command.Execute() + h.AssertNil(t, err) + ref, err := name.ParseReference(node) + h.AssertNil(t, err) + file, err := os.ReadFile(tmpDir+string(os.PathSeparator)+ref.Name()+string(os.PathSeparator)+ref.Name()) + h.AssertNil(t, err) + json.Unmarshal(file, &manifestList) + var hashRef = "sha256:6bbe10dec34e310f581af3d2f4b1bca020b4b4d097063e77a5301e74af5a0196" + img, err := manifestList.Index.Image(hashRef) + h.AssertNotNil(t, err) + h.AssertNil(t, img) + }) }) when("when --arch flags passed", func() { - + it("should not return an error when os flag is not specified", func() { + var node = "traefik:v3.0.0-beta4" + command.SetArgs([]string{node, "--arch", "amd64"}) + err := command.Execute() + h.AssertNil(t, err) + }) + it("should return an error when arch doesn't exists in the imageIndex", func() { + var node = "traefik:v3.0.0-beta4" + command.SetArgs([]string{node, "--arch", "abc"}) + err := command.Execute() + h.AssertNotNil(t, err) + }) + it("should return an error when manifest is passed instead of manifestList", func() { + var node = "traefik:v3.0.0-beta4-windowsservercore-1809" + command.SetArgs([]string{node, "--arch", "arm64"}) + err := command.Execute() + h.AssertNotNil(t, err) + }) + it("should include the manifest of given arch from the image index", func() { + var manifestList fakeBadIndexStruct + var hashes = []string{ + "sha256:777e2106170c66742ddbe77f703badb7dc94d9a5b1dc2c4a01538fad9aef56bb", + "sha256:9909a171ac287c316f771a4f1d1384df96957ed772bc39caf6deb6e3e360316f", + } + var node = "alpine:3.18.4" + command.SetArgs([]string{node, "--arch", "arm"}) + err := command.Execute() + h.AssertNil(t, err) + ref, err := name.ParseReference(node) + h.AssertNil(t, err) + file, err := os.ReadFile(tmpDir+string(os.PathSeparator)+ref.Name()+string(os.PathSeparator)+ref.Name()) + h.AssertNil(t, err) + json.Unmarshal(file, &manifestList) + for _, hash := range hashes { + img, err := manifestList.Index.Image(hash) + h.AssertNotNil(t, err) + h.AssertNil(t, img) + } + }) }) when("when --variant flags passed", func() { - + it("should not return an error when os flag is not specified", func() { + var node = "alpine:3.18" + command.SetArgs([]string{node, "--arch", "arm", "--variant", "v7"}) + err := command.Execute() + h.AssertNil(t, err) + }) + it("should not return an error when arch flag is not specified", func() { + var node = "alpine:3.18" + command.SetArgs([]string{node, "--os", "linux", "--variant", "v7"}) + err := command.Execute() + h.AssertNil(t, err) + }) + it("should return an error when variant doesn't exists in the imageIndex", func() { + var node = "alpine:3.18" + command.SetArgs([]string{node, "--variant", "v9"}) + err := command.Execute() + h.AssertNotNil(t, err) + }) + it("should return an error when manifest is passed instead of manifestList", func() { + var node = "alpine@sha256:777e2106170c66742ddbe77f703badb7dc94d9a5b1dc2c4a01538fad9aef56bb" + command.SetArgs([]string{node, "--variant", "v6"}) + err := command.Execute() + h.AssertNotNil(t, err) + }) + it("should include the manifest of given variant from the image index", func() { + var manifestList fakeBadIndexStruct + var hashes = []string{ + "sha256:777e2106170c66742ddbe77f703badb7dc94d9a5b1dc2c4a01538fad9aef56bb", + } + var node = "alpine:3.18.4" + command.SetArgs([]string{node, "--variant", "v6"}) + err := command.Execute() + h.AssertNil(t, err) + ref, err := name.ParseReference(node) + h.AssertNil(t, err) + file, err := os.ReadFile(tmpDir+string(os.PathSeparator)+ref.Name()+string(os.PathSeparator)+ref.Name()) + h.AssertNil(t, err) + json.Unmarshal(file, &manifestList) + for _, hash := range hashes { + img, err := manifestList.Index.Image(hash) + h.AssertNotNil(t, err) + h.AssertNil(t, img) + } + }) }) when("when --os-version flags passed", func() { - + it("should return an error when os flag is not specified", func() { + var node = "traefik:v3.0.0-beta4-windowsservercore-1809" + command.SetArgs([]string{node, "--os-version", "10.0.14393.1066"}) + err := command.Execute() + h.AssertNotNil(t, err) + }) + it("should include the manifest of given os-version from the image index", func() { + var node = "traefik:v3.0.0-beta4-windowsservercore-1809" + command.SetArgs([]string{node, "--os-version", "10.0.14393.1066"}) + err := command.Execute() + h.AssertNil(t, err) + }) }) - when("when --features flags passed", func() { + // --feature is reserved for future use + // when("when --features flags passed", func() { - }) - when("when --os-features flags passed", func() { - - }) + // }) + // when("when --os-features flags passed", func() { + // it("") + // }) when("when --annotations flags passed", func() { - - }) - when("when multiple flags passed", func() { - - }) - when("when no args passed", func() { - + it("should accept annotations", func() { + + }) }) - when("when manifest list reference is incorrect", func() { - - }) - when("when manifest reference is incorrect", func() { - - }) - when("when manifest passed in-place of manifest list on first arg", func() { - - }) - when("when manifest list is passed on second arg", func() { - + when("should throw an error when", func() { + it("has no args passed", func() { + command.SetArgs([]string{}) + err := command.Execute() + h.AssertNotNil(t, err) + }) + it("has manifest list reference is incorrect", func() { + var node = "traefikWr0!/\ng`" + command.SetArgs([]string{node}) + err := command.Execute() + h.AssertNotNil(t, err) + }) + it("has manifest reference is incorrect", func() { + var node = "traefik:v3.0.0-beta4-windowsservercore-1809" + command.SetArgs([]string{node, node+"!#@:'"}) + err := command.Execute() + h.AssertNotNil(t, err) + }) + it("has manifest passed in-place of manifest list on first arg", func() { + var node = "alpine@sha256:777e2106170c66742ddbe77f703badb7dc94d9a5b1dc2c4a01538fad9aef56bb" + command.SetArgs([]string{node, node}) + err := command.Execute() + h.AssertNotNil(t, err) + }) }) - when("when manifest is passed on second arg with --all option", func() { - + when("should warn when",func() { + it("manifest is passed on second arg with --all option", func() { + var node = "alpine@sha256:777e2106170c66742ddbe77f703badb7dc94d9a5b1dc2c4a01538fad9aef56bb" + command.SetArgs([]string{"alpine:3.18", node, "--all"}) + err := command.Execute() + h.AssertNil(t, err) + h.AssertNotNil(t, outBuf) + }) }) - when("when manifest list in locally available", func() { - - }) - when("when manifest is not locally available", func() { - - }) - when("when manifest is locally available passed", func() { - - }) - when("when multiple manifests passed", func() { - + when("manifest/ImageIndex", func() { + it("manifest locally available", func() { + var node = "alpine@sha256:777e2106170c66742ddbe77f703badb7dc94d9a5b1dc2c4a01538fad9aef56bb" + command.SetArgs([]string{"alpine:3.18", node, "--all"}) + err := command.Execute() + h.AssertNil(t, err) + h.AssertNotNil(t, outBuf) + }) + it("manifest available at registry only", func() { + var node = "alpine@sha256:777e2106170c66742ddbe77f703badb7dc94d9a5b1dc2c4a01538fad9aef56bb" + command.SetArgs([]string{"alpine:3.18", node, "--all"}) + err := command.Execute() + h.AssertNil(t, err) + h.AssertNotNil(t, outBuf) + }) + it("manifest list locally available", func() { + var node = "alpine@sha256:777e2106170c66742ddbe77f703badb7dc94d9a5b1dc2c4a01538fad9aef56bb" + command.SetArgs([]string{"alpine:3.18", node, "--all"}) + err := command.Execute() + h.AssertNil(t, err) + h.AssertNotNil(t, outBuf) + }) + it("manifest list available at registry only", func() { + var node = "alpine@sha256:777e2106170c66742ddbe77f703badb7dc94d9a5b1dc2c4a01538fad9aef56bb" + command.SetArgs([]string{"alpine:3.18", node, "--all"}) + err := command.Execute() + h.AssertNil(t, err) + h.AssertNotNil(t, outBuf) + }) }) }) } From f12c84995bffd573f9c60fbc68efdb1b92e38a44 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Fri, 10 Nov 2023 09:18:59 +0000 Subject: [PATCH 61/62] WIP improved add_manifest client tests Signed-off-by: WYGIN --- pkg/client/add_manifest_test.go | 211 ++++++++++++++++++++++---------- testhelpers/testhelpers.go | 28 +++-- 2 files changed, 165 insertions(+), 74 deletions(-) diff --git a/pkg/client/add_manifest_test.go b/pkg/client/add_manifest_test.go index 374f95a646..cdf7f3471d 100644 --- a/pkg/client/add_manifest_test.go +++ b/pkg/client/add_manifest_test.go @@ -4,6 +4,8 @@ import ( "bytes" "encoding/json" "os" + "path/filepath" + "strings" "testing" "github.com/golang/mock/gomock" @@ -14,6 +16,7 @@ import ( "github.com/sclevine/spec/report" "github.com/spf13/cobra" + "github.com/buildpacks/imgutil/fakes" "github.com/buildpacks/pack/internal/commands" "github.com/buildpacks/pack/internal/commands/testmocks" "github.com/buildpacks/pack/pkg/logging" @@ -35,6 +38,7 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { mockController *gomock.Controller mockClient *testmocks.MockPackClient tmpDir string + manifestListName string ) it.Before(func() { @@ -42,8 +46,6 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { logger = logging.NewLogWithWriters(&outBuf, &outBuf) mockController = gomock.NewController(t) mockClient = testmocks.NewMockPackClient(mockController) - - command = commands.ManifestAdd(logger, mockClient) }) when("#AddManifest", func() { @@ -52,23 +54,38 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { h.AssertNil(t, err) os.Setenv("XDG_RUNTIME_DIR", tmpDir) logger = logging.NewLogWithWriters(&outBuf, &outBuf) + + command = commands.ManifestAdd(logger, mockClient) + manifestCreate := commands.ManifestCreate(logger, mockClient) + manifestListName = "cnbs/sample-package:hello-multiarch-universe" + manifestCreate.SetArgs([]string{manifestListName}) + err = manifestCreate.Execute() + h.AssertNil(t, err) + ref, err := name.ParseReference(manifestListName) + h.AssertNil(t, err) + manifestListFile, err := os.ReadFile(filepath.Join(tmpDir,"manifests", ref.Name(), ref.Identifier())) + h.AssertNil(t, err) + h.AssertNotNil(t, manifestListFile) + }) + it.After(func() { + err := os.RemoveAll(os.Getenv("XDG_RUNTIME_DIR")) + h.AssertNil(t, err) }) when("no flags specified", func() { it("should be able to add manifest list", func() { - command.SetArgs([]string{"node:latest"}) + command.SetArgs([]string{manifestListName, "node:latest"}) err := command.Execute() h.AssertNil(t, err) - os.ReadFile() }) it("should be able to add manifest", func() { - command.SetArgs([]string{"node@sha256:a59381eeb372ade238dcde65dce1fb6ad48c4eda288bf4e3e50b94176ee67d60"}) + command.SetArgs([]string{manifestListName, "node@sha256:a59381eeb372ade238dcde65dce1fb6ad48c4eda288bf4e3e50b94176ee67d60"}) err := command.Execute() h.AssertNil(t, err) }) }) when("when --all flags passed", func() { it("should print a warning if no imageIndex is passed", func() { - command.SetArgs([]string{"node@sha256:a59381eeb372ade238dcde65dce1fb6ad48c4eda288bf4e3e50b94176ee67d60", "--all"}) + command.SetArgs([]string{manifestListName, "node@sha256:a59381eeb372ade238dcde65dce1fb6ad48c4eda288bf4e3e50b94176ee67d60", "--all"}) err := command.Execute() h.AssertNil(t, err) h.AssertEq(t, outBuf.String(), "some warning") @@ -76,7 +93,7 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { it("should add all images in ImageIndex if imageIndex is passed", func() { var manifestList fakeBadIndexStruct var node = "node:18.18.2-slim" - command.SetArgs([]string{node, "--all"}) + command.SetArgs([]string{manifestListName, node, "--all"}) var hashes = []string{ "sha256:ef52e84aa85baadfcdfe6f40162c368c08d4def29751ed1429abe1908316b198", "sha256:a02f6e55993fdb04a45017f1a9bd1876dc0a3fe89a1d88e53393752f80859e22", @@ -87,11 +104,12 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { err := command.Execute() h.AssertNil(t, err) h.AssertEq(t, outBuf.String(), "") - ref, err := name.ParseReference(node) + ref, err := name.ParseReference(manifestListName) h.AssertNil(t, err) - file, err := os.ReadFile(tmpDir+string(os.PathSeparator)+ref.Name()+string(os.PathSeparator)+ref.Name()) + file, err := os.ReadFile(filepath.Join(tmpDir,"manifests", ref.Name(), ref.Identifier())) + h.AssertNil(t, err) + err = json.Unmarshal(file, &manifestList) h.AssertNil(t, err) - json.Unmarshal(file, &manifestList) for _, hash := range hashes { hashRef, err := v1.NewHash(hash) h.AssertNil(t, err) @@ -104,21 +122,22 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { when("when --os flags passed", func() { it("should return an error when ImageIndex is not passed", func() { var node = "node@sha256:ef52e84aa85baadfcdfe6f40162c368c08d4def29751ed1429abe1908316b198" - command.SetArgs([]string{node, "--os", "linux"}) + command.SetArgs([]string{manifestListName, node, "--os", "linux"}) err := command.Execute() h.AssertNotNil(t, err) }) it("should add image with given os to local list if it exists in imageIndex", func() { var manifestList fakeBadIndexStruct var node = "traefik:v3.0.0-beta4-windowsservercore-1809" - command.SetArgs([]string{node, "--os", "windows"}) + command.SetArgs([]string{manifestListName, node, "--os", "windows"}) err := command.Execute() h.AssertNil(t, err) - ref, err := name.ParseReference(node) + ref, err := name.ParseReference(manifestListName) h.AssertNil(t, err) - file, err := os.ReadFile(tmpDir+string(os.PathSeparator)+ref.Name()+string(os.PathSeparator)+ref.Name()) + file, err := os.ReadFile(filepath.Join(tmpDir,"manifests", ref.Name(), ref.Identifier())) + h.AssertNil(t, err) + err = json.Unmarshal(file, &manifestList) h.AssertNil(t, err) - json.Unmarshal(file, &manifestList) var hashRef = "sha256:6bbe10dec34e310f581af3d2f4b1bca020b4b4d097063e77a5301e74af5a0196" img, err := manifestList.Index.Image(hashRef) h.AssertNil(t, err) @@ -127,14 +146,15 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { it("should return an error when given os doesn't exists in the ImageIndex", func() { var manifestList fakeBadIndexStruct var node = "traefik:v3.0.0-beta4-windowsservercore-1809" - command.SetArgs([]string{node, "--os", "linux"}) + command.SetArgs([]string{manifestListName, node, "--os", "linux"}) err := command.Execute() h.AssertNil(t, err) - ref, err := name.ParseReference(node) + ref, err := name.ParseReference(manifestListName) h.AssertNil(t, err) - file, err := os.ReadFile(tmpDir+string(os.PathSeparator)+ref.Name()+string(os.PathSeparator)+ref.Name()) + file, err := os.ReadFile(filepath.Join(tmpDir,"manifests", ref.Name(), ref.Identifier())) + h.AssertNil(t, err) + err = json.Unmarshal(file, &manifestList) h.AssertNil(t, err) - json.Unmarshal(file, &manifestList) var hashRef = "sha256:6bbe10dec34e310f581af3d2f4b1bca020b4b4d097063e77a5301e74af5a0196" img, err := manifestList.Index.Image(hashRef) h.AssertNotNil(t, err) @@ -144,19 +164,19 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { when("when --arch flags passed", func() { it("should not return an error when os flag is not specified", func() { var node = "traefik:v3.0.0-beta4" - command.SetArgs([]string{node, "--arch", "amd64"}) + command.SetArgs([]string{manifestListName, node, "--arch", "amd64"}) err := command.Execute() h.AssertNil(t, err) }) it("should return an error when arch doesn't exists in the imageIndex", func() { var node = "traefik:v3.0.0-beta4" - command.SetArgs([]string{node, "--arch", "abc"}) + command.SetArgs([]string{manifestListName, node, "--arch", "abc"}) err := command.Execute() h.AssertNotNil(t, err) }) it("should return an error when manifest is passed instead of manifestList", func() { var node = "traefik:v3.0.0-beta4-windowsservercore-1809" - command.SetArgs([]string{node, "--arch", "arm64"}) + command.SetArgs([]string{manifestListName, node, "--arch", "arm64"}) err := command.Execute() h.AssertNotNil(t, err) }) @@ -167,14 +187,15 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { "sha256:9909a171ac287c316f771a4f1d1384df96957ed772bc39caf6deb6e3e360316f", } var node = "alpine:3.18.4" - command.SetArgs([]string{node, "--arch", "arm"}) + command.SetArgs([]string{manifestListName, node, "--arch", "arm"}) err := command.Execute() h.AssertNil(t, err) - ref, err := name.ParseReference(node) + ref, err := name.ParseReference(manifestListName) h.AssertNil(t, err) - file, err := os.ReadFile(tmpDir+string(os.PathSeparator)+ref.Name()+string(os.PathSeparator)+ref.Name()) + file, err := os.ReadFile(filepath.Join(tmpDir,"manifests", ref.Name(), ref.Identifier())) + h.AssertNil(t, err) + err = json.Unmarshal(file, &manifestList) h.AssertNil(t, err) - json.Unmarshal(file, &manifestList) for _, hash := range hashes { img, err := manifestList.Index.Image(hash) h.AssertNotNil(t, err) @@ -185,25 +206,25 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { when("when --variant flags passed", func() { it("should not return an error when os flag is not specified", func() { var node = "alpine:3.18" - command.SetArgs([]string{node, "--arch", "arm", "--variant", "v7"}) + command.SetArgs([]string{manifestListName, node, "--arch", "arm", "--variant", "v7"}) err := command.Execute() h.AssertNil(t, err) }) it("should not return an error when arch flag is not specified", func() { var node = "alpine:3.18" - command.SetArgs([]string{node, "--os", "linux", "--variant", "v7"}) + command.SetArgs([]string{manifestListName, node, "--os", "linux", "--variant", "v7"}) err := command.Execute() h.AssertNil(t, err) }) it("should return an error when variant doesn't exists in the imageIndex", func() { var node = "alpine:3.18" - command.SetArgs([]string{node, "--variant", "v9"}) + command.SetArgs([]string{manifestListName, node, "--variant", "v9"}) err := command.Execute() h.AssertNotNil(t, err) }) it("should return an error when manifest is passed instead of manifestList", func() { var node = "alpine@sha256:777e2106170c66742ddbe77f703badb7dc94d9a5b1dc2c4a01538fad9aef56bb" - command.SetArgs([]string{node, "--variant", "v6"}) + command.SetArgs([]string{manifestListName, node, "--variant", "v6"}) err := command.Execute() h.AssertNotNil(t, err) }) @@ -213,14 +234,15 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { "sha256:777e2106170c66742ddbe77f703badb7dc94d9a5b1dc2c4a01538fad9aef56bb", } var node = "alpine:3.18.4" - command.SetArgs([]string{node, "--variant", "v6"}) + command.SetArgs([]string{manifestListName, node, "--variant", "v6"}) err := command.Execute() h.AssertNil(t, err) - ref, err := name.ParseReference(node) + ref, err := name.ParseReference(manifestListName) h.AssertNil(t, err) - file, err := os.ReadFile(tmpDir+string(os.PathSeparator)+ref.Name()+string(os.PathSeparator)+ref.Name()) + file, err := os.ReadFile(filepath.Join(tmpDir,"manifests", ref.Name(), ref.Identifier())) + h.AssertNil(t, err) + err = json.Unmarshal(file, &manifestList) h.AssertNil(t, err) - json.Unmarshal(file, &manifestList) for _, hash := range hashes { img, err := manifestList.Index.Image(hash) h.AssertNotNil(t, err) @@ -229,29 +251,89 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { }) }) when("when --os-version flags passed", func() { - it("should return an error when os flag is not specified", func() { + it("should not return an error when os flag is not specified", func() { var node = "traefik:v3.0.0-beta4-windowsservercore-1809" - command.SetArgs([]string{node, "--os-version", "10.0.14393.1066"}) + command.SetArgs([]string{manifestListName, node, "--os-version", "10.0.14393.1066"}) err := command.Execute() - h.AssertNotNil(t, err) + h.AssertNil(t, err) }) it("should include the manifest of given os-version from the image index", func() { + var manifestList fakeBadIndexStruct + var osVersion = "10.0.14393.1066" var node = "traefik:v3.0.0-beta4-windowsservercore-1809" - command.SetArgs([]string{node, "--os-version", "10.0.14393.1066"}) + command.SetArgs([]string{manifestListName, node, "--os-version", osVersion}) err := command.Execute() h.AssertNil(t, err) + ref, err := name.ParseReference(manifestListName) + h.AssertNil(t, err) + file, err := os.ReadFile(filepath.Join(tmpDir,"manifests", ref.Name(), ref.Identifier())) + h.AssertNil(t, err) + json.Unmarshal(file, &manifestList) + indexManifest, err := manifestList.Index.IndexManifest() + h.AssertNil(t, err) + h.AssertEq(t, indexManifest.Subject.Platform.OSVersion, osVersion) }) }) // --feature is reserved for future use - // when("when --features flags passed", func() { - - // }) - // when("when --os-features flags passed", func() { - // it("") - // }) + when("when --features flags passed", func() { + it("should set expected features", func() { + var manifestList fakeBadIndexStruct + var features = []string{ + "someFeature", + } + var node = "traefik:v3.0.0-beta4-windowsservercore-1809" + command.SetArgs([]string{manifestListName, node, "--features", features[0]}) + err := command.Execute() + h.AssertNil(t, err) + ref, err := name.ParseReference(manifestListName) + h.AssertNil(t, err) + file, err := os.ReadFile(filepath.Join(tmpDir,"manifests", ref.Name(), ref.Identifier())) + h.AssertNil(t, err) + json.Unmarshal(file, &manifestList) + indexManifest, err := manifestList.Index.IndexManifest() + h.AssertNil(t, err) + h.AssertContains(t, indexManifest.Subject.Platform.Features, features[0]) + }) + }) + when("when --os-features flags passed", func() { + it("should set expected os-featres", func() { + var manifestList fakeBadIndexStruct + var osFeatures = []string{ + "win32k.sys", + } + var node = "traefik:v3.0.0-beta4-windowsservercore-1809" + command.SetArgs([]string{manifestListName, node, "--os-features", osFeatures[0]}) + err := command.Execute() + h.AssertNil(t, err) + ref, err := name.ParseReference(manifestListName) + h.AssertNil(t, err) + file, err := os.ReadFile(filepath.Join(tmpDir,"manifests", ref.Name(), ref.Identifier())) + h.AssertNil(t, err) + json.Unmarshal(file, &manifestList) + indexManifest, err := manifestList.Index.IndexManifest() + h.AssertNil(t, err) + h.AssertContains(t, indexManifest.Subject.Platform.OSFeatures, osFeatures[0]) + }) + }) when("when --annotations flags passed", func() { it("should accept annotations", func() { - + var manifestList fakeBadIndexStruct + var annotations = []string{ + "com.example.key1=value1", + "com.example.key2=value2", + } + var node = "traefik:v3.0.0-beta4-windowsservercore-1809" + command.SetArgs([]string{manifestListName, node, "--annotations", strings.Join(annotations, ";")}) + err := command.Execute() + h.AssertNil(t, err) + ref, err := name.ParseReference(manifestListName) + h.AssertNil(t, err) + file, err := os.ReadFile(filepath.Join(tmpDir,"manifests", ref.Name(), ref.Identifier())) + h.AssertNil(t, err) + json.Unmarshal(file, &manifestList) + indexManifest, err := manifestList.Index.IndexManifest() + h.AssertNil(t, err) + h.AssertEq(t, h.MapToStringSlice(indexManifest.Subject.Annotations), annotations) }) }) when("should throw an error when", func() { @@ -268,13 +350,13 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { }) it("has manifest reference is incorrect", func() { var node = "traefik:v3.0.0-beta4-windowsservercore-1809" - command.SetArgs([]string{node, node+"!#@:'"}) + command.SetArgs([]string{manifestListName, node+"!#@:'"}) err := command.Execute() h.AssertNotNil(t, err) }) it("has manifest passed in-place of manifest list on first arg", func() { var node = "alpine@sha256:777e2106170c66742ddbe77f703badb7dc94d9a5b1dc2c4a01538fad9aef56bb" - command.SetArgs([]string{node, node}) + command.SetArgs([]string{node, manifestListName}) err := command.Execute() h.AssertNotNil(t, err) }) @@ -282,40 +364,39 @@ func testManifestAddCommand(t *testing.T, when spec.G, it spec.S) { when("should warn when",func() { it("manifest is passed on second arg with --all option", func() { var node = "alpine@sha256:777e2106170c66742ddbe77f703badb7dc94d9a5b1dc2c4a01538fad9aef56bb" - command.SetArgs([]string{"alpine:3.18", node, "--all"}) + command.SetArgs([]string{manifestListName, node, "--all"}) err := command.Execute() h.AssertNil(t, err) h.AssertNotNil(t, outBuf) }) }) when("manifest/ImageIndex", func() { - it("manifest locally available", func() { + it("manifest list locally available", func() { var node = "alpine@sha256:777e2106170c66742ddbe77f703badb7dc94d9a5b1dc2c4a01538fad9aef56bb" - command.SetArgs([]string{"alpine:3.18", node, "--all"}) - err := command.Execute() + command.SetArgs([]string{manifestListName, node, "--all"}) + ref, err := name.ParseReference(manifestListName) h.AssertNil(t, err) - h.AssertNotNil(t, outBuf) - }) - it("manifest available at registry only", func() { - var node = "alpine@sha256:777e2106170c66742ddbe77f703badb7dc94d9a5b1dc2c4a01538fad9aef56bb" - command.SetArgs([]string{"alpine:3.18", node, "--all"}) - err := command.Execute() + var path = filepath.Join(tmpDir,"manifests", ref.Name(), ref.Identifier()) + _, err = os.Stat(path) h.AssertNil(t, err) - h.AssertNotNil(t, outBuf) - }) - it("manifest list locally available", func() { - var node = "alpine@sha256:777e2106170c66742ddbe77f703badb7dc94d9a5b1dc2c4a01538fad9aef56bb" - command.SetArgs([]string{"alpine:3.18", node, "--all"}) - err := command.Execute() + err = command.Execute() h.AssertNil(t, err) h.AssertNotNil(t, outBuf) + _, err = os.Stat(path) + h.AssertNil(t, err) }) it("manifest list available at registry only", func() { var node = "alpine@sha256:777e2106170c66742ddbe77f703badb7dc94d9a5b1dc2c4a01538fad9aef56bb" + ref, err := name.ParseReference(node) + h.AssertNil(t, err) + var path = filepath.Join(tmpDir,"manifests", ref.Name(), ref.Identifier()) + _, err = os.Stat(path) + h.AssertNotNil(t, err) command.SetArgs([]string{"alpine:3.18", node, "--all"}) - err := command.Execute() + err = command.Execute() + h.AssertNil(t, err) + _, err = os.Stat(path) h.AssertNil(t, err) - h.AssertNotNil(t, outBuf) }) }) }) diff --git a/testhelpers/testhelpers.go b/testhelpers/testhelpers.go index 3eeb4b6431..a38c1a503d 100644 --- a/testhelpers/testhelpers.go +++ b/testhelpers/testhelpers.go @@ -116,18 +116,28 @@ func AssertError(t *testing.T, actual error, expected string) { } } -func AssertContains(t *testing.T, actual, expected string) { - t.Helper() - if !strings.Contains(actual, expected) { - t.Fatalf( - "Expected '%s' to contain '%s'\n\nDiff:%s", - actual, - expected, - cmp.Diff(expected, actual), - ) +func AssertContains(t *testing.T, actual string, expected ...string) { + t.Helper() + for _, e := range expected { + if !strings.Contains(actual, e) { + t.Fatalf( + "Expected '%s' to contain '%s'\n\nDiff:%s", + actual, + e, + cmp.Diff(expected, actual), + ) + } } } +func MapToStringSlice(m map[string]string) []string { + var stringSlice []string + for key, value := range m { + stringSlice = append(stringSlice, fmt.Sprintf("%s=%s", key, value)) + } + return stringSlice +} + func AssertContainsAllInOrder(t *testing.T, actual bytes.Buffer, expected ...string) { t.Helper() From 43268129f94ecb66ef6d866fe81bf7ab32a59230 Mon Sep 17 00:00:00 2001 From: WYGIN Date: Fri, 10 Nov 2023 09:36:31 +0000 Subject: [PATCH 62/62] WIP refactor manifest_add client Signed-off-by: WYGIN --- internal/commands/manifest_add.go | 41 +++++++++++++++++++++++++++++-- pkg/client/add_manifest.go | 4 ++- 2 files changed, 42 insertions(+), 3 deletions(-) diff --git a/internal/commands/manifest_add.go b/internal/commands/manifest_add.go index 05f56a4481..97a4faac65 100644 --- a/internal/commands/manifest_add.go +++ b/internal/commands/manifest_add.go @@ -1,6 +1,9 @@ package commands import ( + "fmt" + "strings" + "github.com/spf13/cobra" "github.com/buildpacks/pack/pkg/client" @@ -9,7 +12,8 @@ import ( // ManifestAddFlags define flags provided to the ManifestAdd type ManifestAddFlags struct { - ManifestAnnotateFlags + OS, OSVersion, OSArch, OSVariant string + OSFeatures, Annotations, Features string all bool } @@ -33,8 +37,21 @@ func ManifestAdd(logger logging.Logger, pack PackClient) *cobra.Command { return err } - imageID, err := pack.AddManifest(cmd.Context(), imageIndex, manifests, client.ManifestAddOptions{ + osFeatures:= strings.Split(flags.OSFeatures, ";") + features:= strings.Split(flags.Features, ";") + annotations, err := stringToKeyValueMap(flags.Annotations) + if err != nil { + return err + } + imageID, err := pack.AddManifest(cmd.Context(), imageIndex, manifests, client.ManifestAddOptions{ + OS: flags.OS, + OSVersion: flags.OSVersion, + OSArch: flags.OSArch, + OSVariant: flags.OSVariant, + OSFeatures: osFeatures, + Features: features, + Annotations: annotations, All: flags.all, }) @@ -59,3 +76,23 @@ func ManifestAdd(logger logging.Logger, pack PackClient) *cobra.Command { func validateManifestAddFlags(flags ManifestAddFlags) error { return nil } + +func stringToKeyValueMap(s string) (map[string]string, error) { + keyValues := strings.Split(s, ";") + + m := map[string]string{} + + for _, keyValue := range keyValues { + parts := strings.Split(keyValue, "=") + if len(parts) != 2 { + return nil, fmt.Errorf("invalid key-value pair: %s", keyValue) + } + + key := parts[0] + value := parts[1] + + m[key] = value + } + + return m, nil + } diff --git a/pkg/client/add_manifest.go b/pkg/client/add_manifest.go index db8be14157..a628688b6c 100644 --- a/pkg/client/add_manifest.go +++ b/pkg/client/add_manifest.go @@ -9,7 +9,9 @@ import ( ) type ManifestAddOptions struct { - ManifestAnnotateOptions + OS, OSVersion, OSArch, OSVariant string + OSFeatures, Features []string + Annotations map[string]string All bool }