Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

copy.Options: add an InstancePlatforms field #1938

Open
wants to merge 1 commit into
base: main
Choose a base branch
from
Open

copy.Options: add an InstancePlatforms field #1938

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions copy/copy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import (
"github.com/containers/image/v5/types"
encconfig "github.com/containers/ocicrypt/config"
digest "github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
"golang.org/x/exp/slices"
"golang.org/x/sync/semaphore"
Expand Down Expand Up @@ -91,8 +92,9 @@ type Options struct {
PreserveDigests bool
// manifest MIME type of image set by user. "" is default and means use the autodetection to the manifest MIME type
ForceManifestMIMEType string
ImageListSelection ImageListSelection // set to either CopySystemImage (the default), CopyAllImages, or CopySpecificImages to control which instances we copy when the source reference is a list; ignored if the source reference is not a list
Instances []digest.Digest // if ImageListSelection is CopySpecificImages, copy only these instances and the list itself
ImageListSelection ImageListSelection // set to either CopySystemImage (the default), CopyAllImages, or CopySpecificImages to control which instances we copy when the source reference is a list; ignored if the source reference is not a list
Instances []digest.Digest // if ImageListSelection is CopySpecificImages, copy only these instances, instances matching the InstancePlatforms list, and the list itself
InstancePlatforms []imgspecv1.Platform // if ImageListSelection is CopySpecificImages, copy only matching instances, instances listed in the Instances list, and the list itself
// Give priority to pulling gzip images if multiple images are present when configured to OptionalBoolTrue,
// prefers the best compression if this is configured as OptionalBoolFalse. Choose automatically (and the choice may change over time)
// if this is set to OptionalBoolUndefined (which is the default behavior, and recommended for most callers).
Expand Down
39 changes: 36 additions & 3 deletions copy/multiple.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@ import (
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/internal/image"
internalManifest "github.com/containers/image/v5/internal/manifest"
"github.com/containers/image/v5/internal/set"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/signature"
"github.com/containers/image/v5/types"
digest "github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/sirupsen/logrus"
"golang.org/x/exp/slices"
)

// copyMultipleImages copies some or all of an image list's instances, using
Expand Down Expand Up @@ -89,15 +91,19 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur
// Copy each image, or just the ones we want to copy, in turn.
instanceDigests := updatedList.Instances()
mtrmac marked this conversation as resolved.
Show resolved Hide resolved
imagesToCopy := len(instanceDigests)
var specificImages *set.Set[digest.Digest]
if options.ImageListSelection == CopySpecificImages {
imagesToCopy = len(options.Instances)
if specificImages, err = determineSpecificImages(options, updatedList); err != nil {
return nil, err
}
imagesToCopy = specificImages.Size()
}
c.Printf("Copying %d of %d images in list\n", imagesToCopy, len(instanceDigests))
updates := make([]manifest.ListUpdate, len(instanceDigests))
instancesCopied := 0
for i, instanceDigest := range instanceDigests {
if options.ImageListSelection == CopySpecificImages &&
!slices.Contains(options.Instances, instanceDigest) {
!specificImages.Contains(instanceDigest) {
update, err := updatedList.Instance(instanceDigest)
if err != nil {
return nil, err
Expand Down Expand Up @@ -196,3 +202,30 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur

return manifestList, nil
}

// determineSpecificImages returns a set of images to copy based on the
// Instances and InstancePlatforms fields of the passed-in options structure
func determineSpecificImages(options *Options, updatedList internalManifest.List) (*set.Set[digest.Digest], error) {
// Start with the instances that were listed by digest.
specificImages := set.New[digest.Digest]()
for _, instanceDigest := range options.Instances {
specificImages.Add(instanceDigest)
}
if len(options.InstancePlatforms) > 0 {
// Choose the best match for each platform we were asked to
// also copy, and add it to the set of instances to copy.
for _, platform := range options.InstancePlatforms {
platformContext := types.SystemContext{
OSChoice: platform.OS,
ArchitectureChoice: platform.Architecture,
VariantChoice: platform.Variant,
}
instanceDigest, err := updatedList.ChooseInstanceByCompression(&platformContext, options.PreferGzipInstances)
if err != nil {
return nil, fmt.Errorf("While choosing the instance for platform spec %v: %w", platform, err)
}
specificImages.Add(instanceDigest)
}
}
return specificImages, nil
}
170 changes: 170 additions & 0 deletions copy/multiple_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,170 @@
package copy

import (
"io/ioutil"
"strings"
"testing"

internalManifest "github.com/containers/image/v5/internal/manifest"
digest "github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func TestDetermineSpecificImages(t *testing.T) {
testCases := []struct {
id string
fixture string
instanceDigests []digest.Digest
instancePlatforms []imgspecv1.Platform
expected []digest.Digest
expectedErrIncludes string
}{
{
id: "no inputs no outputs",
fixture: "../manifest/fixtures/v2list.manifest.json",
},
{
id: "instances only out of order",
fixture: "../manifest/fixtures/v2list.manifest.json",
instanceDigests: []digest.Digest{
"sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2",
"sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783",
},
expected: []digest.Digest{
"sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783",
"sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2",
},
},
{
id: "instances only in order",
fixture: "../manifest/fixtures/v2list.manifest.json",
instanceDigests: []digest.Digest{
"sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783",
"sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2",
},
expected: []digest.Digest{
"sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783",
"sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2",
},
},
{
id: "platforms only in order",
fixture: "../manifest/fixtures/v2list.manifest.json",
instancePlatforms: []imgspecv1.Platform{
{
OS: "linux",
Architecture: "ppc64le",
},
{
OS: "linux",
Architecture: "s390x",
},
},
expected: []digest.Digest{
"sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783",
"sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2",
},
},
{
id: "platforms only out of order",
fixture: "../manifest/fixtures/v2list.manifest.json",
instancePlatforms: []imgspecv1.Platform{
{
OS: "linux",
Architecture: "s390x",
},
{
OS: "linux",
Architecture: "ppc64le",
},
},
expected: []digest.Digest{
"sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783",
"sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2",
},
},
{
id: "mixed without duplicates",
fixture: "../manifest/fixtures/v2list.manifest.json",
instancePlatforms: []imgspecv1.Platform{
{
OS: "linux",
Architecture: "s390x",
},
},
instanceDigests: []digest.Digest{
"sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783",
},
expected: []digest.Digest{
"sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783",
"sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2",
},
},
{
id: "mixed with duplicates",
fixture: "../manifest/fixtures/v2list.manifest.json",
instancePlatforms: []imgspecv1.Platform{
{
OS: "linux",
Architecture: "ppc64le",
},
{
OS: "linux",
Architecture: "s390x",
},
},
instanceDigests: []digest.Digest{
"sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783",
"sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2",
},
expected: []digest.Digest{
"sha256:7820f9a86d4ad15a2c4f0c0e5479298df2aa7c2f6871288e2ef8546f3e7b6783",
"sha256:e4c0df75810b953d6717b8f8f28298d73870e8aa2a0d5e77b8391f16fdfbbbe2",
},
},
{
id: "no such platform",
fixture: "../manifest/fixtures/v2list.manifest.json",
instancePlatforms: []imgspecv1.Platform{
{
OS: "windows",
Architecture: "amd64",
},
{
OS: "darwin",
Architecture: "arm64",
},
},
expectedErrIncludes: "no image found in manifest list for",
},
}
for _, tc := range testCases {
t.Run(tc.id, func(t *testing.T) {
listBlob, err := ioutil.ReadFile(tc.fixture)
require.NoErrorf(t, err, "unexpected error reading fixture %q", tc.fixture)
list, err := internalManifest.ListFromBlob(listBlob, internalManifest.GuessMIMEType(listBlob))
require.NoErrorf(t, err, "unexpected error parsing fixture %q", tc.fixture)
options := Options{
Instances: tc.instanceDigests,
InstancePlatforms: tc.instancePlatforms,
}
specific, err := determineSpecificImages(&options, list)
if err != nil && tc.expectedErrIncludes != "" {
if strings.Contains(err.Error(), tc.expectedErrIncludes) {
// okay
return
}
}
require.NoErrorf(t, err, "unexpected error selecting instances")
var selected []digest.Digest
for _, instanceDigest := range list.Instances() {
if specific.Contains(instanceDigest) {
selected = append(selected, instanceDigest)
}
}
assert.Equalf(t, tc.expected, selected, "given instance list %#v and platforms list %#v", tc.instanceDigests, tc.instancePlatforms)
})
}
}
4 changes: 4 additions & 0 deletions internal/set/set.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,7 @@ func (s *Set[E]) Empty() bool {
func (s *Set[E]) Values() []E {
return maps.Keys(s.m)
}

func (s *Set[E]) Size() int {
return len(s.m)
}