diff --git a/go.mod b/go.mod index b087bb110..cf5558a54 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/containerd/containerd v1.7.2 github.com/containernetworking/cni v1.1.2 github.com/containernetworking/plugins v1.3.0 - github.com/containers/image/v5 v5.26.1 + github.com/containers/image/v5 v5.27.0 github.com/containers/ocicrypt v1.1.7 github.com/containers/storage v1.48.0 github.com/coreos/go-systemd/v22 v22.5.0 diff --git a/go.sum b/go.sum index 7f1724152..4d432f31b 100644 --- a/go.sum +++ b/go.sum @@ -49,8 +49,8 @@ github.com/containernetworking/cni v1.1.2 h1:wtRGZVv7olUHMOqouPpn3cXJWpJgM6+EUl3 github.com/containernetworking/cni v1.1.2/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= github.com/containernetworking/plugins v1.3.0 h1:QVNXMT6XloyMUoO2wUOqWTC1hWFV62Q6mVDp5H1HnjM= github.com/containernetworking/plugins v1.3.0/go.mod h1:Pc2wcedTQQCVuROOOaLBPPxrEXqqXBFt3cZ+/yVg6l0= -github.com/containers/image/v5 v5.26.1 h1:8y3xq8GO/6y8FR+nAedHPsAFiAtOrab9qHTBpbqaX8g= -github.com/containers/image/v5 v5.26.1/go.mod h1:IwlOGzTkGnmfirXxt0hZeJlzv1zVukE03WZQ203Z9GA= +github.com/containers/image/v5 v5.27.0 h1:4jKVWAa4YurTWUyAWMoC71zJkSylBR7pWd0jqGkukYc= +github.com/containers/image/v5 v5.27.0/go.mod h1:IwlOGzTkGnmfirXxt0hZeJlzv1zVukE03WZQ203Z9GA= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01 h1:Qzk5C6cYglewc+UyGf6lc8Mj2UaPTHy/iF2De0/77CA= github.com/containers/libtrust v0.0.0-20230121012942-c1716e8a8d01/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/ocicrypt v1.1.7 h1:thhNr4fu2ltyGz8aMx8u48Ae0Pnbip3ePP9/mzkZ/3U= diff --git a/libimage/manifest_list.go b/libimage/manifest_list.go index 0223fb355..12586f2a4 100644 --- a/libimage/manifest_list.go +++ b/libimage/manifest_list.go @@ -415,6 +415,8 @@ type ManifestListPushOptions struct { ImageListSelection imageCopy.ImageListSelection // Use when selecting only specific imags. Instances []digest.Digest + // Add existing instances with requested compression algorithms to manifest list + AddCompression []string } // Push pushes a manifest to the specified destination. @@ -446,6 +448,7 @@ func (m *ManifestList) Push(ctx context.Context, destination string, options *Ma defer copier.close() pushOptions := manifests.PushOptions{ + AddCompression: options.AddCompression, Store: m.image.runtime.store, SystemContext: copier.systemContext, ImageListSelection: options.ImageListSelection, diff --git a/libimage/manifests/manifests.go b/libimage/manifests/manifests.go index 7a51b8423..68f991485 100644 --- a/libimage/manifests/manifests.go +++ b/libimage/manifests/manifests.go @@ -14,6 +14,7 @@ import ( "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/image" "github.com/containers/image/v5/manifest" + "github.com/containers/image/v5/pkg/compression" "github.com/containers/image/v5/signature" "github.com/containers/image/v5/signature/signer" is "github.com/containers/image/v5/storage" @@ -70,6 +71,7 @@ type PushOptions struct { RemoveSignatures bool // true to discard signatures in images ManifestType string // the format to use when saving the list - possible options are oci, v2s1, and v2s2 SourceFilter LookupReferenceFunc // filter the list source + AddCompression []string // add existing instances with requested compression algorithms to manifest list } // Create creates a new list containing information about the specified image, @@ -239,6 +241,10 @@ func (l *list) Push(ctx context.Context, dest types.ImageReference, options Push return nil, "", err } } + compressionVariants, err := prepareAddWithCompression(options.AddCompression) + if err != nil { + return nil, "", err + } copyOptions := &cp.Options{ ImageListSelection: options.ImageListSelection, Instances: options.Instances, @@ -252,6 +258,7 @@ func (l *list) Push(ctx context.Context, dest types.ImageReference, options Push SignBySigstorePrivateKeyFile: options.SignBySigstorePrivateKeyFile, SignSigstorePrivateKeyPassphrase: options.SignSigstorePrivateKeyPassphrase, ForceManifestMIMEType: singleImageManifestType, + EnsureCompressionVariantsExist: compressionVariants, } // Copy whatever we were asked to copy. @@ -266,6 +273,18 @@ func (l *list) Push(ctx context.Context, dest types.ImageReference, options Push return nil, manifestDigest, nil } +func prepareAddWithCompression(variants []string) ([]cp.OptionCompressionVariant, error) { + res := []cp.OptionCompressionVariant{} + for _, name := range variants { + algo, err := compression.AlgorithmByName(name) + if err != nil { + return nil, fmt.Errorf("requested algorithm %s is not supported for replication: %w", name, err) + } + res = append(res, cp.OptionCompressionVariant{Algorithm: algo}) + } + return res, nil +} + // Add adds information about the specified image to the list, computing the // image's manifest's digest, retrieving OS and architecture information from // the image's configuration, and recording the image's reference so that it diff --git a/libimage/manifests/manifests_test.go b/libimage/manifests/manifests_test.go index 3b8b0e400..06b5ba0b0 100644 --- a/libimage/manifests/manifests_test.go +++ b/libimage/manifests/manifests_test.go @@ -333,4 +333,8 @@ func TestPush(t *testing.T) { options.Instances = append(options.Instances, otherListDigest) _, _, err = list.Push(ctx, destRef, options) assert.Nilf(t, err, "list.Push(four specified)") + + options.AddCompression = []string{"zstd"} + _, _, err = list.Push(ctx, destRef, options) + assert.NoError(t, err, "list.Push(with replication for zstd specified)") } diff --git a/vendor/github.com/containers/image/v5/copy/blob.go b/vendor/github.com/containers/image/v5/copy/blob.go index f45b97f56..8d5580d7c 100644 --- a/vendor/github.com/containers/image/v5/copy/blob.go +++ b/vendor/github.com/containers/image/v5/copy/blob.go @@ -83,12 +83,12 @@ func (ic *imageCopier) copyBlobFromStream(ctx context.Context, srcReader io.Read return types.BlobInfo{}, err } - // === Report progress using the ic.c.progress channel, if required. - if ic.c.progress != nil && ic.c.progressInterval > 0 { + // === Report progress using the ic.c.options.Progress channel, if required. + if ic.c.options.Progress != nil && ic.c.options.ProgressInterval > 0 { progressReader := newProgressReader( stream.reader, - ic.c.progress, - ic.c.progressInterval, + ic.c.options.Progress, + ic.c.options.ProgressInterval, srcInfo, ) defer progressReader.reportDone() diff --git a/vendor/github.com/containers/image/v5/copy/copy.go b/vendor/github.com/containers/image/v5/copy/copy.go index bf8f4015b..ac0e6f2fa 100644 --- a/vendor/github.com/containers/image/v5/copy/copy.go +++ b/vendor/github.com/containers/image/v5/copy/copy.go @@ -17,6 +17,7 @@ import ( "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/blobinfocache" + compression "github.com/containers/image/v5/pkg/compression/types" "github.com/containers/image/v5/signature" "github.com/containers/image/v5/signature/signer" "github.com/containers/image/v5/transports" @@ -126,36 +127,46 @@ type Options struct { // Download layer contents with "nondistributable" media types ("foreign" layers) and translate the layer media type // to not indicate "nondistributable". DownloadForeignLayers bool + + // Contains slice of OptionCompressionVariant, where copy will ensure that for each platform + // in the manifest list, a variant with the requested compression will exist. + // Invalid when copying a non-multi-architecture image. That will probably + // change in the future. + EnsureCompressionVariantsExist []OptionCompressionVariant +} + +// OptionCompressionVariant allows to supply information about +// selected compression algorithm and compression level by the +// end-user. Refer to EnsureCompressionVariantsExist to know +// more about its usage. +type OptionCompressionVariant struct { + Algorithm compression.Algorithm + Level *int // Only used when we are creating a new image instance using the specified algorithm, not when the image already contains such an instance } // copier allows us to keep track of diffID values for blobs, and other // data shared across one or more images in a possible manifest list. // The owner must call close() when done. type copier struct { - dest private.ImageDestination - rawSource private.ImageSource - reportWriter io.Writer - progressOutput io.Writer - progressInterval time.Duration - progress chan types.ProgressProperties + policyContext *signature.PolicyContext + dest private.ImageDestination + rawSource private.ImageSource + options *Options // never nil + + reportWriter io.Writer + progressOutput io.Writer + + unparsedToplevel *image.UnparsedImage // for rawSource blobInfoCache internalblobinfocache.BlobInfoCache2 - ociDecryptConfig *encconfig.DecryptConfig - ociEncryptConfig *encconfig.EncryptConfig concurrentBlobCopiesSemaphore *semaphore.Weighted // Limits the amount of concurrently copied blobs - downloadForeignLayers bool - signers []*signer.Signer // Signers to use to create new signatures for the image - signersToClose []*signer.Signer // Signers that should be closed when this copier is destroyed. + signers []*signer.Signer // Signers to use to create new signatures for the image + signersToClose []*signer.Signer // Signers that should be closed when this copier is destroyed. } // Image copies image from srcRef to destRef, using policyContext to validate // source image admissibility. It returns the manifest which was written to // the new copy of the image. func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, srcRef types.ImageReference, options *Options) (copiedManifest []byte, retErr error) { - // NOTE this function uses an output parameter for the error return value. - // Setting this and returning is the ideal way to return an error. - // - // the defers in this routine will wrap the error return with its own errors - // which can be valuable context in the middle of a multi-streamed copy. if options == nil { options = &Options{} } @@ -209,27 +220,27 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, } c := &copier{ - dest: dest, - rawSource: rawSource, - reportWriter: reportWriter, - progressOutput: progressOutput, - progressInterval: options.ProgressInterval, - progress: options.Progress, + policyContext: policyContext, + dest: dest, + rawSource: rawSource, + options: options, + + reportWriter: reportWriter, + progressOutput: progressOutput, + + unparsedToplevel: image.UnparsedInstance(rawSource, nil), // FIXME? The cache is used for sources and destinations equally, but we only have a SourceCtx and DestinationCtx. // For now, use DestinationCtx (because blob reuse changes the behavior of the destination side more); eventually // we might want to add a separate CommonCtx — or would that be too confusing? - blobInfoCache: internalblobinfocache.FromBlobInfoCache(blobinfocache.DefaultCache(options.DestinationCtx)), - ociDecryptConfig: options.OciDecryptConfig, - ociEncryptConfig: options.OciEncryptConfig, - downloadForeignLayers: options.DownloadForeignLayers, + blobInfoCache: internalblobinfocache.FromBlobInfoCache(blobinfocache.DefaultCache(options.DestinationCtx)), } defer c.close() // Set the concurrentBlobCopiesSemaphore if we can copy layers in parallel. if dest.HasThreadSafePutBlob() && rawSource.HasThreadSafeGetBlob() { - c.concurrentBlobCopiesSemaphore = options.ConcurrentBlobCopiesSemaphore + c.concurrentBlobCopiesSemaphore = c.options.ConcurrentBlobCopiesSemaphore if c.concurrentBlobCopiesSemaphore == nil { - max := options.MaxParallelDownloads + max := c.options.MaxParallelDownloads if max == 0 { max = maxParallelDownloads } @@ -237,33 +248,40 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, } } else { c.concurrentBlobCopiesSemaphore = semaphore.NewWeighted(int64(1)) - if options.ConcurrentBlobCopiesSemaphore != nil { - if err := options.ConcurrentBlobCopiesSemaphore.Acquire(ctx, 1); err != nil { + if c.options.ConcurrentBlobCopiesSemaphore != nil { + if err := c.options.ConcurrentBlobCopiesSemaphore.Acquire(ctx, 1); err != nil { return nil, fmt.Errorf("acquiring semaphore for concurrent blob copies: %w", err) } - defer options.ConcurrentBlobCopiesSemaphore.Release(1) + defer c.options.ConcurrentBlobCopiesSemaphore.Release(1) } } - if err := c.setupSigners(options); err != nil { + if err := c.setupSigners(); err != nil { return nil, err } - unparsedToplevel := image.UnparsedInstance(rawSource, nil) - multiImage, err := isMultiImage(ctx, unparsedToplevel) + multiImage, err := isMultiImage(ctx, c.unparsedToplevel) if err != nil { return nil, fmt.Errorf("determining manifest MIME type for %s: %w", transports.ImageName(srcRef), err) } if !multiImage { + if len(options.EnsureCompressionVariantsExist) > 0 { + return nil, fmt.Errorf("EnsureCompressionVariantsExist is not implemented when not creating a multi-architecture image") + } // The simple case: just copy a single image. - if copiedManifest, _, _, err = c.copySingleImage(ctx, policyContext, options, unparsedToplevel, unparsedToplevel, nil); err != nil { + single, err := c.copySingleImage(ctx, c.unparsedToplevel, nil, copySingleImageOptions{requireCompressionFormatMatch: false}) + if err != nil { return nil, err } - } else if options.ImageListSelection == CopySystemImage { + copiedManifest = single.manifest + } else if c.options.ImageListSelection == CopySystemImage { + if len(options.EnsureCompressionVariantsExist) > 0 { + return nil, fmt.Errorf("EnsureCompressionVariantsExist is not implemented when not creating a multi-architecture image") + } // This is a manifest list, and we weren't asked to copy multiple images. Choose a single image that // matches the current system to copy, and copy it. - mfest, manifestType, err := unparsedToplevel.Manifest(ctx) + mfest, manifestType, err := c.unparsedToplevel.Manifest(ctx) if err != nil { return nil, fmt.Errorf("reading manifest for %s: %w", transports.ImageName(srcRef), err) } @@ -271,34 +289,35 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, if err != nil { return nil, fmt.Errorf("parsing primary manifest as list for %s: %w", transports.ImageName(srcRef), err) } - instanceDigest, err := manifestList.ChooseInstanceByCompression(options.SourceCtx, options.PreferGzipInstances) // try to pick one that matches options.SourceCtx + instanceDigest, err := manifestList.ChooseInstanceByCompression(c.options.SourceCtx, c.options.PreferGzipInstances) // try to pick one that matches c.options.SourceCtx if err != nil { return nil, fmt.Errorf("choosing an image from manifest list %s: %w", transports.ImageName(srcRef), err) } logrus.Debugf("Source is a manifest list; copying (only) instance %s for current system", instanceDigest) unparsedInstance := image.UnparsedInstance(rawSource, &instanceDigest) - - if copiedManifest, _, _, err = c.copySingleImage(ctx, policyContext, options, unparsedToplevel, unparsedInstance, nil); err != nil { + single, err := c.copySingleImage(ctx, unparsedInstance, nil, copySingleImageOptions{requireCompressionFormatMatch: false}) + if err != nil { return nil, fmt.Errorf("copying system image from manifest list: %w", err) } - } else { /* options.ImageListSelection == CopyAllImages or options.ImageListSelection == CopySpecificImages, */ + copiedManifest = single.manifest + } else { /* c.options.ImageListSelection == CopyAllImages or c.options.ImageListSelection == CopySpecificImages, */ // If we were asked to copy multiple images and can't, that's an error. if !supportsMultipleImages(c.dest) { return nil, fmt.Errorf("copying multiple images: destination transport %q does not support copying multiple images as a group", destRef.Transport().Name()) } // Copy some or all of the images. - switch options.ImageListSelection { + switch c.options.ImageListSelection { case CopyAllImages: logrus.Debugf("Source is a manifest list; copying all instances") case CopySpecificImages: logrus.Debugf("Source is a manifest list; copying some instances") } - if copiedManifest, err = c.copyMultipleImages(ctx, policyContext, options, unparsedToplevel); err != nil { + if copiedManifest, err = c.copyMultipleImages(ctx); err != nil { return nil, err } } - if err := c.dest.Commit(ctx, unparsedToplevel); err != nil { + if err := c.dest.Commit(ctx, c.unparsedToplevel); err != nil { return nil, fmt.Errorf("committing the finished image: %w", err) } diff --git a/vendor/github.com/containers/image/v5/copy/encryption.go b/vendor/github.com/containers/image/v5/copy/encryption.go index 86fadff66..b406b0c31 100644 --- a/vendor/github.com/containers/image/v5/copy/encryption.go +++ b/vendor/github.com/containers/image/v5/copy/encryption.go @@ -34,7 +34,7 @@ type bpDecryptionStepData struct { // srcInfo is only used for error messages. // Returns data for other steps; the caller should eventually use updateCryptoOperation. func (ic *imageCopier) blobPipelineDecryptionStep(stream *sourceStream, srcInfo types.BlobInfo) (*bpDecryptionStepData, error) { - if !isOciEncrypted(stream.info.MediaType) || ic.c.ociDecryptConfig == nil { + if !isOciEncrypted(stream.info.MediaType) || ic.c.options.OciDecryptConfig == nil { return &bpDecryptionStepData{ decrypting: false, }, nil @@ -47,7 +47,7 @@ func (ic *imageCopier) blobPipelineDecryptionStep(stream *sourceStream, srcInfo desc := imgspecv1.Descriptor{ Annotations: stream.info.Annotations, } - reader, decryptedDigest, err := ocicrypt.DecryptLayer(ic.c.ociDecryptConfig, stream.reader, desc, false) + reader, decryptedDigest, err := ocicrypt.DecryptLayer(ic.c.options.OciDecryptConfig, stream.reader, desc, false) if err != nil { return nil, fmt.Errorf("decrypting layer %s: %w", srcInfo.Digest, err) } @@ -81,7 +81,7 @@ type bpEncryptionStepData struct { // Returns data for other steps; the caller should eventually call updateCryptoOperationAndAnnotations. func (ic *imageCopier) blobPipelineEncryptionStep(stream *sourceStream, toEncrypt bool, srcInfo types.BlobInfo, decryptionStep *bpDecryptionStepData) (*bpEncryptionStepData, error) { - if !toEncrypt || isOciEncrypted(srcInfo.MediaType) || ic.c.ociEncryptConfig == nil { + if !toEncrypt || isOciEncrypted(srcInfo.MediaType) || ic.c.options.OciEncryptConfig == nil { return &bpEncryptionStepData{ encrypting: false, }, nil @@ -101,7 +101,7 @@ func (ic *imageCopier) blobPipelineEncryptionStep(stream *sourceStream, toEncryp Size: srcInfo.Size, Annotations: annotations, } - reader, finalizer, err := ocicrypt.EncryptLayer(ic.c.ociEncryptConfig, stream.reader, desc) + reader, finalizer, err := ocicrypt.EncryptLayer(ic.c.options.OciEncryptConfig, stream.reader, desc) if err != nil { return nil, fmt.Errorf("encrypting blob %s: %w", srcInfo.Digest, err) } diff --git a/vendor/github.com/containers/image/v5/copy/multiple.go b/vendor/github.com/containers/image/v5/copy/multiple.go index 41ea1b11b..34f2129d6 100644 --- a/vendor/github.com/containers/image/v5/copy/multiple.go +++ b/vendor/github.com/containers/image/v5/copy/multiple.go @@ -5,16 +5,19 @@ import ( "context" "errors" "fmt" + "sort" "strings" "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/internal/image" internalManifest "github.com/containers/image/v5/internal/manifest" + "github.com/containers/image/v5/internal/set" "github.com/containers/image/v5/manifest" - "github.com/containers/image/v5/signature" + "github.com/containers/image/v5/pkg/compression" digest "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/sirupsen/logrus" + "golang.org/x/exp/maps" "golang.org/x/exp/slices" ) @@ -28,30 +31,125 @@ const ( type instanceCopy struct { op instanceCopyKind sourceDigest digest.Digest + + // Fields which can be used by callers when operation + // is `instanceCopyClone` + cloneCompressionVariant OptionCompressionVariant + clonePlatform *imgspecv1.Platform + cloneAnnotations map[string]string +} + +// internal type only to make imgspecv1.Platform comparable +type platformComparable struct { + architecture string + os string + osVersion string + osFeatures string + variant string +} + +// Converts imgspecv1.Platform to a comparable format. +func platformV1ToPlatformComparable(platform *imgspecv1.Platform) platformComparable { + if platform == nil { + return platformComparable{} + } + osFeatures := slices.Clone(platform.OSFeatures) + sort.Strings(osFeatures) + return platformComparable{architecture: platform.Architecture, + os: platform.OS, + // This is strictly speaking ambiguous, fields of OSFeatures can contain a ','. Probably good enough for now. + osFeatures: strings.Join(osFeatures, ","), + osVersion: platform.OSVersion, + variant: platform.Variant, + } +} + +// platformCompressionMap prepares a mapping of platformComparable -> CompressionAlgorithmNames for given digests +func platformCompressionMap(list internalManifest.List, instanceDigests []digest.Digest) (map[platformComparable]*set.Set[string], error) { + res := make(map[platformComparable]*set.Set[string]) + for _, instanceDigest := range instanceDigests { + instanceDetails, err := list.Instance(instanceDigest) + if err != nil { + return nil, fmt.Errorf("getting details for instance %s: %w", instanceDigest, err) + } + platform := platformV1ToPlatformComparable(instanceDetails.ReadOnly.Platform) + platformSet, ok := res[platform] + if !ok { + platformSet = set.New[string]() + res[platform] = platformSet + } + platformSet.AddSlice(instanceDetails.ReadOnly.CompressionAlgorithmNames) + } + return res, nil +} + +func validateCompressionVariantExists(input []OptionCompressionVariant) error { + for _, option := range input { + _, err := compression.AlgorithmByName(option.Algorithm.Name()) + if err != nil { + return fmt.Errorf("invalid algorithm %q in option.EnsureCompressionVariantsExist: %w", option.Algorithm.Name(), err) + } + } + return nil } // prepareInstanceCopies prepares a list of instances which needs to copied to the manifest list. -func prepareInstanceCopies(instanceDigests []digest.Digest, options *Options) []instanceCopy { +func prepareInstanceCopies(list internalManifest.List, instanceDigests []digest.Digest, options *Options) ([]instanceCopy, error) { res := []instanceCopy{} + if options.ImageListSelection == CopySpecificImages && len(options.EnsureCompressionVariantsExist) > 0 { + // List can already contain compressed instance for a compression selected in `EnsureCompressionVariantsExist` + // It’s unclear what it means when `CopySpecificImages` includes an instance in options.Instances, + // EnsureCompressionVariantsExist asks for an instance with some compression, + // an instance with that compression already exists, but is not included in options.Instances. + // We might define the semantics and implement this in the future. + return res, fmt.Errorf("EnsureCompressionVariantsExist is not implemented for CopySpecificImages") + } + err := validateCompressionVariantExists(options.EnsureCompressionVariantsExist) + if err != nil { + return res, err + } + compressionsByPlatform, err := platformCompressionMap(list, instanceDigests) + if err != nil { + return nil, err + } for i, instanceDigest := range instanceDigests { if options.ImageListSelection == CopySpecificImages && !slices.Contains(options.Instances, instanceDigest) { logrus.Debugf("Skipping instance %s (%d/%d)", instanceDigest, i+1, len(instanceDigests)) continue } + instanceDetails, err := list.Instance(instanceDigest) + if err != nil { + return res, fmt.Errorf("getting details for instance %s: %w", instanceDigest, err) + } res = append(res, instanceCopy{ op: instanceCopyCopy, sourceDigest: instanceDigest, }) + platform := platformV1ToPlatformComparable(instanceDetails.ReadOnly.Platform) + compressionList := compressionsByPlatform[platform] + for _, compressionVariant := range options.EnsureCompressionVariantsExist { + if !compressionList.Contains(compressionVariant.Algorithm.Name()) { + res = append(res, instanceCopy{ + op: instanceCopyClone, + sourceDigest: instanceDigest, + cloneCompressionVariant: compressionVariant, + clonePlatform: instanceDetails.ReadOnly.Platform, + cloneAnnotations: maps.Clone(instanceDetails.ReadOnly.Annotations), + }) + // add current compression to the list so that we don’t create duplicate clones + compressionList.Add(compressionVariant.Algorithm.Name()) + } + } } - return res + return res, nil } // copyMultipleImages copies some or all of an image list's instances, using -// policyContext to validate source image admissibility. -func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signature.PolicyContext, options *Options, unparsedToplevel *image.UnparsedImage) (copiedManifest []byte, retErr error) { +// c.policyContext to validate source image admissibility. +func (c *copier) copyMultipleImages(ctx context.Context) (copiedManifest []byte, retErr error) { // Parse the list and get a copy of the original value after it's re-encoded. - manifestList, manifestType, err := unparsedToplevel.Manifest(ctx) + manifestList, manifestType, err := c.unparsedToplevel.Manifest(ctx) if err != nil { return nil, fmt.Errorf("reading manifest list: %w", err) } @@ -61,7 +159,7 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur } updatedList := originalList.CloneInternal() - sigs, err := c.sourceSignatures(ctx, unparsedToplevel, options, + sigs, err := c.sourceSignatures(ctx, c.unparsedToplevel, "Getting image list signatures", "Checking if image list destination supports signatures") if err != nil { @@ -94,12 +192,12 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur if destIsDigestedReference { cannotModifyManifestListReason = "Destination specifies a digest" } - if options.PreserveDigests { + if c.options.PreserveDigests { cannotModifyManifestListReason = "Instructed to preserve digests" } // Determine if we'll need to convert the manifest list to a different format. - forceListMIMEType := options.ForceManifestMIMEType + forceListMIMEType := c.options.ForceManifestMIMEType switch forceListMIMEType { case manifest.DockerV2Schema1MediaType, manifest.DockerV2Schema1SignedMediaType, manifest.DockerV2Schema2MediaType: forceListMIMEType = manifest.DockerV2ListMediaType @@ -119,8 +217,11 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur // Copy each image, or just the ones we want to copy, in turn. instanceDigests := updatedList.Instances() instanceEdits := []internalManifest.ListEdit{} - instanceCopyList := prepareInstanceCopies(instanceDigests, options) - c.Printf("Copying %d of %d images in list\n", len(instanceCopyList), len(instanceDigests)) + instanceCopyList, err := prepareInstanceCopies(updatedList, instanceDigests, c.options) + if err != nil { + return nil, fmt.Errorf("preparing instances for copy: %w", err) + } + c.Printf("Copying %d images generated from %d images in list\n", len(instanceCopyList), len(instanceDigests)) for i, instance := range instanceCopyList { // Update instances to be edited by their `ListOperation` and // populate necessary fields. @@ -129,17 +230,39 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur logrus.Debugf("Copying instance %s (%d/%d)", instance.sourceDigest, i+1, len(instanceCopyList)) c.Printf("Copying image %s (%d/%d)\n", instance.sourceDigest, i+1, len(instanceCopyList)) unparsedInstance := image.UnparsedInstance(c.rawSource, &instanceCopyList[i].sourceDigest) - updatedManifest, updatedManifestType, updatedManifestDigest, err := c.copySingleImage(ctx, policyContext, options, unparsedToplevel, unparsedInstance, &instanceCopyList[i].sourceDigest) + updated, err := c.copySingleImage(ctx, unparsedInstance, &instanceCopyList[i].sourceDigest, copySingleImageOptions{requireCompressionFormatMatch: false}) if err != nil { return nil, fmt.Errorf("copying image %d/%d from manifest list: %w", i+1, len(instanceCopyList), err) } // Record the result of a possible conversion here. instanceEdits = append(instanceEdits, internalManifest.ListEdit{ - ListOperation: internalManifest.ListOpUpdate, - UpdateOldDigest: instance.sourceDigest, - UpdateDigest: updatedManifestDigest, - UpdateSize: int64(len(updatedManifest)), - UpdateMediaType: updatedManifestType}) + ListOperation: internalManifest.ListOpUpdate, + UpdateOldDigest: instance.sourceDigest, + UpdateDigest: updated.manifestDigest, + UpdateSize: int64(len(updated.manifest)), + UpdateCompressionAlgorithms: updated.compressionAlgorithms, + UpdateMediaType: updated.manifestMIMEType}) + case instanceCopyClone: + logrus.Debugf("Replicating instance %s (%d/%d)", instance.sourceDigest, i+1, len(instanceCopyList)) + c.Printf("Replicating image %s (%d/%d)\n", instance.sourceDigest, i+1, len(instanceCopyList)) + unparsedInstance := image.UnparsedInstance(c.rawSource, &instanceCopyList[i].sourceDigest) + updated, err := c.copySingleImage(ctx, unparsedInstance, &instanceCopyList[i].sourceDigest, copySingleImageOptions{ + requireCompressionFormatMatch: true, + compressionFormat: &instance.cloneCompressionVariant.Algorithm, + compressionLevel: instance.cloneCompressionVariant.Level}) + if err != nil { + return nil, fmt.Errorf("replicating image %d/%d from manifest list: %w", i+1, len(instanceCopyList), err) + } + // Record the result of a possible conversion here. + instanceEdits = append(instanceEdits, internalManifest.ListEdit{ + ListOperation: internalManifest.ListOpAdd, + AddDigest: updated.manifestDigest, + AddSize: int64(len(updated.manifest)), + AddMediaType: updated.manifestMIMEType, + AddPlatform: instance.clonePlatform, + AddAnnotations: instance.cloneAnnotations, + AddCompressionAlgorithms: updated.compressionAlgorithms, + }) default: return nil, fmt.Errorf("copying image: invalid copy operation %d", instance.op) } @@ -204,7 +327,7 @@ func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signatur } // Sign the manifest list. - newSigs, err := c.createSignatures(ctx, manifestList, options.SignIdentity) + newSigs, err := c.createSignatures(ctx, manifestList, c.options.SignIdentity) if err != nil { return nil, err } diff --git a/vendor/github.com/containers/image/v5/copy/sign.go b/vendor/github.com/containers/image/v5/copy/sign.go index fd19e18cd..0ec54ded2 100644 --- a/vendor/github.com/containers/image/v5/copy/sign.go +++ b/vendor/github.com/containers/image/v5/copy/sign.go @@ -13,20 +13,20 @@ import ( "github.com/containers/image/v5/transports" ) -// setupSigners initializes c.signers based on options. -func (c *copier) setupSigners(options *Options) error { - c.signers = append(c.signers, options.Signers...) - // c.signersToClose is intentionally not updated with options.Signers. +// setupSigners initializes c.signers. +func (c *copier) setupSigners() error { + c.signers = append(c.signers, c.options.Signers...) + // c.signersToClose is intentionally not updated with c.options.Signers. // We immediately append created signers to c.signers, and we rely on c.close() to clean them up; so we don’t need // to clean up any created signers on failure. - if options.SignBy != "" { + if c.options.SignBy != "" { opts := []simplesigning.Option{ - simplesigning.WithKeyFingerprint(options.SignBy), + simplesigning.WithKeyFingerprint(c.options.SignBy), } - if options.SignPassphrase != "" { - opts = append(opts, simplesigning.WithPassphrase(options.SignPassphrase)) + if c.options.SignPassphrase != "" { + opts = append(opts, simplesigning.WithPassphrase(c.options.SignPassphrase)) } signer, err := simplesigning.NewSigner(opts...) if err != nil { @@ -36,9 +36,9 @@ func (c *copier) setupSigners(options *Options) error { c.signersToClose = append(c.signersToClose, signer) } - if options.SignBySigstorePrivateKeyFile != "" { + if c.options.SignBySigstorePrivateKeyFile != "" { signer, err := sigstore.NewSigner( - sigstore.WithPrivateKeyFile(options.SignBySigstorePrivateKeyFile, options.SignSigstorePrivateKeyPassphrase), + sigstore.WithPrivateKeyFile(c.options.SignBySigstorePrivateKeyFile, c.options.SignSigstorePrivateKeyPassphrase), ) if err != nil { return err @@ -50,13 +50,13 @@ func (c *copier) setupSigners(options *Options) error { return nil } -// sourceSignatures returns signatures from unparsedSource based on options, +// sourceSignatures returns signatures from unparsedSource, // and verifies that they can be used (to avoid copying a large image when we // can tell in advance that it would ultimately fail) -func (c *copier) sourceSignatures(ctx context.Context, unparsed private.UnparsedImage, options *Options, +func (c *copier) sourceSignatures(ctx context.Context, unparsed private.UnparsedImage, gettingSignaturesMessage, checkingDestMessage string) ([]internalsig.Signature, error) { var sigs []internalsig.Signature - if options.RemoveSignatures { + if c.options.RemoveSignatures { sigs = []internalsig.Signature{} } else { c.Printf("%s\n", gettingSignaturesMessage) diff --git a/vendor/github.com/containers/image/v5/copy/single.go b/vendor/github.com/containers/image/v5/copy/single.go index b8569a70c..f40b5f2f7 100644 --- a/vendor/github.com/containers/image/v5/copy/single.go +++ b/vendor/github.com/containers/image/v5/copy/single.go @@ -18,7 +18,6 @@ import ( "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/compression" compressiontypes "github.com/containers/image/v5/pkg/compression/types" - "github.com/containers/image/v5/signature" "github.com/containers/image/v5/transports" "github.com/containers/image/v5/types" digest "github.com/opencontainers/go-digest" @@ -30,40 +29,54 @@ import ( // imageCopier tracks state specific to a single image (possibly an item of a manifest list) type imageCopier struct { - c *copier - manifestUpdates *types.ManifestUpdateOptions - src *image.SourcedImage - diffIDsAreNeeded bool - cannotModifyManifestReason string // The reason the manifest cannot be modified, or an empty string if it can - canSubstituteBlobs bool - compressionFormat *compressiontypes.Algorithm // Compression algorithm to use, if the user explicitly requested one, or nil. - compressionLevel *int - ociEncryptLayers *[]int + c *copier + manifestUpdates *types.ManifestUpdateOptions + src *image.SourcedImage + diffIDsAreNeeded bool + cannotModifyManifestReason string // The reason the manifest cannot be modified, or an empty string if it can + canSubstituteBlobs bool + compressionFormat *compressiontypes.Algorithm // Compression algorithm to use, if the user explicitly requested one, or nil. + compressionLevel *int + requireCompressionFormatMatch bool } -// copySingleImage copies a single (non-manifest-list) image unparsedImage, using policyContext to validate +type copySingleImageOptions struct { + requireCompressionFormatMatch bool + compressionFormat *compressiontypes.Algorithm // Compression algorithm to use, if the user explicitly requested one, or nil. + compressionLevel *int +} + +// copySingleImageResult carries data produced by copySingleImage +type copySingleImageResult struct { + manifest []byte + manifestMIMEType string + manifestDigest digest.Digest + compressionAlgorithms []compressiontypes.Algorithm +} + +// copySingleImage copies a single (non-manifest-list) image unparsedImage, using c.policyContext to validate // source image admissibility. -func (c *copier) copySingleImage(ctx context.Context, policyContext *signature.PolicyContext, options *Options, unparsedToplevel, unparsedImage *image.UnparsedImage, targetInstance *digest.Digest) (retManifest []byte, retManifestType string, retManifestDigest digest.Digest, retErr error) { +func (c *copier) copySingleImage(ctx context.Context, unparsedImage *image.UnparsedImage, targetInstance *digest.Digest, opts copySingleImageOptions) (copySingleImageResult, error) { // The caller is handling manifest lists; this could happen only if a manifest list contains a manifest list. // Make sure we fail cleanly in such cases. multiImage, err := isMultiImage(ctx, unparsedImage) if err != nil { // FIXME FIXME: How to name a reference for the sub-image? - return nil, "", "", fmt.Errorf("determining manifest MIME type for %s: %w", transports.ImageName(unparsedImage.Reference()), err) + return copySingleImageResult{}, fmt.Errorf("determining manifest MIME type for %s: %w", transports.ImageName(unparsedImage.Reference()), err) } if multiImage { - return nil, "", "", fmt.Errorf("Unexpectedly received a manifest list instead of a manifest for a single image") + return copySingleImageResult{}, fmt.Errorf("Unexpectedly received a manifest list instead of a manifest for a single image") } // Please keep this policy check BEFORE reading any other information about the image. // (The multiImage check above only matches the MIME type, which we have received anyway. // Actual parsing of anything should be deferred.) - if allowed, err := policyContext.IsRunningImageAllowed(ctx, unparsedImage); !allowed || err != nil { // Be paranoid and fail if either return value indicates so. - return nil, "", "", fmt.Errorf("Source image rejected: %w", err) + if allowed, err := c.policyContext.IsRunningImageAllowed(ctx, unparsedImage); !allowed || err != nil { // Be paranoid and fail if either return value indicates so. + return copySingleImageResult{}, fmt.Errorf("Source image rejected: %w", err) } - src, err := image.FromUnparsedImage(ctx, options.SourceCtx, unparsedImage) + src, err := image.FromUnparsedImage(ctx, c.options.SourceCtx, unparsedImage) if err != nil { - return nil, "", "", fmt.Errorf("initializing image from source %s: %w", transports.ImageName(c.rawSource.Reference()), err) + return copySingleImageResult{}, fmt.Errorf("initializing image from source %s: %w", transports.ImageName(c.rawSource.Reference()), err) } // If the destination is a digested reference, make a note of that, determine what digest value we're @@ -75,33 +88,33 @@ func (c *copier) copySingleImage(ctx context.Context, policyContext *signature.P destIsDigestedReference = true matches, err := manifest.MatchesDigest(src.ManifestBlob, digested.Digest()) if err != nil { - return nil, "", "", fmt.Errorf("computing digest of source image's manifest: %w", err) + return copySingleImageResult{}, fmt.Errorf("computing digest of source image's manifest: %w", err) } if !matches { - manifestList, _, err := unparsedToplevel.Manifest(ctx) + manifestList, _, err := c.unparsedToplevel.Manifest(ctx) if err != nil { - return nil, "", "", fmt.Errorf("reading manifest from source image: %w", err) + return copySingleImageResult{}, fmt.Errorf("reading manifest from source image: %w", err) } matches, err = manifest.MatchesDigest(manifestList, digested.Digest()) if err != nil { - return nil, "", "", fmt.Errorf("computing digest of source image's manifest: %w", err) + return copySingleImageResult{}, fmt.Errorf("computing digest of source image's manifest: %w", err) } if !matches { - return nil, "", "", errors.New("Digest of source image's manifest would not match destination reference") + return copySingleImageResult{}, errors.New("Digest of source image's manifest would not match destination reference") } } } } - if err := checkImageDestinationForCurrentRuntime(ctx, options.DestinationCtx, src, c.dest); err != nil { - return nil, "", "", err + if err := checkImageDestinationForCurrentRuntime(ctx, c.options.DestinationCtx, src, c.dest); err != nil { + return copySingleImageResult{}, err } - sigs, err := c.sourceSignatures(ctx, src, options, + sigs, err := c.sourceSignatures(ctx, src, "Getting image source signatures", "Checking if image destination supports signatures") if err != nil { - return nil, "", "", err + return copySingleImageResult{}, err } // Determine if we're allowed to modify the manifest. @@ -114,7 +127,7 @@ func (c *copier) copySingleImage(ctx context.Context, policyContext *signature.P if destIsDigestedReference { cannotModifyManifestReason = "Destination specifies a digest" } - if options.PreserveDigests { + if c.options.PreserveDigests { cannotModifyManifestReason = "Instructed to preserve digests" } @@ -123,13 +136,16 @@ func (c *copier) copySingleImage(ctx context.Context, policyContext *signature.P manifestUpdates: &types.ManifestUpdateOptions{InformationOnly: types.ManifestUpdateInformation{Destination: c.dest}}, src: src, // diffIDsAreNeeded is computed later - cannotModifyManifestReason: cannotModifyManifestReason, - ociEncryptLayers: options.OciEncryptLayers, + cannotModifyManifestReason: cannotModifyManifestReason, + requireCompressionFormatMatch: opts.requireCompressionFormatMatch, } - if options.DestinationCtx != nil { + if opts.compressionFormat != nil { + ic.compressionFormat = opts.compressionFormat + ic.compressionLevel = opts.compressionLevel + } else if c.options.DestinationCtx != nil { // Note that compressionFormat and compressionLevel can be nil. - ic.compressionFormat = options.DestinationCtx.CompressionFormat - ic.compressionLevel = options.DestinationCtx.CompressionLevel + ic.compressionFormat = c.options.DestinationCtx.CompressionFormat + ic.compressionLevel = c.options.DestinationCtx.CompressionLevel } // Decide whether we can substitute blobs with semantic equivalents: // - Don’t do that if we can’t modify the manifest at all @@ -142,20 +158,20 @@ func (c *copier) copySingleImage(ctx context.Context, policyContext *signature.P ic.canSubstituteBlobs = ic.cannotModifyManifestReason == "" && len(c.signers) == 0 if err := ic.updateEmbeddedDockerReference(); err != nil { - return nil, "", "", err + return copySingleImageResult{}, err } - destRequiresOciEncryption := (isEncrypted(src) && ic.c.ociDecryptConfig != nil) || options.OciEncryptLayers != nil + destRequiresOciEncryption := (isEncrypted(src) && ic.c.options.OciDecryptConfig != nil) || c.options.OciEncryptLayers != nil manifestConversionPlan, err := determineManifestConversion(determineManifestConversionInputs{ srcMIMEType: ic.src.ManifestMIMEType, destSupportedManifestMIMETypes: ic.c.dest.SupportedManifestMIMETypes(), - forceManifestMIMEType: options.ForceManifestMIMEType, + forceManifestMIMEType: c.options.ForceManifestMIMEType, requiresOCIEncryption: destRequiresOciEncryption, cannotModifyManifestReason: ic.cannotModifyManifestReason, }) if err != nil { - return nil, "", "", err + return copySingleImageResult{}, err } // We set up this part of ic.manifestUpdates quite early, not just around the // code that calls copyUpdatedConfigAndManifest, so that other parts of the copy code @@ -169,27 +185,28 @@ func (c *copier) copySingleImage(ctx context.Context, policyContext *signature.P ic.diffIDsAreNeeded = src.UpdatedImageNeedsLayerDiffIDs(*ic.manifestUpdates) // If enabled, fetch and compare the destination's manifest. And as an optimization skip updating the destination iff equal - if options.OptimizeDestinationImageAlreadyExists { + if c.options.OptimizeDestinationImageAlreadyExists { shouldUpdateSigs := len(sigs) > 0 || len(c.signers) != 0 // TODO: Consider allowing signatures updates only and skipping the image's layers/manifest copy if possible noPendingManifestUpdates := ic.noPendingManifestUpdates() - logrus.Debugf("Checking if we can skip copying: has signatures=%t, OCI encryption=%t, no manifest updates=%t", shouldUpdateSigs, destRequiresOciEncryption, noPendingManifestUpdates) - if !shouldUpdateSigs && !destRequiresOciEncryption && noPendingManifestUpdates { - isSrcDestManifestEqual, retManifest, retManifestType, retManifestDigest, err := compareImageDestinationManifestEqual(ctx, options, src, targetInstance, c.dest) + logrus.Debugf("Checking if we can skip copying: has signatures=%t, OCI encryption=%t, no manifest updates=%t, compression match required for resuing blobs=%t", shouldUpdateSigs, destRequiresOciEncryption, noPendingManifestUpdates, opts.requireCompressionFormatMatch) + if !shouldUpdateSigs && !destRequiresOciEncryption && noPendingManifestUpdates && !ic.requireCompressionFormatMatch { + matchedResult, err := ic.compareImageDestinationManifestEqual(ctx, targetInstance) if err != nil { logrus.Warnf("Failed to compare destination image manifest: %v", err) - return nil, "", "", err + return copySingleImageResult{}, err } - if isSrcDestManifestEqual { + if matchedResult != nil { c.Printf("Skipping: image already present at destination\n") - return retManifest, retManifestType, retManifestDigest, nil + return *matchedResult, nil } } } - if err := ic.copyLayers(ctx); err != nil { - return nil, "", "", err + compressionAlgos, err := ic.copyLayers(ctx) + if err != nil { + return copySingleImageResult{}, err } // With docker/distribution registries we do not know whether the registry accepts schema2 or schema1 only; @@ -197,8 +214,12 @@ func (c *copier) copySingleImage(ctx context.Context, policyContext *signature.P // without actually trying to upload something and getting a types.ManifestTypeRejectedError. // So, try the preferred manifest MIME type with possibly-updated blob digests, media types, and sizes if // we're altering how they're compressed. If the process succeeds, fine… - manifestBytes, retManifestDigest, err := ic.copyUpdatedConfigAndManifest(ctx, targetInstance) - retManifestType = manifestConversionPlan.preferredMIMEType + manifestBytes, manifestDigest, err := ic.copyUpdatedConfigAndManifest(ctx, targetInstance) + wipResult := copySingleImageResult{ + manifest: manifestBytes, + manifestMIMEType: manifestConversionPlan.preferredMIMEType, + manifestDigest: manifestDigest, + } if err != nil { logrus.Debugf("Writing manifest using preferred type %s failed: %v", manifestConversionPlan.preferredMIMEType, err) // … if it fails, and the failure is either because the manifest is rejected by the registry, or @@ -213,14 +234,14 @@ func (c *copier) copySingleImage(ctx context.Context, policyContext *signature.P // We don’t have other options. // In principle the code below would handle this as well, but the resulting error message is fairly ugly. // Don’t bother the user with MIME types if we have no choice. - return nil, "", "", err + return copySingleImageResult{}, err } // If the original MIME type is acceptable, determineManifestConversion always uses it as manifestConversionPlan.preferredMIMEType. // So if we are here, we will definitely be trying to convert the manifest. // With ic.cannotModifyManifestReason != "", that would just be a string of repeated failures for the same reason, // so let’s bail out early and with a better error message. if ic.cannotModifyManifestReason != "" { - return nil, "", "", fmt.Errorf("writing manifest failed and we cannot try conversions: %q: %w", cannotModifyManifestReason, err) + return copySingleImageResult{}, fmt.Errorf("writing manifest failed and we cannot try conversions: %q: %w", cannotModifyManifestReason, err) } // errs is a list of errors when trying various manifest types. Also serves as an "upload succeeded" flag when set to nil. @@ -236,34 +257,37 @@ func (c *copier) copySingleImage(ctx context.Context, policyContext *signature.P } // We have successfully uploaded a manifest. - manifestBytes = attemptedManifest - retManifestDigest = attemptedManifestDigest - retManifestType = manifestMIMEType + wipResult = copySingleImageResult{ + manifest: attemptedManifest, + manifestMIMEType: manifestMIMEType, + manifestDigest: attemptedManifestDigest, + } errs = nil // Mark this as a success so that we don't abort below. break } if errs != nil { - return nil, "", "", fmt.Errorf("Uploading manifest failed, attempted the following formats: %s", strings.Join(errs, ", ")) + return copySingleImageResult{}, fmt.Errorf("Uploading manifest failed, attempted the following formats: %s", strings.Join(errs, ", ")) } } if targetInstance != nil { - targetInstance = &retManifestDigest + targetInstance = &wipResult.manifestDigest } - newSigs, err := c.createSignatures(ctx, manifestBytes, options.SignIdentity) + newSigs, err := c.createSignatures(ctx, wipResult.manifest, c.options.SignIdentity) if err != nil { - return nil, "", "", err + return copySingleImageResult{}, err } sigs = append(sigs, newSigs...) if len(sigs) > 0 { c.Printf("Storing signatures\n") if err := c.dest.PutSignaturesWithFormat(ctx, sigs, targetInstance); err != nil { - return nil, "", "", fmt.Errorf("writing signatures: %w", err) + return copySingleImageResult{}, fmt.Errorf("writing signatures: %w", err) } } - - return manifestBytes, retManifestType, retManifestDigest, nil + wipResult.compressionAlgorithms = compressionAlgos + res := wipResult // We are done + return res, nil } // checkImageDestinationForCurrentRuntime enforces dest.MustMatchRuntimeOS, if necessary. @@ -323,52 +347,68 @@ func (ic *imageCopier) noPendingManifestUpdates() bool { return reflect.DeepEqual(*ic.manifestUpdates, types.ManifestUpdateOptions{InformationOnly: ic.manifestUpdates.InformationOnly}) } -// compareImageDestinationManifestEqual compares the `src` and `dest` image manifests (reading the manifest from the -// (possibly remote) destination). Returning true and the destination's manifest, type and digest if they compare equal. -func compareImageDestinationManifestEqual(ctx context.Context, options *Options, src *image.SourcedImage, targetInstance *digest.Digest, dest types.ImageDestination) (bool, []byte, string, digest.Digest, error) { - srcManifestDigest, err := manifest.Digest(src.ManifestBlob) +// compareImageDestinationManifestEqual compares the source and destination image manifests (reading the manifest from the +// (possibly remote) destination). If they are equal, it returns a full copySingleImageResult, nil otherwise. +func (ic *imageCopier) compareImageDestinationManifestEqual(ctx context.Context, targetInstance *digest.Digest) (*copySingleImageResult, error) { + srcManifestDigest, err := manifest.Digest(ic.src.ManifestBlob) if err != nil { - return false, nil, "", "", fmt.Errorf("calculating manifest digest: %w", err) + return nil, fmt.Errorf("calculating manifest digest: %w", err) } - destImageSource, err := dest.Reference().NewImageSource(ctx, options.DestinationCtx) + destImageSource, err := ic.c.dest.Reference().NewImageSource(ctx, ic.c.options.DestinationCtx) if err != nil { - logrus.Debugf("Unable to create destination image %s source: %v", dest.Reference(), err) - return false, nil, "", "", nil + logrus.Debugf("Unable to create destination image %s source: %v", ic.c.dest.Reference(), err) + return nil, nil } destManifest, destManifestType, err := destImageSource.GetManifest(ctx, targetInstance) if err != nil { logrus.Debugf("Unable to get destination image %s/%s manifest: %v", destImageSource, targetInstance, err) - return false, nil, "", "", nil + return nil, nil } destManifestDigest, err := manifest.Digest(destManifest) if err != nil { - return false, nil, "", "", fmt.Errorf("calculating manifest digest: %w", err) + return nil, fmt.Errorf("calculating manifest digest: %w", err) } logrus.Debugf("Comparing source and destination manifest digests: %v vs. %v", srcManifestDigest, destManifestDigest) if srcManifestDigest != destManifestDigest { - return false, nil, "", "", nil + return nil, nil + } + + compressionAlgos := set.New[string]() + for _, srcInfo := range ic.src.LayerInfos() { + compression := compressionAlgorithmFromMIMEType(srcInfo) + compressionAlgos.Add(compression.Name()) + } + + algos, err := algorithmsByNames(compressionAlgos.Values()) + if err != nil { + return nil, err } // Destination and source manifests, types and digests should all be equivalent - return true, destManifest, destManifestType, destManifestDigest, nil + return ©SingleImageResult{ + manifest: destManifest, + manifestMIMEType: destManifestType, + manifestDigest: srcManifestDigest, + compressionAlgorithms: algos, + }, nil } // copyLayers copies layers from ic.src/ic.c.rawSource to dest, using and updating ic.manifestUpdates if necessary and ic.cannotModifyManifestReason == "". -func (ic *imageCopier) copyLayers(ctx context.Context) error { +func (ic *imageCopier) copyLayers(ctx context.Context) ([]compressiontypes.Algorithm, error) { srcInfos := ic.src.LayerInfos() numLayers := len(srcInfos) updatedSrcInfos, err := ic.src.LayerInfosForCopy(ctx) if err != nil { - return err + return nil, err } srcInfosUpdated := false if updatedSrcInfos != nil && !reflect.DeepEqual(srcInfos, updatedSrcInfos) { if ic.cannotModifyManifestReason != "" { - return fmt.Errorf("Copying this image would require changing layer representation, which we cannot do: %q", ic.cannotModifyManifestReason) + return nil, fmt.Errorf("Copying this image would require changing layer representation, which we cannot do: %q", ic.cannotModifyManifestReason) } srcInfos = updatedSrcInfos srcInfosUpdated = true @@ -384,7 +424,7 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error { // layer is empty. man, err := manifest.FromBlob(ic.src.ManifestBlob, ic.src.ManifestMIMEType) if err != nil { - return err + return nil, err } manifestLayerInfos := man.LayerInfos() @@ -396,7 +436,7 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error { defer ic.c.concurrentBlobCopiesSemaphore.Release(1) defer copyGroup.Done() cld := copyLayerData{} - if !ic.c.downloadForeignLayers && ic.c.dest.AcceptsForeignLayerURLs() && len(srcLayer.URLs) != 0 { + if !ic.c.options.DownloadForeignLayers && ic.c.dest.AcceptsForeignLayerURLs() && len(srcLayer.URLs) != 0 { // DiffIDs are, currently, needed only when converting from schema1. // In which case src.LayerInfos will not have URLs because schema1 // does not support them. @@ -415,10 +455,10 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error { // Decide which layers to encrypt layersToEncrypt := set.New[int]() var encryptAll bool - if ic.ociEncryptLayers != nil { - encryptAll = len(*ic.ociEncryptLayers) == 0 + if ic.c.options.OciEncryptLayers != nil { + encryptAll = len(*ic.c.options.OciEncryptLayers) == 0 totalLayers := len(srcInfos) - for _, l := range *ic.ociEncryptLayers { + for _, l := range *ic.c.options.OciEncryptLayers { // if layer is negative, it is reverse indexed. layersToEncrypt.Add((totalLayers + l) % totalLayers) } @@ -450,14 +490,18 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error { // A call to copyGroup.Wait() is done at this point by the defer above. return nil }(); err != nil { - return err + return nil, err } + compressionAlgos := set.New[string]() destInfos := make([]types.BlobInfo, numLayers) diffIDs := make([]digest.Digest, numLayers) for i, cld := range data { if cld.err != nil { - return cld.err + return nil, cld.err + } + if cld.destInfo.CompressionAlgorithm != nil { + compressionAlgos.Add(cld.destInfo.CompressionAlgorithm.Name()) } destInfos[i] = cld.destInfo diffIDs[i] = cld.diffID @@ -472,7 +516,11 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error { if srcInfosUpdated || layerDigestsDiffer(srcInfos, destInfos) { ic.manifestUpdates.LayerInfos = destInfos } - return nil + algos, err := algorithmsByNames(compressionAlgos.Values()) + if err != nil { + return nil, err + } + return algos, nil } // layerDigestsDiffer returns true iff the digests in a and b differ (ignoring sizes and possible other fields) @@ -577,6 +625,19 @@ type diffIDResult struct { err error } +func compressionAlgorithmFromMIMEType(srcInfo types.BlobInfo) *compressiontypes.Algorithm { + // This MIME type → compression mapping belongs in manifest-specific code in our manifest + // package (but we should preferably replace/change UpdatedImage instead of productizing + // this workaround). + switch srcInfo.MediaType { + case manifest.DockerV2Schema2LayerMediaType, imgspecv1.MediaTypeImageLayerGzip: + return &compression.Gzip + case imgspecv1.MediaTypeImageLayerZstd: + return &compression.Zstd + } + return nil +} + // copyLayer copies a layer with srcInfo (with known Digest and Annotations and possibly known Size) in src to dest, perhaps (de/re/)compressing it, // and returns a complete blobInfo of the copied layer, and a value for LayerDiffIDs if diffIDIsNeeded // srcRef can be used as an additional hint to the destination during checking whether a layer can be reused but srcRef can be nil. @@ -588,17 +649,8 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to // which uses the compression information to compute the updated MediaType values. // (Sadly UpdatedImage() is documented to not update MediaTypes from // ManifestUpdateOptions.LayerInfos[].MediaType, so we are doing it indirectly.) - // - // This MIME type → compression mapping belongs in manifest-specific code in our manifest - // package (but we should preferably replace/change UpdatedImage instead of productizing - // this workaround). if srcInfo.CompressionAlgorithm == nil { - switch srcInfo.MediaType { - case manifest.DockerV2Schema2LayerMediaType, imgspecv1.MediaTypeImageLayerGzip: - srcInfo.CompressionAlgorithm = &compression.Gzip - case imgspecv1.MediaTypeImageLayerZstd: - srcInfo.CompressionAlgorithm = &compression.Zstd - } + srcInfo.CompressionAlgorithm = compressionAlgorithmFromMIMEType(srcInfo) } ic.c.printCopyInfo("blob", srcInfo) @@ -608,7 +660,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to // When encrypting to decrypting, only use the simple code path. We might be able to optimize more // (e.g. if we know the DiffID of an encrypted compressed layer, it might not be necessary to pull, decrypt and decompress again), // but it’s not trivially safe to do such things, so until someone takes the effort to make a comprehensive argument, let’s not. - encryptingOrDecrypting := toEncrypt || (isOciEncrypted(srcInfo.MediaType) && ic.c.ociDecryptConfig != nil) + encryptingOrDecrypting := toEncrypt || (isOciEncrypted(srcInfo.MediaType) && ic.c.options.OciDecryptConfig != nil) canAvoidProcessingCompleteLayer := !diffIDIsNeeded && !encryptingOrDecrypting // Don’t read the layer from the source if we already have the blob, and optimizations are acceptable. @@ -623,12 +675,20 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to // a failure when we eventually try to update the manifest with the digest and MIME type of the reused blob. // Fixing that will probably require passing more information to TryReusingBlob() than the current version of // the ImageDestination interface lets us pass in. + var requiredCompression *compressiontypes.Algorithm + var originalCompression *compressiontypes.Algorithm + if ic.requireCompressionFormatMatch { + requiredCompression = ic.compressionFormat + originalCompression = srcInfo.CompressionAlgorithm + } reused, reusedBlob, err := ic.c.dest.TryReusingBlobWithOptions(ctx, srcInfo, private.TryReusingBlobOptions{ - Cache: ic.c.blobInfoCache, - CanSubstitute: canSubstitute, - EmptyLayer: emptyLayer, - LayerIndex: &layerIndex, - SrcRef: srcRef, + Cache: ic.c.blobInfoCache, + CanSubstitute: canSubstitute, + EmptyLayer: emptyLayer, + LayerIndex: &layerIndex, + SrcRef: srcRef, + RequiredCompression: requiredCompression, + OriginalCompression: originalCompression, }) if err != nil { return types.BlobInfo{}, "", fmt.Errorf("trying to reuse blob %s at destination: %w", srcInfo.Digest, err) @@ -642,8 +702,8 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to }() // Throw an event that the layer has been skipped - if ic.c.progress != nil && ic.c.progressInterval > 0 { - ic.c.progress <- types.ProgressProperties{ + if ic.c.options.Progress != nil && ic.c.options.ProgressInterval > 0 { + ic.c.options.Progress <- types.ProgressProperties{ Event: types.ProgressEventSkipped, Artifact: srcInfo, } @@ -818,3 +878,16 @@ func computeDiffID(stream io.Reader, decompressor compressiontypes.DecompressorF return digest.Canonical.FromReader(stream) } + +// algorithmsByNames returns slice of Algorithms from slice of Algorithm Names +func algorithmsByNames(names []string) ([]compressiontypes.Algorithm, error) { + result := []compressiontypes.Algorithm{} + for _, name := range names { + algo, err := compression.AlgorithmByName(name) + if err != nil { + return nil, err + } + result = append(result, algo) + } + return result, nil +} diff --git a/vendor/github.com/containers/image/v5/directory/directory_dest.go b/vendor/github.com/containers/image/v5/directory/directory_dest.go index 974d23d5f..222723a8f 100644 --- a/vendor/github.com/containers/image/v5/directory/directory_dest.go +++ b/vendor/github.com/containers/image/v5/directory/directory_dest.go @@ -190,6 +190,9 @@ func (d *dirImageDestination) PutBlobWithOptions(ctx context.Context, stream io. // If the blob has been successfully reused, returns (true, info, nil). // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. func (d *dirImageDestination) TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options private.TryReusingBlobOptions) (bool, private.ReusedBlob, error) { + if !impl.OriginalBlobMatchesRequiredCompression(options) { + return false, private.ReusedBlob{}, nil + } if info.Digest == "" { return false, private.ReusedBlob{}, fmt.Errorf("Can not check for a blob with unknown digest") } diff --git a/vendor/github.com/containers/image/v5/docker/docker_image_dest.go b/vendor/github.com/containers/image/v5/docker/docker_image_dest.go index 44e2aea23..63e372d67 100644 --- a/vendor/github.com/containers/image/v5/docker/docker_image_dest.go +++ b/vendor/github.com/containers/image/v5/docker/docker_image_dest.go @@ -321,13 +321,21 @@ func (d *dockerImageDestination) TryReusingBlobWithOptions(ctx context.Context, return false, private.ReusedBlob{}, errors.New("Can not check for a blob with unknown digest") } - // First, check whether the blob happens to already exist at the destination. - haveBlob, reusedInfo, err := d.tryReusingExactBlob(ctx, info, options.Cache) - if err != nil { - return false, private.ReusedBlob{}, err - } - if haveBlob { - return true, reusedInfo, nil + if impl.OriginalBlobMatchesRequiredCompression(options) { + // First, check whether the blob happens to already exist at the destination. + haveBlob, reusedInfo, err := d.tryReusingExactBlob(ctx, info, options.Cache) + if err != nil { + return false, private.ReusedBlob{}, err + } + if haveBlob { + return true, reusedInfo, nil + } + } else { + requiredCompression := "nil" + if options.OriginalCompression != nil { + requiredCompression = options.OriginalCompression.Name() + } + logrus.Debugf("Ignoring exact blob match case due to compression mismatch ( %s vs %s )", options.RequiredCompression.Name(), requiredCompression) } // Then try reusing blobs from other locations. @@ -338,6 +346,19 @@ func (d *dockerImageDestination) TryReusingBlobWithOptions(ctx context.Context, logrus.Debugf("Error parsing BlobInfoCache location reference: %s", err) continue } + compressionOperation, compressionAlgorithm, err := blobinfocache.OperationAndAlgorithmForCompressor(candidate.CompressorName) + if err != nil { + logrus.Debugf("OperationAndAlgorithmForCompressor Failed: %v", err) + continue + } + if !impl.BlobMatchesRequiredCompression(options, compressionAlgorithm) { + requiredCompression := "nil" + if compressionAlgorithm != nil { + requiredCompression = compressionAlgorithm.Name() + } + logrus.Debugf("Ignoring candidate blob %s as reuse candidate due to compression mismatch ( %s vs %s ) in %s", candidate.Digest.String(), options.RequiredCompression.Name(), requiredCompression, candidateRepo.Name()) + continue + } if candidate.CompressorName != blobinfocache.Uncompressed { logrus.Debugf("Trying to reuse cached location %s compressed with %s in %s", candidate.Digest.String(), candidate.CompressorName, candidateRepo.Name()) } else { @@ -388,12 +409,6 @@ func (d *dockerImageDestination) TryReusingBlobWithOptions(ctx context.Context, options.Cache.RecordKnownLocation(d.ref.Transport(), bicTransportScope(d.ref), candidate.Digest, newBICLocationReference(d.ref)) - compressionOperation, compressionAlgorithm, err := blobinfocache.OperationAndAlgorithmForCompressor(candidate.CompressorName) - if err != nil { - logrus.Debugf("... Failed: %v", err) - continue - } - return true, private.ReusedBlob{ Digest: candidate.Digest, Size: size, diff --git a/vendor/github.com/containers/image/v5/docker/internal/tarfile/dest.go b/vendor/github.com/containers/image/v5/docker/internal/tarfile/dest.go index 00e25748b..7507d8559 100644 --- a/vendor/github.com/containers/image/v5/docker/internal/tarfile/dest.go +++ b/vendor/github.com/containers/image/v5/docker/internal/tarfile/dest.go @@ -129,6 +129,9 @@ func (d *Destination) PutBlobWithOptions(ctx context.Context, stream io.Reader, // If the blob has been successfully reused, returns (true, info, nil). // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. func (d *Destination) TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options private.TryReusingBlobOptions) (bool, private.ReusedBlob, error) { + if !impl.OriginalBlobMatchesRequiredCompression(options) { + return false, private.ReusedBlob{}, nil + } if err := d.archive.lock(); err != nil { return false, private.ReusedBlob{}, err } diff --git a/vendor/github.com/containers/image/v5/internal/imagedestination/impl/helpers.go b/vendor/github.com/containers/image/v5/internal/imagedestination/impl/helpers.go new file mode 100644 index 000000000..d5de81a61 --- /dev/null +++ b/vendor/github.com/containers/image/v5/internal/imagedestination/impl/helpers.go @@ -0,0 +1,20 @@ +package impl + +import ( + "github.com/containers/image/v5/internal/private" + compression "github.com/containers/image/v5/pkg/compression/types" +) + +// BlobMatchesRequiredCompression validates if compression is required by the caller while selecting a blob, if it is required +// then function performs a match against the compression requested by the caller and compression of existing blob +// (which can be nil to represent uncompressed or unknown) +func BlobMatchesRequiredCompression(options private.TryReusingBlobOptions, candidateCompression *compression.Algorithm) bool { + if options.RequiredCompression == nil { + return true // no requirement imposed + } + return candidateCompression != nil && (options.RequiredCompression.Name() == candidateCompression.Name()) +} + +func OriginalBlobMatchesRequiredCompression(opts private.TryReusingBlobOptions) bool { + return BlobMatchesRequiredCompression(opts, opts.OriginalCompression) +} diff --git a/vendor/github.com/containers/image/v5/internal/imagedestination/wrapper.go b/vendor/github.com/containers/image/v5/internal/imagedestination/wrapper.go index 41a81628b..17e1870c1 100644 --- a/vendor/github.com/containers/image/v5/internal/imagedestination/wrapper.go +++ b/vendor/github.com/containers/image/v5/internal/imagedestination/wrapper.go @@ -64,6 +64,9 @@ func (w *wrapped) PutBlobWithOptions(ctx context.Context, stream io.Reader, inpu // If the blob has been successfully reused, returns (true, info, nil). // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. func (w *wrapped) TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options private.TryReusingBlobOptions) (bool, private.ReusedBlob, error) { + if options.RequiredCompression != nil { + return false, private.ReusedBlob{}, nil + } reused, blob, err := w.TryReusingBlob(ctx, info, options.Cache, options.CanSubstitute) if !reused || err != nil { return reused, private.ReusedBlob{}, err diff --git a/vendor/github.com/containers/image/v5/internal/manifest/docker_schema2_list.go b/vendor/github.com/containers/image/v5/internal/manifest/docker_schema2_list.go index 516ca7ac9..14a476642 100644 --- a/vendor/github.com/containers/image/v5/internal/manifest/docker_schema2_list.go +++ b/vendor/github.com/containers/image/v5/internal/manifest/docker_schema2_list.go @@ -5,6 +5,7 @@ import ( "fmt" platform "github.com/containers/image/v5/internal/pkg/platform" + compression "github.com/containers/image/v5/pkg/compression/types" "github.com/containers/image/v5/types" "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -57,11 +58,20 @@ func (list *Schema2ListPublic) Instances() []digest.Digest { func (list *Schema2ListPublic) Instance(instanceDigest digest.Digest) (ListUpdate, error) { for _, manifest := range list.Manifests { if manifest.Digest == instanceDigest { - return ListUpdate{ + ret := ListUpdate{ Digest: manifest.Digest, Size: manifest.Size, MediaType: manifest.MediaType, - }, nil + } + ret.ReadOnly.CompressionAlgorithmNames = []string{compression.GzipAlgorithmName} + ret.ReadOnly.Platform = &imgspecv1.Platform{ + OS: manifest.Platform.OS, + Architecture: manifest.Platform.Architecture, + OSVersion: manifest.Platform.OSVersion, + OSFeatures: manifest.Platform.OSFeatures, + Variant: manifest.Platform.Variant, + } + return ret, nil } } return ListUpdate{}, fmt.Errorf("unable to find instance %s passed to Schema2List.Instances", instanceDigest) diff --git a/vendor/github.com/containers/image/v5/internal/manifest/list.go b/vendor/github.com/containers/image/v5/internal/manifest/list.go index 8786324ea..189f1a718 100644 --- a/vendor/github.com/containers/image/v5/internal/manifest/list.go +++ b/vendor/github.com/containers/image/v5/internal/manifest/list.go @@ -68,6 +68,12 @@ type ListUpdate struct { Digest digest.Digest Size int64 MediaType string + // ReadOnly fields: may be set by Instance(), ignored by UpdateInstance() + ReadOnly struct { + Platform *imgspecv1.Platform + Annotations map[string]string + CompressionAlgorithmNames []string + } } type ListOp int diff --git a/vendor/github.com/containers/image/v5/internal/manifest/oci_index.go b/vendor/github.com/containers/image/v5/internal/manifest/oci_index.go index fd251d951..3038d8124 100644 --- a/vendor/github.com/containers/image/v5/internal/manifest/oci_index.go +++ b/vendor/github.com/containers/image/v5/internal/manifest/oci_index.go @@ -53,11 +53,15 @@ func (index *OCI1IndexPublic) Instances() []digest.Digest { func (index *OCI1IndexPublic) Instance(instanceDigest digest.Digest) (ListUpdate, error) { for _, manifest := range index.Manifests { if manifest.Digest == instanceDigest { - return ListUpdate{ + ret := ListUpdate{ Digest: manifest.Digest, Size: manifest.Size, MediaType: manifest.MediaType, - }, nil + } + ret.ReadOnly.Platform = manifest.Platform + ret.ReadOnly.Annotations = manifest.Annotations + ret.ReadOnly.CompressionAlgorithmNames = annotationsToCompressionAlgorithmNames(manifest.Annotations) + return ret, nil } } return ListUpdate{}, fmt.Errorf("unable to find instance %s in OCI1Index", instanceDigest) @@ -78,14 +82,29 @@ func (index *OCI1IndexPublic) UpdateInstances(updates []ListUpdate) error { return index.editInstances(editInstances) } -func addCompressionAnnotations(compressionAlgorithms []compression.Algorithm, annotationsMap map[string]string) { +func annotationsToCompressionAlgorithmNames(annotations map[string]string) []string { + result := make([]string, 0, 1) + if annotations[OCI1InstanceAnnotationCompressionZSTD] == OCI1InstanceAnnotationCompressionZSTDValue { + result = append(result, compression.ZstdAlgorithmName) + } + // No compression was detected, hence assume instance has default compression `Gzip` + if len(result) == 0 { + result = append(result, compression.GzipAlgorithmName) + } + return result +} + +func addCompressionAnnotations(compressionAlgorithms []compression.Algorithm, annotationsMap *map[string]string) { // TODO: This should also delete the algorithm if map already contains an algorithm and compressionAlgorithm // list has a different algorithm. To do that, we would need to modify the callers to always provide a reliable // and full compressionAlghorithms list. + if *annotationsMap == nil && len(compressionAlgorithms) > 0 { + *annotationsMap = map[string]string{} + } for _, algo := range compressionAlgorithms { switch algo.Name() { case compression.ZstdAlgorithmName: - annotationsMap[OCI1InstanceAnnotationCompressionZSTD] = OCI1InstanceAnnotationCompressionZSTDValue + (*annotationsMap)[OCI1InstanceAnnotationCompressionZSTD] = OCI1InstanceAnnotationCompressionZSTDValue default: continue } @@ -130,13 +149,13 @@ func (index *OCI1IndexPublic) editInstances(editInstances []ListEdit) error { maps.Copy(index.Manifests[targetIndex].Annotations, editInstance.UpdateAnnotations) } } - addCompressionAnnotations(editInstance.UpdateCompressionAlgorithms, index.Manifests[targetIndex].Annotations) + addCompressionAnnotations(editInstance.UpdateCompressionAlgorithms, &index.Manifests[targetIndex].Annotations) case ListOpAdd: annotations := map[string]string{} if editInstance.AddAnnotations != nil { annotations = maps.Clone(editInstance.AddAnnotations) } - addCompressionAnnotations(editInstance.AddCompressionAlgorithms, annotations) + addCompressionAnnotations(editInstance.AddCompressionAlgorithms, &annotations) addedEntries = append(addedEntries, imgspecv1.Descriptor{ MediaType: editInstance.AddMediaType, Size: editInstance.AddSize, diff --git a/vendor/github.com/containers/image/v5/internal/private/private.go b/vendor/github.com/containers/image/v5/internal/private/private.go index b1dd4ceb0..95d561fcd 100644 --- a/vendor/github.com/containers/image/v5/internal/private/private.go +++ b/vendor/github.com/containers/image/v5/internal/private/private.go @@ -112,10 +112,11 @@ type TryReusingBlobOptions struct { // Transports, OTOH, MUST support these fields being zero-valued for types.ImageDestination callers // if they use internal/imagedestination/impl.Compat; // in that case, they will all be consistently zero-valued. - - EmptyLayer bool // True if the blob is an "empty"/"throwaway" layer, and may not necessarily be physically represented. - LayerIndex *int // If the blob is a layer, a zero-based index of the layer within the image; nil otherwise. - SrcRef reference.Named // A reference to the source image that contains the input blob. + RequiredCompression *compression.Algorithm // If set, reuse blobs with a matching algorithm as per implementations in internal/imagedestination/impl.helpers.go + OriginalCompression *compression.Algorithm // Must be set if RequiredCompression is set; can be set to nil to indicate “uncompressed” or “unknown”. + EmptyLayer bool // True if the blob is an "empty"/"throwaway" layer, and may not necessarily be physically represented. + LayerIndex *int // If the blob is a layer, a zero-based index of the layer within the image; nil otherwise. + SrcRef reference.Named // A reference to the source image that contains the input blob. } // ReusedBlob is information about a blob reused in a destination. diff --git a/vendor/github.com/containers/image/v5/internal/set/set.go b/vendor/github.com/containers/image/v5/internal/set/set.go index 3e777fe12..acf30343e 100644 --- a/vendor/github.com/containers/image/v5/internal/set/set.go +++ b/vendor/github.com/containers/image/v5/internal/set/set.go @@ -28,6 +28,12 @@ func (s *Set[E]) Add(v E) { s.m[v] = struct{}{} // Possibly writing the same struct{}{} presence marker again. } +func (s *Set[E]) AddSlice(slice []E) { + for _, v := range slice { + s.Add(v) + } +} + func (s *Set[E]) Delete(v E) { delete(s.m, v) } diff --git a/vendor/github.com/containers/image/v5/oci/layout/oci_dest.go b/vendor/github.com/containers/image/v5/oci/layout/oci_dest.go index 0a9e4eab9..8ff43d448 100644 --- a/vendor/github.com/containers/image/v5/oci/layout/oci_dest.go +++ b/vendor/github.com/containers/image/v5/oci/layout/oci_dest.go @@ -172,6 +172,9 @@ func (d *ociImageDestination) PutBlobWithOptions(ctx context.Context, stream io. // If the blob has been successfully reused, returns (true, info, nil). // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. func (d *ociImageDestination) TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options private.TryReusingBlobOptions) (bool, private.ReusedBlob, error) { + if !impl.OriginalBlobMatchesRequiredCompression(options) { + return false, private.ReusedBlob{}, nil + } if info.Digest == "" { return false, private.ReusedBlob{}, errors.New("Can not check for a blob with unknown digest") } diff --git a/vendor/github.com/containers/image/v5/ostree/ostree_dest.go b/vendor/github.com/containers/image/v5/ostree/ostree_dest.go index 48f3ee5a7..d00a0cdf8 100644 --- a/vendor/github.com/containers/image/v5/ostree/ostree_dest.go +++ b/vendor/github.com/containers/image/v5/ostree/ostree_dest.go @@ -335,6 +335,9 @@ func (d *ostreeImageDestination) importConfig(repo *otbuiltin.Repo, blob *blobTo // reflected in the manifest that will be written. // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. func (d *ostreeImageDestination) TryReusingBlobWithOptions(ctx context.Context, info types.BlobInfo, options private.TryReusingBlobOptions) (bool, private.ReusedBlob, error) { + if !impl.OriginalBlobMatchesRequiredCompression(options) { + return false, private.ReusedBlob{}, nil + } if d.repo == nil { repo, err := openRepo(d.ref.repo) if err != nil { diff --git a/vendor/github.com/containers/image/v5/storage/storage_dest.go b/vendor/github.com/containers/image/v5/storage/storage_dest.go index 576d510cc..7bbbf1752 100644 --- a/vendor/github.com/containers/image/v5/storage/storage_dest.go +++ b/vendor/github.com/containers/image/v5/storage/storage_dest.go @@ -307,6 +307,9 @@ func (s *storageImageDestination) PutBlobPartial(ctx context.Context, chunkAcces // If the blob has been successfully reused, returns (true, info, nil). // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. func (s *storageImageDestination) TryReusingBlobWithOptions(ctx context.Context, blobinfo types.BlobInfo, options private.TryReusingBlobOptions) (bool, private.ReusedBlob, error) { + if !impl.OriginalBlobMatchesRequiredCompression(options) { + return false, private.ReusedBlob{}, nil + } reused, info, err := s.tryReusingBlobAsPending(blobinfo.Digest, blobinfo.Size, &options) if err != nil || !reused || options.LayerIndex == nil { return reused, info, err diff --git a/vendor/github.com/containers/image/v5/version/version.go b/vendor/github.com/containers/image/v5/version/version.go index 3c8fc094d..584764c9a 100644 --- a/vendor/github.com/containers/image/v5/version/version.go +++ b/vendor/github.com/containers/image/v5/version/version.go @@ -6,9 +6,9 @@ const ( // VersionMajor is for an API incompatible changes VersionMajor = 5 // VersionMinor is for functionality in a backwards-compatible manner - VersionMinor = 26 + VersionMinor = 27 // VersionPatch is for backwards-compatible bug fixes - VersionPatch = 1 + VersionPatch = 0 // VersionDev indicates development branch. Releases will be empty string. VersionDev = "" diff --git a/vendor/modules.txt b/vendor/modules.txt index 42e0db834..03c779c7a 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -88,7 +88,7 @@ github.com/containernetworking/cni/pkg/version # github.com/containernetworking/plugins v1.3.0 ## explicit; go 1.20 github.com/containernetworking/plugins/pkg/ns -# github.com/containers/image/v5 v5.26.1 +# github.com/containers/image/v5 v5.27.0 ## explicit; go 1.18 github.com/containers/image/v5/copy github.com/containers/image/v5/directory