From 866b54cad2b7bbf405608866867cfc9acb7a9f66 Mon Sep 17 00:00:00 2001 From: Daniel Linsley Date: Mon, 6 Jan 2025 10:32:27 -0800 Subject: [PATCH] do the sha1 hash during download instead of after to reduce local I/O for artifactory and bosh.io release sources. Already implemented for github release source, incompatible with the s3 release source which uses a concurrent download client. --- .../component/artifactory_release_source.go | 18 +++++------------- internal/component/bosh_io_release_source.go | 16 ++++------------ 2 files changed, 9 insertions(+), 25 deletions(-) diff --git a/internal/component/artifactory_release_source.go b/internal/component/artifactory_release_source.go index 4d8435b9..199e1efb 100644 --- a/internal/component/artifactory_release_source.go +++ b/internal/component/artifactory_release_source.go @@ -100,25 +100,17 @@ func (ars *ArtifactoryReleaseSource) DownloadRelease(releaseDir string, remoteRe if err != nil { return Local{}, err } - defer func() { _ = out.Close() }() + defer closeAndIgnoreError(out) - _, err = io.Copy(out, resp.Body) + hash := sha1.New() + + mw := io.MultiWriter(out,hash) + _, err = io.Copy(mw, resp.Body) _ = resp.Body.Close() if err != nil { return Local{}, err } - _, err = out.Seek(0, 0) - if err != nil { - return Local{}, fmt.Errorf("error reseting file cursor: %w", err) // untested - } - - hash := sha1.New() - _, err = io.Copy(hash, out) - if err != nil { - return Local{}, fmt.Errorf("error hashing file contents: %w", err) // untested - } - remoteRelease.SHA1 = hex.EncodeToString(hash.Sum(nil)) return Local{Lock: remoteRelease, LocalPath: filePath}, nil diff --git a/internal/component/bosh_io_release_source.go b/internal/component/bosh_io_release_source.go index 0260de62..de50a2ea 100644 --- a/internal/component/bosh_io_release_source.go +++ b/internal/component/bosh_io_release_source.go @@ -161,23 +161,15 @@ func (src BOSHIOReleaseSource) DownloadRelease(releaseDir string, remoteRelease } defer closeAndIgnoreError(out) - _, err = io.Copy(out, resp.Body) + hash := sha1.New() + + mw := io.MultiWriter(out,hash) + _, err = io.Copy(mw, resp.Body) _ = resp.Body.Close() if err != nil { return Local{}, err } - _, err = out.Seek(0, 0) - if err != nil { - return Local{}, fmt.Errorf("error reseting file cursor: %w", err) // untested - } - - hash := sha1.New() - _, err = io.Copy(hash, out) - if err != nil { - return Local{}, fmt.Errorf("error hashing file contents: %w", err) // untested - } - remoteRelease.SHA1 = hex.EncodeToString(hash.Sum(nil)) return Local{Lock: remoteRelease, LocalPath: filePath}, nil