use simplified distribution

Signed-off-by: Alex Goodman <[email protected]>

go.mod

@@ -8,7 +8,7 @@ require (
 	github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
 	github.com/adrg/xdg v0.5.3
 	github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a
-	github.com/anchore/grype v0.84.1-0.20241120030844-ce4acf74b750
+	github.com/anchore/grype v0.84.1-0.20241120161733-6a7d93a49b1a
 	github.com/anchore/syft v1.16.0
 	github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de
 	github.com/dave/jennifer v1.7.1

go.sum

@@ -252,8 +252,8 @@ github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0v
 github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ=
 github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4 h1:rmZG77uXgE+o2gozGEBoUMpX27lsku+xrMwlmBZJtbg=
 github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E=
-github.com/anchore/grype v0.84.1-0.20241120030844-ce4acf74b750 h1:P6bUmjeHSSEWw2qxIWGNaeohdVJNhD0iPxsv5CFiV5M=
-github.com/anchore/grype v0.84.1-0.20241120030844-ce4acf74b750/go.mod h1:K1L+fDE7YjSaPe6sIMpgaZp/YkFS4SgUSKFqVhKMIXs=
+github.com/anchore/grype v0.84.1-0.20241120161733-6a7d93a49b1a h1:f3OgNF3DsTzsW6Vcd7bes7Kth9I3NXwCz21gGlpTYgY=
+github.com/anchore/grype v0.84.1-0.20241120161733-6a7d93a49b1a/go.mod h1:K1L+fDE7YjSaPe6sIMpgaZp/YkFS4SgUSKFqVhKMIXs=
 github.com/anchore/packageurl-go v0.1.1-0.20241018175412-5c22e6360c4f h1:dAQPIrQ3a5PBqZeZ+B9NGZsGmodk4NO9OjDIsQmQyQM=
 github.com/anchore/packageurl-go v0.1.1-0.20241018175412-5c22e6360c4f/go.mod h1:KoYIv7tdP5+CC9VGkeZV4/vGCKsY55VvoG+5dadg4YI=
 github.com/anchore/stereoscope v0.0.8 h1:ma8A7SnM5WWU0HJ2p6YBq7myN7zKa0JnUQOY/4enekk=

internal/tarutil/writer.go

@@ -192,7 +192,7 @@ func (w *writer) Close() error {
 //	return spy, spy, nil
 //}
 //
-//func sizer(path string) func() int64 {
+// func sizer(path string) func() int64 {
 //	return func() int64 {
 //		stat, err := os.Stat(path)
 //		if err != nil {

pkg/process/package.go

@@ -1,210 +1,169 @@
 package process
 
 import (
+	"errors"
 	"fmt"
-	"net/url"
 	"os"
 	"path"
 	"path/filepath"
 	"strings"
 	"time"
 
-	"github.com/spf13/afero"
-
 	"github.com/anchore/grype-db/internal/log"
 	"github.com/anchore/grype-db/internal/tarutil"
-	"github.com/anchore/grype/grype/db/legacy/distribution"
-	grypeDBLegacy "github.com/anchore/grype/grype/db/v5"
-	grypeDBLegacyStore "github.com/anchore/grype/grype/db/v5/store"
+	grypeDBLegacyDistribution "github.com/anchore/grype/grype/db/legacy/distribution"
 	v6 "github.com/anchore/grype/grype/db/v6"
+	v6Distribution "github.com/anchore/grype/grype/db/v6/distribution"
 )
 
-func secondsSinceEpoch() int64 {
-	return time.Now().UTC().Unix()
-}
-
 func Package(dbDir, publishBaseURL, overrideArchiveExtension string) error {
-	// check if metadata file exists
-	if _, err := os.Stat(filepath.Join(dbDir, distribution.MetadataFileName)); os.IsNotExist(err) {
-		return v6Package(dbDir, overrideArchiveExtension)
+	// check if metadata file exists, if so, then this
+	if _, err := os.Stat(filepath.Join(dbDir, grypeDBLegacyDistribution.MetadataFileName)); os.IsNotExist(err) {
+		return packageDB(dbDir, overrideArchiveExtension)
 	}
-	return legacyPackage(dbDir, publishBaseURL, overrideArchiveExtension)
+	return packageLegacyDB(dbDir, publishBaseURL, overrideArchiveExtension)
 }
 
-func v6Package(dbDir, overrideArchiveExtension string) error {
+func packageDB(dbDir, overrideArchiveExtension string) error {
 	extension, err := resolveV6Extension(overrideArchiveExtension)
 	if err != nil {
 		return err
 	}
 	log.WithFields("from", dbDir, "extension", extension).Info("packaging database")
 
-	s, err := v6.NewReader(v6.Config{DBDirPath: dbDir})
+	tarPath, err := calculateTarPath(dbDir, extension)
 	if err != nil {
-		return fmt.Errorf("unable to open vulnerability store: %w", err)
-	}
-
-	metadata, err := s.GetDBMetadata()
-	if err != nil {
-		return fmt.Errorf("unable to get vulnerability store metadata: %w", err)
-	}
-
-	if metadata.Model != v6.ModelVersion {
-		return fmt.Errorf("metadata model %d does not match vulnerability store model %d", v6.ModelVersion, metadata.Model)
+		return err
 	}
 
-	// TODO: add support for fetching all provider data
-
-	tarName := fmt.Sprintf(
-		"vulnerability-db_v%s_%s_%s.%s",
-		fmt.Sprintf("%d.%d.%d", metadata.Model, metadata.Revision, metadata.Addition),
-		"TODOPROVIDERTIME",
-		metadata.BuildTimestamp.UTC().Format(time.RFC3339),
-		extension,
-	)
-	tarPath := path.Join(dbDir, tarName)
-
-	if err := populate(tarName, dbDir); err != nil {
+	if err := populateTar(tarPath); err != nil {
 		return err
 	}
 
 	log.WithFields("path", tarPath).Info("created database archive")
 
-	// TODO: write out latest.json file
-
-	return nil
+	return writeLatestDocument(tarPath)
 }
 
-func resolveV6Extension(overrideArchiveExtension string) (string, error) {
-	var extension = "tar.xz"
-
-	if overrideArchiveExtension != "" {
-		extension = strings.TrimLeft(overrideArchiveExtension, ".")
+func writeLatestDocument(tarPath string) error {
+	archive, err := v6Distribution.NewArchive(tarPath)
+	if err != nil || archive == nil {
+		return fmt.Errorf("unable to create archive: %w", err)
 	}
 
-	var found bool
-	for _, valid := range []string{"tar.xz", "tar.zst", "tar.gz"} {
-		if valid == extension {
-			found = true
-			break
-		}
+	doc := v6Distribution.NewLatestDocument(*archive)
+	if doc == nil {
+		return errors.New("unable to create latest document")
 	}
 
-	if !found {
-		return "", fmt.Errorf("unsupported archive extension %q", extension)
-	}
-	return extension, nil
-}
+	dbDir := filepath.Dir(tarPath)
 
-func legacyPackage(dbDir, publishBaseURL, overrideArchiveExtension string) error { //nolint:funlen
-	log.WithFields("from", dbDir, "url", publishBaseURL, "extension-override", overrideArchiveExtension).Info("packaging database")
+	latestPath := path.Join(dbDir, v6Distribution.LatestFileName)
 
-	fs := afero.NewOsFs()
-	metadata, err := distribution.NewMetadataFromDir(fs, dbDir)
+	fh, err := os.OpenFile(latestPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
 	if err != nil {
-		return err
+		return fmt.Errorf("unable to create latest file: %w", err)
 	}
 
-	if metadata == nil {
-		return fmt.Errorf("no metadata found in %q", dbDir)
+	if err = doc.Write(fh); err != nil {
+		return fmt.Errorf("unable to write latest document: %w", err)
 	}
+	return nil
+}
 
-	s, err := grypeDBLegacyStore.New(filepath.Join(dbDir, grypeDBLegacy.VulnerabilityStoreFileName), false)
+func calculateTarPath(dbDir string, extension string) (string, error) {
+	s, err := v6.NewReader(v6.Config{DBDirPath: dbDir})
 	if err != nil {
-		return fmt.Errorf("unable to open vulnerability store: %w", err)
+		return "", fmt.Errorf("unable to open vulnerability store: %w", err)
 	}
 
-	id, err := s.GetID()
+	metadata, err := s.GetDBMetadata()
 	if err != nil {
-		return fmt.Errorf("unable to get vulnerability store ID: %w", err)
+		return "", fmt.Errorf("unable to get vulnerability store metadata: %w", err)
 	}
 
-	if id.SchemaVersion != metadata.Version {
-		return fmt.Errorf("metadata version %d does not match vulnerability store version %d", metadata.Version, id.SchemaVersion)
+	if metadata.Model != v6.ModelVersion {
+		return "", fmt.Errorf("metadata model %d does not match vulnerability store model %d", v6.ModelVersion, metadata.Model)
 	}
 
-	u, err := url.Parse(publishBaseURL)
+	providers, err := s.AllProviders()
 	if err != nil {
-		return err
+		return "", fmt.Errorf("unable to get all providers: %w", err)
 	}
 
-	// we need a well-ordered string to append to the archive name to ensure uniqueness (to avoid overwriting
-	// existing archives in the CDN) as well as to ensure that multiple archives created in the same day are
-	// put in the correct order in the listing file. The DB timestamp represents the age of the data in the DB
-	// not when the DB was created. The trailer represents the time the DB was packaged.
-	trailer := fmt.Sprintf("%d", secondsSinceEpoch())
+	if len(providers) == 0 {
+		return "", fmt.Errorf("no providers found in the vulnerability store")
+	}
 
-	// TODO (alex): supporting tar.zst
-	// var extension = "tar.zst"
-	var extension = "tar.gz"
+	eldest := eldestProviderTimestamp(providers)
+	if eldest == nil {
+		return "", errors.New("could not resolve eldest provider timestamp")
+	}
+	// output archive vulnerability-db_VERSION_OLDESTDATADATE_BUILTEPOCH.tar.gz, where:
+	// - VERSION: schema version in the form of v#.#.#
+	// - OLDESTDATADATE: RFC3338 formatted value of the oldest date capture date found for all contained providers
+	// - BUILTEPOCH: linux epoch formatted value of the database metadata built field
+	tarName := fmt.Sprintf(
+		"vulnerability-db_v%s_%s_%d.%s",
+		fmt.Sprintf("%d.%d.%d", metadata.Model, metadata.Revision, metadata.Addition),
+		eldest.UTC().Format(time.RFC3339),
+		metadata.BuildTimestamp.Unix(),
+		extension,
+	)
+
+	return filepath.Join(dbDir, tarName), err
+}
+
+func eldestProviderTimestamp(providers []v6.Provider) *time.Time {
+	var eldest *time.Time
+	for _, p := range providers {
+		if eldest == nil || p.DateCaptured.Before(*eldest) {
+			eldest = p.DateCaptured
+		}
+	}
+	return eldest
+}
+
+func resolveV6Extension(overrideArchiveExtension string) (string, error) {
+	var extension = "tar.xz"
 
 	if overrideArchiveExtension != "" {
 		extension = strings.TrimLeft(overrideArchiveExtension, ".")
 	}
-	// TODO (alex): supporting tar.zst
-	// else if metadata.Version < 5 {
-	// 	extension = "tar.gz"
-	// }
 
 	var found bool
-	for _, valid := range []string{"tar.zst", "tar.gz"} {
+	for _, valid := range []string{"tar.xz", "tar.zst", "tar.gz"} {
 		if valid == extension {
 			found = true
 			break
 		}
 	}
 
 	if !found {
-		return fmt.Errorf("invalid archive extension %q", extension)
-	}
-
-	// we attach a random value at the end of the file name to prevent from overwriting DBs in S3 that are already
-	// cached in the CDN. Ideally this would be based off of the archive checksum but a random string is simpler.
-	tarName := fmt.Sprintf(
-		"vulnerability-db_v%d_%s_%s.%s",
-		metadata.Version,
-		metadata.Built.Format(time.RFC3339),
-		trailer,
-		extension,
-	)
-	tarPath := path.Join(dbDir, tarName)
-
-	if err := populate(tarName, dbDir); err != nil {
-		return err
-	}
-
-	log.WithFields("path", tarPath).Info("created database archive")
-
-	entry, err := distribution.NewListingEntryFromArchive(fs, *metadata, tarPath, u)
-	if err != nil {
-		return fmt.Errorf("unable to create listing entry from archive: %w", err)
-	}
-
-	listing := distribution.NewListing(entry)
-	listingPath := path.Join(dbDir, distribution.ListingFileName)
-	if err = listing.Write(listingPath); err != nil {
-		return err
+		return "", fmt.Errorf("unsupported archive extension %q", extension)
 	}
-
-	log.WithFields("path", listingPath).Debug("created initial listing file")
-
-	return nil
+	return extension, nil
 }
 
-func populate(tarName, dbDir string) error {
+func populateTar(tarPath string) error {
 	originalDir, err := os.Getwd()
 	if err != nil {
 		return fmt.Errorf("unable to get CWD: %w", err)
 	}
 
-	if err = os.Chdir(dbDir); err != nil {
-		return fmt.Errorf("unable to cd to build dir: %w", err)
-	}
+	dbDir, tarName := filepath.Split(tarPath)
 
-	defer func() {
-		if err = os.Chdir(originalDir); err != nil {
-			log.Errorf("unable to cd to original dir: %v", err)
+	if dbDir != "" {
+		if err = os.Chdir(dbDir); err != nil {
+			return fmt.Errorf("unable to cd to build dir: %w", err)
 		}
-	}()
+
+		defer func() {
+			if err = os.Chdir(originalDir); err != nil {
+				log.Errorf("unable to cd to original dir: %v", err)
+			}
+		}()
+	}
 
 	fileInfos, err := os.ReadDir("./")
 	if err != nil {