diff --git a/grype/db/v6/db.go b/grype/db/v6/db.go index ca1e3cc8242..3ae07f39638 100644 --- a/grype/db/v6/db.go +++ b/grype/db/v6/db.go @@ -35,6 +35,14 @@ type Writer interface { io.Closer } +type Curator interface { + Reader() (Reader, error) + Status() Status + Delete() error + Update() (bool, error) + Import(dbArchivePath string) error +} + type Config struct { DBDirPath string } diff --git a/grype/db/v6/description.go b/grype/db/v6/description.go index 18f7456b1f1..8d7e88e6a81 100644 --- a/grype/db/v6/description.go +++ b/grype/db/v6/description.go @@ -1,7 +1,9 @@ package v6 import ( + "encoding/json" "fmt" + "io" "path" "time" @@ -85,3 +87,17 @@ func NewDescriptionFromDir(fs afero.Fs, dir string) (*Description, error) { func (m Description) String() string { return fmt.Sprintf("DB(version=%s built=%s checksum=%s)", m.SchemaVersion, m.Built, m.Checksum) } + +func writeDescription(writer io.Writer, m Description) error { + if m.SchemaVersion == "" { + return fmt.Errorf("missing schema version") + } + + contents, err := json.MarshalIndent(m, "", " ") + if err != nil { + return fmt.Errorf("failed to encode metadata file: %w", err) + } + + _, err = writer.Write(contents) + return err +} diff --git a/grype/db/v6/description_test.go b/grype/db/v6/description_test.go index dc4949f8637..8ad39872c5f 100644 --- a/grype/db/v6/description_test.go +++ b/grype/db/v6/description_test.go @@ -6,6 +6,7 @@ import ( "io" "os" "path" + "strings" "testing" "time" @@ -125,3 +126,24 @@ func TestTime_JSONUnmarshalling(t *testing.T) { }) } } + +func TestDatabaseDescription_Write(t *testing.T) { + + validDescription := Description{ + SchemaVersion: "1.0.0", + Built: Time{Time: time.Date(2023, 9, 26, 12, 2, 3, 0, time.UTC)}, + Checksum: "xxh64:dummychecksum", + } + + sb := strings.Builder{} + + err := writeDescription(&sb, validDescription) + require.NoError(t, err) + + expected := fmt.Sprintf(`{ + "schemaVersion": "1.0.0", + "built": "2023-09-26T12:02:03Z", + "checksum": "xxh64:dummychecksum" +}`) + assert.Equal(t, expected, sb.String()) +} diff --git a/grype/db/v6/distribution/client.go b/grype/db/v6/distribution/client.go index a4b7cf329f4..8995ce706f1 100644 --- a/grype/db/v6/distribution/client.go +++ b/grype/db/v6/distribution/client.go @@ -28,8 +28,7 @@ type Config struct { CACert string // validations - ValidateByHashOnGet bool - RequireUpdateCheck bool + RequireUpdateCheck bool // timeouts CheckTimeout time.Duration @@ -50,11 +49,10 @@ type client struct { func DefaultConfig() Config { return Config{ - LatestURL: "https://grype.anchore.io/databases/latest.json", - ValidateByHashOnGet: true, - RequireUpdateCheck: false, - CheckTimeout: 30 * time.Second, - UpdateTimeout: 300 * time.Second, + LatestURL: "https://grype.anchore.io/databases/latest.json", + RequireUpdateCheck: false, + CheckTimeout: 30 * time.Second, + UpdateTimeout: 300 * time.Second, } } diff --git a/grype/db/v6/installation/curator.go b/grype/db/v6/installation/curator.go new file mode 100644 index 00000000000..440170b7323 --- /dev/null +++ b/grype/db/v6/installation/curator.go @@ -0,0 +1,432 @@ +package installation + +import ( + "encoding/json" + "fmt" + "os" + "path" + "path/filepath" + "strconv" + "time" + + "github.com/adrg/xdg" + "github.com/hako/durafmt" + "github.com/mholt/archiver/v3" + "github.com/spf13/afero" + "github.com/wagoodman/go-partybus" + "github.com/wagoodman/go-progress" + + db "github.com/anchore/grype/grype/db/v6" + "github.com/anchore/grype/grype/db/v6/distribution" + "github.com/anchore/grype/grype/event" + "github.com/anchore/grype/internal/bus" + "github.com/anchore/grype/internal/file" + "github.com/anchore/grype/internal/log" +) + +const lastUpdateCheckFileName = "last_update_check" + +type monitor struct { + *progress.AtomicStage + downloadProgress *progress.Manual + importProgress *progress.Manual +} + +type Config struct { + DBRootDir string + + // validations + ValidateAge bool + ValidateChecksum bool + MaxAllowedBuiltAge time.Duration + UpdateCheckMaxFrequency time.Duration +} + +func DefaultConfig() Config { + return Config{ + DBRootDir: filepath.Join(xdg.CacheHome, "grype", "db"), + ValidateAge: true, + ValidateChecksum: true, + MaxAllowedBuiltAge: time.Hour * 24 * 5, // 5 days + UpdateCheckMaxFrequency: 2 * time.Hour, // 2 hours + } +} + +func (c Config) DBFilePath() string { + return path.Join(c.DBDirectoryPath(), db.VulnerabilityDBFileName) +} + +func (c Config) DBDirectoryPath() string { + return path.Join(c.DBRootDir, strconv.Itoa(db.ModelVersion)) +} + +type curator struct { + fs afero.Fs + client distribution.Client + config Config +} + +func NewCurator(cfg Config, downloader distribution.Client) (db.Curator, error) { + return curator{ + fs: afero.NewOsFs(), + client: downloader, + config: cfg, + }, nil +} + +func (c curator) Reader() (db.Reader, error) { + s, err := db.NewReader( + db.Config{ + DBDirPath: c.config.DBDirectoryPath(), + }, + ) + if err != nil { + return nil, err + } + + return s, c.validate() +} + +func (c curator) Status() db.Status { + dbDir := c.config.DBDirectoryPath() + + d, err := readDatabaseDescription(c.fs, dbDir) + if err != nil { + return db.Status{ + Err: fmt.Errorf("failed to parse database metadata (%s): %w", dbDir, err), + } + } + if d == nil { + return db.Status{ + Err: fmt.Errorf("database metadata not found at %q", dbDir), + } + } + + return db.Status{ + Built: db.Time{Time: d.Built.Time}, + SchemaVersion: d.SchemaVersion.String(), + Location: dbDir, + Checksum: d.Checksum, + Err: c.validate(), + } +} + +// Delete removes the DB and metadata file for this specific schema. +func (c curator) Delete() error { + return c.fs.RemoveAll(c.config.DBDirectoryPath()) +} + +// Update the existing DB, returning an indication if any action was taken. +func (c curator) Update() (bool, error) { + if !c.isUpdateCheckAllowed() { + // we should not notify the user of an update check if the current configuration and state + // indicates we're should be in a low-pass filter mode (and the check frequency is too high). + // this should appear to the user as if we never attempted to check for an update at all. + return false, nil + } + + mon := newMonitor() + defer mon.SetCompleted() + + current, err := readDatabaseDescription(c.fs, c.config.DBDirectoryPath()) + if err != nil { + return false, fmt.Errorf("unable to read current database metadata: %w", err) + } + + mon.Set("checking for update") + update, checkErr := c.client.IsUpdateAvailable(current) + if checkErr != nil { + // we want to continue if possible even if we can't check for an update + log.Warnf("unable to check for vulnerability database update") + log.Debugf("check for vulnerability update failed: %+v", checkErr) + } + + if update == nil { + if checkErr == nil { + // there was no update (or any issue while checking for an update) + c.setLastSuccessfulUpdateCheck() + } + + mon.Set("no update available") + return false, nil + } + + log.Infof("downloading new vulnerability DB") + mon.Set("downloading") + dest, err := c.client.Download(*update, filepath.Dir(c.config.DBRootDir), mon.downloadProgress) + if err != nil { + return false, fmt.Errorf("unable to update vulnerability database: %w", err) + } + + if err := c.activate(dest, mon); err != nil { + return false, fmt.Errorf("unable to activate new vulnerability database: %w", err) + } + + // only set the last successful update check if the update was successful + c.setLastSuccessfulUpdateCheck() + + if current != nil { + log.WithFields( + "from", current.Built.String(), + "to", update.Description.Built.String(), + "version", update.Description.SchemaVersion, + ).Info("updated vulnerability DB") + return true, nil + } + + log.WithFields( + "version", update.Description.SchemaVersion, + "built", update.Description.Built.String(), + ).Info("downloaded new vulnerability DB") + return true, nil +} + +func (c curator) isUpdateCheckAllowed() bool { + if c.config.UpdateCheckMaxFrequency == 0 { + log.Trace("no max-frequency set for update check") + return true + } + + elapsed, err := c.durationSinceUpdateCheck() + if err != nil { + // we had an IO error (or similar) trying to read or parse the file, we should not block the update check. + log.WithFields("error", err).Trace("unable to determine if update check is allowed") + return true + } + if elapsed == nil { + // there was no last check (this is a first run case), we should not block the update check. + return true + } + + return *elapsed > c.config.UpdateCheckMaxFrequency +} + +func (c curator) durationSinceUpdateCheck() (*time.Duration, error) { + // open `$dbDir/last_update_check` file and read the timestamp and do now() - timestamp + + filePath := path.Join(c.config.DBDirectoryPath(), lastUpdateCheckFileName) + + if _, err := c.fs.Stat(filePath); os.IsNotExist(err) { + log.Trace("first-run of DB update") + return nil, nil + } + + fh, err := c.fs.OpenFile(filePath, os.O_RDONLY, 0) + if err != nil { + return nil, fmt.Errorf("unable to read last update check timestamp: %w", err) + } + + defer fh.Close() + + // read and parse rfc3339 timestamp + var lastCheckStr string + _, err = fmt.Fscanf(fh, "%s", &lastCheckStr) + if err != nil { + return nil, fmt.Errorf("unable to read last update check timestamp: %w", err) + } + + lastCheck, err := time.Parse(time.RFC3339, lastCheckStr) + if err != nil { + return nil, fmt.Errorf("unable to parse last update check timestamp: %w", err) + } + + if lastCheck.IsZero() { + return nil, fmt.Errorf("empty update check timestamp") + } + + elapsed := time.Since(lastCheck) + return &elapsed, nil +} + +func (c curator) setLastSuccessfulUpdateCheck() { + // note: we should always assume the DB dir actually exists, otherwise let this operation fail (since having a DB + // is a prerequisite for a successful update). + + filePath := path.Join(c.config.DBDirectoryPath(), lastUpdateCheckFileName) + fh, err := c.fs.OpenFile(filePath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + log.WithFields("error", err).Trace("unable to write last update check timestamp") + return + } + + defer fh.Close() + + _, _ = fmt.Fprintf(fh, "%s", time.Now().UTC().Format(time.RFC3339)) +} + +// validate checks the current database to ensure file integrity and if it can be used by this version of the application. +func (c curator) validate() error { + metadata, err := c.validateIntegrity(c.config.DBDirectoryPath()) + if err != nil { + return err + } + + return c.ensureNotStale(metadata) +} + +// Import takes a DB archive file and imports it into the final DB location. +func (c curator) Import(dbArchivePath string) error { + mon := newMonitor() + mon.Set("unarchiving") + defer mon.SetCompleted() + + // note: the temp directory is persisted upon download/validation/activation failure to allow for investigation + tempDir, err := os.MkdirTemp(c.config.DBRootDir, fmt.Sprintf("tmp-v%v-import", db.ModelVersion)) + if err != nil { + return fmt.Errorf("unable to create db temp dir: %w", err) + } + + err = archiver.Unarchive(dbArchivePath, tempDir) + if err != nil { + return err + } + + mon.downloadProgress.SetCompleted() + + err = c.activate(tempDir, mon) + if err != nil { + removeAllOrLog(c.fs, tempDir) + return err + } + + return nil +} + +// activate swaps over the downloaded db to the application directory +func (c curator) activate(dbDirPath string, mon monitor) error { + defer mon.importProgress.SetCompleted() + + mon.Set("validating DB integrity") + + if _, err := c.validateIntegrity(dbDirPath); err != nil { + return err + } + + mon.Set("activating") + + dbDir := c.config.DBDirectoryPath() + _, err := c.fs.Stat(dbDir) + if !os.IsNotExist(err) { + // remove any previous databases + err = c.Delete() + if err != nil { + return fmt.Errorf("failed to purge existing database: %w", err) + } + } + + // activate the new db cache by moving the temp dir to final location + return os.Rename(dbDirPath, dbDir) +} + +func (c curator) validateIntegrity(dbDirPath string) (*db.Description, error) { + // check that the disk checksum still matches the db payload + metadata, err := readDatabaseDescription(c.fs, dbDirPath) + if err != nil { + return nil, fmt.Errorf("failed to parse database metadata (%s): %w", dbDirPath, err) + } + if metadata == nil { + return nil, fmt.Errorf("database metadata not found: %s", dbDirPath) + } + + if c.config.ValidateChecksum { + dbPath := path.Join(dbDirPath, db.VulnerabilityDBFileName) + valid, actualHash, err := file.ValidateByHash(c.fs, dbPath, metadata.Checksum) + if err != nil { + return nil, err + } + if !valid { + return nil, fmt.Errorf("bad db checksum (%s): %q vs %q", dbPath, metadata.Checksum, actualHash) + } + } + + gotModel, ok := metadata.SchemaVersion.Model() + if !ok || gotModel != db.ModelVersion { + return nil, fmt.Errorf("unsupported database version: have=%d want=%d", gotModel, db.ModelVersion) + } + + // TODO: add version checks here to ensure this version of the application can use this database version (relative to what the DB says, not JUST the metadata!) + + return metadata, nil +} + +// ensureNotStale ensures the vulnerability database has not passed +// the max allowed age, calculated from the time it was built until now. +func (c curator) ensureNotStale(m *db.Description) error { + if m == nil { + return fmt.Errorf("no metadata to validate") + } + + if !c.config.ValidateAge { + return nil + } + + // built time is defined in UTC, + // we should compare it against UTC + now := time.Now().UTC() + + age := now.Sub(m.Built.Time) + if age > c.config.MaxAllowedBuiltAge { + return fmt.Errorf("the vulnerability database was built %s ago (max allowed age is %s)", durafmt.ParseShort(age), durafmt.ParseShort(c.config.MaxAllowedBuiltAge)) + } + + return nil +} + +func removeAllOrLog(fs afero.Fs, dir string) { + if err := fs.RemoveAll(dir); err != nil { + log.WithFields("error", err).Warnf("failed to remove path %q", dir) + } +} + +func newMonitor() monitor { + // let consumers know of a monitorable event (download + import stages) + importProgress := progress.NewManual(1) + stage := progress.NewAtomicStage("") + downloadProgress := progress.NewManual(1) + aggregateProgress := progress.NewAggregator(progress.DefaultStrategy, downloadProgress, importProgress) + + bus.Publish(partybus.Event{ + Type: event.UpdateVulnerabilityDatabase, + Value: progress.StagedProgressable(&struct { + progress.Stager + progress.Progressable + }{ + Stager: progress.Stager(stage), + Progressable: progress.Progressable(aggregateProgress), + }), + }) + + return monitor{ + AtomicStage: stage, + downloadProgress: downloadProgress, + importProgress: importProgress, + } +} + +func (m monitor) SetCompleted() { + m.downloadProgress.SetCompleted() + m.importProgress.SetCompleted() +} + +func readDatabaseDescription(fs afero.Fs, dir string) (*db.Description, error) { + metadataFilePath := path.Join(dir, db.DescriptionFileName) + exists, err := file.Exists(fs, metadataFilePath) + if err != nil { + return nil, fmt.Errorf("unable to check if DB metadata path exists (%s): %w", metadataFilePath, err) + } + if !exists { + return nil, nil + } + f, err := fs.Open(metadataFilePath) + if err != nil { + return nil, fmt.Errorf("unable to open DB metadata path (%s): %w", metadataFilePath, err) + } + defer f.Close() + + var m db.Description + err = json.NewDecoder(f).Decode(&m) + if err != nil { + return nil, fmt.Errorf("unable to parse DB metadata (%s): %w", metadataFilePath, err) + } + return &m, nil +} diff --git a/grype/db/v6/installation/curator_test.go b/grype/db/v6/installation/curator_test.go new file mode 100644 index 00000000000..9981fa8d237 --- /dev/null +++ b/grype/db/v6/installation/curator_test.go @@ -0,0 +1,552 @@ +package installation + +import ( + "bytes" + "encoding/json" + "errors" + "os" + "path" + "path/filepath" + "testing" + "time" + + "github.com/OneOfOne/xxhash" + "github.com/spf13/afero" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + "github.com/wagoodman/go-progress" + + "github.com/anchore/grype/grype/db/internal/schemaver" + db "github.com/anchore/grype/grype/db/v6" + "github.com/anchore/grype/grype/db/v6/distribution" + "github.com/anchore/grype/internal/file" +) + +type mockClient struct { + mock.Mock +} + +func (m *mockClient) IsUpdateAvailable(current *db.Description) (*distribution.Archive, error) { + args := m.Called(current) + + err := args.Error(1) + + if err != nil { + return nil, err + } + + return args.Get(0).(*distribution.Archive), nil +} + +func (m *mockClient) Download(archive distribution.Archive, dest string, downloadProgress *progress.Manual) (string, error) { + args := m.Called(archive, dest, downloadProgress) + return args.String(0), args.Error(1) +} + +func newTestCurator(t *testing.T) curator { + tempDir := t.TempDir() + cfg := DefaultConfig() + cfg.DBRootDir = tempDir + + ci, err := NewCurator(cfg, new(mockClient)) + require.NoError(t, err) + + c := ci.(curator) + return c +} + +type setupConfig struct { + badDbChecksum bool + workingUpdate bool +} + +type setupOption func(*setupConfig) + +func withBadDBChecksum() setupOption { + return func(c *setupConfig) { + c.badDbChecksum = true + } +} + +func withWorkingUpdateIntegrations() setupOption { + return func(c *setupConfig) { + c.workingUpdate = true + } +} + +func setupCuratorForUpdate(t *testing.T, opts ...setupOption) curator { + cfg := setupConfig{} + + for _, o := range opts { + o(&cfg) + } + + c := newTestCurator(t) + + dbDir := c.config.DBDirectoryPath() + stageDir := filepath.Join(c.config.DBRootDir, "staged") + + // populate metadata into the downloaded dir + oldDescription := db.Description{ + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + Built: db.Time{Time: time.Now().Add(-48 * time.Hour)}, + Checksum: writeTestDB(t, c.fs, dbDir), + } + + newDescription := oldDescription + newDescription.Built = db.Time{Time: time.Now()} + newDescription.Checksum = writeTestDB(t, c.fs, stageDir) + if cfg.badDbChecksum { + newDescription.Checksum = "xxh64:badchecksum" + } + + writeTestMetadata(t, c.fs, dbDir, oldDescription) + writeTestMetadata(t, c.fs, stageDir, newDescription) + + if cfg.workingUpdate { + mc := c.client.(*mockClient) + + // ensure the update "works" + mc.On("IsUpdateAvailable", mock.Anything).Return(&distribution.Archive{}, nil) + mc.On("Download", mock.Anything, mock.Anything, mock.Anything).Return(stageDir, nil) + } + + return c +} + +func writeTestMetadata(t *testing.T, fs afero.Fs, dir string, description db.Description) { + require.NoError(t, fs.MkdirAll(dir, 0755)) + + descriptionJSON, err := json.Marshal(description) + require.NoError(t, err) + + metadataFilePath := path.Join(dir, db.DescriptionFileName) + require.NoError(t, afero.WriteFile(fs, metadataFilePath, descriptionJSON, 0644)) +} + +func writeTestDB(t *testing.T, fs afero.Fs, dir string) string { + require.NoError(t, fs.MkdirAll(dir, 0755)) + + content := []byte(dir) + p := path.Join(dir, db.VulnerabilityDBFileName) + require.NoError(t, afero.WriteFile(fs, p, content, 0644)) + h, err := file.HashReader(bytes.NewReader(content), xxhash.New64()) + require.NoError(t, err) + return "xxh64:" + h +} + +func TestCurator_Update(t *testing.T) { + + t.Run("happy path: successful update", func(t *testing.T) { + c := setupCuratorForUpdate(t, withWorkingUpdateIntegrations()) + mc := c.client.(*mockClient) + + updated, err := c.Update() + + require.NoError(t, err) + require.True(t, updated) + require.FileExists(t, filepath.Join(c.config.DBDirectoryPath(), lastUpdateCheckFileName)) + + mc.AssertExpectations(t) + }) + + t.Run("error checking for updates", func(t *testing.T) { + c := setupCuratorForUpdate(t) + mc := c.client.(*mockClient) + + mc.On("IsUpdateAvailable", mock.Anything).Return(nil, errors.New("check failed")) + + updated, err := c.Update() + + require.NoError(t, err) + require.False(t, updated) + require.NoFileExists(t, filepath.Join(c.config.DBDirectoryPath(), lastUpdateCheckFileName)) + + mc.AssertExpectations(t) + }) + + t.Run("error during download", func(t *testing.T) { + c := setupCuratorForUpdate(t) + mc := c.client.(*mockClient) + + mc.On("IsUpdateAvailable", mock.Anything).Return(&distribution.Archive{}, nil) + mc.On("Download", mock.Anything, mock.Anything, mock.Anything).Return("", errors.New("download failed")) + + updated, err := c.Update() + + require.ErrorContains(t, err, "download failed") + require.False(t, updated) + require.NoFileExists(t, filepath.Join(c.config.DBDirectoryPath(), lastUpdateCheckFileName)) + + mc.AssertExpectations(t) + }) + + t.Run("error during activation: bad checksum", func(t *testing.T) { + c := setupCuratorForUpdate(t, withBadDBChecksum(), withWorkingUpdateIntegrations()) + mc := c.client.(*mockClient) + + updated, err := c.Update() + + require.ErrorContains(t, err, "bad db checksum") + require.False(t, updated) + require.NoFileExists(t, filepath.Join(c.config.DBDirectoryPath(), lastUpdateCheckFileName)) + + mc.AssertExpectations(t) + }) + + t.Run("error during activation: cannot move dir", func(t *testing.T) { + c := setupCuratorForUpdate(t, withWorkingUpdateIntegrations()) + mc := c.client.(*mockClient) + + // simulate not being able to move the staged dir to the db dir + c.fs = afero.NewReadOnlyFs(c.fs) + + updated, err := c.Update() + + require.ErrorContains(t, err, "operation not permitted") + require.False(t, updated) + require.NoFileExists(t, filepath.Join(c.config.DBDirectoryPath(), lastUpdateCheckFileName)) + + mc.AssertExpectations(t) + }) +} + +func TestReadDatabaseDescription(t *testing.T) { + fs := afero.NewMemMapFs() + + description := db.Description{ + SchemaVersion: "1.0.0", + Built: db.Time{Time: time.Date(2021, 1, 2, 3, 4, 5, 6, time.UTC)}, + Checksum: "xxh64:dummychecksum", + } + + descriptionJSON, err := json.Marshal(description) + require.NoError(t, err) + + metadataFilePath := path.Join("someDir", db.DescriptionFileName) + require.NoError(t, afero.WriteFile(fs, metadataFilePath, descriptionJSON, 0644)) + + result, err := readDatabaseDescription(fs, "someDir") + require.NoError(t, err) + + require.NotNil(t, result) + require.Equal(t, + db.Description{ + SchemaVersion: "1.0.0", + Built: db.Time{Time: description.Built.Time.Truncate(time.Second)}, + Checksum: "xxh64:dummychecksum", + }, + *result, + ) +} + +func TestCurator_IsUpdateCheckAllowed(t *testing.T) { + + newCurator := func(t *testing.T) curator { + tempDir := t.TempDir() + + cfg := DefaultConfig() + cfg.UpdateCheckMaxFrequency = 10 * time.Minute + cfg.DBRootDir = tempDir + + ci, err := NewCurator(cfg, nil) + require.NoError(t, err) + + c := ci.(curator) + return c + } + + writeLastCheckContents := func(t *testing.T, cfg Config, contents string) { + require.NoError(t, os.MkdirAll(cfg.DBDirectoryPath(), 0755)) + p := filepath.Join(cfg.DBDirectoryPath(), lastUpdateCheckFileName) + err := os.WriteFile(p, []byte(contents), 0644) + require.NoError(t, err) + } + + writeLastCheckTime := func(t *testing.T, cfg Config, lastCheckTime time.Time) { + writeLastCheckContents(t, cfg, lastCheckTime.Format(time.RFC3339)) + } + + t.Run("first run check (no last check file)", func(t *testing.T) { + c := newCurator(t) + require.True(t, c.isUpdateCheckAllowed()) + }) + + t.Run("check not allowed due to frequency", func(t *testing.T) { + c := newCurator(t) + writeLastCheckTime(t, c.config, time.Now().Add(-5*time.Minute)) + + require.False(t, c.isUpdateCheckAllowed()) + }) + + t.Run("check allowed after the frequency period", func(t *testing.T) { + c := newCurator(t) + writeLastCheckTime(t, c.config, time.Now().Add(-20*time.Minute)) + + require.True(t, c.isUpdateCheckAllowed()) + }) + + t.Run("error reading last check file", func(t *testing.T) { + c := newCurator(t) + + // simulate a situation where the last check file exists but is corrupted + writeLastCheckContents(t, c.config, "invalid timestamp") + + allowed := c.isUpdateCheckAllowed() + require.True(t, allowed) // should return true since an error is encountered + }) + +} + +func TestCurator_DurationSinceUpdateCheck(t *testing.T) { + newCurator := func(t *testing.T) curator { + tempDir := t.TempDir() + + cfg := DefaultConfig() + cfg.DBRootDir = tempDir + + ci, err := NewCurator(cfg, nil) + require.NoError(t, err) + + c := ci.(curator) + return c + } + + writeLastCheckContents := func(t *testing.T, cfg Config, contents string) { + require.NoError(t, os.MkdirAll(cfg.DBDirectoryPath(), 0755)) + p := filepath.Join(cfg.DBDirectoryPath(), lastUpdateCheckFileName) + err := os.WriteFile(p, []byte(contents), 0644) + require.NoError(t, err) + } + + t.Run("no last check file", func(t *testing.T) { + c := newCurator(t) + elapsed, err := c.durationSinceUpdateCheck() + require.NoError(t, err) + require.Nil(t, elapsed) // should be nil since no file exists + }) + + t.Run("valid last check file", func(t *testing.T) { + c := newCurator(t) + writeLastCheckContents(t, c.config, time.Now().Add(-5*time.Minute).Format(time.RFC3339)) + + elapsed, err := c.durationSinceUpdateCheck() + require.NoError(t, err) + require.NotNil(t, elapsed) + require.True(t, *elapsed >= 5*time.Minute) // should be at least 5 minutes + }) + + t.Run("malformed last check file", func(t *testing.T) { + c := newCurator(t) + writeLastCheckContents(t, c.config, "invalid timestamp") + + _, err := c.durationSinceUpdateCheck() + require.Error(t, err) + require.Contains(t, err.Error(), "unable to parse last update check timestamp") + }) +} + +func TestCurator_SetLastSuccessfulUpdateCheck(t *testing.T) { + newCurator := func(t *testing.T) curator { + tempDir := t.TempDir() + + cfg := DefaultConfig() + cfg.DBRootDir = tempDir + + ci, err := NewCurator(cfg, nil) + require.NoError(t, err) + + c := ci.(curator) + + require.NoError(t, c.fs.MkdirAll(c.config.DBDirectoryPath(), 0755)) + + return c + } + + t.Run("set last successful update check", func(t *testing.T) { + c := newCurator(t) + + c.setLastSuccessfulUpdateCheck() + + data, err := afero.ReadFile(c.fs, filepath.Join(c.config.DBDirectoryPath(), lastUpdateCheckFileName)) + require.NoError(t, err) + + lastCheckTime, err := time.Parse(time.RFC3339, string(data)) + require.NoError(t, err) + require.WithinDuration(t, time.Now().UTC(), lastCheckTime, time.Second) + }) + + t.Run("error writing last successful update check", func(t *testing.T) { + c := newCurator(t) + + // make the file system read-only to simulate a write error + readonlyFs := afero.NewReadOnlyFs(c.fs) + c.fs = readonlyFs + + c.setLastSuccessfulUpdateCheck() + + require.NoFileExists(t, filepath.Join(c.config.DBDirectoryPath(), lastUpdateCheckFileName)) + }) + + t.Run("ensure last successful update check file is created", func(t *testing.T) { + c := newCurator(t) + + c.setLastSuccessfulUpdateCheck() + + require.FileExists(t, filepath.Join(c.config.DBDirectoryPath(), lastUpdateCheckFileName)) + }) +} + +func TestCurator_EnsureNotStale(t *testing.T) { + newCurator := func(t *testing.T) curator { + tempDir := t.TempDir() + cfg := DefaultConfig() + cfg.DBRootDir = tempDir + cfg.MaxAllowedBuiltAge = 48 * time.Hour // set max age to 48 hours + + ci, err := NewCurator(cfg, new(mockClient)) + require.NoError(t, err) + + return ci.(curator) + } + + hoursAgo := func(h int) db.Time { + return db.Time{Time: time.Now().UTC().Add(-time.Duration(h) * time.Hour)} + } + + tests := []struct { + name string + description *db.Description + wantErr require.ErrorAssertionFunc + modifyConfig func(*Config) + }{ + { + name: "valid metadata within age limit", + description: &db.Description{ + Built: hoursAgo(24), + }, + }, + { + name: "stale metadata exactly at age limit", + description: &db.Description{ + Built: hoursAgo(48), + }, + wantErr: func(t require.TestingT, err error, msgAndArgs ...interface{}) { + require.ErrorContains(t, err, "the vulnerability database was built") + }, + }, + { + name: "stale metadata", + description: &db.Description{ + Built: hoursAgo(50), + }, + wantErr: func(t require.TestingT, err error, msgAndArgs ...interface{}) { + require.ErrorContains(t, err, "the vulnerability database was built") + }, + }, + { + name: "no metadata", + description: nil, + wantErr: func(t require.TestingT, err error, msgAndArgs ...interface{}) { + require.ErrorContains(t, err, "no metadata to validate") + }, + }, + { + name: "age validation disabled", + description: &db.Description{ + Built: hoursAgo(50), + }, + modifyConfig: func(cfg *Config) { + cfg.ValidateAge = false + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if tt.wantErr == nil { + tt.wantErr = require.NoError + } + + c := newCurator(t) + + if tt.modifyConfig != nil { + tt.modifyConfig(&c.config) + } + + err := c.ensureNotStale(tt.description) + tt.wantErr(t, err) + }) + } +} + +func TestCurator_ValidateIntegrity(t *testing.T) { + newCurator := func(t *testing.T) curator { + tempDir := t.TempDir() + cfg := DefaultConfig() + cfg.DBRootDir = tempDir + + ci, err := NewCurator(cfg, new(mockClient)) + require.NoError(t, err) + + return ci.(curator) + } + + t.Run("valid metadata with correct checksum", func(t *testing.T) { + c := newCurator(t) + dbDir := c.config.DBDirectoryPath() + + metadata := db.Description{ + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + Built: db.Time{Time: time.Now()}, + Checksum: writeTestDB(t, c.fs, dbDir), + } + + writeTestMetadata(t, c.fs, dbDir, metadata) + + result, err := c.validateIntegrity(dbDir) + require.NoError(t, err) + require.NotNil(t, result) + }) + + t.Run("invalid metadata (not found)", func(t *testing.T) { + c := newCurator(t) + + _, err := c.validateIntegrity("non/existent/path") + require.ErrorContains(t, err, "database metadata not found") + }) + + t.Run("invalid checksum", func(t *testing.T) { + c := newCurator(t) + dbDir := c.config.DBDirectoryPath() + + metadata := db.Description{ + SchemaVersion: schemaver.New(db.ModelVersion, db.Revision, db.Addition), + Built: db.Time{Time: time.Now()}, + Checksum: "xxh64:invalidchecksum", + } + + writeTestMetadata(t, c.fs, dbDir, metadata) + + writeTestDB(t, c.fs, dbDir) + + _, err := c.validateIntegrity(dbDir) + require.ErrorContains(t, err, "bad db checksum") + }) + + t.Run("unsupported database version", func(t *testing.T) { + c := newCurator(t) + dbDir := c.config.DBDirectoryPath() + + metadata := db.Description{ + SchemaVersion: schemaver.New(9999, 0, 0), // invalid version + Built: db.Time{Time: time.Now()}, + Checksum: writeTestDB(t, c.fs, dbDir), + } + + writeTestMetadata(t, c.fs, dbDir, metadata) + + _, err := c.validateIntegrity(dbDir) + require.ErrorContains(t, err, "unsupported database version") + }) +} diff --git a/grype/db/v6/status.go b/grype/db/v6/status.go new file mode 100644 index 00000000000..3d275bf51ea --- /dev/null +++ b/grype/db/v6/status.go @@ -0,0 +1,9 @@ +package v6 + +type Status struct { + Built Time `json:"built"` + SchemaVersion string `json:"schemaVersion"` + Location string `json:"location"` + Checksum string `json:"checksum"` + Err error `json:"error"` +} diff --git a/grype/db/v6/store.go b/grype/db/v6/store.go index a95ffe9bf21..0df09d339e4 100644 --- a/grype/db/v6/store.go +++ b/grype/db/v6/store.go @@ -2,7 +2,10 @@ package v6 import ( "fmt" + "os" + "path/filepath" + "github.com/spf13/afero" "gorm.io/gorm" "github.com/anchore/grype/grype/db/internal/gormadapter" @@ -47,5 +50,21 @@ func (s *store) Close() error { return fmt.Errorf("failed to vacuum: %w", err) } - return nil + fs := afero.NewOsFs() + + desc, err := NewDescriptionFromDir(fs, s.config.DBDirPath) + if err != nil { + return fmt.Errorf("failed to create description from dir: %w", err) + } + + if desc == nil { + return fmt.Errorf("unable to describe the database") + } + + fh, err := fs.OpenFile(filepath.Join(s.config.DBDirPath, DescriptionFileName), os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("failed to open description file: %w", err) + } + + return writeDescription(fh, *desc) } diff --git a/internal/file/hasher.go b/internal/file/hasher.go index c9af0c0b69e..bf1b2b4f4db 100644 --- a/internal/file/hasher.go +++ b/internal/file/hasher.go @@ -8,6 +8,7 @@ import ( "io" "strings" + "github.com/OneOfOne/xxhash" "github.com/spf13/afero" ) @@ -18,6 +19,9 @@ func ValidateByHash(fs afero.Fs, path, hashStr string) (bool, string, error) { case strings.HasPrefix(hashStr, "sha256:"): hashFn = "sha256" hasher = sha256.New() + case strings.HasPrefix(hashStr, "xxh64:"): + hashFn = "xxh64" + hasher = xxhash.New64() default: return false, "", fmt.Errorf("hasher not supported or specified (given: %s)", hashStr) } @@ -39,8 +43,12 @@ func HashFile(fs afero.Fs, path string, hasher hash.Hash) (string, error) { } defer f.Close() - if _, err := io.Copy(hasher, f); err != nil { - return "", fmt.Errorf("failed to hash file '%s': %w", path, err) + return HashReader(f, hasher) +} + +func HashReader(reader io.Reader, hasher hash.Hash) (string, error) { + if _, err := io.Copy(hasher, reader); err != nil { + return "", fmt.Errorf("failed to hash reader: %w", err) } return hex.EncodeToString(hasher.Sum(nil)), nil