diff --git a/go.mod b/go.mod index 928423919..bf8d19e51 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 require ( github.com/Masterminds/semver v1.5.0 github.com/doug-martin/goqu/v8 v8.6.0 + github.com/evanphx/json-patch/v5 v5.6.0 github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 github.com/google/uuid v1.3.0 @@ -21,6 +22,7 @@ require ( github.com/quay/claircore/updater/driver v1.0.0 github.com/quay/goval-parser v0.8.8 github.com/quay/zlog v1.1.5 + github.com/regclient/regclient v0.5.0 github.com/remind101/migrate v0.0.0-20170729031349-52c1edff7319 github.com/rs/zerolog v1.29.0 github.com/ulikunitz/xz v0.5.11 @@ -36,6 +38,7 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/davecgh/go-spew v1.1.1 // indirect + github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect github.com/dustin/go-humanize v1.0.1 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect @@ -48,10 +51,13 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.16 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect + github.com/pkg/errors v0.9.1 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.10.1 // indirect github.com/quay/claircore/toolkit v1.0.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + github.com/sirupsen/logrus v1.9.3 // indirect go.opentelemetry.io/otel v1.11.0 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/sys v0.10.0 // indirect @@ -70,3 +76,5 @@ require ( replace github.com/quay/claircore/toolkit => ./toolkit replace github.com/quay/claircore/updater/driver => ./updater/driver + +replace github.com/regclient/regclient => github.com/hdonnay/regclient v0.0.0-20230802204448-895786ac70b4 diff --git a/go.sum b/go.sum index 4edda3a9a..fbe90f6c6 100644 --- a/go.sum +++ b/go.sum @@ -18,10 +18,14 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= +github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/doug-martin/goqu/v8 v8.6.0 h1:KWuDGL135poBgY+SceArvOtIIEpieNKgIZCvgerI228= github.com/doug-martin/goqu/v8 v8.6.0/go.mod h1:wiiYWkiguNXK5d4kGIkYmOxBScEL37d9Cfv9tXhPsTk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/evanphx/json-patch/v5 v5.6.0 h1:b91NhWfaz02IuVxO9faSllyAtNXHMPkC5J8sJCLunww= +github.com/evanphx/json-patch/v5 v5.6.0/go.mod h1:G79N1coSVB93tBe7j6PhzjmR3/2VvlbKOFpnXhI9Bw4= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -44,6 +48,8 @@ github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbu github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hdonnay/regclient v0.0.0-20230802204448-895786ac70b4 h1:o49DXtiX6S+68U3c6q7U6G1BBoRcVXZvrZlrPxssCrA= +github.com/hdonnay/regclient v0.0.0-20230802204448-895786ac70b4/go.mod h1:rdhLqry6esQAJE+HfEmcjWaLcQ3iX4pDAd1bndKH6yE= github.com/jackc/chunkreader v1.0.0/go.mod h1:RT6O25fNZIuasFJRyZ4R/Y2BbhasbmZXF9QQ7T3kePo= github.com/jackc/chunkreader/v2 v2.0.0/go.mod h1:odVSm741yZoC3dpHEUXIqA9tQRhFrgOHwnPIn9lDKlk= github.com/jackc/chunkreader/v2 v2.0.1 h1:i+RDz65UE+mmpjTfyz0MoVTnzeYxroil2G82ki7MGG8= @@ -93,6 +99,7 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -131,6 +138,8 @@ github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsO github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -168,6 +177,8 @@ github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXY github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= @@ -255,6 +266,7 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/test/fixture/fixture.go b/test/fixture/fixture.go new file mode 100644 index 000000000..8cb53a9b9 --- /dev/null +++ b/test/fixture/fixture.go @@ -0,0 +1,141 @@ +package fixture + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/regclient/regclient" + "github.com/regclient/regclient/scheme" + "github.com/regclient/regclient/scheme/ocidir" + oci "github.com/regclient/regclient/types/oci/v1" + "github.com/regclient/regclient/types/ref" + + "github.com/quay/claircore/test/integration" +) + +func lookupCaller(t *testing.T) string { + const module = `github.com/quay/claircore/` + pc, _, _, ok := runtime.Caller(2) + if !ok { + t.Fatal("unable to get caller") + } + info := runtime.FuncForPC(pc) + name := info.Name() + idx := strings.LastIndexByte(name, '.') + if idx == -1 { + t.Fatalf("weird name: %q", name) + } + return strings.TrimPrefix(name[:idx], module) +} + +var ( + // Alternatively, use ghcr.io ? + Registry = `quay.io` + Namespace = `projectquay` + Tag = `latest` +) + +const ( + // VulnerabilitiesType is the Artifact Type that indicates this artifact should + // be used to build a vulnerability database. + // + // The "yolo" type indicates this should be passed into an unknown, + // per-package processing function. The type is functionally + // "application/octet-stream". + VulnerabilitiesType = `application/vnd.claircore-test.vulnerabilities.yolo.layer.v1` + // VulnerabilitiesType is the Artifact Type that indicates this artifact should + // be used to build an advisory database. + // + // The blob should be of type "application/zip". The artifact may indicate + // via TBD named annotations what Matcher is expected to handle the data. + AdvisoriesType = `application/vnd.claircore-test.advisories.zip.layer.v1` + // VerifyType is the Artifact Type that indicates .. + VerifyType = `application/vnd.claircore-test.verify.jsonpatch.layer.v1` + + MatcherConfigType = `application/vnd.claircore-test.matcher.configuration.layer.v1+json` + UpdaterConfigType = `application/vnd.claircore-test.updater.configuration.layer.v1+json` + IndexerConfigType = `application/vnd.claircore-test.indexer.configuration.layer.v1+json` +) + +// Fetch ... +func Fetch[V Value, K Kind[V]](ctx context.Context, t *testing.T) []V { + t.Helper() + repo := lookupCaller(t) + dir := filepath.Join(integration.PackageCacheDir(t), "fixtures") + if err := os.Mkdir(dir, 0755); err != nil && !errors.Is(err, fs.ErrExist) { + t.Fatal(err) + } + sys := regclient.NewDirFS(dir) + var elem K + + c := regclient.New( + regclient.WithFS(sys), + regclient.WithDockerCreds(), + ) + tgt, err := ref.New("ocidir://" + Tag) + if err != nil { + t.Fatal(err) + } + t.Run("FetchFixtures", func(t *testing.T) { + integration.Skip(t) + name := fmt.Sprintf("%s/%s/%s:%s", Registry, Namespace, repo, Tag) + t.Logf("pulling fixtues referencing %q", name) + remote, err := ref.New(name) + if err != nil { + t.Fatal(err) + } + defer c.Close(ctx, remote) + if err := c.ImageCopy(ctx, remote, tgt); err != nil { + t.Error(err) + } + }) + + switch _, err := fs.Stat(sys, path.Join(Tag, "oci-layout")); { + case errors.Is(err, nil): // OK + case errors.Is(err, fs.ErrNotExist): + t.Skip("skipping integration test: need integration tag at least once to populate fixtures") + default: + t.Fatalf("unexpected error with local files: %v", err) + } + + at := elem.ArtifactType() + list, err := c.ReferrerList(ctx, tgt, scheme.WithReferrerAT(at)) + if err != nil { + t.Fatal(err) + } + if list.IsEmpty() { + t.Logf("no manifests of type %q for %s", at, tgt.CommonName()) + return nil + } + + out := make([]V, len(list.Descriptors)) + local := ocidir.New(ocidir.WithFS(sys)) + for i, d := range list.Descriptors { + rd, err := c.BlobGet(ctx, tgt, d) + if err != nil { + t.Fatalf("error fetching blob: %v", err) + } + t.Logf("found descriptor: %v", d) + // Don't worry too much about the Reader; all tests are transitory, man. + var m oci.Manifest + if err := json.NewDecoder(rd).Decode(&m); err != nil { + t.Fatalf("unexpected error decoding descriptor data: %v", err) + } + rd.Close() + var k K = &out[i] + if err := k.Load(ctx, t, dir, local, tgt, &m); err != nil { + t.Fatalf("tk: %v", err) + } + } + + return out +} diff --git a/test/fixture/fixture_test.go b/test/fixture/fixture_test.go new file mode 100644 index 000000000..3b4a22f3e --- /dev/null +++ b/test/fixture/fixture_test.go @@ -0,0 +1,27 @@ +package fixture + +import ( + "context" + "testing" +) + +func TestLookup(t *testing.T) { + const want = `test/fixture` + var got string + // Simulate getting called from a top-level test: + func() { + got = lookupCaller(t) + }() + t.Logf("got: %q, want: %q", got, want) + if got != want { + t.Fail() + } +} + +func TestFetch(t *testing.T) { + ctx := context.Background() + tcs := Fetch[Indexer](ctx, t) + for _, tc := range tcs { + t.Logf("got layer: %v", tc.Manifest.Hash) + } +} diff --git a/test/fixture/kinds.go b/test/fixture/kinds.go new file mode 100644 index 000000000..e8c8158e9 --- /dev/null +++ b/test/fixture/kinds.go @@ -0,0 +1,138 @@ +package fixture + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "path/filepath" + "testing" + + jsonpatch "github.com/evanphx/json-patch/v5" + "github.com/quay/claircore" + "github.com/regclient/regclient/scheme/ocidir" + "github.com/regclient/regclient/types" + oci "github.com/regclient/regclient/types/oci/v1" + "github.com/regclient/regclient/types/ref" +) + +const artifactTmpl = `application/vnd.claircore.test-fixture.%s.config.v1` + +type Kind[T Value] interface { + *T + // Load takes an OCIDir so that we can _know_ the path of blobs. + Load(context.Context, *testing.T, string, *ocidir.OCIDir, ref.Ref, *oci.Manifest) error + ArtifactType() string +} + +type Value interface { + Indexer | Matcher | Updater | MatcherFlow | Integration +} + +var ErrBadArtifactType = errors.New("bad artifact type") + +type testTransform struct { + p *jsonpatch.Patch +} + +var opTest = json.RawMessage(`"test"`) + +func (t *testTransform) UnmarshalJSON(b []byte) error { + var op jsonpatch.Operation + if err := json.Unmarshal(b, &op); err != nil { + return err + } + op["op"] = &opTest + *t.p = append(*t.p, op) + return nil +} + +type Indexer struct { + Manifest claircore.Manifest + Verify jsonpatch.Patch +} + +func (*Indexer) ArtifactType() string { return fmt.Sprintf(artifactTmpl, `indexer`) } +func (i *Indexer) Load(ctx context.Context, t *testing.T, root string, dir *ocidir.OCIDir, r ref.Ref, m *oci.Manifest) error { + if m.ArtifactType != i.ArtifactType() { + return ErrBadArtifactType + } + for _, l := range m.Layers { + switch l.ArtifactType { + case types.MediaTypeOCI1Manifest: + d := l.Digest.String() + i.Manifest.Hash = claircore.MustParseDigest(d) + r := r + r.Tag = "" + r.Digest = d + om, err := dir.ManifestGet(ctx, r) + if err != nil { + return err + } + ls, err := om.GetLayers() + if err != nil { + return err + } + for _, d := range ls { + i.Manifest.Layers = append(i.Manifest.Layers, + &claircore.Layer{ + Hash: claircore.MustParseDigest(d.Digest.String()), + URI: (&url.URL{ + Scheme: "file", + Opaque: filepath.Join(root, "blobs", d.Digest.Algorithm().String(), d.Digest.Encoded()), + }).String(), + }) + } + case VerifyType: + rc, err := dir.BlobGet(ctx, r, l) + if err != nil { + return err + } + defer rc.Close() + if err := json.NewDecoder(rc).Decode(&testTransform{&i.Verify}); err != nil { + return err + } + default: // Skip + t.Logf("skipping artifact: %s", l.ArtifactType) + } + } + return nil +} + +type Matcher struct{} + +func (*Matcher) ArtifactType() string { return fmt.Sprintf(artifactTmpl, `matcher`) } +func (ma *Matcher) Load(ctx context.Context, t *testing.T, root string, dir *ocidir.OCIDir, r ref.Ref, m *oci.Manifest) error { + panic("TODO: implement") +} + +type Updater struct{} + +func (*Updater) ArtifactType() string { return fmt.Sprintf(artifactTmpl, `updater`) } +func (u *Updater) Load(ctx context.Context, t *testing.T, root string, dir *ocidir.OCIDir, r ref.Ref, m *oci.Manifest) error { + panic("TODO: implement") +} + +type MatcherFlow struct{} + +func (*MatcherFlow) ArtifactType() string { return fmt.Sprintf(artifactTmpl, `matcher-flow`) } +func (mf *MatcherFlow) Load(ctx context.Context, t *testing.T, root string, dir *ocidir.OCIDir, r ref.Ref, m *oci.Manifest) error { + panic("TODO: implement") +} + +type Integration struct{} + +func (*Integration) ArtifactType() string { return fmt.Sprintf(artifactTmpl, `integration`) } +func (i *Integration) Load(ctx context.Context, t *testing.T, root string, dir *ocidir.OCIDir, r ref.Ref, m *oci.Manifest) error { + panic("TODO: implement") +} + +// Compile-time type checks: +var ( + _ func(context.Context, *testing.T) []Indexer = Fetch[Indexer] + _ func(context.Context, *testing.T) []Matcher = Fetch[Matcher] + _ func(context.Context, *testing.T) []Updater = Fetch[Updater] + _ func(context.Context, *testing.T) []MatcherFlow = Fetch[MatcherFlow] + _ func(context.Context, *testing.T) []Integration = Fetch[Integration] +)