diff --git a/go.mod b/go.mod index f6045127..ca7f47d8 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/anchore/grype-db -go 1.23.1 +go 1.23.2 require ( github.com/Masterminds/semver/v3 v3.3.1 @@ -8,10 +8,11 @@ require ( github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d github.com/adrg/xdg v0.5.3 github.com/anchore/go-logger v0.0.0-20230725134548-c21dafa1ec5a - github.com/anchore/grype v0.86.0 - github.com/anchore/syft v1.18.0 + github.com/anchore/grype v0.86.1-0.20241216215412-e795ae550360 + github.com/anchore/syft v1.18.1-0.20241210183324-561ed50c2d20 github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de github.com/dave/jennifer v1.7.1 + github.com/dustin/go-humanize v1.0.1 github.com/glebarez/sqlite v1.11.0 github.com/go-test/deep v1.1.1 github.com/google/go-cmp v0.6.0 @@ -62,6 +63,7 @@ require ( github.com/Microsoft/hcsshim v0.11.7 // indirect github.com/ProtonMail/go-crypto v1.1.2 // indirect github.com/acobaugh/osrelease v0.1.0 // indirect + github.com/anchore/archiver/v3 v3.5.3-0.20241210171143-5b1d8d1c7c51 // indirect github.com/anchore/clio v0.0.0-20241115144204-29e89f9fa837 // indirect github.com/anchore/fangs v0.0.0-20241014225144-4e1713cafd77 // indirect github.com/anchore/go-collections v0.0.0-20240216171411-9321230ce537 // indirect @@ -70,7 +72,7 @@ require ( github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4 // indirect github.com/anchore/packageurl-go v0.1.1-0.20241018175412-5c22e6360c4f // indirect github.com/anchore/stereoscope v0.0.10 // indirect - github.com/andybalholm/brotli v1.0.4 // indirect + github.com/andybalholm/brotli v1.1.1 // indirect github.com/aquasecurity/go-pep440-version v0.0.0-20210121094942-22b2f8951d46 // indirect github.com/aquasecurity/go-version v0.0.0-20210121072130-637058cfe492 // indirect github.com/aws/aws-sdk-go v1.44.288 // indirect @@ -99,13 +101,12 @@ require ( github.com/distribution/reference v0.6.0 // indirect github.com/docker/cli v27.3.1+incompatible // indirect github.com/docker/distribution v2.8.3+incompatible // indirect - github.com/docker/docker v27.3.1+incompatible // indirect + github.com/docker/docker v27.4.0+incompatible // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dsnet/compress v0.0.2-0.20210315054119-f66993602bf5 // indirect - github.com/dustin/go-humanize v1.0.1 // indirect github.com/edsrzf/mmap-go v1.1.0 // indirect github.com/elliotchance/phpserialize v1.4.0 // indirect github.com/emirpasic/gods v1.18.1 // indirect @@ -145,7 +146,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/kastenhq/goversion v0.0.0-20230811215019-93b2f8823953 // indirect github.com/kevinburke/ssh_config v1.2.0 // indirect - github.com/klauspost/pgzip v1.2.5 // indirect + github.com/klauspost/pgzip v1.2.6 // indirect github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f // indirect github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d // indirect github.com/knqyf263/go-rpmdb v0.1.1 // indirect @@ -156,7 +157,6 @@ require ( github.com/mattn/go-isatty v0.0.20 // indirect github.com/mattn/go-runewidth v0.0.16 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect - github.com/mholt/archiver/v3 v3.5.1 // indirect github.com/microsoft/go-rustaudit v0.0.0-20220730194248-4b17361d90a5 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect @@ -171,7 +171,7 @@ require ( github.com/moby/sys/userns v0.1.0 // indirect github.com/muesli/termenv v0.15.2 // indirect github.com/ncruces/go-strftime v0.1.9 // indirect - github.com/nwaples/rardecode v1.1.0 // indirect + github.com/nwaples/rardecode v1.1.3 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0 // indirect @@ -182,7 +182,7 @@ require ( github.com/pborman/indent v1.2.1 // indirect github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect - github.com/pierrec/lz4/v4 v4.1.19 // indirect + github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pjbgf/sha1cd v0.3.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect @@ -220,7 +220,7 @@ require ( go.opentelemetry.io/otel/trace v1.24.0 // indirect go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect - golang.org/x/crypto v0.30.0 // indirect + golang.org/x/crypto v0.31.0 // indirect golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect golang.org/x/mod v0.22.0 // indirect golang.org/x/net v0.32.0 // indirect diff --git a/go.sum b/go.sum index 84e5be3b..f9a2853c 100644 --- a/go.sum +++ b/go.sum @@ -234,8 +234,8 @@ github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuy github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/anchore/archiver/v3 v3.5.2 h1:Bjemm2NzuRhmHy3m0lRe5tNoClB9A4zYyDV58PaB6aA= -github.com/anchore/archiver/v3 v3.5.2/go.mod h1:e3dqJ7H78uzsRSEACH1joayhuSyhnonssnDhppzS1L4= +github.com/anchore/archiver/v3 v3.5.3-0.20241210171143-5b1d8d1c7c51 h1:yhk+P8lF3ZiROjmaVRao9WGTRo4b/wYjoKEiAHWrKwc= +github.com/anchore/archiver/v3 v3.5.3-0.20241210171143-5b1d8d1c7c51/go.mod h1:nwuGSd7aZp0rtYt79YggCGafz1RYsclE7pi3fhLwvuw= github.com/anchore/clio v0.0.0-20241115144204-29e89f9fa837 h1:bIG3WsfosZsJ5LMC7PB9J/ekFM3a0j0ZEDvN3ID6GTI= github.com/anchore/clio v0.0.0-20241115144204-29e89f9fa837/go.mod h1:tRQVKkjYeejrh9AdM0s1esbwtMU7rdHAHSQWkv4qskE= github.com/anchore/fangs v0.0.0-20241014225144-4e1713cafd77 h1:h7+GCqazHVS5GDJYYS6wjjglYi8xFnVWMdSUukoImTM= @@ -252,18 +252,17 @@ github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04 h1:VzprUTpc0v github.com/anchore/go-testutils v0.0.0-20200925183923-d5f45b0d3c04/go.mod h1:6dK64g27Qi1qGQZ67gFmBFvEHScy0/C8qhQhNe5B5pQ= github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4 h1:rmZG77uXgE+o2gozGEBoUMpX27lsku+xrMwlmBZJtbg= github.com/anchore/go-version v1.2.2-0.20210903204242-51efa5b487c4/go.mod h1:Bkc+JYWjMCF8OyZ340IMSIi2Ebf3uwByOk6ho4wne1E= -github.com/anchore/grype v0.86.0 h1:jNKf2QbMLCtyZF5bUgcTQG1t2pI807Dz3Os3KgyaDyw= -github.com/anchore/grype v0.86.0/go.mod h1:savAXQzmEw40ei1KHni2t6sSHpNhivOvaxuz/VQrZ4U= +github.com/anchore/grype v0.86.1-0.20241216215412-e795ae550360 h1:kvW6K0msph6b30vkTtJRIrP02vJCYTOytlM6jGYQKKI= +github.com/anchore/grype v0.86.1-0.20241216215412-e795ae550360/go.mod h1:zhoftUcm7GpNLfll6/f7GRmGroCD0un8jp0LZ7JkPSw= github.com/anchore/packageurl-go v0.1.1-0.20241018175412-5c22e6360c4f h1:dAQPIrQ3a5PBqZeZ+B9NGZsGmodk4NO9OjDIsQmQyQM= github.com/anchore/packageurl-go v0.1.1-0.20241018175412-5c22e6360c4f/go.mod h1:KoYIv7tdP5+CC9VGkeZV4/vGCKsY55VvoG+5dadg4YI= github.com/anchore/stereoscope v0.0.10 h1:FjvYNLeRlCHpbrnCOR8fwIa1W2YHTwoW4Peu5iCqKWE= github.com/anchore/stereoscope v0.0.10/go.mod h1:BikvtLTXRzjZnbCdk1tQJBTVZZdilR1JCCpFRYYeaNU= -github.com/anchore/syft v1.18.0 h1:z77YRJ0RHseEvNWhacgrf6WNMhfGDu/e/KQAZIGgMb8= -github.com/anchore/syft v1.18.0/go.mod h1:lvzj34iwa9YR7Q6Vaq4yr4929Sz64yaYp/J/4+4SOCI= +github.com/anchore/syft v1.18.1-0.20241210183324-561ed50c2d20 h1:qKBgKYqf9371QpceC0Fmf1GycKjkn7/a+2uj7Jj6jsg= +github.com/anchore/syft v1.18.1-0.20241210183324-561ed50c2d20/go.mod h1:hfZPkKBBaj8zVYQGwxVO0cR23/Ml7VoC3Hwg/sArROA= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= -github.com/andybalholm/brotli v1.0.1/go.mod h1:loMXtMfwqflxFJPmdbJO0a3KNoPuLBgiu3qAvBg8x/Y= -github.com/andybalholm/brotli v1.0.4 h1:V7DdXeJtZscaqfNuAdSRuRFzuiKlHSC/Zh3zl9qY3JY= -github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= +github.com/andybalholm/brotli v1.1.1 h1:PR2pgnyFznKEugtsUo0xLdDop5SKXd5Qf5ysW+7XdTA= +github.com/andybalholm/brotli v1.1.1/go.mod h1:05ib4cKhjx3OQYUY22hTVd34Bc8upXjOLL2rKwwZBoA= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be h1:9AeTilPcZAjCFIImctFaOjnTIavg87rW78vTPkQqLI8= github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be/go.mod h1:ySMOLuWl6zY27l47sB3qLNK6tF2fkHG55UZxx8oIVo4= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= @@ -382,8 +381,8 @@ github.com/docker/cli v27.3.1+incompatible h1:qEGdFBF3Xu6SCvCYhc7CzaQTlBmqDuzxPD github.com/docker/cli v27.3.1+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8= github.com/docker/distribution v2.8.3+incompatible h1:AtKxIZ36LoNK51+Z6RpzLpddBirtxJnzDrHLEKxTAYk= github.com/docker/distribution v2.8.3+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v27.3.1+incompatible h1:KttF0XoteNTicmUtBO0L2tP+J7FGRFTjaEF4k6WdhfI= -github.com/docker/docker v27.3.1+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v27.4.0+incompatible h1:I9z7sQ5qyzO0BfAb9IMOawRkAGxhYsidKiTMcm0DU+A= +github.com/docker/docker v27.4.0+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.7.0 h1:xtCHsjxogADNZcdv1pKUHXryefjlVRqWqIhk/uXJp0A= github.com/docker/docker-credential-helpers v0.7.0/go.mod h1:rETQfLdHNT3foU5kuNkFR1R1V12OJRRO5lzt2D1b5X0= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= @@ -511,7 +510,6 @@ github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golang/snappy v0.0.2/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -681,13 +679,12 @@ github.com/kevinburke/ssh_config v1.2.0/go.mod h1:CT57kijsi8u/K/BOFA39wgDQJ9CxiF github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.4.1/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= -github.com/klauspost/compress v1.11.4/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc= github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= -github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= -github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f h1:GvCU5GXhHq+7LeOzx/haG7HSIZokl3/0GkoUFzsRJjg= github.com/knqyf263/go-apk-version v0.0.0-20200609155635-041fdbb8563f/go.mod h1:q59u9px8b7UTj0nIjEjvmTWekazka6xIt6Uogz5Dm+8= github.com/knqyf263/go-deb-version v0.0.0-20190517075300-09fca494f03d h1:X4cedH4Kn3JPupAwwWuo4AzYp16P0OyLO9d7OnMZc/c= @@ -796,8 +793,8 @@ github.com/muesli/termenv v0.15.2/go.mod h1:Epx+iuz8sNs7mNKhxzH4fWXGNpZwUaJKRS1n github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4= github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= -github.com/nwaples/rardecode v1.1.0 h1:vSxaY8vQhOcVr4mm5e8XllHWTiM4JF507A0Katqw7MQ= -github.com/nwaples/rardecode v1.1.0/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= +github.com/nwaples/rardecode v1.1.3 h1:cWCaZwfM5H7nAD6PyEdcVnczzV8i/JtotnyW/dD9lEc= +github.com/nwaples/rardecode v1.1.3/go.mod h1:5DzqNKiOdpKKBH87u8VlvAnPZMXcGRhxWkRpHbbfGS0= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/gomega v1.34.1 h1:EUMJIKUjM8sKjYbtxQI9A4z2o+rruxnzNvpknOXie6k= @@ -823,9 +820,8 @@ github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3v github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= -github.com/pierrec/lz4/v4 v4.1.2/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= -github.com/pierrec/lz4/v4 v4.1.19 h1:tYLzDnjDXh9qIxSTKHwXwOYmm9d887Y7Y1ZkyXYHAN4= -github.com/pierrec/lz4/v4 v4.1.19/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= +github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pjbgf/sha1cd v0.3.0 h1:4D5XXmUUBUl/xQ6IjCkEAbqXskkq/4O7LmGn0AqMDs4= github.com/pjbgf/sha1cd v0.3.0/go.mod h1:nZ1rrWOcGJ5uZgEEVL1VUM9iRQiZvWdbZjkKyFzPPsI= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -960,7 +956,6 @@ github.com/therootcompany/xz v1.0.1 h1:CmOtsn1CbtmyYiusbfmhmkpAAETj0wBIH6kCYaX+x github.com/therootcompany/xz v1.0.1/go.mod h1:3K3UH1yCKgBneZYhuQUvJ9HPD19UEXEI0BWbMn8qNMY= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/ulikunitz/xz v0.5.8/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= -github.com/ulikunitz/xz v0.5.9/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/ulikunitz/xz v0.5.12 h1:37Nm15o69RwBkXM0J6A5OlE67RZTfzUxTj8fB3dfcsc= github.com/ulikunitz/xz v0.5.12/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= @@ -990,6 +985,8 @@ github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofm github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU= +github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1048,8 +1045,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY= -golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= diff --git a/internal/tarutil/reader_entry.go b/internal/tarutil/reader_entry.go index eda3326f..4bf74b79 100644 --- a/internal/tarutil/reader_entry.go +++ b/internal/tarutil/reader_entry.go @@ -33,6 +33,8 @@ func (t ReaderEntry) writeEntry(tw lowLevelWriter) error { } func writeEntry(tw lowLevelWriter, filename string, fileInfo os.FileInfo, opener func() (io.Reader, error)) error { + log.WithFields("path", filename).Trace("adding file to archive") + header, err := tar.FileInfoHeader(fileInfo, "") if err != nil { return err diff --git a/pkg/process/build.go b/pkg/process/build.go index 9feb629f..00fd73f9 100644 --- a/pkg/process/build.go +++ b/pkg/process/build.go @@ -5,6 +5,8 @@ import ( "fmt" "time" + "github.com/dustin/go-humanize" + "github.com/anchore/grype-db/internal/log" "github.com/anchore/grype-db/pkg/data" v3 "github.com/anchore/grype-db/pkg/process/v3" @@ -104,17 +106,39 @@ func getWriter(schemaVersion int, dataAge time.Time, directory string, states pr func build(results []providerResults, writer data.Writer, processors ...data.Processor) error { lastUpdate := time.Now() + var totalRecords int + for _, result := range results { + totalRecords += int(result.count) + } + log.WithFields("total", humanize.Comma(int64(totalRecords))).Info("processing all records") + + var recordsProcessed int + + // for exponential moving average, choose an alpha between 0 and 1, where 1 biases towards the most recent sample + // and 0 biases towards the average of all samples. + rateWindow := newEMA(0.4) + for _, result := range results { - log.WithFields("provider", result.provider.Provider, "count", result.count).Info("processing provider records") - idx := 0 + log.WithFields("provider", result.provider.Provider, "total", humanize.Comma(result.count)).Info("processing provider records") + providerRecordsProcessed := 0 + recordsProcessedInStatusCycle := 0 for opener := range result.openers { - idx++ - log.WithFields("entry", opener.String()).Tracef("processing") + providerRecordsProcessed++ + recordsProcessed++ + recordsProcessedInStatusCycle++ var processor data.Processor if time.Since(lastUpdate) > 3*time.Second { - log.WithFields("provider", result.provider.Provider, "count", result.count, "processed", idx).Debug("processing provider records") + r := recordsPerSecond(recordsProcessedInStatusCycle, lastUpdate) + rateWindow.Add(r) + + log.WithFields( + "provider", fmt.Sprintf("%q %1.0f/s (%1.2f%%)", result.provider.Provider, r, percent(providerRecordsProcessed, int(result.count))), + "overall", fmt.Sprintf("%1.2f%%", percent(recordsProcessed, totalRecords)), + "eta", eta(recordsProcessed, totalRecords, rateWindow.Average()).String(), + ).Debug("status") lastUpdate = time.Now() + recordsProcessedInStatusCycle = 0 } f, err := opener.Open() @@ -129,7 +153,6 @@ func build(results []providerResults, writer data.Writer, processors ...data.Pro for _, candidate := range processors { if candidate.IsSupported(envelope.Schema) { processor = candidate - log.WithFields("schema", envelope.Schema).Trace("matched with processor") break } } @@ -153,3 +176,48 @@ func build(results []providerResults, writer data.Writer, processors ...data.Pro return nil } + +type expMovingAverage struct { + alpha float64 + value float64 + count int +} + +func newEMA(alpha float64) *expMovingAverage { + return &expMovingAverage{alpha: alpha} +} + +func (e *expMovingAverage) Add(sample float64) { + if e.count == 0 { + e.value = sample // initialize with the first sample + } else { + e.value = e.alpha*sample + (1-e.alpha)*e.value + } + e.count++ +} + +func (e *expMovingAverage) Average() float64 { + return e.value +} + +func recordsPerSecond(idx int, lastUpdate time.Time) float64 { + sec := time.Since(lastUpdate).Seconds() + if sec == 0 { + return 0 + } + return float64(idx) / sec +} + +func percent(idx, total int) float64 { + if total == 0 { + return 0 + } + return float64(idx) / float64(total) * 100 +} + +func eta(idx, total int, rate float64) time.Duration { + if rate == 0 { + return 0 + } + return time.Duration(float64(total-idx)/rate) * time.Second +} diff --git a/pkg/process/package.go b/pkg/process/package.go index 2a8d369c..fae49d78 100644 --- a/pkg/process/package.go +++ b/pkg/process/package.go @@ -1,8 +1,8 @@ package process import ( - "errors" "fmt" + "net/url" "os" "path" "path/filepath" @@ -10,116 +10,116 @@ import ( "time" "github.com/scylladb/go-set/strset" + "github.com/spf13/afero" "github.com/anchore/grype-db/internal/log" "github.com/anchore/grype-db/internal/tarutil" - "github.com/anchore/grype-db/pkg/provider" + v6process "github.com/anchore/grype-db/pkg/process/v6" grypeDBLegacyDistribution "github.com/anchore/grype/grype/db/legacy/distribution" - v6 "github.com/anchore/grype/grype/db/v6" - v6Distribution "github.com/anchore/grype/grype/db/v6/distribution" + grypeDBLegacy "github.com/anchore/grype/grype/db/v5" + grypeDBLegacyStore "github.com/anchore/grype/grype/db/v5/store" ) +// listingFiles is a set of files that should not be included in the archive +var listingFiles = strset.New("listing.json", "latest.json", "history.json") + func Package(dbDir, publishBaseURL, overrideArchiveExtension string) error { // check if metadata file exists, if so, then this if _, err := os.Stat(filepath.Join(dbDir, grypeDBLegacyDistribution.MetadataFileName)); os.IsNotExist(err) { - return packageDB(dbDir, overrideArchiveExtension) + return v6process.CreateArchive(dbDir, overrideArchiveExtension) } return packageLegacyDB(dbDir, publishBaseURL, overrideArchiveExtension) } -func packageDB(dbDir, overrideArchiveExtension string) error { - extension, err := resolveExtension(overrideArchiveExtension) +func packageLegacyDB(dbDir, publishBaseURL, overrideArchiveExtension string) error { //nolint:funlen + log.WithFields("from", dbDir, "url", publishBaseURL, "extension-override", overrideArchiveExtension).Info("packaging database") + + fs := afero.NewOsFs() + metadata, err := grypeDBLegacyDistribution.NewMetadataFromDir(fs, dbDir) if err != nil { return err } - log.WithFields("from", dbDir, "extension", extension).Info("packaging database") - s, err := v6.NewReader(v6.Config{DBDirPath: dbDir}) + if metadata == nil { + return fmt.Errorf("no metadata found in %q", dbDir) + } + + s, err := grypeDBLegacyStore.New(filepath.Join(dbDir, grypeDBLegacy.VulnerabilityStoreFileName), false) if err != nil { return fmt.Errorf("unable to open vulnerability store: %w", err) } - metadata, err := s.GetDBMetadata() - if err != nil || metadata == nil { - return fmt.Errorf("unable to get vulnerability store metadata: %w", err) + id, err := s.GetID() + if err != nil { + return fmt.Errorf("unable to get vulnerability store ID: %w", err) } - if metadata.Model != v6.ModelVersion { - return fmt.Errorf("metadata model %d does not match vulnerability store model %d", v6.ModelVersion, metadata.Model) + if id.SchemaVersion != metadata.Version { + return fmt.Errorf("metadata version %d does not match vulnerability store version %d", metadata.Version, id.SchemaVersion) } - providerModels, err := s.AllProviders() + u, err := url.Parse(publishBaseURL) if err != nil { - return fmt.Errorf("unable to get all providers: %w", err) + return err } - if len(providerModels) == 0 { - return fmt.Errorf("no providers found in the vulnerability store") + // we need a well-ordered string to append to the archive name to ensure uniqueness (to avoid overwriting + // existing archives in the CDN) as well as to ensure that multiple archives created in the same day are + // put in the correct order in the listing file. The DB timestamp represents the age of the data in the DB + // not when the DB was created. The trailer represents the time the DB was packaged. + trailer := fmt.Sprintf("%d", secondsSinceEpoch()) + + var extension = "tar.gz" + if overrideArchiveExtension != "" { + extension = strings.TrimLeft(overrideArchiveExtension, ".") } - eldest, err := toProviders(providerModels).EarliestTimestamp() - if err != nil { - return err + var found bool + for _, valid := range []string{"tar.zst", "tar.gz"} { + if valid == extension { + found = true + break + } + } + + if !found { + return fmt.Errorf("invalid archive extension %q", extension) } - // output archive vulnerability-db_VERSION_OLDESTDATADATE_BUILTEPOCH.tar.gz, where: - // - VERSION: schema version in the form of v#.#.# - // - OLDESTDATADATE: RFC3338 formatted value of the oldest date capture date found for all contained providers - // - BUILTEPOCH: linux epoch formatted value of the database metadata built field + // we attach a random value at the end of the file name to prevent from overwriting DBs in S3 that are already + // cached in the CDN. Ideally this would be based off of the archive checksum but a random string is simpler. tarName := fmt.Sprintf( - "vulnerability-db_v%s_%s_%d.%s", - fmt.Sprintf("%d.%d.%d", metadata.Model, metadata.Revision, metadata.Addition), - eldest.UTC().Format(time.RFC3339), - metadata.BuildTimestamp.Unix(), + "vulnerability-db_v%d_%s_%s.%s", + metadata.Version, + metadata.Built.Format(time.RFC3339), + trailer, extension, ) + tarPath := path.Join(dbDir, tarName) - tarPath := filepath.Join(dbDir, tarName) - - if err := populateTar(tarPath); err != nil { + if err := populateLegacyTar(tarPath); err != nil { return err } log.WithFields("path", tarPath).Info("created database archive") - return writeLatestDocument(tarPath, *metadata) -} - -func toProviders(states []v6.Provider) provider.States { - var result provider.States - for _, state := range states { - result = append(result, provider.State{ - Provider: state.ID, - Timestamp: *state.DateCaptured, - }) + entry, err := grypeDBLegacyDistribution.NewListingEntryFromArchive(fs, *metadata, tarPath, u) + if err != nil { + return fmt.Errorf("unable to create listing entry from archive: %w", err) } - return result -} - -func resolveExtension(overrideArchiveExtension string) (string, error) { - var extension = "tar.zst" - if overrideArchiveExtension != "" { - extension = strings.TrimLeft(overrideArchiveExtension, ".") + listing := grypeDBLegacyDistribution.NewListing(entry) + listingPath := path.Join(dbDir, grypeDBLegacyDistribution.ListingFileName) + if err = listing.Write(listingPath); err != nil { + return err } - var found bool - for _, valid := range []string{"tar.zst", "tar.xz", "tar.gz"} { - if valid == extension { - found = true - break - } - } + log.WithFields("path", listingPath).Debug("created initial listing file") - if !found { - return "", fmt.Errorf("unsupported archive extension %q", extension) - } - return extension, nil + return nil } -var listingFiles = strset.New("listing.json", "latest.json", "history.json") - -func populateTar(tarPath string) error { +func populateLegacyTar(tarPath string) error { originalDir, err := os.Getwd() if err != nil { return fmt.Errorf("unable to get CWD: %w", err) @@ -158,28 +158,6 @@ func populateTar(tarPath string) error { return nil } -func writeLatestDocument(tarPath string, metadata v6.DBMetadata) error { - archive, err := v6Distribution.NewArchive(tarPath, *metadata.BuildTimestamp, metadata.Model, metadata.Revision, metadata.Addition) - if err != nil || archive == nil { - return fmt.Errorf("unable to create archive: %w", err) - } - - doc := v6Distribution.NewLatestDocument(*archive) - if doc == nil { - return errors.New("unable to create latest document") - } - - dbDir := filepath.Dir(tarPath) - - latestPath := path.Join(dbDir, v6Distribution.LatestFileName) - - fh, err := os.OpenFile(latestPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) - if err != nil { - return fmt.Errorf("unable to create latest file: %w", err) - } - - if err = doc.Write(fh); err != nil { - return fmt.Errorf("unable to write latest document: %w", err) - } - return nil +func secondsSinceEpoch() int64 { + return time.Now().UTC().Unix() } diff --git a/pkg/process/package_legacy.go b/pkg/process/package_legacy.go deleted file mode 100644 index 6672fe19..00000000 --- a/pkg/process/package_legacy.go +++ /dev/null @@ -1,116 +0,0 @@ -package process - -import ( - "fmt" - "net/url" - "path" - "path/filepath" - "strings" - "time" - - "github.com/spf13/afero" - - "github.com/anchore/grype-db/internal/log" - "github.com/anchore/grype/grype/db/legacy/distribution" - grypeDBLegacy "github.com/anchore/grype/grype/db/v5" - grypeDBLegacyStore "github.com/anchore/grype/grype/db/v5/store" -) - -func packageLegacyDB(dbDir, publishBaseURL, overrideArchiveExtension string) error { //nolint:funlen - log.WithFields("from", dbDir, "url", publishBaseURL, "extension-override", overrideArchiveExtension).Info("packaging database") - - fs := afero.NewOsFs() - metadata, err := distribution.NewMetadataFromDir(fs, dbDir) - if err != nil { - return err - } - - if metadata == nil { - return fmt.Errorf("no metadata found in %q", dbDir) - } - - s, err := grypeDBLegacyStore.New(filepath.Join(dbDir, grypeDBLegacy.VulnerabilityStoreFileName), false) - if err != nil { - return fmt.Errorf("unable to open vulnerability store: %w", err) - } - - id, err := s.GetID() - if err != nil { - return fmt.Errorf("unable to get vulnerability store ID: %w", err) - } - - if id.SchemaVersion != metadata.Version { - return fmt.Errorf("metadata version %d does not match vulnerability store version %d", metadata.Version, id.SchemaVersion) - } - - u, err := url.Parse(publishBaseURL) - if err != nil { - return err - } - - // we need a well-ordered string to append to the archive name to ensure uniqueness (to avoid overwriting - // existing archives in the CDN) as well as to ensure that multiple archives created in the same day are - // put in the correct order in the listing file. The DB timestamp represents the age of the data in the DB - // not when the DB was created. The trailer represents the time the DB was packaged. - trailer := fmt.Sprintf("%d", secondsSinceEpoch()) - - // TODO (alex): supporting tar.zst - // var extension = "tar.zst" - var extension = "tar.gz" - - if overrideArchiveExtension != "" { - extension = strings.TrimLeft(overrideArchiveExtension, ".") - } - // TODO (alex): supporting tar.zst - // else if metadata.Version < 5 { - // extension = "tar.gz" - // } - - var found bool - for _, valid := range []string{"tar.zst", "tar.gz"} { - if valid == extension { - found = true - break - } - } - - if !found { - return fmt.Errorf("invalid archive extension %q", extension) - } - - // we attach a random value at the end of the file name to prevent from overwriting DBs in S3 that are already - // cached in the CDN. Ideally this would be based off of the archive checksum but a random string is simpler. - tarName := fmt.Sprintf( - "vulnerability-db_v%d_%s_%s.%s", - metadata.Version, - metadata.Built.Format(time.RFC3339), - trailer, - extension, - ) - tarPath := path.Join(dbDir, tarName) - - if err := populateTar(tarPath); err != nil { - return err - } - - log.WithFields("path", tarPath).Info("created database archive") - - entry, err := distribution.NewListingEntryFromArchive(fs, *metadata, tarPath, u) - if err != nil { - return fmt.Errorf("unable to create listing entry from archive: %w", err) - } - - listing := distribution.NewListing(entry) - listingPath := path.Join(dbDir, distribution.ListingFileName) - if err = listing.Write(listingPath); err != nil { - return err - } - - log.WithFields("path", listingPath).Debug("created initial listing file") - - return nil -} - -func secondsSinceEpoch() int64 { - return time.Now().UTC().Unix() -} diff --git a/pkg/process/v6/archive.go b/pkg/process/v6/archive.go new file mode 100644 index 00000000..58965636 --- /dev/null +++ b/pkg/process/v6/archive.go @@ -0,0 +1,166 @@ +package v6 + +import ( + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" + "time" + + "github.com/anchore/grype-db/internal/log" + "github.com/anchore/grype-db/internal/tarutil" + "github.com/anchore/grype-db/pkg/provider" + v6 "github.com/anchore/grype/grype/db/v6" + v6Distribution "github.com/anchore/grype/grype/db/v6/distribution" +) + +func CreateArchive(dbDir, overrideArchiveExtension string) error { + extension, err := resolveExtension(overrideArchiveExtension) + if err != nil { + return err + } + log.WithFields("from", dbDir, "extension", extension).Info("packaging database") + + cfg := v6.Config{DBDirPath: dbDir} + s, err := v6.NewReader(cfg) + if err != nil { + return fmt.Errorf("unable to open vulnerability store: %w", err) + } + + metadata, err := s.GetDBMetadata() + if err != nil || metadata == nil { + return fmt.Errorf("unable to get vulnerability store metadata: %w", err) + } + + if metadata.Model != v6.ModelVersion { + return fmt.Errorf("metadata model %d does not match vulnerability store model %d", v6.ModelVersion, metadata.Model) + } + + providerModels, err := s.AllProviders() + if err != nil { + return fmt.Errorf("unable to get all providers: %w", err) + } + + if len(providerModels) == 0 { + return fmt.Errorf("no providers found in the vulnerability store") + } + + eldest, err := toProviders(providerModels).EarliestTimestamp() + if err != nil { + return err + } + + // output archive vulnerability-db_VERSION_OLDESTDATADATE_BUILTEPOCH.tar.gz, where: + // - VERSION: schema version in the form of v#.#.# + // - OLDESTDATADATE: RFC3338 formatted value of the oldest date capture date found for all contained providers + // - BUILTEPOCH: linux epoch formatted value of the database metadata built field + tarName := fmt.Sprintf( + "vulnerability-db_v%s_%s_%d.%s", + fmt.Sprintf("%d.%d.%d", metadata.Model, metadata.Revision, metadata.Addition), + eldest.UTC().Format(time.RFC3339), + metadata.BuildTimestamp.Unix(), + extension, + ) + + tarPath := filepath.Join(dbDir, tarName) + + if err := populateTar(dbDir, tarName, v6.VulnerabilityDBFileName); err != nil { + return err + } + + log.WithFields("path", tarPath).Info("created database archive") + + return writeLatestDocument(tarPath, *metadata) +} + +func toProviders(states []v6.Provider) provider.States { + var result provider.States + for _, state := range states { + result = append(result, provider.State{ + Provider: state.ID, + Timestamp: *state.DateCaptured, + }) + } + return result +} + +func resolveExtension(overrideArchiveExtension string) (string, error) { + var extension = "tar.zst" + + if overrideArchiveExtension != "" { + extension = strings.TrimLeft(overrideArchiveExtension, ".") + } + + var found bool + for _, valid := range []string{"tar.zst", "tar.xz", "tar.gz"} { + if valid == extension { + found = true + break + } + } + + if !found { + return "", fmt.Errorf("unsupported archive extension %q", extension) + } + return extension, nil +} + +func populateTar(dbDir, tarName string, files ...string) error { + originalDir, err := os.Getwd() + if err != nil { + return fmt.Errorf("unable to get CWD: %w", err) + } + + if dbDir != "" { + if err = os.Chdir(dbDir); err != nil { + return fmt.Errorf("unable to cd to build dir: %w", err) + } + + defer func() { + if err = os.Chdir(originalDir); err != nil { + log.Errorf("unable to cd to original dir: %v", err) + } + }() + } + + for _, f := range files { + _, err := os.Stat(f) + if err != nil { + return fmt.Errorf("unable to stat file %q: %w", f, err) + } + } + + if err = tarutil.PopulateWithPaths(tarName, files...); err != nil { + return fmt.Errorf("unable to create db archive: %w", err) + } + + return nil +} + +func writeLatestDocument(tarPath string, metadata v6.DBMetadata) error { + archive, err := v6Distribution.NewArchive(tarPath, *metadata.BuildTimestamp, metadata.Model, metadata.Revision, metadata.Addition) + if err != nil || archive == nil { + return fmt.Errorf("unable to create archive: %w", err) + } + + doc := v6Distribution.NewLatestDocument(*archive) + if doc == nil { + return errors.New("unable to create latest document") + } + + dbDir := filepath.Dir(tarPath) + + latestPath := path.Join(dbDir, v6Distribution.LatestFileName) + + fh, err := os.OpenFile(latestPath, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("unable to create latest file: %w", err) + } + + if err = doc.Write(fh); err != nil { + return fmt.Errorf("unable to write latest document: %w", err) + } + return nil +} diff --git a/pkg/process/v6/transformers/github/transform.go b/pkg/process/v6/transformers/github/transform.go index d263aeef..ece7e38a 100644 --- a/pkg/process/v6/transformers/github/transform.go +++ b/pkg/process/v6/transformers/github/transform.go @@ -28,6 +28,7 @@ func Transform(vulnerability unmarshal.GitHubAdvisory, state provider.State) ([] func getVulnerability(vuln unmarshal.GitHubAdvisory, state provider.State) grypeDB.VulnerabilityHandle { return grypeDB.VulnerabilityHandle{ Name: vuln.Advisory.GhsaID, + ProviderID: state.Provider, Provider: internal.ProviderModel(state), ModifiedDate: internal.ParseTime(vuln.Advisory.Updated), PublishedDate: internal.ParseTime(vuln.Advisory.Published), diff --git a/pkg/process/v6/transformers/github/transform_test.go b/pkg/process/v6/transformers/github/transform_test.go index 462a3e57..0882932f 100644 --- a/pkg/process/v6/transformers/github/transform_test.go +++ b/pkg/process/v6/transformers/github/transform_test.go @@ -78,7 +78,8 @@ func TestGetVulnerability(t *testing.T) { name: "test-fixtures/GHSA-2wgc-48g2-cj5w.json", expected: []grypeDB.VulnerabilityHandle{ { - Name: "GHSA-2wgc-48g2-cj5w", + Name: "GHSA-2wgc-48g2-cj5w", + ProviderID: "github", Provider: &grypeDB.Provider{ ID: "github", Version: "1", @@ -119,7 +120,8 @@ func TestGetVulnerability(t *testing.T) { name: "test-fixtures/GHSA-3x74-v64j-qc3f.json", expected: []grypeDB.VulnerabilityHandle{ { - Name: "GHSA-3x74-v64j-qc3f", + Name: "GHSA-3x74-v64j-qc3f", + ProviderID: "github", Provider: &grypeDB.Provider{ ID: "github", Version: "1", @@ -160,7 +162,8 @@ func TestGetVulnerability(t *testing.T) { name: "test-fixtures/github-github-npm-0.json", expected: []grypeDB.VulnerabilityHandle{ { - Name: "GHSA-vc9j-fhvv-8vrf", + Name: "GHSA-vc9j-fhvv-8vrf", + ProviderID: "github", Provider: &grypeDB.Provider{ ID: "github", Version: "1", @@ -201,7 +204,8 @@ func TestGetVulnerability(t *testing.T) { name: "test-fixtures/github-github-python-0.json", expected: []grypeDB.VulnerabilityHandle{ { - Name: "GHSA-6cwv-x26c-w2q4", + Name: "GHSA-6cwv-x26c-w2q4", + ProviderID: "github", Provider: &grypeDB.Provider{ ID: "github", Version: "1", @@ -228,7 +232,8 @@ func TestGetVulnerability(t *testing.T) { }, }, { - Name: "GHSA-p5wr-vp8g-q5p4", + Name: "GHSA-p5wr-vp8g-q5p4", + ProviderID: "github", Provider: &grypeDB.Provider{ ID: "github", Version: "1", @@ -258,7 +263,8 @@ func TestGetVulnerability(t *testing.T) { name: "test-fixtures/github-withdrawn.json", expected: []grypeDB.VulnerabilityHandle{ { - Name: "GHSA-6cwv-x26c-w2q4", + Name: "GHSA-6cwv-x26c-w2q4", + ProviderID: "github", Provider: &grypeDB.Provider{ ID: "github", Version: "1", @@ -291,7 +297,8 @@ func TestGetVulnerability(t *testing.T) { name: "test-fixtures/multiple-fixed-in-names.json", expected: []grypeDB.VulnerabilityHandle{ { - Name: "GHSA-p5wr-vp8g-q5p4", + Name: "GHSA-p5wr-vp8g-q5p4", + ProviderID: "github", Provider: &grypeDB.Provider{ ID: "github", Version: "1", diff --git a/pkg/process/v6/transformers/nvd/transform.go b/pkg/process/v6/transformers/nvd/transform.go index 5ea3f839..4dfa3125 100644 --- a/pkg/process/v6/transformers/nvd/transform.go +++ b/pkg/process/v6/transformers/nvd/transform.go @@ -41,6 +41,7 @@ func transform(cfg Config, vulnerability unmarshal.NVDVulnerability, state provi in := []any{ grypeDB.VulnerabilityHandle{ Name: vulnerability.ID, + ProviderID: state.Provider, Provider: internal.ProviderModel(state), ModifiedDate: internal.ParseTime(vulnerability.LastModified), PublishedDate: internal.ParseTime(vulnerability.Published), diff --git a/pkg/process/v6/transformers/nvd/transform_test.go b/pkg/process/v6/transformers/nvd/transform_test.go index b308b29e..53a438fd 100644 --- a/pkg/process/v6/transformers/nvd/transform_test.go +++ b/pkg/process/v6/transformers/nvd/transform_test.go @@ -58,6 +58,7 @@ func TestTransform(t *testing.T) { { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ Name: "CVE-2018-5487", + ProviderID: "nvd", Provider: expectedProvider("nvd"), ModifiedDate: timeRef(time.Date(2018, 7, 5, 13, 52, 30, 627000000, time.UTC)), PublishedDate: timeRef(time.Date(2018, 5, 24, 14, 29, 0, 390000000, time.UTC)), @@ -133,6 +134,7 @@ func TestTransform(t *testing.T) { { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ Name: "CVE-2018-1000222", + ProviderID: "nvd", Provider: expectedProvider("nvd"), ModifiedDate: timeRef(time.Date(2020, 3, 31, 2, 15, 12, 667000000, time.UTC)), PublishedDate: timeRef(time.Date(2018, 8, 20, 20, 29, 1, 347000000, time.UTC)), @@ -220,6 +222,7 @@ func TestTransform(t *testing.T) { { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ Name: "CVE-2018-10189", + ProviderID: "nvd", Provider: expectedProvider("nvd"), ModifiedDate: timeRef(time.Date(2018, 5, 23, 14, 41, 49, 73000000, time.UTC)), PublishedDate: timeRef(time.Date(2018, 4, 17, 20, 29, 0, 410000000, time.UTC)), @@ -302,6 +305,7 @@ func TestTransform(t *testing.T) { { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ Name: "CVE-2015-8978", + ProviderID: "nvd", Provider: expectedProvider("nvd"), ModifiedDate: timeRef(time.Date(2016, 11, 28, 19, 50, 59, 600000000, time.UTC)), PublishedDate: timeRef(time.Date(2016, 11, 22, 17, 59, 0, 180000000, time.UTC)), @@ -360,6 +364,7 @@ func TestTransform(t *testing.T) { { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ Name: "CVE-2022-26488", + ProviderID: "nvd", Provider: expectedProvider("nvd"), ModifiedDate: timeRef(time.Date(2022, 9, 3, 3, 34, 19, 933000000, time.UTC)), PublishedDate: timeRef(time.Date(2022, 3, 10, 17, 47, 45, 383000000, time.UTC)), @@ -466,6 +471,7 @@ func TestTransform(t *testing.T) { { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ Name: "CVE-2022-0543", + ProviderID: "nvd", Provider: expectedProvider("nvd"), ModifiedDate: timeRef(time.Date(2023, 9, 29, 15, 55, 24, 533000000, time.UTC)), PublishedDate: timeRef(time.Date(2022, 2, 18, 20, 15, 17, 583000000, time.UTC)), @@ -560,6 +566,7 @@ func TestTransform(t *testing.T) { { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ Name: "CVE-2020-10729", + ProviderID: "nvd", Provider: expectedProvider("nvd"), ModifiedDate: timeRef(time.Date(2021, 12, 10, 19, 57, 6, 357000000, time.UTC)), PublishedDate: timeRef(time.Date(2021, 5, 27, 19, 15, 7, 880000000, time.UTC)), @@ -650,6 +657,7 @@ func TestTransform(t *testing.T) { { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ Name: "CVE-2023-38733", + ProviderID: "nvd", Provider: expectedProvider("nvd"), ModifiedDate: timeRef(time.Date(2023, 8, 26, 2, 25, 42, 957000000, time.UTC)), PublishedDate: timeRef(time.Date(2023, 8, 22, 22, 15, 8, 460000000, time.UTC)), @@ -737,6 +745,7 @@ func TestTransform(t *testing.T) { { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ Name: "CVE-2023-45283", + ProviderID: "nvd", Provider: expectedProvider("nvd"), ModifiedDate: timeRef(time.Date(2023, 12, 14, 10, 15, 7, 947000000, time.UTC)), PublishedDate: timeRef(time.Date(2023, 11, 9, 17, 15, 8, 757000000, time.UTC)), @@ -847,6 +856,7 @@ func TestTransform(t *testing.T) { { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ Name: "CVE-2023-45283", + ProviderID: "nvd", Provider: expectedProvider("nvd"), ModifiedDate: timeRef(time.Date(2023, 12, 14, 10, 15, 7, 947000000, time.UTC)), PublishedDate: timeRef(time.Date(2023, 11, 9, 17, 15, 8, 757000000, time.UTC)), diff --git a/pkg/process/v6/transformers/os/transform.go b/pkg/process/v6/transformers/os/transform.go index a7b0a895..f86792f4 100644 --- a/pkg/process/v6/transformers/os/transform.go +++ b/pkg/process/v6/transformers/os/transform.go @@ -25,6 +25,7 @@ func Transform(vulnerability unmarshal.OSVulnerability, state provider.State) ([ in := []any{ grypeDB.VulnerabilityHandle{ Name: vulnerability.Vulnerability.Name, + ProviderID: state.Provider, Provider: internal.ProviderModel(state), Status: string(grypeDB.VulnerabilityActive), ModifiedDate: internal.ParseTime(vulnerability.Vulnerability.Metadata.Updated), diff --git a/pkg/process/v6/transformers/os/transform_test.go b/pkg/process/v6/transformers/os/transform_test.go index 6e2b44f7..f496a64e 100644 --- a/pkg/process/v6/transformers/os/transform_test.go +++ b/pkg/process/v6/transformers/os/transform_test.go @@ -83,9 +83,10 @@ func TestTransform(t *testing.T) { want: []transformers.RelatedEntries{ { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ - Name: "CVE-2018-19967", - Status: "active", - Provider: expectedProvider("alpine"), + Name: "CVE-2018-19967", + Status: "active", + ProviderID: "alpine", + Provider: expectedProvider("alpine"), BlobValue: &grypeDB.VulnerabilityBlob{ ID: "CVE-2018-19967", References: []grypeDB.Reference{ @@ -125,9 +126,10 @@ func TestTransform(t *testing.T) { want: []transformers.RelatedEntries{ { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ - Name: "ALAS-2018-1106", - Provider: expectedProvider("amazon"), - Status: "active", + Name: "ALAS-2018-1106", + ProviderID: "amazon", + Provider: expectedProvider("amazon"), + Status: "active", BlobValue: &grypeDB.VulnerabilityBlob{ ID: "ALAS-2018-1106", References: []grypeDB.Reference{ @@ -236,9 +238,10 @@ func TestTransform(t *testing.T) { want: []transformers.RelatedEntries{ { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ - Name: "ALAS-2021-1704", - Provider: expectedProvider("amazon"), - Status: "active", + Name: "ALAS-2021-1704", + ProviderID: "amazon", + Provider: expectedProvider("amazon"), + Status: "active", BlobValue: &grypeDB.VulnerabilityBlob{ ID: "ALAS-2021-1704", @@ -288,9 +291,10 @@ func TestTransform(t *testing.T) { }, { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ - Name: "ALASKERNEL-5.4-2022-007", - Provider: expectedProvider("amazon"), - Status: "active", + Name: "ALASKERNEL-5.4-2022-007", + ProviderID: "amazon", + Provider: expectedProvider("amazon"), + Status: "active", BlobValue: &grypeDB.VulnerabilityBlob{ ID: "ALASKERNEL-5.4-2022-007", References: []grypeDB.Reference{ @@ -339,9 +343,10 @@ func TestTransform(t *testing.T) { }, { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ - Name: "ALASKERNEL-5.10-2022-005", - Provider: expectedProvider("amazon"), - Status: "active", + Name: "ALASKERNEL-5.10-2022-005", + ProviderID: "amazon", + Provider: expectedProvider("amazon"), + Status: "active", BlobValue: &grypeDB.VulnerabilityBlob{ ID: "ALASKERNEL-5.10-2022-005", References: []grypeDB.Reference{ @@ -396,9 +401,10 @@ func TestTransform(t *testing.T) { want: []transformers.RelatedEntries{ { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ - Name: "CVE-2023-29403", - Provider: expectedProvider("mariner"), - Status: "active", + Name: "CVE-2023-29403", + ProviderID: "mariner", + Provider: expectedProvider("mariner"), + Status: "active", BlobValue: &grypeDB.VulnerabilityBlob{ ID: "CVE-2023-29403", Description: "CVE-2023-29403 affecting package golang for versions less than 1.20.7-1. A patched version of the package is available.", @@ -439,9 +445,10 @@ func TestTransform(t *testing.T) { want: []transformers.RelatedEntries{ { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ - Name: "CVE-2008-7220", - Provider: expectedProvider("debian"), - Status: "active", + Name: "CVE-2008-7220", + ProviderID: "debian", + Provider: expectedProvider("debian"), + Status: "active", BlobValue: &grypeDB.VulnerabilityBlob{ ID: "CVE-2008-7220", References: []grypeDB.Reference{ @@ -517,9 +524,10 @@ func TestTransform(t *testing.T) { want: []transformers.RelatedEntries{ { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ - Name: "CVE-2011-4623", - Provider: expectedProvider("debian"), - Status: "active", + Name: "CVE-2011-4623", + ProviderID: "debian", + Provider: expectedProvider("debian"), + Status: "active", BlobValue: &grypeDB.VulnerabilityBlob{ ID: "CVE-2011-4623", References: []grypeDB.Reference{ @@ -553,9 +561,10 @@ func TestTransform(t *testing.T) { }, { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ - Name: "CVE-2008-5618", - Provider: expectedProvider("debian"), - Status: "active", + Name: "CVE-2008-5618", + ProviderID: "debian", + Provider: expectedProvider("debian"), + Status: "active", BlobValue: &grypeDB.VulnerabilityBlob{ ID: "CVE-2008-5618", References: []grypeDB.Reference{ @@ -595,9 +604,10 @@ func TestTransform(t *testing.T) { want: []transformers.RelatedEntries{ { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ - Name: "CVE-2021-37621", - Provider: expectedProvider("mariner"), - Status: "active", + Name: "CVE-2021-37621", + ProviderID: "mariner", + Provider: expectedProvider("mariner"), + Status: "active", BlobValue: &grypeDB.VulnerabilityBlob{ ID: "CVE-2021-37621", Description: "CVE-2021-37621 affecting package exiv2 for versions less than 0.27.5-1. An upgraded version of the package is available that resolves this issue.", @@ -640,9 +650,10 @@ func TestTransform(t *testing.T) { want: []transformers.RelatedEntries{ { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ - Name: "CVE-2023-29404", - Provider: expectedProvider("mariner"), - Status: "active", + Name: "CVE-2023-29404", + ProviderID: "mariner", + Provider: expectedProvider("mariner"), + Status: "active", BlobValue: &grypeDB.VulnerabilityBlob{ ID: "CVE-2023-29404", Description: "CVE-2023-29404 affecting package golang for versions less than 1.20.7-1. A patched version of the package is available.", @@ -684,6 +695,7 @@ func TestTransform(t *testing.T) { { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ Name: "ELSA-2020-2550", + ProviderID: "oracle", Provider: expectedProvider("oracle"), Status: "active", PublishedDate: timeRef(time.Date(2020, 6, 15, 0, 0, 0, 0, time.UTC)), @@ -757,9 +769,10 @@ func TestTransform(t *testing.T) { want: []transformers.RelatedEntries{ { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ - Name: "CVE-2020-14350", - Provider: expectedProvider("oracle"), - Status: "active", + Name: "CVE-2020-14350", + ProviderID: "oracle", + Provider: expectedProvider("oracle"), + Status: "active", BlobValue: &grypeDB.VulnerabilityBlob{ ID: "CVE-2020-14350", Description: "A flaw was found in PostgreSQL, where some PostgreSQL extensions did not use the search_path safely in their installation script. This flaw allows an attacker with sufficient privileges to trick an administrator into executing a specially crafted script during the extension's installation or update. The highest threat from this vulnerability is to confidentiality, integrity, as well as system availability.", @@ -852,9 +865,10 @@ func TestTransform(t *testing.T) { want: []transformers.RelatedEntries{ { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ - Name: "CVE-2020-6819", - Provider: expectedProvider("redhat"), - Status: "active", + Name: "CVE-2020-6819", + ProviderID: "redhat", + Provider: expectedProvider("redhat"), + Status: "active", BlobValue: &grypeDB.VulnerabilityBlob{ ID: "CVE-2020-6819", Description: "A flaw was found in Mozilla Firefox. A race condition can occur while running the nsDocShell destructor causing a use-after-free memory issue. The highest threat from this vulnerability is to data confidentiality and integrity as well as system availability.", @@ -942,9 +956,10 @@ func TestTransform(t *testing.T) { want: []transformers.RelatedEntries{ { VulnerabilityHandle: grypeDB.VulnerabilityHandle{ - Name: "CVE-2020-14350", - Provider: expectedProvider("redhat"), - Status: "active", + Name: "CVE-2020-14350", + ProviderID: "redhat", + Provider: expectedProvider("redhat"), + Status: "active", BlobValue: &grypeDB.VulnerabilityBlob{ ID: "CVE-2020-14350", Description: "A flaw was found in PostgreSQL, where some PostgreSQL extensions did not use the search_path safely in their installation script. This flaw allows an attacker with sufficient privileges to trick an administrator into executing a specially crafted script during the extension's installation or update. The highest threat from this vulnerability is to confidentiality, integrity, as well as system availability.", diff --git a/pkg/process/v6/writer.go b/pkg/process/v6/writer.go index 40358889..dff56c05 100644 --- a/pkg/process/v6/writer.go +++ b/pkg/process/v6/writer.go @@ -49,7 +49,6 @@ func NewWriter(directory string, states provider.States) (data.Writer, error) { } func (w writer) Write(entries ...data.Entry) error { - log.WithFields("records", len(entries)).Trace("writing records to DB") for _, entry := range entries { if entry.DBSchemaVersion != grypeDB.ModelVersion { return fmt.Errorf("wrong schema version: want %+v got %+v", grypeDB.ModelVersion, entry.DBSchemaVersion) @@ -57,6 +56,7 @@ func (w writer) Write(entries ...data.Entry) error { switch row := entry.Data.(type) { case transformers.RelatedEntries: + log.WithFields("vuln", row.VulnerabilityHandle.Name, "affected-packages", len(row.Related)).Trace("writing") if err := w.writeEntry(row); err != nil { return fmt.Errorf("unable to write entry to store: %w", err) }