diff --git a/pkg/dnfjson/dnfjson.go b/pkg/dnfjson/dnfjson.go index f92d5a7262..f45eb6806d 100644 --- a/pkg/dnfjson/dnfjson.go +++ b/pkg/dnfjson/dnfjson.go @@ -586,6 +586,8 @@ func (result depsolveResult) toRPMMD(rhsm map[string]bool) ([]rpmmd.PackageSpec, rpmDependencies[i].RemoteLocation = dep.RemoteLocation rpmDependencies[i].Checksum = dep.Checksum rpmDependencies[i].CheckGPG = repo.GPGCheck + rpmDependencies[i].RepoID = dep.RepoID + rpmDependencies[i].Path = dep.Path if verify := repo.SSLVerify; verify != nil { rpmDependencies[i].IgnoreSSL = !*verify } diff --git a/pkg/manifest/manifest.go b/pkg/manifest/manifest.go index 1ae10c5a0f..fa7b2cdec1 100644 --- a/pkg/manifest/manifest.go +++ b/pkg/manifest/manifest.go @@ -41,6 +41,14 @@ const ( DISTRO_FEDORA ) +// Inputs specifies the inputs for manifest instantiating +type Inputs struct { + PackageSets map[string][]rpmmd.PackageSpec + ContainerSpecs map[string][]container.Spec + OstreeCommits map[string][]ostree.CommitSpec + RpmRepos map[string][]rpmmd.RepoConfig +} + // An OSBuildManifest is an opaque JSON object, which is a valid input to osbuild type OSBuildManifest []byte @@ -138,19 +146,39 @@ func (m Manifest) GetOSTreeSourceSpecs() map[string][]ostree.SourceSpec { return ostreeSpecs } +// TODO: remove once all callers are fixed func (m Manifest) Serialize(packageSets map[string][]rpmmd.PackageSpec, containerSpecs map[string][]container.Spec, ostreeCommits map[string][]ostree.CommitSpec, rpmRepos map[string][]rpmmd.RepoConfig) (OSBuildManifest, error) { + inputs := &Inputs{ + PackageSets: packageSets, + ContainerSpecs: containerSpecs, + OstreeCommits: ostreeCommits, + RpmRepos: rpmRepos, + } + return m.SerializeFull(inputs, nil) +} + +// Options contains the (optional) configs for the manifest instantiating +type Options struct { + RpmDownloader osbuild.RpmDownloader +} + +// TODO: rename to Serialize() onces callers are fixed +func (m Manifest) SerializeFull(inputs *Inputs, opts *Options) (OSBuildManifest, error) { + if opts == nil { + opts = &Options{} + } pipelines := make([]osbuild.Pipeline, 0) packages := make([]rpmmd.PackageSpec, 0) commits := make([]ostree.CommitSpec, 0) inline := make([]string, 0) containers := make([]container.Spec, 0) for _, pipeline := range m.pipelines { - pipeline.serializeStart(packageSets[pipeline.Name()], containerSpecs[pipeline.Name()], ostreeCommits[pipeline.Name()], rpmRepos[pipeline.Name()]) + pipeline.serializeStart(inputs.PackageSets[pipeline.Name()], inputs.ContainerSpecs[pipeline.Name()], inputs.OstreeCommits[pipeline.Name()], inputs.RpmRepos[pipeline.Name()]) } for _, pipeline := range m.pipelines { commits = append(commits, pipeline.getOSTreeCommits()...) pipelines = append(pipelines, pipeline.serialize()) - packages = append(packages, packageSets[pipeline.Name()]...) + packages = append(packages, inputs.PackageSets[pipeline.Name()]...) inline = append(inline, pipeline.getInline()...) containers = append(containers, pipeline.getContainerSpecs()...) } @@ -158,7 +186,7 @@ func (m Manifest) Serialize(packageSets map[string][]rpmmd.PackageSpec, containe pipeline.serializeEnd() } - sources, err := osbuild.GenSources(packages, commits, inline, containers) + sources, err := osbuild.GenSources(packages, commits, inline, containers, inputs.RpmRepos, opts.RpmDownloader) if err != nil { return nil, err } diff --git a/pkg/osbuild/librepo_source.go b/pkg/osbuild/librepo_source.go new file mode 100644 index 0000000000..c13e9a001b --- /dev/null +++ b/pkg/osbuild/librepo_source.go @@ -0,0 +1,120 @@ +package osbuild + +import ( + "fmt" + "regexp" + + "github.com/osbuild/images/pkg/rpmmd" +) + +var librepoDigestPattern = regexp.MustCompile(`(sha256|sha384|sha512):[0-9a-f]{32,128}`) + +type LibrepoSource struct { + Items map[string]*LibrepoSourceItem `json:"items"` + Options *LibrepoSourceOptions `json:"options"` +} + +func (LibrepoSource) isSource() {} + +func NewLibrepoSource() *LibrepoSource { + return &LibrepoSource{ + Items: make(map[string]*LibrepoSourceItem), + Options: &LibrepoSourceOptions{ + Mirrors: make(map[string]*LibrepoSourceMirror), + }, + } +} + +type LibrepoSourceItem struct { + Path string `json:"path"` + MirrorID string `json:"mirror"` +} + +func findRepoById(repos map[string][]rpmmd.RepoConfig, repoID string) *rpmmd.RepoConfig { + for _, repos := range repos { + for _, repo := range repos { + if repo.Id == repoID { + return &repo + } + } + } + return nil +} + +func mirrorFromRepo(repo *rpmmd.RepoConfig) (*LibrepoSourceMirror, error) { + // XXX: add support for secrets + switch { + case repo.Metalink != "": + return &LibrepoSourceMirror{ + URL: repo.Metalink, + Type: "metalink", + }, nil + case repo.MirrorList != "": + return &LibrepoSourceMirror{ + URL: repo.MirrorList, + Type: "mirrorlist", + }, nil + case len(repo.BaseURLs) > 0: + return &LibrepoSourceMirror{ + // XXX: should we pick a random one instead? + URL: repo.BaseURLs[0], + Type: "baseurl", + }, nil + } + + return nil, fmt.Errorf("cannot find metalink, mirrorlist or baseurl for %+v", repo) +} + +func (source *LibrepoSource) AddPackage(pkg rpmmd.PackageSpec, repos map[string][]rpmmd.RepoConfig) error { + pkgRepo := findRepoById(repos, pkg.RepoID) + if pkgRepo == nil { + return fmt.Errorf("cannot find repo-id %v for %v in %+v", pkg.RepoID, pkg.Name, repos) + } + if _, ok := source.Options.Mirrors[pkgRepo.Id]; !ok { + mirror, err := mirrorFromRepo(pkgRepo) + if err != nil { + return err + } + source.Options.Mirrors[pkgRepo.Id] = mirror + } + mirror := source.Options.Mirrors[pkgRepo.Id] + // XXX: should we error here if one package requests IgnoreSSL + // and one does not for the same mirror? + if pkg.IgnoreSSL { + mirror.Insecure = true + } + if pkg.Secrets == "org.osbuild.rhsm" { + mirror.Secrets = &URLSecrets{ + Name: "org.osbuild.rhsm", + } + } else if pkg.Secrets == "org.osbuild.mtls" { + mirror.Secrets = &URLSecrets{ + Name: "org.osbuild.mtls", + } + } + + item := &LibrepoSourceItem{ + Path: pkg.Path, + MirrorID: pkgRepo.Id, + } + source.Items[pkg.Checksum] = item + return nil +} + +type LibrepoSourceOptions struct { + Mirrors map[string]*LibrepoSourceMirror `json:"mirrors"` +} + +type LibrepoSourceMirror struct { + URL string `json:"url"` + Type string `json:"type"` + + Insecure bool `json:"insecure,omitempty"` + Secrets *URLSecrets `json:"secrets,omitempty"` + + // XXX: should we expose those? if so we need a way to set them, + // current this is done in manifest.GenSources which cannot take + // options. + // MaxParallels *int `json:"max-parallels,omitempty"` + // FastestMirror bool `json:"fastest-mirror,omitempty"` +} diff --git a/pkg/osbuild/librepo_source_test.go b/pkg/osbuild/librepo_source_test.go new file mode 100644 index 0000000000..ffafc878c8 --- /dev/null +++ b/pkg/osbuild/librepo_source_test.go @@ -0,0 +1,125 @@ +package osbuild_test + +import ( + "encoding/json" + "fmt" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/osbuild/images/pkg/osbuild" + "github.com/osbuild/images/pkg/rpmmd" +) + +var opensslPkg = rpmmd.PackageSpec{ + Name: "openssl-libs", + Epoch: 1, + Version: "3.0.1", + Release: "5.el9", + Arch: "x86_64", + RemoteLocation: "https://example.com/repo/Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm", + Checksum: "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666", + Path: "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm", + RepoID: "repo_id", +} + +var fakeRepos = map[string][]rpmmd.RepoConfig{ + "build": []rpmmd.RepoConfig{ + { + Id: "repo_id", + Metalink: "http://example.com/metalink", + }, + }, +} + +func TestLibrepoSimple(t *testing.T) { + pkg := opensslPkg + + sources := osbuild.NewLibrepoSource() + err := sources.AddPackage(pkg, fakeRepos) + assert.NoError(t, err) + + expectedJSON := `{ + "items": { + "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666": { + "path": "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm", + "mirror": "repo_id" + } + }, + "options": { + "mirrors": { + "repo_id": { + "url": "http://example.com/metalink", + "type": "metalink" + } + } + } +}` + b, err := json.MarshalIndent(sources, "", " ") + assert.NoError(t, err) + assert.Equal(t, expectedJSON, string(b)) +} + +func TestLibrepoInsecure(t *testing.T) { + pkg := opensslPkg + pkg.IgnoreSSL = true + + sources := osbuild.NewLibrepoSource() + err := sources.AddPackage(pkg, fakeRepos) + assert.NoError(t, err) + + expectedJSON := `{ + "items": { + "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666": { + "path": "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm", + "mirror": "repo_id" + } + }, + "options": { + "mirrors": { + "repo_id": { + "url": "http://example.com/metalink", + "type": "metalink", + "insecure": true + } + } + } +}` + b, err := json.MarshalIndent(sources, "", " ") + assert.NoError(t, err) + assert.Equal(t, expectedJSON, string(b)) +} + +func TestLibrepoSecrets(t *testing.T) { + for _, secret := range []string{"org.osbuild.rhsm", "org.osbuild.mtls"} { + pkg := opensslPkg + pkg.Secrets = secret + + sources := osbuild.NewLibrepoSource() + err := sources.AddPackage(pkg, fakeRepos) + assert.NoError(t, err) + + expectedJSON := fmt.Sprintf(`{ + "items": { + "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666": { + "path": "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm", + "mirror": "repo_id" + } + }, + "options": { + "mirrors": { + "repo_id": { + "url": "http://example.com/metalink", + "type": "metalink", + "secrets": { + "name": "%s" + } + } + } + } +}`, secret) + b, err := json.MarshalIndent(sources, "", " ") + assert.NoError(t, err) + assert.Equal(t, expectedJSON, string(b)) + } +} diff --git a/pkg/osbuild/source.go b/pkg/osbuild/source.go index ce31dfac8f..209a26208d 100644 --- a/pkg/osbuild/source.go +++ b/pkg/osbuild/source.go @@ -3,6 +3,7 @@ package osbuild import ( "encoding/json" "errors" + "fmt" "github.com/osbuild/images/pkg/container" "github.com/osbuild/images/pkg/ostree" @@ -54,19 +55,60 @@ func (sources *Sources) UnmarshalJSON(data []byte) error { return nil } -func GenSources(packages []rpmmd.PackageSpec, ostreeCommits []ostree.CommitSpec, inlineData []string, containers []container.Spec) (Sources, error) { +func addPackagesCurl(sources Sources, packages []rpmmd.PackageSpec) error { + curl := NewCurlSource() + for _, pkg := range packages { + err := curl.AddPackage(pkg) + if err != nil { + return err + } + } + sources["org.osbuild.curl"] = curl + return nil +} + +func addPackagesLibrepo(sources Sources, packages []rpmmd.PackageSpec, rpmRepos map[string][]rpmmd.RepoConfig) error { + librepo := NewLibrepoSource() + for _, pkg := range packages { + err := librepo.AddPackage(pkg, rpmRepos) + if err != nil { + return err + } + } + sources["org.osbuild.librepo"] = librepo + return nil +} + +// RpmDownloader specifies what backend to use for rpm downloads +// Note that the librepo backend requires a newer osbuild. +type RpmDownloader uint64 + +const ( + RpmDownloaderCurl = iota + RpmDownloaderLibrepo = iota +) + +func GenSources(packages []rpmmd.PackageSpec, ostreeCommits []ostree.CommitSpec, inlineData []string, containers []container.Spec, rpmRepos map[string][]rpmmd.RepoConfig, rpmDownloader RpmDownloader) (Sources, error) { + // The signature of this functionis already relatively long, + // if we need to add more options, refactor into "struct + // Inputs" (rpm,ostree,etc) and "struct Options" + // (rpmDownloader) sources := Sources{} // collect rpm package sources if len(packages) > 0 { - curl := NewCurlSource() - for _, pkg := range packages { - err := curl.AddPackage(pkg) - if err != nil { - return nil, err - } + var err error + switch rpmDownloader { + case RpmDownloaderCurl: + err = addPackagesCurl(sources, packages) + case RpmDownloaderLibrepo: + err = addPackagesLibrepo(sources, packages, rpmRepos) + default: + err = fmt.Errorf("unknown rpm downloader %v", rpmDownloader) + } + if err != nil { + return nil, err } - sources["org.osbuild.curl"] = curl } // collect ostree commit sources diff --git a/pkg/osbuild/source_test.go b/pkg/osbuild/source_test.go index f7929a2508..aa4f612ee5 100644 --- a/pkg/osbuild/source_test.go +++ b/pkg/osbuild/source_test.go @@ -119,7 +119,7 @@ func TestSource_UnmarshalJSON(t *testing.T) { } func TestGenSourcesTrivial(t *testing.T) { - sources, err := GenSources(nil, nil, nil, nil) + sources, err := GenSources(nil, nil, nil, nil, nil, 0) assert.NoError(t, err) jsonOutput, err := json.MarshalIndent(sources, "", " ") @@ -135,7 +135,7 @@ func TestGenSourcesContainerStorage(t *testing.T) { LocalStorage: true, }, } - sources, err := GenSources(nil, nil, nil, containers) + sources, err := GenSources(nil, nil, nil, containers, nil, 0) assert.NoError(t, err) jsonOutput, err := json.MarshalIndent(sources, "", " ") @@ -159,7 +159,7 @@ func TestGenSourcesSkopeo(t *testing.T) { ImageID: imageID, }, } - sources, err := GenSources(nil, nil, nil, containers) + sources, err := GenSources(nil, nil, nil, containers, nil, 0) assert.NoError(t, err) jsonOutput, err := json.MarshalIndent(sources, "", " ") @@ -190,7 +190,7 @@ func TestGenSourcesWithSkopeoIndex(t *testing.T) { ImageID: imageID, }, } - sources, err := GenSources(nil, nil, nil, containers) + sources, err := GenSources(nil, nil, nil, containers, nil, 0) assert.NoError(t, err) jsonOutput, err := json.MarshalIndent(sources, "", " ") diff --git a/pkg/rpmmd/repository.go b/pkg/rpmmd/repository.go index e897a0a56b..8da421bad3 100644 --- a/pkg/rpmmd/repository.go +++ b/pkg/rpmmd/repository.go @@ -165,6 +165,9 @@ type PackageSpec struct { Secrets string `json:"secrets,omitempty"` CheckGPG bool `json:"check_gpg,omitempty"` IgnoreSSL bool `json:"ignore_ssl,omitempty"` + + Path string `json:"path,omitempty"` + RepoID string `json:"repo_id,omitempty"` } type PackageSource struct {