many: enable support for librepo based rpm downloading

pkg/dnfjson/dnfjson.go

@@ -586,6 +586,8 @@ func (result depsolveResult) toRPMMD(rhsm map[string]bool) ([]rpmmd.PackageSpec,
 		rpmDependencies[i].RemoteLocation = dep.RemoteLocation
 		rpmDependencies[i].Checksum = dep.Checksum
 		rpmDependencies[i].CheckGPG = repo.GPGCheck
+		rpmDependencies[i].RepoID = dep.RepoID
+		rpmDependencies[i].Path = dep.Path
 		if verify := repo.SSLVerify; verify != nil {
 			rpmDependencies[i].IgnoreSSL = !*verify
 		}

pkg/manifest/manifest.go

@@ -41,6 +41,14 @@ const (
 	DISTRO_FEDORA
 )
 
+// Inputs specifies the inputs for manifest instantiating
+type Inputs struct {
+	PackageSets    map[string][]rpmmd.PackageSpec
+	ContainerSpecs map[string][]container.Spec
+	OstreeCommits  map[string][]ostree.CommitSpec
+	RpmRepos       map[string][]rpmmd.RepoConfig
+}
+
 // An OSBuildManifest is an opaque JSON object, which is a valid input to osbuild
 type OSBuildManifest []byte
 
@@ -138,27 +146,47 @@ func (m Manifest) GetOSTreeSourceSpecs() map[string][]ostree.SourceSpec {
 	return ostreeSpecs
 }
 
+// TODO: remove once all callers are fixed
 func (m Manifest) Serialize(packageSets map[string][]rpmmd.PackageSpec, containerSpecs map[string][]container.Spec, ostreeCommits map[string][]ostree.CommitSpec, rpmRepos map[string][]rpmmd.RepoConfig) (OSBuildManifest, error) {
+	inputs := &Inputs{
+		PackageSets:    packageSets,
+		ContainerSpecs: containerSpecs,
+		OstreeCommits:  ostreeCommits,
+		RpmRepos:       rpmRepos,
+	}
+	return m.SerializeFull(inputs, nil)
+}
+
+// Options contains the (optional) configs for the manifest instantiating
+type Options struct {
+	RpmDownloader osbuild.RpmDownloader
+}
+
+// TODO: rename to Serialize() onces callers are fixed
+func (m Manifest) SerializeFull(inputs *Inputs, opts *Options) (OSBuildManifest, error) {
+	if opts == nil {
+		opts = &Options{}
+	}
 	pipelines := make([]osbuild.Pipeline, 0)
 	packages := make([]rpmmd.PackageSpec, 0)
 	commits := make([]ostree.CommitSpec, 0)
 	inline := make([]string, 0)
 	containers := make([]container.Spec, 0)
 	for _, pipeline := range m.pipelines {
-		pipeline.serializeStart(packageSets[pipeline.Name()], containerSpecs[pipeline.Name()], ostreeCommits[pipeline.Name()], rpmRepos[pipeline.Name()])
+		pipeline.serializeStart(inputs.PackageSets[pipeline.Name()], inputs.ContainerSpecs[pipeline.Name()], inputs.OstreeCommits[pipeline.Name()], inputs.RpmRepos[pipeline.Name()])
 	}
 	for _, pipeline := range m.pipelines {
 		commits = append(commits, pipeline.getOSTreeCommits()...)
 		pipelines = append(pipelines, pipeline.serialize())
-		packages = append(packages, packageSets[pipeline.Name()]...)
+		packages = append(packages, inputs.PackageSets[pipeline.Name()]...)
 		inline = append(inline, pipeline.getInline()...)
 		containers = append(containers, pipeline.getContainerSpecs()...)
 	}
 	for _, pipeline := range m.pipelines {
 		pipeline.serializeEnd()
 	}
 
-	sources, err := osbuild.GenSources(packages, commits, inline, containers)
+	sources, err := osbuild.GenSources(packages, commits, inline, containers, inputs.RpmRepos, opts.RpmDownloader)
 	if err != nil {
 		return nil, err
 	}

pkg/osbuild/librepo_source.go

@@ -0,0 +1,120 @@
+package osbuild
+
+import (
+	"fmt"
+	"regexp"
+
+	"github.com/osbuild/images/pkg/rpmmd"
+)
+
+var librepoDigestPattern = regexp.MustCompile(`(sha256|sha384|sha512):[0-9a-f]{32,128}`)
+
+type LibrepoSource struct {
+	Items   map[string]*LibrepoSourceItem `json:"items"`
+	Options *LibrepoSourceOptions         `json:"options"`
+}
+
+func (LibrepoSource) isSource() {}
+
+func NewLibrepoSource() *LibrepoSource {
+	return &LibrepoSource{
+		Items: make(map[string]*LibrepoSourceItem),
+		Options: &LibrepoSourceOptions{
+			Mirrors: make(map[string]*LibrepoSourceMirror),
+		},
+	}
+}
+
+type LibrepoSourceItem struct {
+	Path     string `json:"path"`
+	MirrorID string `json:"mirror"`
+}
+
+func findRepoById(repos map[string][]rpmmd.RepoConfig, repoID string) *rpmmd.RepoConfig {
+	for _, repos := range repos {
+		for _, repo := range repos {
+			if repo.Id == repoID {
+				return &repo
+			}
+		}
+	}
+	return nil
+}
+
+func mirrorFromRepo(repo *rpmmd.RepoConfig) (*LibrepoSourceMirror, error) {
+	// XXX: add support for secrets
+	switch {
+	case repo.Metalink != "":
+		return &LibrepoSourceMirror{
+			URL:  repo.Metalink,
+			Type: "metalink",
+		}, nil
+	case repo.MirrorList != "":
+		return &LibrepoSourceMirror{
+			URL:  repo.MirrorList,
+			Type: "mirrorlist",
+		}, nil
+	case len(repo.BaseURLs) > 0:
+		return &LibrepoSourceMirror{
+			// XXX: should we pick a random one instead?
+			URL:  repo.BaseURLs[0],
+			Type: "baseurl",
+		}, nil
+	}
+
+	return nil, fmt.Errorf("cannot find metalink, mirrorlist or baseurl for %+v", repo)
+}
+
+func (source *LibrepoSource) AddPackage(pkg rpmmd.PackageSpec, repos map[string][]rpmmd.RepoConfig) error {
+	pkgRepo := findRepoById(repos, pkg.RepoID)
+	if pkgRepo == nil {
+		return fmt.Errorf("cannot find repo-id %v for %v in %+v", pkg.RepoID, pkg.Name, repos)
+	}
+	if _, ok := source.Options.Mirrors[pkgRepo.Id]; !ok {
+		mirror, err := mirrorFromRepo(pkgRepo)
+		if err != nil {
+			return err
+		}
+		source.Options.Mirrors[pkgRepo.Id] = mirror
+	}
+	mirror := source.Options.Mirrors[pkgRepo.Id]
+	// XXX: should we error here if one package requests IgnoreSSL
+	// and one does not for the same mirror?
+	if pkg.IgnoreSSL {
+		mirror.Insecure = true
+	}
+	if pkg.Secrets == "org.osbuild.rhsm" {
+		mirror.Secrets = &URLSecrets{
+			Name: "org.osbuild.rhsm",
+		}
+	} else if pkg.Secrets == "org.osbuild.mtls" {
+		mirror.Secrets = &URLSecrets{
+			Name: "org.osbuild.mtls",
+		}
+	}
+
+	item := &LibrepoSourceItem{
+		Path:     pkg.Path,
+		MirrorID: pkgRepo.Id,
+	}
+	source.Items[pkg.Checksum] = item
+	return nil
+}
+
+type LibrepoSourceOptions struct {
+	Mirrors map[string]*LibrepoSourceMirror `json:"mirrors"`
+}
+
+type LibrepoSourceMirror struct {
+	URL  string `json:"url"`
+	Type string `json:"type"`
+
+	Insecure bool        `json:"insecure,omitempty"`
+	Secrets  *URLSecrets `json:"secrets,omitempty"`
+
+	// XXX: should we expose those? if so we need a way to set them,
+	// current this is done in manifest.GenSources which cannot take
+	// options.
+	// MaxParallels  *int `json:"max-parallels,omitempty"`
+	// FastestMirror bool `json:"fastest-mirror,omitempty"`
+}

pkg/osbuild/librepo_source_test.go

@@ -0,0 +1,125 @@
+package osbuild_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/osbuild/images/pkg/osbuild"
+	"github.com/osbuild/images/pkg/rpmmd"
+)
+
+var opensslPkg = rpmmd.PackageSpec{
+	Name:           "openssl-libs",
+	Epoch:          1,
+	Version:        "3.0.1",
+	Release:        "5.el9",
+	Arch:           "x86_64",
+	RemoteLocation: "https://example.com/repo/Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm",
+	Checksum:       "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666",
+	Path:           "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm",
+	RepoID:         "repo_id",
+}
+
+var fakeRepos = map[string][]rpmmd.RepoConfig{
+	"build": []rpmmd.RepoConfig{
+		{
+			Id:       "repo_id",
+			Metalink: "http://example.com/metalink",
+		},
+	},
+}
+
+func TestLibrepoSimple(t *testing.T) {
+	pkg := opensslPkg
+
+	sources := osbuild.NewLibrepoSource()
+	err := sources.AddPackage(pkg, fakeRepos)
+	assert.NoError(t, err)
+
+	expectedJSON := `{
+  "items": {
+    "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666": {
+      "path": "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm",
+      "mirror": "repo_id"
+    }
+  },
+  "options": {
+    "mirrors": {
+      "repo_id": {
+        "url": "http://example.com/metalink",
+        "type": "metalink"
+      }
+    }
+  }
+}`
+	b, err := json.MarshalIndent(sources, "", "  ")
+	assert.NoError(t, err)
+	assert.Equal(t, expectedJSON, string(b))
+}
+
+func TestLibrepoInsecure(t *testing.T) {
+	pkg := opensslPkg
+	pkg.IgnoreSSL = true
+
+	sources := osbuild.NewLibrepoSource()
+	err := sources.AddPackage(pkg, fakeRepos)
+	assert.NoError(t, err)
+
+	expectedJSON := `{
+  "items": {
+    "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666": {
+      "path": "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm",
+      "mirror": "repo_id"
+    }
+  },
+  "options": {
+    "mirrors": {
+      "repo_id": {
+        "url": "http://example.com/metalink",
+        "type": "metalink",
+        "insecure": true
+      }
+    }
+  }
+}`
+	b, err := json.MarshalIndent(sources, "", "  ")
+	assert.NoError(t, err)
+	assert.Equal(t, expectedJSON, string(b))
+}
+
+func TestLibrepoSecrets(t *testing.T) {
+	for _, secret := range []string{"org.osbuild.rhsm", "org.osbuild.mtls"} {
+		pkg := opensslPkg
+		pkg.Secrets = secret
+
+		sources := osbuild.NewLibrepoSource()
+		err := sources.AddPackage(pkg, fakeRepos)
+		assert.NoError(t, err)
+
+		expectedJSON := fmt.Sprintf(`{
+  "items": {
+    "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666": {
+      "path": "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm",
+      "mirror": "repo_id"
+    }
+  },
+  "options": {
+    "mirrors": {
+      "repo_id": {
+        "url": "http://example.com/metalink",
+        "type": "metalink",
+        "secrets": {
+          "name": "%s"
+        }
+      }
+    }
+  }
+}`, secret)
+		b, err := json.MarshalIndent(sources, "", "  ")
+		assert.NoError(t, err)
+		assert.Equal(t, expectedJSON, string(b))
+	}
+}

pkg/osbuild/source.go

@@ -3,6 +3,7 @@ package osbuild
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 
 	"github.com/osbuild/images/pkg/container"
 	"github.com/osbuild/images/pkg/ostree"
@@ -54,19 +55,60 @@ func (sources *Sources) UnmarshalJSON(data []byte) error {
 	return nil
 }
 
-func GenSources(packages []rpmmd.PackageSpec, ostreeCommits []ostree.CommitSpec, inlineData []string, containers []container.Spec) (Sources, error) {
+func addPackagesCurl(sources Sources, packages []rpmmd.PackageSpec) error {
+	curl := NewCurlSource()
+	for _, pkg := range packages {
+		err := curl.AddPackage(pkg)
+		if err != nil {
+			return err
+		}
+	}
+	sources["org.osbuild.curl"] = curl
+	return nil
+}
+
+func addPackagesLibrepo(sources Sources, packages []rpmmd.PackageSpec, rpmRepos map[string][]rpmmd.RepoConfig) error {
+	librepo := NewLibrepoSource()
+	for _, pkg := range packages {
+		err := librepo.AddPackage(pkg, rpmRepos)
+		if err != nil {
+			return err
+		}
+	}
+	sources["org.osbuild.librepo"] = librepo
+	return nil
+}
+
+// RpmDownloader specifies what backend to use for rpm downloads
+// Note that the librepo backend requires a newer osbuild.
+type RpmDownloader uint64
+
+const (
+	RpmDownloaderCurl    = iota
+	RpmDownloaderLibrepo = iota
+)
+
+func GenSources(packages []rpmmd.PackageSpec, ostreeCommits []ostree.CommitSpec, inlineData []string, containers []container.Spec, rpmRepos map[string][]rpmmd.RepoConfig, rpmDownloader RpmDownloader) (Sources, error) {
+	// The signature of this functionis already relatively long,
+	// if we need to add more options, refactor into "struct
+	// Inputs" (rpm,ostree,etc) and "struct Options"
+	// (rpmDownloader)
 	sources := Sources{}
 
 	// collect rpm package sources
 	if len(packages) > 0 {
-		curl := NewCurlSource()
-		for _, pkg := range packages {
-			err := curl.AddPackage(pkg)
-			if err != nil {
-				return nil, err
-			}
+		var err error
+		switch rpmDownloader {
+		case RpmDownloaderCurl:
+			err = addPackagesCurl(sources, packages)
+		case RpmDownloaderLibrepo:
+			err = addPackagesLibrepo(sources, packages, rpmRepos)
+		default:
+			err = fmt.Errorf("unknown rpm downloader %v", rpmDownloader)
+		}
+		if err != nil {
+			return nil, err
 		}
-		sources["org.osbuild.curl"] = curl
 	}
 
 	// collect ostree commit sources