many: enable support for librepo based rpm downloading

pkg/dnfjson/dnfjson.go

@@ -586,6 +586,8 @@ func (result depsolveResult) toRPMMD(rhsm map[string]bool) ([]rpmmd.PackageSpec,
 		rpmDependencies[i].RemoteLocation = dep.RemoteLocation
 		rpmDependencies[i].Checksum = dep.Checksum
 		rpmDependencies[i].CheckGPG = repo.GPGCheck
+		rpmDependencies[i].RepoID = dep.RepoID
+		rpmDependencies[i].Path = dep.Path
 		if verify := repo.SSLVerify; verify != nil {
 			rpmDependencies[i].IgnoreSSL = !*verify
 		}

pkg/dnfjson/dnfjson_test.go

@@ -134,7 +134,8 @@ func TestDepsolver(t *testing.T) {
 				require.NotNil(t, res)
 			}
 
-			assert.Equal(expectedResult(s.RepoConfig), res.Packages)
+			assert.Equal(len(res.Repos), 1)
+			assert.Equal(expectedResult(res.Repos[0]), res.Packages)
 
 			if tc.sbomType != sbom.StandardTypeNone {
 				require.NotNil(t, res.SBOM)
@@ -725,6 +726,8 @@ func expectedResult(repo rpmmd.RepoConfig) []rpmmd.PackageSpec {
 	for idx := range exp {
 		urlTemplate := exp[idx].RemoteLocation
 		exp[idx].RemoteLocation = fmt.Sprintf(urlTemplate, strings.Join(repo.BaseURLs, ","))
+		exp[idx].Path = strings.TrimPrefix(urlTemplate, "%s/")
+		exp[idx].RepoID = repo.Id
 	}
 	return exp
 }

pkg/manifest/manifest.go

@@ -41,6 +41,14 @@ const (
 	DISTRO_FEDORA
 )
 
+// Inputs specifies the inputs for manifest instantiating
+type Inputs struct {
+	PackageSets    map[string][]rpmmd.PackageSpec
+	ContainerSpecs map[string][]container.Spec
+	OstreeCommits  map[string][]ostree.CommitSpec
+	RpmRepos       map[string][]rpmmd.RepoConfig
+}
+
 // An OSBuildManifest is an opaque JSON object, which is a valid input to osbuild
 type OSBuildManifest []byte
 
@@ -138,27 +146,47 @@ func (m Manifest) GetOSTreeSourceSpecs() map[string][]ostree.SourceSpec {
 	return ostreeSpecs
 }
 
+// TODO: remove once all callers are fixed
 func (m Manifest) Serialize(packageSets map[string][]rpmmd.PackageSpec, containerSpecs map[string][]container.Spec, ostreeCommits map[string][]ostree.CommitSpec, rpmRepos map[string][]rpmmd.RepoConfig) (OSBuildManifest, error) {
+	inputs := &Inputs{
+		PackageSets:    packageSets,
+		ContainerSpecs: containerSpecs,
+		OstreeCommits:  ostreeCommits,
+		RpmRepos:       rpmRepos,
+	}
+	return m.SerializeFull(inputs, nil)
+}
+
+// Options contains the (optional) configs for the manifest instantiating
+type Options struct {
+	RpmDownloader osbuild.RpmDownloader
+}
+
+// TODO: rename to Serialize() onces callers are fixed
+func (m Manifest) SerializeFull(inputs *Inputs, opts *Options) (OSBuildManifest, error) {
+	if opts == nil {
+		opts = &Options{}
+	}
 	pipelines := make([]osbuild.Pipeline, 0)
 	packages := make([]rpmmd.PackageSpec, 0)
 	commits := make([]ostree.CommitSpec, 0)
 	inline := make([]string, 0)
 	containers := make([]container.Spec, 0)
 	for _, pipeline := range m.pipelines {
-		pipeline.serializeStart(packageSets[pipeline.Name()], containerSpecs[pipeline.Name()], ostreeCommits[pipeline.Name()], rpmRepos[pipeline.Name()])
+		pipeline.serializeStart(inputs.PackageSets[pipeline.Name()], inputs.ContainerSpecs[pipeline.Name()], inputs.OstreeCommits[pipeline.Name()], inputs.RpmRepos[pipeline.Name()])
 	}
 	for _, pipeline := range m.pipelines {
 		commits = append(commits, pipeline.getOSTreeCommits()...)
 		pipelines = append(pipelines, pipeline.serialize())
-		packages = append(packages, packageSets[pipeline.Name()]...)
+		packages = append(packages, inputs.PackageSets[pipeline.Name()]...)
 		inline = append(inline, pipeline.getInline()...)
 		containers = append(containers, pipeline.getContainerSpecs()...)
 	}
 	for _, pipeline := range m.pipelines {
 		pipeline.serializeEnd()
 	}
 
-	sources, err := osbuild.GenSources(packages, commits, inline, containers)
+	sources, err := osbuild.GenSources(packages, commits, inline, containers, inputs.RpmRepos, opts.RpmDownloader)
 	if err != nil {
 		return nil, err
 	}

pkg/osbuild/librepo_source.go

@@ -0,0 +1,120 @@
+package osbuild
+
+import (
+	"fmt"
+	"regexp"
+
+	"github.com/osbuild/images/pkg/rpmmd"
+)
+
+var librepoDigestPattern = regexp.MustCompile(`(sha256|sha384|sha512):[0-9a-f]{32,128}`)
+
+type LibrepoSource struct {
+	Items   map[string]*LibrepoSourceItem `json:"items"`
+	Options *LibrepoSourceOptions         `json:"options"`
+}
+
+func (LibrepoSource) isSource() {}
+
+func NewLibrepoSource() *LibrepoSource {
+	return &LibrepoSource{
+		Items: make(map[string]*LibrepoSourceItem),
+		Options: &LibrepoSourceOptions{
+			Mirrors: make(map[string]*LibrepoSourceMirror),
+		},
+	}
+}
+
+type LibrepoSourceItem struct {
+	Path     string `json:"path"`
+	MirrorID string `json:"mirror"`
+}
+
+func findRepoById(repos map[string][]rpmmd.RepoConfig, repoID string) *rpmmd.RepoConfig {
+	for _, repos := range repos {
+		for _, repo := range repos {
+			if repo.Id == repoID {
+				return &repo
+			}
+		}
+	}
+	return nil
+}
+
+func mirrorFromRepo(repo *rpmmd.RepoConfig) (*LibrepoSourceMirror, error) {
+	// XXX: add support for secrets
+	switch {
+	case repo.Metalink != "":
+		return &LibrepoSourceMirror{
+			URL:  repo.Metalink,
+			Type: "metalink",
+		}, nil
+	case repo.MirrorList != "":
+		return &LibrepoSourceMirror{
+			URL:  repo.MirrorList,
+			Type: "mirrorlist",
+		}, nil
+	case len(repo.BaseURLs) > 0:
+		return &LibrepoSourceMirror{
+			// XXX: should we pick a random one instead?
+			URL:  repo.BaseURLs[0],
+			Type: "baseurl",
+		}, nil
+	}
+
+	return nil, fmt.Errorf("cannot find metalink, mirrorlist or baseurl for %+v", repo)
+}
+
+func (source *LibrepoSource) AddPackage(pkg rpmmd.PackageSpec, repos map[string][]rpmmd.RepoConfig) error {
+	pkgRepo := findRepoById(repos, pkg.RepoID)
+	if pkgRepo == nil {
+		return fmt.Errorf("cannot find repo-id %v for %v in %+v", pkg.RepoID, pkg.Name, repos)
+	}
+	if _, ok := source.Options.Mirrors[pkgRepo.Id]; !ok {
+		mirror, err := mirrorFromRepo(pkgRepo)
+		if err != nil {
+			return err
+		}
+		source.Options.Mirrors[pkgRepo.Id] = mirror
+	}
+	mirror := source.Options.Mirrors[pkgRepo.Id]
+	// XXX: should we error here if one package requests IgnoreSSL
+	// and one does not for the same mirror?
+	if pkg.IgnoreSSL {
+		mirror.Insecure = true
+	}
+	if pkg.Secrets == "org.osbuild.rhsm" {
+		mirror.Secrets = &URLSecrets{
+			Name: "org.osbuild.rhsm",
+		}
+	} else if pkg.Secrets == "org.osbuild.mtls" {
+		mirror.Secrets = &URLSecrets{
+			Name: "org.osbuild.mtls",
+		}
+	}
+
+	item := &LibrepoSourceItem{
+		Path:     pkg.Path,
+		MirrorID: pkgRepo.Id,
+	}
+	source.Items[pkg.Checksum] = item
+	return nil
+}
+
+type LibrepoSourceOptions struct {
+	Mirrors map[string]*LibrepoSourceMirror `json:"mirrors"`
+}
+
+type LibrepoSourceMirror struct {
+	URL  string `json:"url"`
+	Type string `json:"type"`
+
+	Insecure bool        `json:"insecure,omitempty"`
+	Secrets  *URLSecrets `json:"secrets,omitempty"`
+
+	// XXX: should we expose those? if so we need a way to set them,
+	// current this is done in manifest.GenSources which cannot take
+	// options.
+	// MaxParallels  *int `json:"max-parallels,omitempty"`
+	// FastestMirror bool `json:"fastest-mirror,omitempty"`
+}

pkg/osbuild/librepo_source_test.go

@@ -0,0 +1,125 @@
+package osbuild_test
+
+import (
+	"encoding/json"
+	"fmt"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/osbuild/images/pkg/osbuild"
+	"github.com/osbuild/images/pkg/rpmmd"
+)
+
+var opensslPkg = rpmmd.PackageSpec{
+	Name:           "openssl-libs",
+	Epoch:          1,
+	Version:        "3.0.1",
+	Release:        "5.el9",
+	Arch:           "x86_64",
+	RemoteLocation: "https://example.com/repo/Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm",
+	Checksum:       "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666",
+	Path:           "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm",
+	RepoID:         "repo_id",
+}
+
+var fakeRepos = map[string][]rpmmd.RepoConfig{
+	"build": []rpmmd.RepoConfig{
+		{
+			Id:       "repo_id",
+			Metalink: "http://example.com/metalink",
+		},
+	},
+}
+
+func TestLibrepoSimple(t *testing.T) {
+	pkg := opensslPkg
+
+	sources := osbuild.NewLibrepoSource()
+	err := sources.AddPackage(pkg, fakeRepos)
+	assert.NoError(t, err)
+
+	expectedJSON := `{
+  "items": {
+    "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666": {
+      "path": "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm",
+      "mirror": "repo_id"
+    }
+  },
+  "options": {
+    "mirrors": {
+      "repo_id": {
+        "url": "http://example.com/metalink",
+        "type": "metalink"
+      }
+    }
+  }
+}`
+	b, err := json.MarshalIndent(sources, "", "  ")
+	assert.NoError(t, err)
+	assert.Equal(t, expectedJSON, string(b))
+}
+
+func TestLibrepoInsecure(t *testing.T) {
+	pkg := opensslPkg
+	pkg.IgnoreSSL = true
+
+	sources := osbuild.NewLibrepoSource()
+	err := sources.AddPackage(pkg, fakeRepos)
+	assert.NoError(t, err)
+
+	expectedJSON := `{
+  "items": {
+    "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666": {
+      "path": "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm",
+      "mirror": "repo_id"
+    }
+  },
+  "options": {
+    "mirrors": {
+      "repo_id": {
+        "url": "http://example.com/metalink",
+        "type": "metalink",
+        "insecure": true
+      }
+    }
+  }
+}`
+	b, err := json.MarshalIndent(sources, "", "  ")
+	assert.NoError(t, err)
+	assert.Equal(t, expectedJSON, string(b))
+}
+
+func TestLibrepoSecrets(t *testing.T) {
+	for _, secret := range []string{"org.osbuild.rhsm", "org.osbuild.mtls"} {
+		pkg := opensslPkg
+		pkg.Secrets = secret
+
+		sources := osbuild.NewLibrepoSource()
+		err := sources.AddPackage(pkg, fakeRepos)
+		assert.NoError(t, err)
+
+		expectedJSON := fmt.Sprintf(`{
+  "items": {
+    "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666": {
+      "path": "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm",
+      "mirror": "repo_id"
+    }
+  },
+  "options": {
+    "mirrors": {
+      "repo_id": {
+        "url": "http://example.com/metalink",
+        "type": "metalink",
+        "secrets": {
+          "name": "%s"
+        }
+      }
+    }
+  }
+}`, secret)
+		b, err := json.MarshalIndent(sources, "", "  ")
+		assert.NoError(t, err)
+		assert.Equal(t, expectedJSON, string(b))
+	}
+}