From 4e2567aa2edd63e8014d84bd6075f4270c32e390 Mon Sep 17 00:00:00 2001 From: Michael Vogt Date: Tue, 7 Jan 2025 12:13:48 +0100 Subject: [PATCH] manifest,sources: add librepo support to Serialize/GenSources This commit enables librepo sources generation via a new osbuild.RpmDownloader iota that is passed to to `manifest.Serialize()` and `osbuild.GenSources()`. We also need to pass the resolved repoConfig to `manifest.Serialize()`. This is currently not type-safe, ideally we would look into how to do this in a type-safe way. Serialize should also take less args ideally. --- cmd/build/main.go | 2 +- cmd/gen-manifests/main.go | 2 +- cmd/osbuild-playground/playground.go | 2 +- pkg/distro/distro_test.go | 2 +- pkg/image/bootc_disk_test.go | 2 +- pkg/image/installer_image_test.go | 2 +- pkg/manifest/manifest.go | 31 ++++----- pkg/osbuild/source.go | 81 ++++++++++++++++++----- pkg/osbuild/source_test.go | 98 ++++++++++++++++++++++++++-- 9 files changed, 179 insertions(+), 43 deletions(-) diff --git a/cmd/build/main.go b/cmd/build/main.go index cbcff4c177..12e22956a9 100644 --- a/cmd/build/main.go +++ b/cmd/build/main.go @@ -82,7 +82,7 @@ func makeManifest( return nil, fmt.Errorf("[ERROR] ostree commit resolution failed: %w", err) } - mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs, nil) + mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs, repoConfigs, 0) if err != nil { return nil, fmt.Errorf("[ERROR] manifest serialization failed: %w", err) } diff --git a/cmd/gen-manifests/main.go b/cmd/gen-manifests/main.go index 694697dd4e..6c09a8f1b8 100644 --- a/cmd/gen-manifests/main.go +++ b/cmd/gen-manifests/main.go @@ -251,7 +251,7 @@ func makeManifestJob( commitSpecs = mockResolveCommits(manifest.GetOSTreeSourceSpecs()) } - mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs, repoConfigs) + mf, err := manifest.Serialize(packageSpecs, containerSpecs, commitSpecs, repoConfigs, 0) if err != nil { return fmt.Errorf("[%s] manifest serialization failed: %s", filename, err.Error()) } diff --git a/cmd/osbuild-playground/playground.go b/cmd/osbuild-playground/playground.go index b4df119362..1c76353a8b 100644 --- a/cmd/osbuild-playground/playground.go +++ b/cmd/osbuild-playground/playground.go @@ -49,7 +49,7 @@ func RunPlayground(img image.ImageKind, d distro.Distro, arch distro.Arch, repos fmt.Fprintf(os.Stderr, "could not clean dnf cache: %s", err.Error()) } - bytes, err := manifest.Serialize(packageSpecs, nil, nil, nil) + bytes, err := manifest.Serialize(packageSpecs, nil, nil, nil, 0) if err != nil { panic("failed to serialize manifest: " + err.Error()) } diff --git a/pkg/distro/distro_test.go b/pkg/distro/distro_test.go index 726c8514cf..799db74997 100644 --- a/pkg/distro/distro_test.go +++ b/pkg/distro/distro_test.go @@ -134,7 +134,7 @@ func TestImageTypePipelineNames(t *testing.T) { } commits[name] = commitSpecs } - mf, err := m.Serialize(packageSets, containers, commits, repoSets) + mf, err := m.Serialize(packageSets, containers, commits, repoSets, 0) assert.NoError(err) pm := new(manifest) err = json.Unmarshal(mf, pm) diff --git a/pkg/image/bootc_disk_test.go b/pkg/image/bootc_disk_test.go index 61cb3d8d67..19fed3c602 100644 --- a/pkg/image/bootc_disk_test.go +++ b/pkg/image/bootc_disk_test.go @@ -88,7 +88,7 @@ func makeBootcDiskImageOsbuildManifest(t *testing.T, opts *bootcDiskImageTestOpt "image": []container.Spec{{Source: "other-src", Digest: makeFakeDigest(t), ImageID: makeFakeDigest(t)}}, } - osbuildManifest, err := m.Serialize(nil, fakeSourceSpecs, nil, nil) + osbuildManifest, err := m.Serialize(nil, fakeSourceSpecs, nil, nil, 0) require.Nil(t, err) return osbuildManifest diff --git a/pkg/image/installer_image_test.go b/pkg/image/installer_image_test.go index 147cd78ad4..eda5e08042 100644 --- a/pkg/image/installer_image_test.go +++ b/pkg/image/installer_image_test.go @@ -364,7 +364,7 @@ func instantiateAndSerialize(t *testing.T, img image.ImageKind, packages map[str _, err := img.InstantiateManifest(&mf, nil, &runner.CentOS{Version: 9}, rng) assert.NoError(t, err) - mfs, err := mf.Serialize(packages, containers, commits, nil) + mfs, err := mf.Serialize(packages, containers, commits, nil, 0) assert.NoError(t, err) return string(mfs) diff --git a/pkg/manifest/manifest.go b/pkg/manifest/manifest.go index bd71d1a631..b8f21ed861 100644 --- a/pkg/manifest/manifest.go +++ b/pkg/manifest/manifest.go @@ -41,12 +41,7 @@ const ( DISTRO_FEDORA ) -type Inputs struct { - Packages []rpmmd.PackageSpec - Containers []container.Spec - Commits []ostree.CommitSpec - RpmRepos []rpmmd.RepoConfig -} +type Inputs osbuild.SourceInputs // An OSBuildManifest is an opaque JSON object, which is a valid input to osbuild type OSBuildManifest []byte @@ -149,32 +144,32 @@ func (m Manifest) GetOSTreeSourceSpecs() map[string][]ostree.SourceSpec { // only depsolved PackageSpecs/RepoConfigs are passed so that we // have a valid mapping of pkg.RepoID<->repo.Id which will be important // for librepo -func (m Manifest) Serialize(packageSets map[string][]rpmmd.PackageSpec, containerSpecs map[string][]container.Spec, ostreeCommits map[string][]ostree.CommitSpec, rpmRepos map[string][]rpmmd.RepoConfig) (OSBuildManifest, error) { - pipelines := make([]osbuild.Pipeline, 0) - packages := make([]rpmmd.PackageSpec, 0) - commits := make([]ostree.CommitSpec, 0) - inline := make([]string, 0) - containers := make([]container.Spec, 0) +func (m Manifest) Serialize(packageSets map[string][]rpmmd.PackageSpec, containerSpecs map[string][]container.Spec, ostreeCommits map[string][]ostree.CommitSpec, resolvedRpmRepos map[string][]rpmmd.RepoConfig, rpmDownloader osbuild.RpmDownloader) (OSBuildManifest, error) { for _, pipeline := range m.pipelines { pipeline.serializeStart(Inputs{ Packages: packageSets[pipeline.Name()], Containers: containerSpecs[pipeline.Name()], Commits: ostreeCommits[pipeline.Name()], - RpmRepos: rpmRepos[pipeline.Name()], + RpmRepos: resolvedRpmRepos[pipeline.Name()], }) } + + var pipelines []osbuild.Pipeline + var mergedInputs osbuild.SourceInputs for _, pipeline := range m.pipelines { - commits = append(commits, pipeline.getOSTreeCommits()...) pipelines = append(pipelines, pipeline.serialize()) - packages = append(packages, packageSets[pipeline.Name()]...) - inline = append(inline, pipeline.getInline()...) - containers = append(containers, pipeline.getContainerSpecs()...) + + mergedInputs.Commits = append(mergedInputs.Commits, pipeline.getOSTreeCommits()...) + mergedInputs.Packages = append(mergedInputs.Packages, packageSets[pipeline.Name()]...) + mergedInputs.RpmRepos = append(mergedInputs.RpmRepos, resolvedRpmRepos[pipeline.Name()]...) + mergedInputs.Containers = append(mergedInputs.Containers, pipeline.getContainerSpecs()...) + mergedInputs.InlineData = append(mergedInputs.InlineData, pipeline.getInline()...) } for _, pipeline := range m.pipelines { pipeline.serializeEnd() } - sources, err := osbuild.GenSources(packages, commits, inline, containers) + sources, err := osbuild.GenSources(mergedInputs, rpmDownloader) if err != nil { return nil, err } diff --git a/pkg/osbuild/source.go b/pkg/osbuild/source.go index ce31dfac8f..50c1c7cbab 100644 --- a/pkg/osbuild/source.go +++ b/pkg/osbuild/source.go @@ -3,12 +3,33 @@ package osbuild import ( "encoding/json" "errors" + "fmt" "github.com/osbuild/images/pkg/container" "github.com/osbuild/images/pkg/ostree" "github.com/osbuild/images/pkg/rpmmd" ) +// RpmDownloader specifies what backend to use for rpm downloads +// Note that the librepo backend requires a newer osbuild. +type RpmDownloader uint64 + +const ( + RpmDownloaderCurl = iota + RpmDownloaderLibrepo = iota +) + +// SourceInputs contains the inputs to generate osbuild.Sources +// Note that for Packages/RpmRepos the depsolve resolved results +// must be passed +type SourceInputs struct { + Packages []rpmmd.PackageSpec + Containers []container.Spec + Commits []ostree.CommitSpec + RpmRepos []rpmmd.RepoConfig + InlineData []string +} + // A Sources map contains all the sources made available to an osbuild run type Sources map[string]Source @@ -54,25 +75,55 @@ func (sources *Sources) UnmarshalJSON(data []byte) error { return nil } -func GenSources(packages []rpmmd.PackageSpec, ostreeCommits []ostree.CommitSpec, inlineData []string, containers []container.Spec) (Sources, error) { +func addPackagesCurl(sources Sources, packages []rpmmd.PackageSpec) error { + curl := NewCurlSource() + for _, pkg := range packages { + err := curl.AddPackage(pkg) + if err != nil { + return err + } + } + sources["org.osbuild.curl"] = curl + return nil +} + +func addPackagesLibrepo(sources Sources, packages []rpmmd.PackageSpec, rpmRepos []rpmmd.RepoConfig) error { + librepo := NewLibrepoSource() + for _, pkg := range packages { + err := librepo.AddPackage(pkg, rpmRepos) + if err != nil { + return err + } + } + sources["org.osbuild.librepo"] = librepo + return nil +} + +// GenSources generates the Sources from the given inputs. Note that +// the packages and rpmRepos need to come from the *resolved* set. +func GenSources(inputs SourceInputs, rpmDownloader RpmDownloader) (Sources, error) { sources := Sources{} // collect rpm package sources - if len(packages) > 0 { - curl := NewCurlSource() - for _, pkg := range packages { - err := curl.AddPackage(pkg) - if err != nil { - return nil, err - } + if len(inputs.Packages) > 0 { + var err error + switch rpmDownloader { + case RpmDownloaderCurl: + err = addPackagesCurl(sources, inputs.Packages) + case RpmDownloaderLibrepo: + err = addPackagesLibrepo(sources, inputs.Packages, inputs.RpmRepos) + default: + err = fmt.Errorf("unknown rpm downloader %v", rpmDownloader) + } + if err != nil { + return nil, err } - sources["org.osbuild.curl"] = curl } // collect ostree commit sources - if len(ostreeCommits) > 0 { + if len(inputs.Commits) > 0 { ostree := NewOSTreeSource() - for _, commit := range ostreeCommits { + for _, commit := range inputs.Commits { ostree.AddItem(commit) } if len(ostree.Items) > 0 { @@ -81,9 +132,9 @@ func GenSources(packages []rpmmd.PackageSpec, ostreeCommits []ostree.CommitSpec, } // collect inline data sources - if len(inlineData) > 0 { + if len(inputs.InlineData) > 0 { ils := NewInlineSource() - for _, data := range inlineData { + for _, data := range inputs.InlineData { ils.AddItem(data) } @@ -91,11 +142,11 @@ func GenSources(packages []rpmmd.PackageSpec, ostreeCommits []ostree.CommitSpec, } // collect skopeo and local container sources - if len(containers) > 0 { + if len(inputs.Containers) > 0 { skopeo := NewSkopeoSource() skopeoIndex := NewSkopeoIndexSource() localContainers := NewContainersStorageSource() - for _, c := range containers { + for _, c := range inputs.Containers { if c.LocalStorage { localContainers.AddItem(c.ImageID) } else { diff --git a/pkg/osbuild/source_test.go b/pkg/osbuild/source_test.go index f7929a2508..6dcef4207a 100644 --- a/pkg/osbuild/source_test.go +++ b/pkg/osbuild/source_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/osbuild/images/pkg/container" + "github.com/osbuild/images/pkg/rpmmd" ) func TestSource_UnmarshalJSON(t *testing.T) { @@ -119,7 +120,7 @@ func TestSource_UnmarshalJSON(t *testing.T) { } func TestGenSourcesTrivial(t *testing.T) { - sources, err := GenSources(nil, nil, nil, nil) + sources, err := GenSources(SourceInputs{}, 0) assert.NoError(t, err) jsonOutput, err := json.MarshalIndent(sources, "", " ") @@ -135,7 +136,7 @@ func TestGenSourcesContainerStorage(t *testing.T) { LocalStorage: true, }, } - sources, err := GenSources(nil, nil, nil, containers) + sources, err := GenSources(SourceInputs{Containers: containers}, 0) assert.NoError(t, err) jsonOutput, err := json.MarshalIndent(sources, "", " ") @@ -159,7 +160,7 @@ func TestGenSourcesSkopeo(t *testing.T) { ImageID: imageID, }, } - sources, err := GenSources(nil, nil, nil, containers) + sources, err := GenSources(SourceInputs{Containers: containers}, 0) assert.NoError(t, err) jsonOutput, err := json.MarshalIndent(sources, "", " ") @@ -190,7 +191,7 @@ func TestGenSourcesWithSkopeoIndex(t *testing.T) { ImageID: imageID, }, } - sources, err := GenSources(nil, nil, nil, containers) + sources, err := GenSources(SourceInputs{Containers: containers}, 0) assert.NoError(t, err) jsonOutput, err := json.MarshalIndent(sources, "", " ") @@ -217,3 +218,92 @@ func TestGenSourcesWithSkopeoIndex(t *testing.T) { } }`) } + +// TODO: move into a common "rpmtest" package +var opensslPkg = rpmmd.PackageSpec{ + Name: "openssl-libs", + RemoteLocation: "https://example.com/repo/Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm", + Checksum: "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666", + Path: "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm", + RepoID: "repo_id_metalink", +} + +var fakeRepos = []rpmmd.RepoConfig{ + { + Id: "repo_id_metalink", + Metalink: "http://example.com/metalink", + }, +} + +func TestGenSourcesRpmDefaultRpmDownloaderIsCurl(t *testing.T) { + inputs := SourceInputs{ + Packages: []rpmmd.PackageSpec{opensslPkg}, + RpmRepos: fakeRepos, + } + var defaultRpmDownloader RpmDownloader + sources, err := GenSources(inputs, defaultRpmDownloader) + assert.NoError(t, err) + + assert.NotNil(t, sources["org.osbuild.curl"]) + assert.Nil(t, sources["org.osbuild.librepo"]) +} + +func TestGenSourcesRpmWithLibcurl(t *testing.T) { + inputs := SourceInputs{ + Packages: []rpmmd.PackageSpec{opensslPkg}, + RpmRepos: fakeRepos, + } + sources, err := GenSources(inputs, RpmDownloaderCurl) + assert.NoError(t, err) + + jsonOutput, err := json.MarshalIndent(sources, "", " ") + assert.NoError(t, err) + assert.Equal(t, string(jsonOutput), `{ + "org.osbuild.curl": { + "items": { + "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666": { + "url": "https://example.com/repo/Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm" + } + } + } +}`) +} + +func TestGenSourcesRpmWithLibrepo(t *testing.T) { + inputs := SourceInputs{ + Packages: []rpmmd.PackageSpec{opensslPkg}, + RpmRepos: fakeRepos, + } + sources, err := GenSources(inputs, RpmDownloaderLibrepo) + assert.NoError(t, err) + + jsonOutput, err := json.MarshalIndent(sources, "", " ") + assert.NoError(t, err) + assert.Equal(t, string(jsonOutput), `{ + "org.osbuild.librepo": { + "items": { + "sha256:fcf2515ec9115551c99d552da721803ecbca23b7ae5a974309975000e8bef666": { + "path": "Packages/openssl-libs-3.0.1-5.el9.x86_64.rpm", + "mirror": "repo_id_metalink" + } + }, + "options": { + "mirrors": { + "repo_id_metalink": { + "url": "http://example.com/metalink", + "type": "metalink" + } + } + } + } +}`) +} + +func TestGenSourcesRpmBad(t *testing.T) { + inputs := SourceInputs{ + Packages: []rpmmd.PackageSpec{opensslPkg}, + RpmRepos: fakeRepos, + } + _, err := GenSources(inputs, 99) + assert.EqualError(t, err, "unknown rpm downloader 99") +}