From 97ff99e53b51e56ca0843c79f4fd38a39f7b3048 Mon Sep 17 00:00:00 2001 From: Kim Mason Date: Fri, 13 Sep 2024 15:00:24 -0600 Subject: [PATCH] Plumb through layout files (e.g. layers) to OCI Image Layout. OCI Image Layout needs all layers, as well as the base image files. --- go/cmd/ocitool/BUILD.bazel | 2 +- go/cmd/ocitool/imagelayout_cmd.go | 55 ++++++++++++++++++------------- go/cmd/ocitool/main.go | 4 +-- go/pkg/ociutil/push.go | 2 +- oci/defs.bzl | 2 +- oci/image.bzl | 19 ++--------- oci/oci_image_layout.bzl | 23 ++++++------- oci/providers.bzl | 8 ----- 8 files changed, 51 insertions(+), 64 deletions(-) diff --git a/go/cmd/ocitool/BUILD.bazel b/go/cmd/ocitool/BUILD.bazel index 6b92c70..127df6b 100644 --- a/go/cmd/ocitool/BUILD.bazel +++ b/go/cmd/ocitool/BUILD.bazel @@ -5,10 +5,10 @@ go_library( srcs = [ "appendlayer_cmd.go", "createlayer_cmd.go", - "imagelayout_cmd.go", "desc_helpers.go", "digest_cmd.go", "gen_cmd.go", + "imagelayout_cmd.go", "index_cmd.go", "main.go", "manifest_cmd.go", diff --git a/go/cmd/ocitool/imagelayout_cmd.go b/go/cmd/ocitool/imagelayout_cmd.go index b1e21b8..c197721 100644 --- a/go/cmd/ocitool/imagelayout_cmd.go +++ b/go/cmd/ocitool/imagelayout_cmd.go @@ -13,33 +13,42 @@ import ( "github.com/urfave/cli/v2" ) -// Given a slice of baseLayoutPaths, where each path contains an OCI Image Format, -// return an index that maps sha256 values to paths. -// If relPath is non-empty, it is prepended to all baseLayoutPaths. -func getBaseLayoutBlobIndex(baseLayoutPaths []string, relPath string) (blob.Index, error) { +// Given a slice of layoutFilePaths, where each path contains a file that may +// be used within an OCI Image Format, return an index that maps sha256 values +// to paths. +// If relPath is non-empty, it is prepended to all layoutFilePaths. +func getLayoutFilesBlobIndex(layoutFilePaths []string, relPath string) (blob.Index, error) { var result blob.Index result.Blobs = make(map[digest.Digest]string) - - for _, p := range baseLayoutPaths { + for _, p := range layoutFilePaths { if len(strings.TrimSpace(p)) == 0 { // Ignore empty paths. continue } - blobsDir := path.Join(p, "blobs", "sha256") if relPath != "" { - blobsDir = path.Join(relPath, blobsDir) - } - entries, err := os.ReadDir(blobsDir) - if err != nil { - return blob.Index{}, fmt.Errorf("unable to read OCI Image Format blobs dir. Base layout paths: %v, Relpath: %s, Path: %s, Error: %w", baseLayoutPaths, relPath, blobsDir, err) + p = path.Join(relPath, p) } - for _, entry := range entries { - if !entry.Type().IsRegular() { - continue + // Use an immediately invoked function here so that defer closes the + // file at a suitable time. + err := func() error { + f, err := os.Open(p) + if err != nil { + return err } - name := entry.Name() - result.Blobs[digest.Digest(name)] = path.Join(blobsDir, name) + defer f.Close() + digester := digest.SHA256.Digester() + _, err = f.WriteTo(digester.Hash()) + if err != nil { + return err + } + digest := digester.Digest() + result.Blobs[digest] = p + return nil + }() + if err != nil { + return blob.Index{}, err } + } return result, nil @@ -50,20 +59,20 @@ func getBaseLayoutBlobIndex(baseLayoutPaths []string, relPath string) (blob.Inde // for the structure of OCI Image Layout directories. func CreateOciImageLayoutCmd(c *cli.Context) error { relPath := c.String("layout-relative") - baseLayoutBlobIdx, err := getBaseLayoutBlobIndex(c.StringSlice("base-image-layouts"), relPath) + // Load providers that read local files, and create a multiprovider that + // contains all of them, as well as providers for base image blobs. + providers, err := LoadLocalProviders(c.StringSlice("layout"), relPath) if err != nil { return err } - // Load providers that read local files, and create a multiprovider that - // contains all of them, as well as providers for base image blobs. - providers, err := LoadLocalProviders(c.StringSlice("layout"), relPath) + layoutFilesBlobIdx, err := getLayoutFilesBlobIndex(c.StringSlice("layout-files"), relPath) if err != nil { return err } - if len(baseLayoutBlobIdx.Blobs) > 0 { - providers = append(providers, &baseLayoutBlobIdx) + if len(layoutFilesBlobIdx.Blobs) > 0 { + providers = append(providers, &layoutFilesBlobIdx) } multiProvider := ociutil.MultiProvider(providers...) diff --git a/go/cmd/ocitool/main.go b/go/cmd/ocitool/main.go index 87c86d4..21dd325 100644 --- a/go/cmd/ocitool/main.go +++ b/go/cmd/ocitool/main.go @@ -215,8 +215,8 @@ as described in https://github.com/opencontainers/image-spec/blob/main/image-lay Name: "desc", }, &cli.StringSliceFlag{ - Name: "base-image-layouts", - Usage: "A comma separated list of directory paths, each path containing an OCI Image Layout.", + Name: "layout-files", + Usage: "A comma separated list of blob files to be placed in the OCI Image Layout (e.g. image layers).", }, &cli.StringFlag{ Name: "out-dir", diff --git a/go/pkg/ociutil/push.go b/go/pkg/ociutil/push.go index 1b67e82..e2605ad 100644 --- a/go/pkg/ociutil/push.go +++ b/go/pkg/ociutil/push.go @@ -281,7 +281,7 @@ func CopyContent(ctx context.Context, from content.Provider, to content.Ingester reader, err := from.ReaderAt(ctx, desc) if err != nil { - return fmt.Errorf("failed to create reader from provider: %w", err) + return fmt.Errorf("failed to create reader from provider. Descriptor: %+v; Error: %w", desc, err) } ref := desc.Digest.String() diff --git a/oci/defs.bzl b/oci/defs.bzl index ae8767e..5f461f4 100755 --- a/oci/defs.bzl +++ b/oci/defs.bzl @@ -1,9 +1,9 @@ """ public API """ load(":image.bzl", _oci_image = "oci_image", _oci_image_index = "oci_image_index", _oci_image_layer = "oci_image_layer") +load(":oci_image_layout.bzl", _oci_image_layout = "oci_image_layout") load(":pull.bzl", _oci_pull = "oci_pull") load(":push.bzl", _oci_push = "oci_push") -load(":oci_image_layout.bzl", _oci_image_layout = "oci_image_layout") oci_pull = _oci_pull oci_push = _oci_push diff --git a/oci/image.bzl b/oci/image.bzl index e046bcc..8727bf1 100755 --- a/oci/image.bzl +++ b/oci/image.bzl @@ -1,5 +1,6 @@ """ image """ -load("@com_github_datadog_rules_oci//oci:providers.bzl", "OCIDescriptor", "OCIImageLayoutInfo", "OCILayout") + +load("@com_github_datadog_rules_oci//oci:providers.bzl", "OCIDescriptor", "OCILayout") # buildifier: disable=function-docstring def get_descriptor_file(ctx, desc): @@ -110,8 +111,6 @@ def _oci_image_index_impl(ctx): outputs = outputs, ) - oci_layouts = [m[OCIImageLayoutInfo].oci_image_layout_dirs for m in ctx.attr.manifests] - return [ OCIDescriptor( descriptor_file = index_desc_file, @@ -120,10 +119,6 @@ def _oci_image_index_impl(ctx): blob_index = layout_file, files = depset(direct = [index_file, layout_file], transitive = [layout_files]), ), - # Pass through any OCIImageLayoutInfo data from the manifests. - OCIImageLayoutInfo( - oci_image_layout_dirs = depset(transitive = oci_layouts), - ), DefaultInfo( files = depset(outputs), ), @@ -228,9 +223,6 @@ def _oci_image_impl(ctx): transitive = [base_layout.files], ), ), - OCIImageLayoutInfo( - oci_image_layout_dirs = depset(ctx.files.pulled_base if ctx.attr.pulled_base != None else []), - ), DefaultInfo( files = depset([ entrypoint_config_file, @@ -256,13 +248,6 @@ oci_image = rule( OCILayout, ], ), - "pulled_base": attr.label( - doc = """A directory that contains the base image in OCI Image Layout format. - See https://github.com/opencontainers/image-spec/blob/main/image-layout.md for a description - of the OCI Image Layout format. This is optional, and if present, is passed through as an output of oci_image, - by the OCIImageLayoutInfo provider.""", - allow_single_file = True, - ), "entrypoint": attr.string_list( doc = """A list of entrypoints for the image; these will be inserted into the generated OCI image config""", diff --git a/oci/oci_image_layout.bzl b/oci/oci_image_layout.bzl index e331728..91af319 100644 --- a/oci/oci_image_layout.bzl +++ b/oci/oci_image_layout.bzl @@ -1,14 +1,15 @@ -load("@com_github_datadog_rules_oci//oci:providers.bzl", "OCIDescriptor", "OCIImageLayoutInfo", "OCILayout") +"""A rule to create a directory in OCI Image Layout format.""" + load("@com_github_datadog_rules_oci//oci:debug_flag.bzl", "DebugInfo") +load("@com_github_datadog_rules_oci//oci:providers.bzl", "OCIDescriptor", "OCILayout") def _oci_image_layout_impl(ctx): toolchain = ctx.toolchains["@com_github_datadog_rules_oci//oci:toolchain"] layout = ctx.attr.manifest[OCILayout] - base_layouts = ctx.attr.manifest[OCIImageLayoutInfo] - base_layout_dirs = "" - if base_layouts != None: - base_layout_dirs = ",".join([p.path for p in base_layouts.oci_image_layout_dirs.to_list()]) + + # layout_files contains all available blobs for the image. + layout_files = ",".join([p.path for p in layout.files.to_list()]) descriptor = ctx.attr.manifest[OCIDescriptor] out_dir = ctx.actions.declare_directory(ctx.label.name) @@ -27,14 +28,14 @@ def _oci_image_layout_impl(ctx): # up 3 levels from the bin directory. "--layout-relative={root}".format(root = ctx.bin_dir.path + "/../../../"), "--desc={desc}".format(desc = descriptor.descriptor_file.path), - "--base-image-layouts={base_layouts}".format(base_layouts = base_layout_dirs), + "--layout-files={layout_files}".format(layout_files = layout_files), "--out-dir={out_dir}".format(out_dir = out_dir.path), ], inputs = - depset(direct = ctx.files.manifest + [layout.blob_index], transitive = [ - base_layouts.oci_image_layout_dirs, - layout.files, - ]), + depset( + direct = ctx.files.manifest + [layout.blob_index], + transitive = [layout.files], + ), outputs = [ out_dir, ], @@ -73,7 +74,7 @@ oci_image_layout = rule( doc = """ An OCILayout index to be written to the OCI Image Format directory. """, - providers = [OCILayout, OCIImageLayoutInfo], + providers = [OCILayout], ), "_debug": attr.label( default = "//oci:debug", diff --git a/oci/providers.bzl b/oci/providers.bzl index 29b70eb..7b992cd 100755 --- a/oci/providers.bzl +++ b/oci/providers.bzl @@ -64,11 +64,3 @@ OCIPlatform = provider( "variant": "Variant is an optional field specifying a variant of the CPU", }, ) - -OCIImageLayoutInfo = provider( - doc = "This provider represents a list of directories that each contain an OCI Image Layout." + - "See https://github.com/opencontainers/image-spec/blob/main/image-layout.md for a description of the format.", - fields = { - "oci_image_layout_dirs": "A list of directories, each containing an OCI Image Layout." - }, -)