From 045a79e72779cdc003b7322ecb5d54fa8fe83ce0 Mon Sep 17 00:00:00 2001 From: Kim Mason <133598994+captainreality@users.noreply.github.com> Date: Wed, 30 Oct 2024 15:18:25 -0600 Subject: [PATCH] Add an OCI Image Layout command to the ociutil tool, and a corresponding Bazel rule. (#61) * Add an OCI Image Layout command to the ociutil tool. The spec at https://github.com/opencontainers/image-spec/blob/main/image-layout.md describes an OCI Image Layout directory structure. This commit updates the ociutil tool with a new command to produce such directories based on an input layout, and a list of directories containing OCI Image Layouts for base images. * Create oci_image_layout bazel rule. This commit creates a bazel rule that produces an OCI Image Format directory based on a provided OCI Image index. The OCI Image Layout is a standardized OCI format described in https://github.com/opencontainers/image-spec/blob/main/image-layout.md. * Add required dependencies to _oci_image_layout_impl action inputs, so that the runfiles are correctly populated * Plumb through layout files (e.g. layers) to OCI Image Layout. OCI Image Layout needs all layers, as well as the base image files. * Update docs and docstrings. * Minor code style cleanup. --------- Co-authored-by: Guy Bolton King --- docs/docs.md | 25 +++++ go/cmd/ocitool/BUILD.bazel | 1 + go/cmd/ocitool/imagelayout_cmd.go | 107 ++++++++++++++++++ go/cmd/ocitool/main.go | 22 ++++ go/pkg/ociutil/BUILD.bazel | 1 + go/pkg/ociutil/multiprovider.go | 2 +- go/pkg/ociutil/ociimagelayout.go | 175 ++++++++++++++++++++++++++++++ go/pkg/ociutil/push.go | 2 +- oci/BUILD.bazel | 11 ++ oci/defs.bzl | 2 + oci/image.bzl | 1 + oci/oci_image_layout.bzl | 74 +++++++++++++ oci/providers.bzl | 2 +- 13 files changed, 422 insertions(+), 3 deletions(-) create mode 100644 go/cmd/ocitool/imagelayout_cmd.go create mode 100644 go/pkg/ociutil/ociimagelayout.go create mode 100644 oci/oci_image_layout.bzl diff --git a/docs/docs.md b/docs/docs.md index d525450..4678429 100644 --- a/docs/docs.md +++ b/docs/docs.md @@ -72,6 +72,31 @@ Create a tarball and an OCI descriptor for it | symlinks | Dictionary of symlink -> target entries to place in the tarball | Dictionary: String -> String | optional | `{}` | + + +## oci_image_layout + +
+oci_image_layout(name, manifest)
+
+ +Writes an OCI Image Index and related blobs to an OCI Image Format +directory. See https://github.com/opencontainers/image-spec/blob/main/image-layout.md +for the specification of the OCI Image Format directory. + +All blobs must be provided in the manifest's OCILayout provider, in the +files attribute. If blobs are missing, creation of the OCI Image Layout +will fail. + +**ATTRIBUTES** + + +| Name | Description | Type | Mandatory | Default | +| :------------- | :------------- | :------------- | :------------- | :------------- | +| name | A unique name for this target. | Name | required | | +| manifest | An OCILayout index to be written to the OCI Image Format directory. | Label | optional | `None` | + + ## oci_push diff --git a/go/cmd/ocitool/BUILD.bazel b/go/cmd/ocitool/BUILD.bazel index 0835bc7..127df6b 100644 --- a/go/cmd/ocitool/BUILD.bazel +++ b/go/cmd/ocitool/BUILD.bazel @@ -8,6 +8,7 @@ go_library( "desc_helpers.go", "digest_cmd.go", "gen_cmd.go", + "imagelayout_cmd.go", "index_cmd.go", "main.go", "manifest_cmd.go", diff --git a/go/cmd/ocitool/imagelayout_cmd.go b/go/cmd/ocitool/imagelayout_cmd.go new file mode 100644 index 0000000..cc6a462 --- /dev/null +++ b/go/cmd/ocitool/imagelayout_cmd.go @@ -0,0 +1,107 @@ +package main + +import ( + "fmt" + "os" + "path" + "strings" + + "github.com/DataDog/rules_oci/go/pkg/blob" + "github.com/DataDog/rules_oci/go/pkg/ociutil" + "github.com/containerd/containerd/images" + "github.com/opencontainers/go-digest" + "github.com/urfave/cli/v2" +) + +// Given a slice of layoutFilePaths, where each path contains a file that may +// be used within an OCI Image Format, return an index that maps sha256 values +// to paths. +// If relPath is non-empty, it is prepended to all layoutFilePaths. +func getLayoutFilesBlobIndex(layoutFilePaths []string, relPath string) (blob.Index, error) { + var result blob.Index + result.Blobs = make(map[digest.Digest]string) + for _, p := range layoutFilePaths { + if len(strings.TrimSpace(p)) == 0 { + // Ignore empty paths. + continue + } + if relPath != "" { + p = path.Join(relPath, p) + } + // Use an immediately invoked function here so that defer closes the + // file at a suitable time. + err := func() error { + f, err := os.Open(p) + if err != nil { + return err + } + defer f.Close() + digester := digest.SHA256.Digester() + _, err = f.WriteTo(digester.Hash()) + if err != nil { + return err + } + digest := digester.Digest() + result.Blobs[digest] = p + return nil + }() + if err != nil { + return blob.Index{}, err + } + } + + return result, nil +} + +// This command creates an OCI Image Layout directory based on the layout parameter. +// The layout-files parameter contains a list of files that are used as blobs +// when referenced by desriptors in the layout parameter. +// See https://github.com/opencontainers/image-spec/blob/main/image-layout.md +// for the structure of OCI Image Layout directories. +func CreateOciImageLayoutCmd(c *cli.Context) error { + relPath := c.String("layout-relative") + // Load providers that read local files, and create a multiprovider that + // contains all of them, as well as providers for base image blobs. + providers, err := LoadLocalProviders(c.StringSlice("layout"), relPath) + if err != nil { + return err + } + + layoutFilesBlobIdx, err := getLayoutFilesBlobIndex(c.StringSlice("layout-files"), relPath) + if err != nil { + return err + } + + if len(layoutFilesBlobIdx.Blobs) > 0 { + providers = append(providers, &layoutFilesBlobIdx) + } + + multiProvider := ociutil.MultiProvider(providers...) + + descriptorFile := c.String("desc") + baseDesc, err := ociutil.ReadDescriptorFromFile(descriptorFile) + if err != nil { + return fmt.Errorf("failed to read base descriptor: %w", err) + } + + outDir := c.String("out-dir") + ociIngester, err := ociutil.NewOciImageLayoutIngester(outDir) + if err != nil { + return err + } + + // Copy the children first; leave the parent (index) to last. + imagesHandler := images.ChildrenHandler(multiProvider) + err = ociutil.CopyChildrenFromHandler(c.Context, imagesHandler, multiProvider, &ociIngester, baseDesc) + if err != nil { + return fmt.Errorf("failed to copy child content to OCI Image Layout: %w", err) + } + + // copy the parent last (in case of image index) + err = ociutil.CopyContent(c.Context, multiProvider, &ociIngester, baseDesc) + if err != nil { + return fmt.Errorf("failed to copy parent content to OCI Image Layout: %w", err) + } + + return nil +} diff --git a/go/cmd/ocitool/main.go b/go/cmd/ocitool/main.go index 71831a4..21dd325 100644 --- a/go/cmd/ocitool/main.go +++ b/go/cmd/ocitool/main.go @@ -202,6 +202,28 @@ var app = &cli.App{ }, }, }, + { + Name: "create-oci-image-layout", + Description: `Creates a directory containing an OCI Image Layout based on the input layout, +as described in https://github.com/opencontainers/image-spec/blob/main/image-layout.md.`, + Action: CreateOciImageLayoutCmd, + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "layout-relative", + }, + &cli.StringFlag{ + Name: "desc", + }, + &cli.StringSliceFlag{ + Name: "layout-files", + Usage: "A comma separated list of blob files to be placed in the OCI Image Layout (e.g. image layers).", + }, + &cli.StringFlag{ + Name: "out-dir", + Usage: "The directory that the OCI Image Layout will be written to.", + }, + }, + }, { Name: "push-blob", Hidden: true, diff --git a/go/pkg/ociutil/BUILD.bazel b/go/pkg/ociutil/BUILD.bazel index 56b1a0a..68da8f6 100644 --- a/go/pkg/ociutil/BUILD.bazel +++ b/go/pkg/ociutil/BUILD.bazel @@ -13,6 +13,7 @@ go_library( "json.go", "manifest.go", "multiprovider.go", + "ociimagelayout.go", "platforms.go", "provider.go", "push.go", diff --git a/go/pkg/ociutil/multiprovider.go b/go/pkg/ociutil/multiprovider.go index 64a31a6..d380ed5 100644 --- a/go/pkg/ociutil/multiprovider.go +++ b/go/pkg/ociutil/multiprovider.go @@ -8,7 +8,7 @@ import ( ocispec "github.com/opencontainers/image-spec/specs-go/v1" ) -// MultiProvider will read from the first provider that can read the requrested +// MultiProvider will read from the first provider that can read the requested // descriptor. func MultiProvider(providers ...content.Provider) content.Provider { return &multiProvider{ diff --git a/go/pkg/ociutil/ociimagelayout.go b/go/pkg/ociutil/ociimagelayout.go new file mode 100644 index 0000000..6647f09 --- /dev/null +++ b/go/pkg/ociutil/ociimagelayout.go @@ -0,0 +1,175 @@ +package ociutil + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "time" + + "github.com/containerd/containerd/content" + "github.com/opencontainers/go-digest" +) + +const BlobsFolderName = "blobs" +const OciImageIndexMediaType = "application/vnd.oci.image.index.v1+json" +const OciLayoutFileName = "oci-layout" +const OciLayoutFileContent = `{ + "imageLayoutVersion": "1.0.0" +}` +const OciIndexFileName = "index.json" +const ContentFileMode = 0755 + +// OciImageLayoutIngester implements functionality to write data to an OCI +// Image Layout directory (https://github.com/opencontainers/image-spec/blob/main/image-layout.md) +type OciImageLayoutIngester struct { + // The path of the directory containing the OCI Image Layout. + Path string +} + +func NewOciImageLayoutIngester(path string) (OciImageLayoutIngester, error) { + if err := os.MkdirAll(path, ContentFileMode); err != nil { + return OciImageLayoutIngester{}, fmt.Errorf("error creating directory for OciImageLayoutIngester: %v, Err: %w", path, err) + } + return OciImageLayoutIngester{Path: path}, nil +} + +// writer returns a Writer object that will write one entity to the OCI Image Layout. +// Examples are OCI Image Index, an OCI Image Manifest, an OCI Image Config, +// and OCI image TAR/GZIP files. +func (ing *OciImageLayoutIngester) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) { + // Initialize the writer options (for those unfamiliar with this pattern, it's known as the + // "functional options pattern"). + var wOpts content.WriterOpts + for _, o := range opts { + if err := o(&wOpts); err != nil { + return nil, fmt.Errorf("unable to apply WriterOpt to WriterOpts. Err: %w", err) + } + } + status := content.Status{ + Ref: wOpts.Ref, + Offset: 0, + Total: wOpts.Desc.Size, + Expected: wOpts.Desc.Digest, + } + return &OciImageLayoutWriter{Path: ing.Path, Opts: wOpts, Stat: status}, nil +} + +type OciImageLayoutWriter struct { + Path string + Opts content.WriterOpts + Dig digest.Digest + Stat content.Status +} + +// Writes bytes to a file, using the provided write flags. +func writeFile(filePath string, writeFlags int, b []byte) error { + f, err := os.OpenFile(filePath, writeFlags, ContentFileMode) + if err != nil { + return fmt.Errorf("error opening file for write: %v, Err: %w", filePath, err) + } + defer f.Close() + + if _, err = f.Write(b); err != nil { + return fmt.Errorf("error writing file: %v, Err: %w", filePath, err) + } + return nil +} + +func (w *OciImageLayoutWriter) Write(b []byte) (n int, err error) { + firstWrite := w.Stat.StartedAt.IsZero() + if firstWrite { + w.Stat.StartedAt = time.Now() + } + + // A function to get the OS flags used to create a writeable file. + getWriteFlags := func(filePath string) (int, error) { + _, err := os.Stat(filePath) + switch { + case firstWrite && err == nil: + // The file exists and it's first write; Truncate it. + return os.O_WRONLY | os.O_TRUNC, nil + case err == nil: + // The file exists and it's not first write; append to it. + return os.O_WRONLY | os.O_APPEND, nil + case errors.Is(err, os.ErrNotExist): + // The file doesn't exist. Create it. + return os.O_WRONLY | os.O_CREATE, nil + default: + // Something went wrong! + return 0, err + } + } + + var filePath string + if w.Opts.Desc.MediaType == OciImageIndexMediaType { + // This is an OCI Image Index. It gets written to the top level index.json file. + // Write the oci-layout file (a simple canned file required by the standard). + layoutFile := path.Join(w.Path, OciLayoutFileName) + // It's possible (but unlikely) for Write to be called repeatedly for an index file. + // In that case, we'll repeatedly rewrite the oci-layout file, which doesn't hurt, + // because the content is always identical. + if err := os.WriteFile(layoutFile, []byte(OciLayoutFileContent), ContentFileMode); err != nil { + return 0, fmt.Errorf("error writing oci-layout file: %v, Err: %w", layoutFile, err) + } + + // Now write the index.json file. + filePath = path.Join(w.Path, OciIndexFileName) + writeFlags, err := getWriteFlags(filePath) + if err != nil { + return 0, fmt.Errorf("error stat'ing file: %v, Err: %w", filePath, err) + } + err = writeFile(filePath, writeFlags, b) + if err != nil { + return 0, err + } + } else { + // This is a blob. Write it to the blobs folder. + algo := w.Opts.Desc.Digest.Algorithm() + blobDir := path.Join(w.Path, BlobsFolderName, algo.String()) + if err := os.MkdirAll(blobDir, ContentFileMode); err != nil { + return 0, fmt.Errorf("error creating blobDir: %v, Err: %w", blobDir, err) + } + filePath = path.Join(blobDir, w.Opts.Desc.Digest.Encoded()) + + writeFlags, err := getWriteFlags(filePath) + if err != nil { + return 0, fmt.Errorf("error stat'ing file: %v, Err: %w", filePath, err) + } + + // Now write the blob file. + err = writeFile(filePath, writeFlags, b) + if err != nil { + return 0, err + } + } + fInfo, err := os.Stat(filePath) + if err != nil { + return 0, fmt.Errorf("error retrieving FileInfo for file: %v, Err: %w", filePath, err) + } + w.Stat.UpdatedAt = fInfo.ModTime() + return len(b), nil +} + +func (w *OciImageLayoutWriter) Close() error { + return nil +} + +// Returns an empty digest until after Commit is called. +func (w *OciImageLayoutWriter) Digest() digest.Digest { + return w.Dig +} + +func (w *OciImageLayoutWriter) Commit(ctx context.Context, size int64, expected digest.Digest, opts ...content.Opt) error { + w.Dig = w.Opts.Desc.Digest + return nil +} + +func (w *OciImageLayoutWriter) Status() (content.Status, error) { + return w.Stat, nil +} + +func (w *OciImageLayoutWriter) Truncate(size int64) error { + return errors.New("truncation is unsupported") +} diff --git a/go/pkg/ociutil/push.go b/go/pkg/ociutil/push.go index 274b293..e2605ad 100644 --- a/go/pkg/ociutil/push.go +++ b/go/pkg/ociutil/push.go @@ -281,7 +281,7 @@ func CopyContent(ctx context.Context, from content.Provider, to content.Ingester reader, err := from.ReaderAt(ctx, desc) if err != nil { - return fmt.Errorf("failed to create reader from ingestor: %w", err) + return fmt.Errorf("failed to create reader from provider. Descriptor: %+v; Error: %w", desc, err) } ref := desc.Digest.String() diff --git a/oci/BUILD.bazel b/oci/BUILD.bazel index 6443160..eff474d 100755 --- a/oci/BUILD.bazel +++ b/oci/BUILD.bazel @@ -38,6 +38,7 @@ bzl_library( visibility = ["//visibility:public"], deps = [ ":image", + ":oci_image_layout", ":pull", ":push", ], @@ -64,6 +65,16 @@ bzl_library( deps = ["@com_github_datadog_rules_oci//oci:providers"], ) +bzl_library( + name = "oci_image_layout", + srcs = ["oci_image_layout.bzl"], + visibility = ["//visibility:public"], + deps = [ + "@com_github_datadog_rules_oci//oci:debug_flag", + "@com_github_datadog_rules_oci//oci:providers", + ], +) + bzl_library( name = "push", srcs = ["push.bzl"], diff --git a/oci/defs.bzl b/oci/defs.bzl index c7f87df..5f461f4 100755 --- a/oci/defs.bzl +++ b/oci/defs.bzl @@ -1,6 +1,7 @@ """ public API """ load(":image.bzl", _oci_image = "oci_image", _oci_image_index = "oci_image_index", _oci_image_layer = "oci_image_layer") +load(":oci_image_layout.bzl", _oci_image_layout = "oci_image_layout") load(":pull.bzl", _oci_pull = "oci_pull") load(":push.bzl", _oci_push = "oci_push") @@ -10,3 +11,4 @@ oci_push = _oci_push oci_image = _oci_image oci_image_index = _oci_image_index oci_image_layer = _oci_image_layer +oci_image_layout = _oci_image_layout diff --git a/oci/image.bzl b/oci/image.bzl index a5953cd..8727bf1 100755 --- a/oci/image.bzl +++ b/oci/image.bzl @@ -48,6 +48,7 @@ def _oci_image_layer_impl(ctx): return [ OCIDescriptor( descriptor_file = descriptor_file, + file = ctx.outputs.layer, ), ] diff --git a/oci/oci_image_layout.bzl b/oci/oci_image_layout.bzl new file mode 100644 index 0000000..ce631d1 --- /dev/null +++ b/oci/oci_image_layout.bzl @@ -0,0 +1,74 @@ +"""A rule to create a directory in OCI Image Layout format.""" + +load("@com_github_datadog_rules_oci//oci:debug_flag.bzl", "DebugInfo") +load("@com_github_datadog_rules_oci//oci:providers.bzl", "OCIDescriptor", "OCILayout") + +def _oci_image_layout_impl(ctx): + toolchain = ctx.toolchains["@com_github_datadog_rules_oci//oci:toolchain"] + + layout = ctx.attr.manifest[OCILayout] + + # layout_files contains all available blobs for the image. + layout_files = ",".join([p.path for p in layout.files.to_list()]) + + descriptor = ctx.attr.manifest[OCIDescriptor] + out_dir = ctx.actions.declare_directory(ctx.label.name) + + ctx.actions.run( + executable = toolchain.sdk.ocitool, + arguments = [ + "--layout={layout}".format(layout = layout.blob_index.path), + "--debug={debug}".format(debug = str(ctx.attr._debug[DebugInfo].debug)), + "create-oci-image-layout", + # We need to use the directory one level above bazel-out for the + # layout-relative directory. This is because the paths in + # oci_image_index's index.layout.json are of the form: + # "bazel-out/os_arch-fastbuild/bin/...". Unfortunately, bazel + # provides no direct way to access this directory, so here we traverse + # up 3 levels from the bin directory. + "--layout-relative={root}".format(root = ctx.bin_dir.path + "/../../../"), + "--desc={desc}".format(desc = descriptor.descriptor_file.path), + "--layout-files={layout_files}".format(layout_files = layout_files), + "--out-dir={out_dir}".format(out_dir = out_dir.path), + ], + inputs = + depset( + direct = ctx.files.manifest + [layout.blob_index], + transitive = [layout.files], + ), + outputs = [ + out_dir, + ], + use_default_shell_env = True, + ) + + return DefaultInfo(files = depset([out_dir])) + +oci_image_layout = rule( + doc = """ + Writes an OCI Image Index and related blobs to an OCI Image Format + directory. See https://github.com/opencontainers/image-spec/blob/main/image-layout.md + for the specification of the OCI Image Format directory. + + All blobs must be provided in the manifest's OCILayout provider, in the + files attribute. If blobs are missing, creation of the OCI Image Layout + will fail. + """, + implementation = _oci_image_layout_impl, + attrs = { + "manifest": attr.label( + doc = """ + An OCILayout index to be written to the OCI Image Format directory. + """, + providers = [OCILayout], + ), + "_debug": attr.label( + default = "//oci:debug", + providers = [DebugInfo], + ), + }, + provides = [ + DefaultInfo, + ], + toolchains = ["@com_github_datadog_rules_oci//oci:toolchain"], +) diff --git a/oci/providers.bzl b/oci/providers.bzl index 47ae6aa..7b992cd 100755 --- a/oci/providers.bzl +++ b/oci/providers.bzl @@ -48,7 +48,7 @@ OCIImageManifest = provider( OCIImageIndexManifest = provider( doc = "", fields = { - "manifests": "List of desciptors", + "manifests": "List of descriptors", "annotations": "String map of arbitrary metadata", }, )