Plumb through layout files (e.g. layers) to OCI Image Layout.

OCI Image Layout needs all layers, as well as the base image files.

go/cmd/ocitool/BUILD.bazel

@@ -5,10 +5,10 @@ go_library(
     srcs = [
         "appendlayer_cmd.go",
         "createlayer_cmd.go",
-        "imagelayout_cmd.go",
         "desc_helpers.go",
         "digest_cmd.go",
         "gen_cmd.go",
+        "imagelayout_cmd.go",
         "index_cmd.go",
         "main.go",
         "manifest_cmd.go",

go/cmd/ocitool/imagelayout_cmd.go

@@ -13,33 +13,42 @@ import (
 	"github.com/urfave/cli/v2"
 )
 
-// Given a slice of baseLayoutPaths, where each path contains an OCI Image Format,
-// return an index that maps sha256 values to paths.
-// If relPath is non-empty, it is prepended to all baseLayoutPaths.
-func getBaseLayoutBlobIndex(baseLayoutPaths []string, relPath string) (blob.Index, error) {
+// Given a slice of layoutFilePaths, where each path contains a file that may
+// be used within an OCI Image Format, return an index that maps sha256 values
+// to paths.
+// If relPath is non-empty, it is prepended to all layoutFilePaths.
+func getLayoutFilesBlobIndex(layoutFilePaths []string, relPath string) (blob.Index, error) {
 	var result blob.Index
 	result.Blobs = make(map[digest.Digest]string)
-
-	for _, p := range baseLayoutPaths {
+	for _, p := range layoutFilePaths {
 		if len(strings.TrimSpace(p)) == 0 {
 			// Ignore empty paths.
 			continue
 		}
-		blobsDir := path.Join(p, "blobs", "sha256")
 		if relPath != "" {
-			blobsDir = path.Join(relPath, blobsDir)
-		}
-		entries, err := os.ReadDir(blobsDir)
-		if err != nil {
-			return blob.Index{}, fmt.Errorf("unable to read OCI Image Format blobs dir. Base layout paths: %v, Relpath: %s, Path: %s, Error: %w", baseLayoutPaths, relPath, blobsDir, err)
+			p = path.Join(relPath, p)
 		}
-		for _, entry := range entries {
-			if !entry.Type().IsRegular() {
-				continue
+		// Use an immediately invoked function here so that defer closes the
+		// file at a suitable time.
+		err := func() error {
+			f, err := os.Open(p)
+			if err != nil {
+				return err
 			}
-			name := entry.Name()
-			result.Blobs[digest.Digest(name)] = path.Join(blobsDir, name)
+			defer f.Close()
+			digester := digest.SHA256.Digester()
+			_, err = f.WriteTo(digester.Hash())
+			if err != nil {
+				return err
+			}
+			digest := digester.Digest()
+			result.Blobs[digest] = p
+			return nil
+		}()
+		if err != nil {
+			return blob.Index{}, err
 		}
+
 	}
 
 	return result, nil
@@ -50,20 +59,20 @@ func getBaseLayoutBlobIndex(baseLayoutPaths []string, relPath string) (blob.Inde
 // for the structure of OCI Image Layout directories.
 func CreateOciImageLayoutCmd(c *cli.Context) error {
 	relPath := c.String("layout-relative")
-	baseLayoutBlobIdx, err := getBaseLayoutBlobIndex(c.StringSlice("base-image-layouts"), relPath)
+	// Load providers that read local files, and create a multiprovider that
+	// contains all of them, as well as providers for base image blobs.
+	providers, err := LoadLocalProviders(c.StringSlice("layout"), relPath)
 	if err != nil {
 		return err
 	}
 
-	// Load providers that read local files, and create a multiprovider that
-	// contains all of them, as well as providers for base image blobs.
-	providers, err := LoadLocalProviders(c.StringSlice("layout"), relPath)
+	layoutFilesBlobIdx, err := getLayoutFilesBlobIndex(c.StringSlice("layout-files"), relPath)
 	if err != nil {
 		return err
 	}
 
-	if len(baseLayoutBlobIdx.Blobs) > 0 {
-		providers = append(providers, &baseLayoutBlobIdx)
+	if len(layoutFilesBlobIdx.Blobs) > 0 {
+		providers = append(providers, &layoutFilesBlobIdx)
 	}
 
 	multiProvider := ociutil.MultiProvider(providers...)

go/cmd/ocitool/main.go

@@ -215,8 +215,8 @@ as described in https://github.com/opencontainers/image-spec/blob/main/image-lay
 					Name: "desc",
 				},
 				&cli.StringSliceFlag{
-					Name:  "base-image-layouts",
-					Usage: "A comma separated list of directory paths, each path containing an OCI Image Layout.",
+					Name:  "layout-files",
+					Usage: "A comma separated list of blob files to be placed in the OCI Image Layout (e.g. image layers).",
 				},
 				&cli.StringFlag{
 					Name:  "out-dir",

go/pkg/ociutil/push.go

@@ -281,7 +281,7 @@ func CopyContent(ctx context.Context, from content.Provider, to content.Ingester
 
 	reader, err := from.ReaderAt(ctx, desc)
 	if err != nil {
-		return fmt.Errorf("failed to create reader from provider: %w", err)
+		return fmt.Errorf("failed to create reader from provider. Descriptor: %+v; Error: %w", desc, err)
 	}
 
 	ref := desc.Digest.String()

oci/defs.bzl

@@ -1,9 +1,9 @@
 """ public API """
 
 load(":image.bzl", _oci_image = "oci_image", _oci_image_index = "oci_image_index", _oci_image_layer = "oci_image_layer")
+load(":oci_image_layout.bzl", _oci_image_layout = "oci_image_layout")
 load(":pull.bzl", _oci_pull = "oci_pull")
 load(":push.bzl", _oci_push = "oci_push")
-load(":oci_image_layout.bzl", _oci_image_layout = "oci_image_layout")
 
 oci_pull = _oci_pull
 oci_push = _oci_push

oci/image.bzl

@@ -1,5 +1,6 @@
 """ image """
-load("@com_github_datadog_rules_oci//oci:providers.bzl", "OCIDescriptor", "OCIImageLayoutInfo", "OCILayout")
+
+load("@com_github_datadog_rules_oci//oci:providers.bzl", "OCIDescriptor", "OCILayout")
 
 # buildifier: disable=function-docstring
 def get_descriptor_file(ctx, desc):
@@ -110,8 +111,6 @@ def _oci_image_index_impl(ctx):
         outputs = outputs,
     )
 
-    oci_layouts = [m[OCIImageLayoutInfo].oci_image_layout_dirs for m in ctx.attr.manifests]
-
     return [
         OCIDescriptor(
             descriptor_file = index_desc_file,
@@ -120,10 +119,6 @@ def _oci_image_index_impl(ctx):
             blob_index = layout_file,
             files = depset(direct = [index_file, layout_file], transitive = [layout_files]),
         ),
-        # Pass through any OCIImageLayoutInfo data from the manifests.
-        OCIImageLayoutInfo(
-            oci_image_layout_dirs = depset(transitive = oci_layouts),
-        ),
         DefaultInfo(
             files = depset(outputs),
         ),
@@ -228,9 +223,6 @@ def _oci_image_impl(ctx):
                 transitive = [base_layout.files],
             ),
         ),
-        OCIImageLayoutInfo(
-            oci_image_layout_dirs = depset(ctx.files.pulled_base if ctx.attr.pulled_base != None else []),
-        ),
         DefaultInfo(
             files = depset([
                 entrypoint_config_file,
@@ -256,13 +248,6 @@ oci_image = rule(
                 OCILayout,
             ],
         ),
-        "pulled_base": attr.label(
-            doc = """A directory that contains the base image in OCI Image Layout format.
-            See https://github.com/opencontainers/image-spec/blob/main/image-layout.md for a description
-            of the OCI Image Layout format. This is optional, and if present, is passed through as an output of oci_image,
-            by the OCIImageLayoutInfo provider.""",
-            allow_single_file = True,
-        ),
         "entrypoint": attr.string_list(
             doc = """A list of entrypoints for the image; these will be inserted into the generated
             OCI image config""",

oci/oci_image_layout.bzl

@@ -1,14 +1,15 @@
-load("@com_github_datadog_rules_oci//oci:providers.bzl", "OCIDescriptor", "OCIImageLayoutInfo", "OCILayout")
+"""A rule to create a directory in OCI Image Layout format."""
+
 load("@com_github_datadog_rules_oci//oci:debug_flag.bzl", "DebugInfo")
+load("@com_github_datadog_rules_oci//oci:providers.bzl", "OCIDescriptor", "OCILayout")
 
 def _oci_image_layout_impl(ctx):
     toolchain = ctx.toolchains["@com_github_datadog_rules_oci//oci:toolchain"]
 
     layout = ctx.attr.manifest[OCILayout]
-    base_layouts = ctx.attr.manifest[OCIImageLayoutInfo]
-    base_layout_dirs = ""
-    if base_layouts != None:
-        base_layout_dirs = ",".join([p.path for p in base_layouts.oci_image_layout_dirs.to_list()])
+
+    # layout_files contains all available blobs for the image.
+    layout_files = ",".join([p.path for p in layout.files.to_list()])
 
     descriptor = ctx.attr.manifest[OCIDescriptor]
     out_dir = ctx.actions.declare_directory(ctx.label.name)
@@ -27,14 +28,14 @@ def _oci_image_layout_impl(ctx):
             # up 3 levels from the bin directory.
             "--layout-relative={root}".format(root = ctx.bin_dir.path + "/../../../"),
             "--desc={desc}".format(desc = descriptor.descriptor_file.path),
-            "--base-image-layouts={base_layouts}".format(base_layouts = base_layout_dirs),
+            "--layout-files={layout_files}".format(layout_files = layout_files),
             "--out-dir={out_dir}".format(out_dir = out_dir.path),
         ],
         inputs =
-            depset(direct = ctx.files.manifest + [layout.blob_index], transitive = [
-                base_layouts.oci_image_layout_dirs,
-                layout.files,
-            ]),
+            depset(
+                direct = ctx.files.manifest + [layout.blob_index],
+                transitive = [layout.files],
+            ),
         outputs = [
             out_dir,
         ],
@@ -73,7 +74,7 @@ oci_image_layout = rule(
             doc = """
                 An OCILayout index to be written to the OCI Image Format directory.
             """,
-            providers = [OCILayout, OCIImageLayoutInfo],
+            providers = [OCILayout],
         ),
         "_debug": attr.label(
             default = "//oci:debug",

oci/providers.bzl

@@ -64,11 +64,3 @@ OCIPlatform = provider(
         "variant": "Variant is an optional field specifying a variant of the CPU",
     },
 )
-
-OCIImageLayoutInfo = provider(
-    doc = "This provider represents a list of directories that each contain an OCI Image Layout." +
-    "See https://github.com/opencontainers/image-spec/blob/main/image-layout.md for a description of the format.",
-    fields = {
-        "oci_image_layout_dirs": "A list of directories, each containing an OCI Image Layout."
-    },
-)