From 49db64dd0cb428cc6859dfc4f6dfbb13548143b7 Mon Sep 17 00:00:00 2001
From: Hardy Jones <88002054+joneshf-dd@users.noreply.github.com>
Date: Mon, 6 Nov 2023 07:42:04 -0800
Subject: [PATCH] LANGTOOLS-2195 Start allowing configuration files (#56)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

## Context

Jira Issue: https://datadoghq.atlassian.net/browse/LANGTOOLS-2195

We've found that sometimes we want to create layers that have many
arguments. Like when we have a ton of files to map. This ends up causing
an issue where the underlying system cannot support so many arguments
and it ends up failing.

We're changing the `ocitool create-layer` command to accept a
configuration file (in JSON format) that can be used to supply arguments
to the command.

There is technically a way to solve this problem without using a JSON
file and making `ocitool create-layer` understand the different syntaxes
that Bazel uses for param files:
https://bazel.build/rules/lib/builtins/Args#use_param_file. Those
syntaxes are hard to deal with, at best. They're written for a very
specific type of CLI, and we don't seem to have an easy way to support
that.

Instead, we choose JSON as our format as it's ubiquitous, supported
natively by Bazel (so we can say `json.encode(…)` in a rule), and
supported by the Go stdlib (so we don't have to try hard to parse it).
It's not a hard requirement, but it makes implemnting support trivial.
---
 go/cmd/ocitool/createlayer_cmd.go | 62 +++++++++++++++++++++++++++----
 go/cmd/ocitool/main.go            |  7 +++-
 2 files changed, 60 insertions(+), 9 deletions(-)

diff --git a/go/cmd/ocitool/createlayer_cmd.go b/go/cmd/ocitool/createlayer_cmd.go
index eda46b6..9ba4c27 100644
--- a/go/cmd/ocitool/createlayer_cmd.go
+++ b/go/cmd/ocitool/createlayer_cmd.go
@@ -3,6 +3,7 @@ package main
 import (
 	"archive/tar"
 	"compress/gzip"
+	"encoding/json"
 	"fmt"
 	"io"
 	"os"
@@ -18,10 +19,15 @@ import (
 )
 
 func CreateLayerCmd(c *cli.Context) error {
-	dir := c.String("dir")
-	files := c.StringSlice("file")
+	config, err := parseConfig(c)
+	if err != nil {
+		return fmt.Errorf("problem parsing config: %w", err)
+	}
+
+	dir := config.Directory
+	files := config.Files
 
-	out, err := os.Create(c.String("out"))
+	out, err := os.Create(config.OutputLayer)
 	if err != nil {
 		return err
 	}
@@ -42,14 +48,14 @@ func CreateLayerCmd(c *cli.Context) error {
 		}
 	}
 
-	for filePath, storePath := range c.Generic("file-map").(*flagutil.KeyValueFlag).Map {
+	for filePath, storePath := range config.FileMapping {
 		err = tarutil.AppendFileToTarWriter(filePath, storePath, tw)
 		if err != nil {
 			return err
 		}
 	}
 
-	for k, v := range c.Generic("symlink").(*flagutil.KeyValueFlag).Map {
+	for k, v := range config.SymlinkMapping {
 		err = tw.WriteHeader(&tar.Header{
 			Typeflag: tar.TypeSymlink,
 			Name:     k,
@@ -71,7 +77,7 @@ func CreateLayerCmd(c *cli.Context) error {
 		Digest:    digester.Digest(),
 	}
 
-	bazelLabel := c.String("bazel-label")
+	bazelLabel := config.BazelLabel
 	if bazelLabel != "" {
 		desc.Annotations = map[string]string{
 			// This will also be added to the image config layer history by append-layers
@@ -79,10 +85,52 @@ func CreateLayerCmd(c *cli.Context) error {
 		}
 	}
 
-	err = ociutil.WriteDescriptorToFile(c.String("outd"), desc)
+	err = ociutil.WriteDescriptorToFile(config.Descriptor, desc)
 	if err != nil {
 		return err
 	}
 
 	return nil
 }
+
+type createLayerConfig struct {
+	BazelLabel     string            `json:"bazel-label" toml:"bazel-label" yaml:"bazel-label"`
+	Descriptor     string            `json:"outd" toml:"outd" yaml:"outd"`
+	Directory      string            `json:"dir" toml:"dir" yaml:"dir"`
+	Files          []string          `json:"file" toml:"file" yaml:"file"`
+	FileMapping    map[string]string `json:"file-map" toml:"file-map" yaml:"file-map"`
+	OutputLayer    string            `json:"out" toml:"out" yaml:"out"`
+	SymlinkMapping map[string]string `json:"symlink" toml:"symlink" yaml:"symlink"`
+}
+
+func newCreateLayerConfig(c *cli.Context) *createLayerConfig {
+	return &createLayerConfig{
+		BazelLabel:     c.String("bazel-label"),
+		Directory:      c.String("dir"),
+		Files:          c.StringSlice("file"),
+		FileMapping:    c.Generic("file-map").(*flagutil.KeyValueFlag).Map,
+		OutputLayer:    c.String("out"),
+		Descriptor:     c.String("outd"),
+		SymlinkMapping: c.Generic("symlink").(*flagutil.KeyValueFlag).Map,
+	}
+}
+
+func parseConfig(c *cli.Context) (*createLayerConfig, error) {
+	configFile := c.Path("configuration-file")
+	if configFile == "" {
+		return newCreateLayerConfig(c), nil
+	}
+
+	file, err := os.ReadFile(configFile)
+	if err != nil {
+		return nil, fmt.Errorf("problem reading config file: %w", err)
+	}
+
+	var config createLayerConfig
+	err = json.Unmarshal(file, &config)
+	if err != nil {
+		return nil, fmt.Errorf("problem parsing config file as JSON: %w", err)
+	}
+
+	return &config, nil
+}
diff --git a/go/cmd/ocitool/main.go b/go/cmd/ocitool/main.go
index 3a5973d..71831a4 100644
--- a/go/cmd/ocitool/main.go
+++ b/go/cmd/ocitool/main.go
@@ -43,9 +43,12 @@ var app = &cli.App{
 			Name:   "create-layer",
 			Action: CreateLayerCmd,
 			Flags: []cli.Flag{
+				&cli.PathFlag{
+					Name:  "configuration-file",
+					Usage: "Path to a configuration file. Useful when there are too many flags to pass at once.",
+				},
 				&cli.StringFlag{
-					Name:     "dir",
-					Required: true,
+					Name: "dir",
 				},
 				&cli.StringSliceFlag{
 					Name: "file",