diff --git a/client/client_test.go b/client/client_test.go index 1912721..6792231 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -367,7 +367,7 @@ func TestDeleteResourceShouldWork(t *testing.T) { responder, ) - resource, err := resource.FromYamlByte([]byte(`{"apiVersion":"v2","kind":"Topic","metadata":{"name":"toto","cluster":"local"},"spec":{}}`)) + resource, err := resource.FromYamlByte([]byte(`{"apiVersion":"v2","kind":"Topic","metadata":{"name":"toto","cluster":"local"},"spec":{}}`), true) if err != nil { t.Error(err) } diff --git a/cmd/apply.go b/cmd/apply.go index 852999a..04df3d1 100644 --- a/cmd/apply.go +++ b/cmd/apply.go @@ -11,20 +11,20 @@ import ( var dryRun *bool -func resourceForPath(path string) ([]resource.Resource, error) { +func resourceForPath(path string, strict bool) ([]resource.Resource, error) { directory, err := isDirectory(path) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) os.Exit(1) } if directory { - return resource.FromFolder(path) + return resource.FromFolder(path, strict) } else { - return resource.FromFile(path) + return resource.FromFile(path, strict) } } -func initApply(kinds schema.KindCatalog) { +func initApply(kinds schema.KindCatalog, strict bool) { // applyCmd represents the apply command var filePath *[]string var applyCmd = &cobra.Command{ @@ -32,7 +32,7 @@ func initApply(kinds schema.KindCatalog) { Short: "Upsert a resource on Conduktor", Long: ``, Run: func(cmd *cobra.Command, args []string) { - resources := loadResourceFromFileFlag(*filePath) + resources := loadResourceFromFileFlag(*filePath, strict) schema.SortResourcesForApply(kinds, resources, *debug) allSuccess := true for _, resource := range resources { diff --git a/cmd/delete.go b/cmd/delete.go index 1dc5496..0f45f50 100644 --- a/cmd/delete.go +++ b/cmd/delete.go @@ -8,7 +8,7 @@ import ( "github.com/spf13/cobra" ) -func initDelete(kinds schema.KindCatalog) { +func initDelete(kinds schema.KindCatalog, strict bool) { var filePath *[]string var deleteCmd = &cobra.Command{ Use: "delete", @@ -17,7 +17,7 @@ func initDelete(kinds schema.KindCatalog) { Args: cobra.NoArgs, Run: func(cmd *cobra.Command, args []string) { // Root command does nothing - resources := loadResourceFromFileFlag(*filePath) + resources := loadResourceFromFileFlag(*filePath, strict) schema.SortResourcesForDelete(kinds, resources, *debug) allSuccess := true for _, resource := range resources { diff --git a/cmd/load_resource_utils.go b/cmd/load_resource_utils.go index 0fbcd72..fc59694 100644 --- a/cmd/load_resource_utils.go +++ b/cmd/load_resource_utils.go @@ -7,10 +7,10 @@ import ( "github.com/conduktor/ctl/resource" ) -func loadResourceFromFileFlag(filePath []string) []resource.Resource { +func loadResourceFromFileFlag(filePath []string, strict bool) []resource.Resource { var resources = make([]resource.Resource, 0) for _, path := range filePath { - r, err := resourceForPath(path) + r, err := resourceForPath(path, strict) if err != nil { fmt.Fprintf(os.Stderr, "%s\n", err) os.Exit(1) diff --git a/cmd/root.go b/cmd/root.go index 618a6ea..f619ae2 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -78,9 +78,10 @@ func init() { kinds[k] = v } debug = rootCmd.PersistentFlags().BoolP("verbose", "v", false, "show more information for debugging") + var permissive = rootCmd.PersistentFlags().Bool("permissive", false, "permissive mode, allow undefined environment variables") initGet(kinds) - initDelete(kinds) - initApply(kinds) + initDelete(kinds, !*permissive) + initApply(kinds, !*permissive) initConsoleMkKind() initGatewayMkKind() initPrintKind(kinds) diff --git a/resource/resource.go b/resource/resource.go index 099a402..a529f68 100644 --- a/resource/resource.go +++ b/resource/resource.go @@ -7,6 +7,7 @@ import ( "io" "os" "path/filepath" + "regexp" "strings" gabs "github.com/Jeffail/gabs/v2" @@ -65,16 +66,16 @@ type forParsingStruct struct { Spec map[string]interface{} } -func FromFile(path string) ([]Resource, error) { +func FromFile(path string, strict bool) ([]Resource, error) { data, err := os.ReadFile(path) if err != nil { return nil, err } - return FromYamlByte(data) + return FromYamlByte(data, strict) } -func FromFolder(path string) ([]Resource, error) { +func FromFolder(path string, strict bool) ([]Resource, error) { dirEntry, err := os.ReadDir(path) if err != nil { return nil, err @@ -82,7 +83,7 @@ func FromFolder(path string) ([]Resource, error) { var result = make([]Resource, 0) for _, entry := range dirEntry { if !entry.IsDir() && (strings.HasSuffix(entry.Name(), ".yml") || strings.HasSuffix(entry.Name(), ".yaml")) { - resources, err := FromFile(filepath.Join(path, entry.Name())) + resources, err := FromFile(filepath.Join(path, entry.Name()), strict) result = append(result, resources...) if err != nil { return nil, err @@ -93,7 +94,8 @@ func FromFolder(path string) ([]Resource, error) { return result, nil } -func FromYamlByte(data []byte) ([]Resource, error) { +func FromYamlByte(data []byte, strict bool) ([]Resource, error) { + data = expandEnvVars(data, strict) reader := bytes.NewReader(data) var yamlData interface{} results := make([]Resource, 0, 2) @@ -118,6 +120,47 @@ func FromYamlByte(data []byte) ([]Resource, error) { return results, nil } +var envVarRegex = regexp.MustCompile(`\$\{([^}]+)\}`) + +// expandEnv replaces ${var} or $var in config according to the values of the current environment variables. +// The replacement is case-sensitive. References to undefined variables are replaced by the empty string. +// A default value can be given by using the form ${var:-default value}. +func expandEnvVars(input []byte, strict bool) []byte { + missingEnvVars := make([]string, 0) + result := envVarRegex.ReplaceAllFunc(input, func(match []byte) []byte { + varName := string(match[2 : len(match)-1]) + defaultValue := "" + if strings.Contains(varName, ":-") { + parts := strings.SplitN(varName, ":-", 2) + varName = parts[0] + defaultValue = parts[1] + } + value, isFound := os.LookupEnv(varName) + + // use default value + if (!isFound || value == "") && defaultValue != "" { + return []byte(defaultValue) + } + + if strict { + if (!isFound || value == "") && defaultValue == "" { + missingEnvVars = append(missingEnvVars, varName) + return []byte("") + } + } else { + if !isFound && defaultValue == "" { + missingEnvVars = append(missingEnvVars, varName) + return []byte("") + } + } + return []byte(value) + }) + if len(missingEnvVars) > 0 { + panic(fmt.Sprintf("Missing environment variables: %s", strings.Join(missingEnvVars, ", "))) + } + return result +} + func extractKeyFromMetadataMap(m map[string]interface{}, key string) (string, error) { if val, ok := m[key]; ok { if str, ok := val.(string); ok { diff --git a/resource/resource_test.go b/resource/resource_test.go index 4bb0c10..d5c9f2c 100644 --- a/resource/resource_test.go +++ b/resource/resource_test.go @@ -76,7 +76,7 @@ metadata: name: cg1 `) - results, err := FromYamlByte(yamlByte) + results, err := FromYamlByte(yamlByte, true) spew.Dump(results) if err != nil { t.Error(err) @@ -104,7 +104,7 @@ metadata: } func TestFromFolder(t *testing.T) { - resources, err := FromFolder("yamls") + resources, err := FromFolder("yamls", true) if err != nil { t.Fatal(err) } @@ -149,6 +149,53 @@ func TestFromFolder(t *testing.T) { }) } +func TestResourceExpansionVariableEnv(t *testing.T) { + topicDesc, err := os.CreateTemp("/tmp", "topic.md") + if err != nil { + t.Fatal(err) + } + defer topicDesc.Close() + defer os.Remove(topicDesc.Name()) + if _, err := topicDesc.Write([]byte(`This topic is awesome`)); err != nil { + log.Fatal(err) + } + + yamlByte := []byte(` +# comment +--- +apiVersion: v1 +kind: Topic +metadata: + cluster: ${CLUSTER_NAME} + name: ${TOPIC_NAME:-toto} + labels: + conduktor.io/descriptionFile: ` + topicDesc.Name() + ` +spec: + replicationFactor: 2 + partition: 3 +`) + os.Setenv("CLUSTER_NAME", "cluster-a") + + results, err := FromYamlByte(yamlByte, true) + spew.Dump(results) + if err != nil { + t.Error(err) + } + + if len(results) != 1 { + t.Errorf("results expected of length 1, got length %d", len(results)) + } + + checkResourceWithoutJsonOrder(t, results[0], Resource{ + Version: "v1", + Kind: "Topic", + Name: "toto", + Metadata: map[string]interface{}{"cluster": "cluster-a", "name": "toto", "labels": map[string]interface{}{"conduktor.io/description": "This topic is awesome"}}, + Spec: map[string]interface{}{"replicationFactor": 2.0, "partition": 3.0}, + Json: []byte(`{"apiVersion":"v1","kind":"Topic","metadata":{"cluster":"cluster-a","name":"toto","labels":{"conduktor.io/description":"This topic is awesome"}},"spec":{"replicationFactor":2,"partition":3}}`), + }) +} + func TestResourceExpansionForTopic(t *testing.T) { topicDesc, err := os.CreateTemp("/tmp", "topic.md") if err != nil { @@ -175,7 +222,7 @@ spec: partition: 3 `) - results, err := FromYamlByte(yamlByte) + results, err := FromYamlByte(yamlByte, true) spew.Dump(results) if err != nil { t.Error(err) @@ -266,7 +313,7 @@ spec: schemaFile: ` + jsonSchema.Name() + ` `) - results, err := FromYamlByte(yamlByte) + results, err := FromYamlByte(yamlByte, true) spew.Dump(results) if err != nil { t.Error(err)