From 494a53f33ed4a7b28c31e5d9d4e9b6f2f0d3d811 Mon Sep 17 00:00:00 2001 From: Martin Schuppert Date: Fri, 11 Oct 2024 11:40:13 +0200 Subject: [PATCH] Add additional template functions Adds templat functions which: - to exec an inline template - indent a template - remove n continuous empty lines from a template An example would be a template like: ``` {{define "my-template"}}new template content with empty lines to remove {{end}} Some other template content and add the rendered from my-template {{$var := execTempl "my-template" . | removeNewLines 1}} {{$var}} ``` Co-authored-by: auniyal61 Signed-off-by: Martin Schuppert --- .pre-commit-config.yaml | 6 +- modules/common/util/template_util.go | 98 ++++++- modules/common/util/template_util_test.go | 259 ++++++++++++++++++ .../templates/testservice/config/bar.conf | 18 ++ 4 files changed, 374 insertions(+), 7 deletions(-) create mode 100644 modules/common/util/testdata/templates/testservice/config/bar.conf diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 1e3c10c0..4d8b44d5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -40,4 +40,8 @@ repos: exclude: ^vendor - id: no-commit-to-branch - id: trailing-whitespace - exclude: ^vendor + exclude: | + (?x)( + ^vendor| + ^modules/common/util/template_util_test.go + ) diff --git a/modules/common/util/template_util.go b/modules/common/util/template_util.go index a641bb89..0b15e832 100644 --- a/modules/common/util/template_util.go +++ b/modules/common/util/template_util.go @@ -17,6 +17,7 @@ limitations under the License. package util import ( + "bufio" "bytes" "fmt" "os" @@ -138,6 +139,82 @@ func ExecuteTemplate(templateFile string, data interface{}) (string, error) { return renderedTemplate, nil } +// template functions +var tmpl *template.Template + +// template function which allows to execute a template from within +// a template file. +// name - name of the template as defined with with `{{define "some-template"}}your template{{end}} +// data - data to pass into to render the template for all can use `.` +func execTempl(name string, data interface{}) (string, error) { + buf := &bytes.Buffer{} + err := tmpl.ExecuteTemplate(buf, name, data) + return buf.String(), err +} + +// template function to indent the template with n tabs +func indent(n int, in string) string { + var out string + s := bufio.NewScanner(bytes.NewReader([]byte(in))) + for s.Scan() { + line := strings.TrimSpace(s.Text()) + for i := 0; i < n; i++ { + line = "\t" + line + } + out += line + "\n" + } + return out +} + +// template function to remove empty lines if there are > n continuous empty lines +func removeNewLines(n int, in string) string { + var out string + s := bufio.NewScanner(bytes.NewReader([]byte(in))) + + // Variable to keep track of consecutive empty lines + emptyLineCount := 0 + for s.Scan() { + line := s.Text() + + if strings.TrimSpace(line) == "" { + emptyLineCount++ + // If we have already seen more then n empty lines, skip this one + if emptyLineCount > n { + continue + } + } else { + // Reset the empty line counter when we encounter a non-empty line + emptyLineCount = 0 + } + + out += line + "\n" + } + return out +} + +// This function removes extra space and new-lines from conf data. +func removeNewLinesInSections(in string) string { + var out string + s := bufio.NewScanner(bytes.NewReader([]byte(in))) + + for s.Scan() { + line := strings.TrimSpace(s.Text()) + + if line != "" { + if strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]") { + // new section-header + if len(out) > 0 { + out += "\n" + } + } + + out += line + "\n" + } + } + + return out +} + // template function to increment an int func add(x, y int) int { return x + y @@ -153,11 +230,16 @@ func lower(s string) string { func ExecuteTemplateData(templateData string, data interface{}) (string, error) { var buff bytes.Buffer + var err error funcs := template.FuncMap{ - "add": add, - "lower": lower, + "add": add, + "execTempl": execTempl, + "indent": indent, + "lower": lower, + "removeNewLines": removeNewLines, + "removeNewLinesInSections": removeNewLinesInSections, } - tmpl, err := template.New("tmp").Option("missingkey=error").Funcs(funcs).Parse(templateData) + tmpl, err = template.New("tmp").Option("missingkey=error").Funcs(funcs).Parse(templateData) if err != nil { return "", err } @@ -193,10 +275,14 @@ func ExecuteTemplateFile(filename string, data interface{}) (string, error) { file := string(b) var buff bytes.Buffer funcs := template.FuncMap{ - "add": add, - "lower": lower, + "add": add, + "execTempl": execTempl, + "indent": indent, + "lower": lower, + "removeNewLines": removeNewLines, + "removeNewLinesInSections": removeNewLinesInSections, } - tmpl, err := template.New("tmp").Option("missingkey=error").Funcs(funcs).Parse(file) + tmpl, err = template.New("tmp").Option("missingkey=error").Funcs(funcs).Parse(file) if err != nil { return "", err } diff --git a/modules/common/util/template_util_test.go b/modules/common/util/template_util_test.go index ef0a8989..3b3e3619 100644 --- a/modules/common/util/template_util_test.go +++ b/modules/common/util/template_util_test.go @@ -37,6 +37,243 @@ func TestLower(t *testing.T) { }) } +func TestIndent(t *testing.T) { + + t.Run("Indent string", func(t *testing.T) { + g := NewWithT(t) + const in = `foo +bar` + // 5 tabs and line break + const expct = ` foo + bar +` + + s := indent(5, in) + + g.Expect(s).To(BeIdenticalTo(expct)) + }) +} + +func TestRemoveNewLines(t *testing.T) { + + t.Run("Remove duplicate new lines", func(t *testing.T) { + g := NewWithT(t) + const in = ` foo + + bar + + +foo + + + + +bar` + + const expct = ` foo + + bar + +foo + +bar +` + + s := removeNewLines(1, in) + + g.Expect(s).To(BeIdenticalTo(expct)) + }) +} + +func TestExecTempl(t *testing.T) { + + t.Run("ExecTempl", func(t *testing.T) { + g := NewWithT(t) + const myTmpl = `{{define "my-template"}}my-template + + +content + + + +with empty lines + + + +to +remove +{{end}} +See result: +{{$var := execTempl "my-template" . | removeNewLines 1}} +{{$var}}` + + // render template using execTempl and remove more then 1 continuous empty lines + const expct = ` +See result: + +my-template + +content + +with empty lines + +to +remove +` + renderedTemplate, err := ExecuteTemplateData(myTmpl, "") + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(renderedTemplate).To(BeIdenticalTo(expct)) + }) +} + +func TestRemoveNewLinesInSections(t *testing.T) { + tests := []struct { + name string + raw string + cleaned string + }{ + { + name: "Empty input", + raw: "", + cleaned: "", + }, + { + name: "Single empty line", + raw: "\n", + cleaned: "", + }, + { + name: "Two empty lines", + raw: "\n\n", + cleaned: "", + }, + { + name: "Insert newline at end of file", + raw: "foo", + cleaned: "foo\n", + }, + { + name: "Remove starting empty line", + raw: "\nfoo", + cleaned: "foo\n", + }, + { + name: "Remove starting empty lines", + raw: "\n\nfoo", + cleaned: "foo\n", + }, + { + name: "Remove extra empty line at the end", + raw: "foo\n\n", + cleaned: "foo\n", + }, + { + name: "Remove extra empty lines at the end", + raw: "foo\n\n\n", + cleaned: "foo\n", + }, + { + name: "Keep subsequent data lines", + raw: "foo\nbar", + cleaned: "foo\nbar\n", + }, + { + name: "Remove empty line between subsequent data", + raw: "foo\n\nbar", + cleaned: "foo\nbar\n", + }, + { + name: "Extra spaces around data lines are not kept", + raw: "\n\n foo \n\n bar ", + cleaned: "foo\nbar\n", + }, + { + name: "Extra spaces around section lines are not kept", + raw: "\n\n [foo] \n\n [bar] ", + cleaned: "[foo]\n\n[bar]\n", + }, + { + name: "Remove extra lines with spaces only", + raw: " \n \nfoo\n \nbar\n \n ", + cleaned: "foo\nbar\n", + }, + { + name: "Remove starting empty line from section header", + raw: "\n[foo]", + cleaned: "[foo]\n", + }, + { + name: "Remove starting empty lines from section header", + raw: "\n\n[foo]", + cleaned: "[foo]\n", + }, + { + name: "Remove extra empty line after section header", + raw: "[foo]\n\n", + cleaned: "[foo]\n", + }, + { + name: "Remove extra empty lines after section header", + raw: "[foo]\n\n\n", + cleaned: "[foo]\n", + }, + { + name: "Insert empty line after section header at the end", + raw: "[foo]", + cleaned: "[foo]\n", + }, + { + name: "Keep one empty line between section headers", + raw: "[foo]\n\n[bar]", + cleaned: "[foo]\n\n[bar]\n", + }, + { + name: "Insert one empty line between section headers", + raw: "[foo]\n[bar]", + cleaned: "[foo]\n\n[bar]\n", + }, + { + name: "Remove more empty lines between section headers", + raw: "[foo]\n\n\n[bar]", + cleaned: "[foo]\n\n[bar]\n", + }, + { + name: "Remove extra empty line between section header and data", + raw: "[foo]\n\nbar", + cleaned: "[foo]\nbar\n", + }, + { + name: "Remove extra empty lines between section header and data", + raw: "[foo]\n\n\nbar", + cleaned: "[foo]\nbar\n", + }, + { + name: "Insert extra line between sections", + raw: "[foo]\nbar\n[goo]\nbaz", + cleaned: "[foo]\nbar\n\n[goo]\nbaz\n", + }, + { + name: "Remove extra lines between sections", + raw: "[foo]\nbar\n\n\n[goo]\nbaz", + cleaned: "[foo]\nbar\n\n[goo]\nbaz\n", + }, + { + name: "Insert no new line when there is a parameter value which brackets", + raw: "[foo]\nkey=[value]\n[bar]", + cleaned: "[foo]\nkey=[value]\n\n[bar]\n", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + + cleaned := removeNewLinesInSections(tt.raw) + g.Expect(cleaned).To(Equal(tt.cleaned)) + }) + } +} + func TestGetTemplatesPath(t *testing.T) { // set the env var used to specify the template path in the container case os.Setenv("OPERATOR_TEMPLATES", templatePath) @@ -74,6 +311,7 @@ func TestGetAllTemplates(t *testing.T) { tmplType: TemplateTypeConfig, version: "", want: []string{ + filepath.Join(path.Dir(filename), templatePath, "testservice", "config", "bar.conf"), filepath.Join(path.Dir(filename), templatePath, "testservice", "config", "config.json"), filepath.Join(path.Dir(filename), templatePath, "testservice", "config", "foo.conf"), }, @@ -138,6 +376,7 @@ func TestGetTemplateData(t *testing.T) { AdditionalTemplate: map[string]string{}, }, want: map[string]string{ + "bar.conf": "[DEFAULT]\nstate_path = /var/lib/nova\ndebug=true\nsome_parameter_with_brackets=[test]\ncompute_driver = libvirt.LibvirtDriver\n\n[oslo_concurrency]\nlock_path = /var/lib/nova/tmp\n", "config.json": "{\n \"command\": \"/usr/sbin/httpd -DFOREGROUND\",\n}\n", "foo.conf": "username = foo\ncount = 1\nadd = 3\nlower = bar\n", }, @@ -175,6 +414,7 @@ func TestGetTemplateData(t *testing.T) { AdditionalTemplate: map[string]string{"common.sh": "/common/common.sh"}, }, want: map[string]string{ + "bar.conf": "[DEFAULT]\nstate_path = /var/lib/nova\ndebug=true\nsome_parameter_with_brackets=[test]\ncompute_driver = libvirt.LibvirtDriver\n\n[oslo_concurrency]\nlock_path = /var/lib/nova/tmp\n", "config.json": "{\n \"command\": \"/usr/sbin/httpd -DFOREGROUND\",\n}\n", "foo.conf": "username = foo\ncount = 1\nadd = 3\nlower = bar\n", "common.sh": "#!/bin/bash\nset -e\n\nfunction common_func {\n echo some common func\n}\n", @@ -261,3 +501,22 @@ func TestGetTemplateData(t *testing.T) { }) } } + +// Run the new line section cleaning twice on an input and ensure that the second cleaning +// does nothing as the first run cleaned everything +// This was failing due to empty line handling between sections is unstable. +func TestRemoveNewLinesInSectionsIsStable(t *testing.T) { + g := NewWithT(t) + + input := ` +[foo] +boo=1 +bar=1 +[goo] +baz=1 +` + cleaned := removeNewLinesInSections(input) + cleaned2 := removeNewLinesInSections(cleaned) + + g.Expect(cleaned2).To(Equal(cleaned)) +} diff --git a/modules/common/util/testdata/templates/testservice/config/bar.conf b/modules/common/util/testdata/templates/testservice/config/bar.conf new file mode 100644 index 00000000..5bfc2e11 --- /dev/null +++ b/modules/common/util/testdata/templates/testservice/config/bar.conf @@ -0,0 +1,18 @@ +{{define "bar-template"}} +[DEFAULT] +state_path = /var/lib/nova + + +debug=true + +some_parameter_with_brackets=[test] +compute_driver = libvirt.LibvirtDriver + + + + +[oslo_concurrency] +lock_path = /var/lib/nova/tmp +{{end}} +{{- $var := execTempl "bar-template" . | removeNewLinesInSections -}} +{{$var -}}