From ca2e366644a0d3d40e75f022aca3565b83924fd5 Mon Sep 17 00:00:00 2001 From: Guillaume Dedrie Date: Wed, 1 Jan 2025 15:21:17 +0100 Subject: [PATCH 1/5] fix(yaml): handle when templates are used as Yaml keys --- .../__fixtures__/docker-compose.4.yml.j2 | 26 +++++ .../manager/docker-compose/extract.spec.ts | 18 ++++ lib/util/yaml.spec.ts | 100 +++++++++++++++--- lib/util/yaml.ts | 20 +++- 4 files changed, 147 insertions(+), 17 deletions(-) create mode 100644 lib/modules/manager/docker-compose/__fixtures__/docker-compose.4.yml.j2 diff --git a/lib/modules/manager/docker-compose/__fixtures__/docker-compose.4.yml.j2 b/lib/modules/manager/docker-compose/__fixtures__/docker-compose.4.yml.j2 new file mode 100644 index 00000000000000..8cb81b431a622b --- /dev/null +++ b/lib/modules/manager/docker-compose/__fixtures__/docker-compose.4.yml.j2 @@ -0,0 +1,26 @@ +--- +version: "3" + +networks: + {{ network_name }}: + external: true + frontend: + external: true + +services: + web: + image: "node:42.0.0" + networks: + - {{ network_name }} + - frontend + ports: + - "80:8000" + deploy: + replicas: 2 + update_config: + parallelism: 2 + delay: 10s + placement: + {{ placement | indent(8) }} + restart_policy: + condition: on-failure diff --git a/lib/modules/manager/docker-compose/extract.spec.ts b/lib/modules/manager/docker-compose/extract.spec.ts index a089c8b465a612..1c3a7ab0272886 100644 --- a/lib/modules/manager/docker-compose/extract.spec.ts +++ b/lib/modules/manager/docker-compose/extract.spec.ts @@ -6,6 +6,7 @@ const yamlFile1 = Fixtures.get('docker-compose.1.yml'); const yamlFile3 = Fixtures.get('docker-compose.3.yml'); const yamlFile3NoVersion = Fixtures.get('docker-compose.3-no-version.yml'); const yamlFile3DefaultValue = Fixtures.get('docker-compose.3-default-val.yml'); +const yamlFile4Templated = Fixtures.get('docker-compose.4.yml.j2'); describe('modules/manager/docker-compose/extract', () => { describe('extractPackageFile()', () => { @@ -56,6 +57,23 @@ describe('modules/manager/docker-compose/extract', () => { expect(res?.deps).toHaveLength(1); }); + it('extracts can parse a templated yaml file', () => { + const res = extractPackageFile(yamlFile4Templated, '', {}); + expect(res).toEqual({ + deps: [ + { + depName: 'node', + currentValue: '42.0.0', + currentDigest: undefined, + replaceString: 'node:42.0.0', + autoReplaceStringTemplate: + '{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}', + datasource: 'docker', + }, + ], + }); + }); + it('extracts can parse yaml tags for version 3', () => { const compose = codeBlock` web: diff --git a/lib/util/yaml.spec.ts b/lib/util/yaml.spec.ts index d45ab5d3d613e0..483ef21016f582 100644 --- a/lib/util/yaml.spec.ts +++ b/lib/util/yaml.spec.ts @@ -181,11 +181,11 @@ describe('util/yaml', () => { ).toEqual([ { myObject: { - aString: null, + aString: "d1ddada", }, }, { - foo: null, + foo: "06de5ba", }, ]); }); @@ -273,25 +273,97 @@ services: ).toThrow(); }); - it('should parse content with template', () => { + it('should parse content with basic templates', () => { expect( parseSingleYaml( codeBlock` - myObject: - aString: {{value}} - {% if test.enabled %} - myNestedObject: - aNestedString: {{value}} - {% endif %} - `, + object: + string: {{value}} + number: 42 + stringTemplated: {{\`value\`}} + numberTemplated: {{42}} + list: + {% if test.enabled %} + - {{value}} + {% endif %} + - listEntry + `, { removeTemplates: true }, ), ).toEqual({ - myObject: { - aString: null, - myNestedObject: { - aNestedString: null, + "object": { + // Hash for `{{value}}` + "string": "f00c233", + "number": 42, + // Hash for `{{\`value\`}}` + "stringTemplated": "7379d56", + // Hash for `{{42}}` + "numberTemplated": "df35179", + "list": [ + "f00c233", + "listEntry", + ], + }, + }); + }); + + it('should parse content with templated yaml keys', () => { + expect( + parseSingleYaml( + codeBlock` + object: + string: "string" + {{aKey}}: + string: "string" + number: 12 + {{anotherKey}}: + string: "another string" + number: 30 + number: 42 + `, + { removeTemplates: true }, + ), + ).toEqual({ + "object": { + "string": "string", + // Hash for `{{aKey}}` + "94fdf85": { + "string": "string", + "number": 12, + }, + // Hash for `{{anotherKey}}` + "8904e7f": { + "string": "another string", + "number": 30, + }, + "number": 42, + }, + }); + }); + + it('should parse content with templated value in objects', () => { + expect( + parseSingleYaml( + codeBlock` + object: + string: "string" + childObject: + {{value}} + key: value + anotherChildObject: + {{value}} + number: 42 + `, + { removeTemplates: true }, + ), + ).toEqual({ + "object": { + "string": "string", + "childObject": { + "key": "value", }, + "anotherChildObject": null, + "number": 42, }, }); }); diff --git a/lib/util/yaml.ts b/lib/util/yaml.ts index e8d5ef50ae2eae..792a64e3bb5f13 100644 --- a/lib/util/yaml.ts +++ b/lib/util/yaml.ts @@ -1,3 +1,4 @@ +import crypto from 'crypto'; import type { CreateNodeOptions, DocumentOptions, @@ -140,12 +141,25 @@ export function dump(obj: any, opts?: DumpOptions): string { return stringify(obj, opts); } +function getShortHash(data: any): string { + + return crypto + .createHash('sha256') + .update(data) + .digest('hex') + .substring(0,7); +} + function massageContent(content: string, options?: YamlOptions): string { if (options?.removeTemplates) { return content - .replace(regEx(/\s+{{.+?}}:.+/gs), '') - .replace(regEx(/{{`.+?`}}/gs), '') - .replace(regEx(/{{.+?}}/gs), '') + // NOTE: It seems safe to empty a line that only + // contains a Jinja2 tag entry. + .replace(regEx(/^(\s*)?{{.+?}}$/gm), '') + // NOTE: In order to keep a proper Yaml syntax before + // the parsing, we're remplacing each of the remaining + // Jinja2 by a hash of the whole matched tag. + .replace(regEx(/{{.+?}}/g), getShortHash) .replace(regEx(/{%`.+?`%}/gs), '') .replace(regEx(/{%.+?%}/g), ''); } From 1ad4f56a4b1638164af7bfcf02fcc31583cc73e7 Mon Sep 17 00:00:00 2001 From: Guillaume Dedrie Date: Wed, 1 Jan 2025 16:02:04 +0100 Subject: [PATCH 2/5] fixup! fix(yaml): handle when templates are used as Yaml keys --- lib/util/yaml.spec.ts | 10 +++++----- lib/util/yaml.ts | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/lib/util/yaml.spec.ts b/lib/util/yaml.spec.ts index 483ef21016f582..5fa0a4928d63b8 100644 --- a/lib/util/yaml.spec.ts +++ b/lib/util/yaml.spec.ts @@ -278,9 +278,9 @@ services: parseSingleYaml( codeBlock` object: - string: {{value}} + string: "string" number: 42 - stringTemplated: {{\`value\`}} + stringTemplated: "{{value}}" numberTemplated: {{42}} list: {% if test.enabled %} @@ -293,10 +293,10 @@ services: ).toEqual({ "object": { // Hash for `{{value}}` - "string": "f00c233", + "string": "string", "number": 42, - // Hash for `{{\`value\`}}` - "stringTemplated": "7379d56", + // Hash for `{{value}}` + "stringTemplated": "f00c233", // Hash for `{{42}}` "numberTemplated": "df35179", "list": [ diff --git a/lib/util/yaml.ts b/lib/util/yaml.ts index 792a64e3bb5f13..2b9cd742eb66d5 100644 --- a/lib/util/yaml.ts +++ b/lib/util/yaml.ts @@ -156,6 +156,7 @@ function massageContent(content: string, options?: YamlOptions): string { // NOTE: It seems safe to empty a line that only // contains a Jinja2 tag entry. .replace(regEx(/^(\s*)?{{.+?}}$/gm), '') + .replace(regEx(/{{`.+?`}}/gs), '') // NOTE: In order to keep a proper Yaml syntax before // the parsing, we're remplacing each of the remaining // Jinja2 by a hash of the whole matched tag. From 83c920a88ff9f209e76977aaccf6ef142aeb20f1 Mon Sep 17 00:00:00 2001 From: Guillaume Dedrie Date: Wed, 1 Jan 2025 16:07:44 +0100 Subject: [PATCH 3/5] fixup! fixup! fix(yaml): handle when templates are used as Yaml keys --- lib/util/yaml.spec.ts | 49 ++++++++++++++++++++----------------------- lib/util/yaml.ts | 30 +++++++++++++------------- 2 files changed, 37 insertions(+), 42 deletions(-) diff --git a/lib/util/yaml.spec.ts b/lib/util/yaml.spec.ts index 5fa0a4928d63b8..0edd6f4122756e 100644 --- a/lib/util/yaml.spec.ts +++ b/lib/util/yaml.spec.ts @@ -181,11 +181,11 @@ describe('util/yaml', () => { ).toEqual([ { myObject: { - aString: "d1ddada", + aString: 'd1ddada', }, }, { - foo: "06de5ba", + foo: '06de5ba', }, ]); }); @@ -291,18 +291,15 @@ services: { removeTemplates: true }, ), ).toEqual({ - "object": { + object: { // Hash for `{{value}}` - "string": "string", - "number": 42, + string: 'string', + number: 42, // Hash for `{{value}}` - "stringTemplated": "f00c233", + stringTemplated: 'f00c233', // Hash for `{{42}}` - "numberTemplated": "df35179", - "list": [ - "f00c233", - "listEntry", - ], + numberTemplated: 'df35179', + list: ['f00c233', 'listEntry'], }, }); }); @@ -324,19 +321,19 @@ services: { removeTemplates: true }, ), ).toEqual({ - "object": { - "string": "string", + object: { + string: 'string', // Hash for `{{aKey}}` - "94fdf85": { - "string": "string", - "number": 12, + '94fdf85': { + string: 'string', + number: 12, }, // Hash for `{{anotherKey}}` - "8904e7f": { - "string": "another string", - "number": 30, + '8904e7f': { + string: 'another string', + number: 30, }, - "number": 42, + number: 42, }, }); }); @@ -357,13 +354,13 @@ services: { removeTemplates: true }, ), ).toEqual({ - "object": { - "string": "string", - "childObject": { - "key": "value", + object: { + string: 'string', + childObject: { + key: 'value', }, - "anotherChildObject": null, - "number": 42, + anotherChildObject: null, + number: 42, }, }); }); diff --git a/lib/util/yaml.ts b/lib/util/yaml.ts index 2b9cd742eb66d5..33267ab6df8fe2 100644 --- a/lib/util/yaml.ts +++ b/lib/util/yaml.ts @@ -143,26 +143,24 @@ export function dump(obj: any, opts?: DumpOptions): string { function getShortHash(data: any): string { - return crypto - .createHash('sha256') - .update(data) - .digest('hex') - .substring(0,7); + return crypto.createHash('sha256').update(data).digest('hex').substring(0,7); } function massageContent(content: string, options?: YamlOptions): string { if (options?.removeTemplates) { - return content - // NOTE: It seems safe to empty a line that only - // contains a Jinja2 tag entry. - .replace(regEx(/^(\s*)?{{.+?}}$/gm), '') - .replace(regEx(/{{`.+?`}}/gs), '') - // NOTE: In order to keep a proper Yaml syntax before - // the parsing, we're remplacing each of the remaining - // Jinja2 by a hash of the whole matched tag. - .replace(regEx(/{{.+?}}/g), getShortHash) - .replace(regEx(/{%`.+?`%}/gs), '') - .replace(regEx(/{%.+?%}/g), ''); + return ( + content + // NOTE: It seems safe to empty a line that only + // contains a Jinja2 tag entry. + .replace(regEx(/^(\s*)?{{.+?}}$/gm), '') + .replace(regEx(/{{`.+?`}}/gs), '') + // NOTE: In order to keep a proper Yaml syntax before + // the parsing, we're remplacing each of the remaining + // Jinja2 by a hash of the whole matched tag. + .replace(regEx(/{{.+?}}/g), getShortHash) + .replace(regEx(/{%`.+?`%}/gs), '') + .replace(regEx(/{%.+?%}/g), '') + ); } return content; From f17fb04f3eb580493b80da8ff2610884455e9dd5 Mon Sep 17 00:00:00 2001 From: Guillaume Dedrie Date: Wed, 1 Jan 2025 16:09:25 +0100 Subject: [PATCH 4/5] fixup! fixup! fixup! fix(yaml): handle when templates are used as Yaml keys --- lib/util/yaml.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/util/yaml.ts b/lib/util/yaml.ts index 33267ab6df8fe2..f97ea6ab866e08 100644 --- a/lib/util/yaml.ts +++ b/lib/util/yaml.ts @@ -143,7 +143,7 @@ export function dump(obj: any, opts?: DumpOptions): string { function getShortHash(data: any): string { - return crypto.createHash('sha256').update(data).digest('hex').substring(0,7); + return crypto.createHash('sha256').update(data).digest('hex').substring(0, 7); } function massageContent(content: string, options?: YamlOptions): string { From 99d4b5659bcba3a5769a3c5f05660dfa9c4d1693 Mon Sep 17 00:00:00 2001 From: Guillaume Dedrie Date: Wed, 1 Jan 2025 16:11:24 +0100 Subject: [PATCH 5/5] fixup! fixup! fixup! fixup! fix(yaml): handle when templates are used as Yaml keys --- lib/util/yaml.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/lib/util/yaml.ts b/lib/util/yaml.ts index f97ea6ab866e08..8c0d226d96fe92 100644 --- a/lib/util/yaml.ts +++ b/lib/util/yaml.ts @@ -142,7 +142,6 @@ export function dump(obj: any, opts?: DumpOptions): string { } function getShortHash(data: any): string { - return crypto.createHash('sha256').update(data).digest('hex').substring(0, 7); }