From 48f067c5ed1d4ce5abe5f76383b97871406fb768 Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Fri, 29 Nov 2024 19:34:55 +0800 Subject: [PATCH 1/9] ci: run test on 3.7.0 --- .github/workflows/e2e.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 3c68f49..6612b93 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - version: [3.8.1, 3.9.1, 3.10.0, 3.11.0] + version: [3.7.0, 3.8.1, 3.9.1, 3.10.0, 3.11.0] env: BACKEND_APISIX_VERSION: ${{ matrix.version }} BACKEND_APISIX_IMAGE: ${{ matrix.version }}-debian From 3e0b38e9a0dd4b7589e273e271e124c9b30e3d34 Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Fri, 29 Nov 2024 20:04:39 +0800 Subject: [PATCH 2/9] fix: 3.7.0 schema error --- libs/backend-apisix/src/operator.ts | 17 ++++++--- libs/backend-apisix/src/transformer.ts | 53 +++++++++++--------------- 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/libs/backend-apisix/src/operator.ts b/libs/backend-apisix/src/operator.ts index 2c2a83b..4457214 100644 --- a/libs/backend-apisix/src/operator.ts +++ b/libs/backend-apisix/src/operator.ts @@ -1,7 +1,7 @@ import * as ADCSDK from '@api7/adc-sdk'; import { Axios } from 'axios'; import { ListrTask } from 'listr2'; -import { SemVer, lt as semVerLT } from 'semver'; +import { SemVer, gte as semVerGTE, lt as semVerLT } from 'semver'; import { FromADC } from './transformer'; import * as typing from './typing'; @@ -32,7 +32,7 @@ export class Operator { const resp = await this.client.put( `/apisix/admin/consumers/${event.parentId}/credentials/${event.resourceId}`, - this.fromADC(event), + this.fromADC(event, ctx.apisixVersion), { validateStatus: () => true, }, @@ -43,7 +43,7 @@ export class Operator { } else { const resp = await this.client.put( `/apisix/admin/${resourceTypeToAPIName(event.resourceType)}/${event.resourceId}`, - this.fromADC(event), + this.fromADC(event, ctx.apisixVersion), { validateStatus: () => true, }, @@ -102,7 +102,7 @@ export class Operator { )} ${event.resourceType}: "${event.resourceName}"`; } - private fromADC(event: ADCSDK.Event) { + private fromADC(event: ADCSDK.Event, version: SemVer) { const fromADC = new FromADC(); switch (event.resourceType) { case ADCSDK.ResourceType.CONSUMER: @@ -127,13 +127,16 @@ export class Operator { return event.newValue; case ADCSDK.ResourceType.ROUTE: { (event.newValue as ADCSDK.Route).id = event.resourceId; - const route = fromADC.transformRoute(event.newValue as ADCSDK.Route); + const route = fromADC.transformRoute( + event.newValue as ADCSDK.Route, + event.parentId, + ); if (event.parentId) route.service_id = event.parentId; return route; } case ADCSDK.ResourceType.SERVICE: (event.newValue as ADCSDK.Service).id = event.resourceId; - return fromADC.transformService(event.newValue as ADCSDK.Service)[0]; + return fromADC.transformService(event.newValue as ADCSDK.Service); case ADCSDK.ResourceType.SSL: (event.newValue as ADCSDK.SSL).id = event.resourceId; return fromADC.transformSSL(event.newValue as ADCSDK.SSL); @@ -141,6 +144,8 @@ export class Operator { (event.newValue as ADCSDK.StreamRoute).id = event.resourceId; const route = fromADC.transformStreamRoute( event.newValue as ADCSDK.StreamRoute, + event.parentId, + semVerGTE(version, '3.8.0'), ); if (event.parentId) route.service_id = event.parentId; return route; diff --git a/libs/backend-apisix/src/transformer.ts b/libs/backend-apisix/src/transformer.ts index 3e9d1e0..5c29422 100644 --- a/libs/backend-apisix/src/transformer.ts +++ b/libs/backend-apisix/src/transformer.ts @@ -242,7 +242,7 @@ export class FromADC { }, {}); } - public transformRoute(route: ADCSDK.Route): typing.Route { + public transformRoute(route: ADCSDK.Route, parentId: string): typing.Route { return ADCSDK.utils.recursiveOmitUndefined({ id: route.id, name: route.name, @@ -255,7 +255,7 @@ export class FromADC { vars: route.vars, filter_func: route.filter_func, - //service_id: '', + service_id: parentId, enable_websocket: route.enable_websocket, plugins: route.plugins, priority: route.priority, @@ -264,31 +264,16 @@ export class FromADC { }); } - public transformService( - service: ADCSDK.Service, - ): [typing.Service, Array, Array] { - const serviceId = ADCSDK.utils.generateId(service.name); - const routes: Array = - service.routes - ?.map(this.transformRoute) - .map((route) => ({ ...route, service_id: serviceId })) ?? []; - const streamRoutes: Array = - service.stream_routes - ?.map(this.transformStreamRoute) - .map((route) => ({ ...route, service_id: serviceId })) ?? []; - return [ - ADCSDK.utils.recursiveOmitUndefined({ - id: service.id, - name: service.name, - desc: service.description, - labels: FromADC.transformLabels(service.labels), - upstream: service.upstream, - plugins: service.plugins, - hosts: service.hosts, - }), - routes, - streamRoutes, - ]; + public transformService(service: ADCSDK.Service): typing.Service { + return ADCSDK.utils.recursiveOmitUndefined({ + id: service.id, + name: service.name, + desc: service.description, + labels: FromADC.transformLabels(service.labels), + upstream: service.upstream, + plugins: service.plugins, + hosts: service.hosts, + }); } public transformConsumer(consumer: ADCSDK.Consumer): typing.Consumer { @@ -400,19 +385,25 @@ export class FromADC { public transformStreamRoute( streamRoute: ADCSDK.StreamRoute, + parentId: string, + injectName = true, ): typing.StreamRoute { + const labels = FromADC.transformLabels(streamRoute.labels); return ADCSDK.utils.recursiveOmitUndefined({ id: undefined, desc: streamRoute.description, - labels: { - ...FromADC.transformLabels(streamRoute.labels), - __ADC_NAME: streamRoute.name, - }, + labels: injectName + ? labels + : { + ...labels, + __ADC_NAME: streamRoute.name, + }, plugins: streamRoute.plugins, remote_addr: streamRoute.remote_addr, server_addr: streamRoute.server_addr, server_port: streamRoute.server_port, sni: streamRoute.sni, + service_id: parentId, } as typing.StreamRoute); } From 480827e4a5d0b0bc6e6a472ac4d6805bed4bc5a2 Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Fri, 29 Nov 2024 21:47:05 +0800 Subject: [PATCH 3/9] fix --- .../e2e/sync-and-dump-1.e2e-spec.ts | 15 ++++++++++++++- libs/backend-apisix/src/transformer.ts | 6 +++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/libs/backend-apisix/e2e/sync-and-dump-1.e2e-spec.ts b/libs/backend-apisix/e2e/sync-and-dump-1.e2e-spec.ts index 8d854a4..7a6283c 100644 --- a/libs/backend-apisix/e2e/sync-and-dump-1.e2e-spec.ts +++ b/libs/backend-apisix/e2e/sync-and-dump-1.e2e-spec.ts @@ -2,13 +2,16 @@ import * as ADCSDK from '@api7/adc-sdk'; import { unset } from 'lodash'; import { readFileSync } from 'node:fs'; import { join } from 'node:path'; +import { gte, lt } from 'semver'; import { BackendAPISIX } from '../src'; import { server, token } from './support/constants'; import { + conditionalIt, createEvent, deleteEvent, dumpConfiguration, + semverCondition, syncEvents, updateEvent, } from './support/utils'; @@ -195,7 +198,17 @@ describe('Sync and Dump - 1', () => { ), ])); - it('Dump', async () => { + conditionalIt(semverCondition(lt, '3.8.0'))('Dump (<3.8.0)', async () => { + const result = (await dumpConfiguration(backend)) as ADCSDK.Configuration; + expect(result.services).toHaveLength(1); + expect(result.services[0]).toMatchObject(service); + expect(result.services[0].stream_routes).toHaveLength(1); + const route = structuredClone(route1); + route.name = result.services[0].stream_routes[0].id; + expect(result.services[0].stream_routes[0]).toMatchObject(route); + }); + + conditionalIt(semverCondition(gte, '3.8.0'))('Dump (>=3.8.0)', async () => { const result = (await dumpConfiguration(backend)) as ADCSDK.Configuration; expect(result.services).toHaveLength(1); expect(result.services[0]).toMatchObject(service); diff --git a/libs/backend-apisix/src/transformer.ts b/libs/backend-apisix/src/transformer.ts index 5c29422..1b1b157 100644 --- a/libs/backend-apisix/src/transformer.ts +++ b/libs/backend-apisix/src/transformer.ts @@ -393,11 +393,11 @@ export class FromADC { id: undefined, desc: streamRoute.description, labels: injectName - ? labels - : { + ? { ...labels, __ADC_NAME: streamRoute.name, - }, + } + : labels, plugins: streamRoute.plugins, remote_addr: streamRoute.remote_addr, server_addr: streamRoute.server_addr, From 24997c4440ed7742f642eaa69666fe8366acd9da Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Fri, 29 Nov 2024 21:47:35 +0800 Subject: [PATCH 4/9] ci: test on apisix 3.6.0 --- .github/workflows/e2e.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 6612b93..41ae33c 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - version: [3.7.0, 3.8.1, 3.9.1, 3.10.0, 3.11.0] + version: [3.6.0, 3.7.0, 3.8.1, 3.9.1, 3.10.0, 3.11.0] env: BACKEND_APISIX_VERSION: ${{ matrix.version }} BACKEND_APISIX_IMAGE: ${{ matrix.version }}-debian From 9ddf441263fc0020a15890b16fadac1f9154dc21 Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Fri, 29 Nov 2024 22:53:32 +0800 Subject: [PATCH 5/9] fix --- libs/backend-apisix/src/operator.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/libs/backend-apisix/src/operator.ts b/libs/backend-apisix/src/operator.ts index 4457214..ed8ace5 100644 --- a/libs/backend-apisix/src/operator.ts +++ b/libs/backend-apisix/src/operator.ts @@ -1,7 +1,7 @@ import * as ADCSDK from '@api7/adc-sdk'; import { Axios } from 'axios'; import { ListrTask } from 'listr2'; -import { SemVer, gte as semVerGTE, lt as semVerLT } from 'semver'; +import { SemVer, lt, gte as semVerGTE, lt as semVerLT } from 'semver'; import { FromADC } from './transformer'; import * as typing from './typing'; @@ -26,6 +26,13 @@ export class Operator { public updateResource(event: ADCSDK.Event): OperateTask { return { title: this.generateTaskName(event), + skip: (ctx) => { + if ( + lt(ctx.apisixVersion, '3.7.0') && + event.resourceType === ADCSDK.ResourceType.STREAM_ROUTE + ) + return 'The stream routes on versions below 3.7.0 are not supported as they are not supported configured on the service.'; + }, task: async (ctx, task) => { if (event.resourceType === ADCSDK.ResourceType.CONSUMER_CREDENTIAL) { if (semVerLT(ctx.apisixVersion, '3.11.0')) return; From d1d738690e1d566bb612476ad85523ee36bdf5f6 Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Fri, 29 Nov 2024 22:57:17 +0800 Subject: [PATCH 6/9] fix: remove stream route test below 3.7 --- .../e2e/sync-and-dump-1.e2e-spec.ts | 163 ++++++++++-------- 1 file changed, 91 insertions(+), 72 deletions(-) diff --git a/libs/backend-apisix/e2e/sync-and-dump-1.e2e-spec.ts b/libs/backend-apisix/e2e/sync-and-dump-1.e2e-spec.ts index 7a6283c..8a316d5 100644 --- a/libs/backend-apisix/e2e/sync-and-dump-1.e2e-spec.ts +++ b/libs/backend-apisix/e2e/sync-and-dump-1.e2e-spec.ts @@ -7,6 +7,7 @@ import { gte, lt } from 'semver'; import { BackendAPISIX } from '../src'; import { server, token } from './support/constants'; import { + conditionalDescribe, conditionalIt, createEvent, deleteEvent, @@ -166,78 +167,96 @@ describe('Sync and Dump - 1', () => { }); }); - describe('Sync and dump service with stream route', () => { - const serviceName = 'test'; - const service = { - name: serviceName, - upstream: { - scheme: 'tcp', - nodes: [ - { - host: '1.1.1.1', - port: 5432, - weight: 100, - }, - ], - }, - } as ADCSDK.Service; - const route1Name = 'postgres'; - const route1 = { - name: route1Name, - server_port: 54320, - } as ADCSDK.StreamRoute; - - it('Create resources', async () => - syncEvents(backend, [ - createEvent(ADCSDK.ResourceType.SERVICE, serviceName, service), - createEvent( - ADCSDK.ResourceType.STREAM_ROUTE, - route1Name, - route1, - serviceName, - ), - ])); - - conditionalIt(semverCondition(lt, '3.8.0'))('Dump (<3.8.0)', async () => { - const result = (await dumpConfiguration(backend)) as ADCSDK.Configuration; - expect(result.services).toHaveLength(1); - expect(result.services[0]).toMatchObject(service); - expect(result.services[0].stream_routes).toHaveLength(1); - const route = structuredClone(route1); - route.name = result.services[0].stream_routes[0].id; - expect(result.services[0].stream_routes[0]).toMatchObject(route); - }); - - conditionalIt(semverCondition(gte, '3.8.0'))('Dump (>=3.8.0)', async () => { - const result = (await dumpConfiguration(backend)) as ADCSDK.Configuration; - expect(result.services).toHaveLength(1); - expect(result.services[0]).toMatchObject(service); - expect(result.services[0].stream_routes).toHaveLength(1); - expect(result.services[0].stream_routes[0]).toMatchObject(route1); - }); - - it('Delete stream route', async () => - syncEvents(backend, [ - deleteEvent(ADCSDK.ResourceType.STREAM_ROUTE, route1Name, serviceName), - ])); - - it('Dump again (non-route)', async () => { - const result = (await dumpConfiguration(backend)) as ADCSDK.Configuration; - expect(result.services).toHaveLength(1); - expect(result.services[0]).toMatchObject(service); - expect(result.services[0].stream_routes).toBeUndefined(); - }); - - it('Delete service', async () => - syncEvents(backend, [ - deleteEvent(ADCSDK.ResourceType.SERVICE, serviceName), - ])); - - it('Dump again (service should not exist)', async () => { - const result = (await dumpConfiguration(backend)) as ADCSDK.Configuration; - expect(result.services).toHaveLength(0); - }); - }); + conditionalDescribe(semverCondition(gte, '3.7.0'))( + 'Sync and dump service with stream route', + () => { + const serviceName = 'test'; + const service = { + name: serviceName, + upstream: { + scheme: 'tcp', + nodes: [ + { + host: '1.1.1.1', + port: 5432, + weight: 100, + }, + ], + }, + } as ADCSDK.Service; + const route1Name = 'postgres'; + const route1 = { + name: route1Name, + server_port: 54320, + } as ADCSDK.StreamRoute; + + it('Create resources', async () => + syncEvents(backend, [ + createEvent(ADCSDK.ResourceType.SERVICE, serviceName, service), + createEvent( + ADCSDK.ResourceType.STREAM_ROUTE, + route1Name, + route1, + serviceName, + ), + ])); + + conditionalIt(semverCondition(lt, '3.8.0'))('Dump (<3.8.0)', async () => { + const result = (await dumpConfiguration( + backend, + )) as ADCSDK.Configuration; + expect(result.services).toHaveLength(1); + expect(result.services[0]).toMatchObject(service); + expect(result.services[0].stream_routes).toHaveLength(1); + const route = structuredClone(route1); + route.name = result.services[0].stream_routes[0].id; + expect(result.services[0].stream_routes[0]).toMatchObject(route); + }); + + conditionalIt(semverCondition(gte, '3.8.0'))( + 'Dump (>=3.8.0)', + async () => { + const result = (await dumpConfiguration( + backend, + )) as ADCSDK.Configuration; + expect(result.services).toHaveLength(1); + expect(result.services[0]).toMatchObject(service); + expect(result.services[0].stream_routes).toHaveLength(1); + expect(result.services[0].stream_routes[0]).toMatchObject(route1); + }, + ); + + it('Delete stream route', async () => + syncEvents(backend, [ + deleteEvent( + ADCSDK.ResourceType.STREAM_ROUTE, + route1Name, + serviceName, + ), + ])); + + it('Dump again (non-route)', async () => { + const result = (await dumpConfiguration( + backend, + )) as ADCSDK.Configuration; + expect(result.services).toHaveLength(1); + expect(result.services[0]).toMatchObject(service); + expect(result.services[0].stream_routes).toBeUndefined(); + }); + + it('Delete service', async () => + syncEvents(backend, [ + deleteEvent(ADCSDK.ResourceType.SERVICE, serviceName), + ])); + + it('Dump again (service should not exist)', async () => { + const result = (await dumpConfiguration( + backend, + )) as ADCSDK.Configuration; + expect(result.services).toHaveLength(0); + }); + }, + ); describe('Sync and dump consumers', () => { const consumer1Name = 'consumer1'; From 66311a97cd5c76012969d086af8505b5d2f6989b Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Fri, 29 Nov 2024 22:58:58 +0800 Subject: [PATCH 7/9] ci: test on apisix 3.5.0 --- .github/workflows/e2e.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 41ae33c..f50ada9 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -12,7 +12,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - version: [3.6.0, 3.7.0, 3.8.1, 3.9.1, 3.10.0, 3.11.0] + version: [3.5.0, 3.6.0, 3.7.0, 3.8.1, 3.9.1, 3.10.0, 3.11.0] env: BACKEND_APISIX_VERSION: ${{ matrix.version }} BACKEND_APISIX_IMAGE: ${{ matrix.version }}-debian From c51c82969c614fbb9795be1eacf222498a4b6d73 Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Fri, 29 Nov 2024 23:01:11 +0800 Subject: [PATCH 8/9] ci: test on apisix 3.2 3.3 3.4 --- .github/workflows/e2e.yaml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index f50ada9..337bc31 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -12,7 +12,17 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - version: [3.5.0, 3.6.0, 3.7.0, 3.8.1, 3.9.1, 3.10.0, 3.11.0] + version: + - 3.2.2 + - 3.3.0 + - 3.4.0 + - 3.5.0 + - 3.6.0 + - 3.7.0 + - 3.8.1 + - 3.9.1 + - 3.10.0 + - 3.11.0 env: BACKEND_APISIX_VERSION: ${{ matrix.version }} BACKEND_APISIX_IMAGE: ${{ matrix.version }}-debian From 681c75bc573494add665ced44f847c7397e7bc12 Mon Sep 17 00:00:00 2001 From: bzp2010 Date: Fri, 29 Nov 2024 23:20:46 +0800 Subject: [PATCH 9/9] docs: update --- libs/backend-apisix/README.md | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/libs/backend-apisix/README.md b/libs/backend-apisix/README.md index 3cfa8a0..77cbb4a 100644 --- a/libs/backend-apisix/README.md +++ b/libs/backend-apisix/README.md @@ -6,8 +6,28 @@ | Features | Supported | | ------------- | --------- | -| Dump to ADC | ✅ | -| Sync from ADC | ✅ | +| Dump to ADC | ✅ | +| Sync from ADC | ✅ | + +## Supported Versions + +> Versions not listed below are untested. + +| Versions | Supported | Status | +| -------- | --------- | ----------- | +| 3.2.x | ✅ | Partial [1] | +| 3.3.x | ✅ | Partial [1] | +| 3.4.x | ✅ | Partial [1] | +| 3.5.x | ✅ | Partial [1] | +| 3.6.x | ✅ | Partial [1] | +| 3.7.x | ✅ | Partial [2] | +| 3.8.x | ✅ | Full | +| 3.9.x | ✅ | Full | +| 3.10.x | ✅ | Full | +| 3.11.x | ✅ | Full | + +1. The stream routes will be skipped during synchronization because they cannot be associated to the service on these versions. +2. The `name` field is lost when the stream route is dumped and synchronized, because it is not defined in the APISIX schema. ## Known Issues/Limitations