From 8dec4bc8900fb6c149a834a99d0a949ce798098c Mon Sep 17 00:00:00 2001 From: euanmillar Date: Tue, 19 Nov 2024 22:08:18 +0000 Subject: [PATCH 1/7] Save SVG cert to Minio correctly --- .../src/features/uploadDocument/handler.ts | 37 +++++++++++++++---- yarn.lock | 33 ++--------------- 2 files changed, 33 insertions(+), 37 deletions(-) diff --git a/packages/documents/src/features/uploadDocument/handler.ts b/packages/documents/src/features/uploadDocument/handler.ts index fc860f6021f..a0028fb0689 100644 --- a/packages/documents/src/features/uploadDocument/handler.ts +++ b/packages/documents/src/features/uploadDocument/handler.ts @@ -31,15 +31,36 @@ export async function documentUploadHandler( const payload = request.payload as IDocumentPayload const ref = uuid() try { - const base64String = payload.fileData.split(',')[1] - const base64Decoded = Buffer.from(base64String, 'base64') - const fileType = (await fromBuffer(base64Decoded)) as IFileInfo - const generateFileName = `${ref}.${fileType.ext}` + // payload is an image + const regex = /^data:image/ + let generateFileName + if (regex.test(payload.fileData)) { + const base64String = payload.fileData.split(',')[1] + const base64Decoded = Buffer.from(base64String, 'base64') + const fileType = (await fromBuffer(base64Decoded)) as IFileInfo + generateFileName = `${ref}.${fileType.ext}` - await minioClient.putObject(MINIO_BUCKET, generateFileName, base64Decoded, { - 'content-type': fileType.mime, - ...payload.metaData - }) + await minioClient.putObject( + MINIO_BUCKET, + generateFileName, + base64Decoded, + { + 'content-type': fileType.mime, + ...payload.metaData + } + ) + } else { + // payload is an svg cert + const generateFileName = `${ref}.CERTIFICATE` + await minioClient.putObject( + MINIO_BUCKET, + generateFileName, + payload.fileData, + { + 'content-type': 'image/svg+xml' + } + ) + } return h .response({ refUrl: `/${MINIO_BUCKET}/${generateFileName}` }) diff --git a/yarn.lock b/yarn.lock index b01c3745588..168111a46b6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -23025,16 +23025,7 @@ string-similarity@^4.0.1: resolved "https://registry.npmjs.org/string-similarity/-/string-similarity-4.0.4.tgz" integrity sha512-/q/8Q4Bl4ZKAPjj8WerIBJWALKkaPRfrvhfF8k/B23i4nzrlRj2/go1m90In7nG/3XDSbOo0+pu6RvCTM9RGMQ== -"string-width-cjs@npm:string-width@^4.2.0": - version "4.2.3" - resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" - integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== - dependencies: - emoji-regex "^8.0.0" - is-fullwidth-code-point "^3.0.0" - strip-ansi "^6.0.1" - -"string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: +"string-width-cjs@npm:string-width@^4.2.0", "string-width@^1.0.2 || 2 || 3 || 4", string-width@^4.1.0, string-width@^4.2.0, string-width@^4.2.3: version "4.2.3" resolved "https://registry.npmjs.org/string-width/-/string-width-4.2.3.tgz" integrity sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g== @@ -23194,7 +23185,7 @@ stringify-object@^3.3.0: is-obj "^1.0.1" is-regexp "^1.0.0" -"strip-ansi-cjs@npm:strip-ansi@^6.0.1": +"strip-ansi-cjs@npm:strip-ansi@^6.0.1", strip-ansi@^6.0.0, strip-ansi@^6.0.1: version "6.0.1" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== @@ -23215,13 +23206,6 @@ strip-ansi@^5.0.0, strip-ansi@^5.1.0, strip-ansi@^5.2.0: dependencies: ansi-regex "^4.1.0" -strip-ansi@^6.0.0, strip-ansi@^6.0.1: - version "6.0.1" - resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.1.tgz" - integrity sha512-Y38VPSHcqkFrCpFnQ9vuSXmquuv5oXOKpGeT6aGrr3o3Gc9AlVa6JBfUSOCnbxGGZF+/0ooI7KrPuUSztUdU5A== - dependencies: - ansi-regex "^5.0.1" - strip-ansi@^7.0.1, strip-ansi@^7.1.0: version "7.1.0" resolved "https://registry.npmjs.org/strip-ansi/-/strip-ansi-7.1.0.tgz" @@ -24351,7 +24335,7 @@ typescript@4.9.5: resolved "https://registry.yarnpkg.com/typescript/-/typescript-4.9.5.tgz#095979f9bcc0d09da324d58d03ce8f8374cbe65a" integrity sha512-1FXk9E2Hm+QzZQ7z+McJiHL4NW1F2EzMu9Nq9i3zAaGqibafqYwCVU6WyWAuyQRRzOlxou8xZSyXLEN8oKj24g== -typescript@5.6.3, "typescript@>=3 < 6", "typescript@^4.7 || 5": +typescript@5.6.3, "typescript@>=3 < 6", "typescript@^4.7 || 5", typescript@^5.6.3: version "5.6.3" resolved "https://registry.yarnpkg.com/typescript/-/typescript-5.6.3.tgz#5f3449e31c9d94febb17de03cc081dd56d81db5b" integrity sha512-hjcS1mhfuyi4WW8IWtjP7brDrG2cuDZukyrYrSauoXGNgx0S7zceP07adYkJycEr56BOUTNPzbInooiN3fn1qw== @@ -25475,7 +25459,7 @@ workbox-window@7.1.0, workbox-window@^7.1.0: "@types/trusted-types" "^2.0.2" workbox-core "7.1.0" -"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0": +"wrap-ansi-cjs@npm:wrap-ansi@^7.0.0", wrap-ansi@^7.0.0: version "7.0.0" resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz" integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== @@ -25502,15 +25486,6 @@ wrap-ansi@^6.0.1, wrap-ansi@^6.2.0: string-width "^4.1.0" strip-ansi "^6.0.0" -wrap-ansi@^7.0.0: - version "7.0.0" - resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-7.0.0.tgz" - integrity sha512-YVGIj2kamLSTxw6NsZjoBxfSwsn0ycdesmc4p+Q21c5zPuZ1pl+NfxVdxPtdHvmNVOQ6XSYG4AUtyt/Fi7D16Q== - dependencies: - ansi-styles "^4.0.0" - string-width "^4.1.0" - strip-ansi "^6.0.0" - wrap-ansi@^8.0.1, wrap-ansi@^8.1.0: version "8.1.0" resolved "https://registry.npmjs.org/wrap-ansi/-/wrap-ansi-8.1.0.tgz" From 51c51ea05a71bfd61efa49612820f33461e1ec8f Mon Sep 17 00:00:00 2001 From: euanmillar Date: Wed, 20 Nov 2024 11:52:15 +0000 Subject: [PATCH 2/7] Refactor approach to sending files from workflow to documents --- .../src/features/uploadDocument/handler.ts | 41 +++---------- packages/gateway/src/utils/documents.ts | 31 ---------- packages/workflow/src/documents.ts | 59 +++++++------------ .../src/records/handler/correction/request.ts | 6 +- 4 files changed, 34 insertions(+), 103 deletions(-) delete mode 100644 packages/gateway/src/utils/documents.ts diff --git a/packages/documents/src/features/uploadDocument/handler.ts b/packages/documents/src/features/uploadDocument/handler.ts index a0028fb0689..cce0f134b72 100644 --- a/packages/documents/src/features/uploadDocument/handler.ts +++ b/packages/documents/src/features/uploadDocument/handler.ts @@ -31,40 +31,17 @@ export async function documentUploadHandler( const payload = request.payload as IDocumentPayload const ref = uuid() try { - // payload is an image - const regex = /^data:image/ - let generateFileName - if (regex.test(payload.fileData)) { - const base64String = payload.fileData.split(',')[1] - const base64Decoded = Buffer.from(base64String, 'base64') - const fileType = (await fromBuffer(base64Decoded)) as IFileInfo - generateFileName = `${ref}.${fileType.ext}` + const base64String = payload.fileData.split(',')[1] + const base64Decoded = Buffer.from(base64String, 'base64') + const fileType = (await fromBuffer(base64Decoded)) as IFileInfo + const generateFileName = `${ref}.${fileType.ext}` - await minioClient.putObject( - MINIO_BUCKET, - generateFileName, - base64Decoded, - { - 'content-type': fileType.mime, - ...payload.metaData - } - ) - } else { - // payload is an svg cert - const generateFileName = `${ref}.CERTIFICATE` - await minioClient.putObject( - MINIO_BUCKET, - generateFileName, - payload.fileData, - { - 'content-type': 'image/svg+xml' - } - ) - } + await minioClient.putObject(MINIO_BUCKET, generateFileName, base64Decoded, { + 'content-type': fileType.mime, + ...payload.metaData + }) - return h - .response({ refUrl: `/${MINIO_BUCKET}/${generateFileName}` }) - .code(200) + return h.response({ refUrl: `/${MINIO_BUCKET}/${generateFileName}` }) } catch (error) { return Promise.reject(new Error(`request failed: ${error.message}`)) } diff --git a/packages/gateway/src/utils/documents.ts b/packages/gateway/src/utils/documents.ts deleted file mode 100644 index 7a08a3ed2fa..00000000000 --- a/packages/gateway/src/utils/documents.ts +++ /dev/null @@ -1,31 +0,0 @@ -/* - * This Source Code Form is subject to the terms of the Mozilla Public - * License, v. 2.0. If a copy of the MPL was not distributed with this - * file, You can obtain one at https://mozilla.org/MPL/2.0/. - * - * OpenCRVS is also distributed under the terms of the Civil Registration - * & Healthcare Disclaimer located at http://opencrvs.org/license. - * - * Copyright (C) The OpenCRVS Authors located at https://github.com/opencrvs/opencrvs-core/blob/master/AUTHORS. - */ -import fetch from '@gateway/fetch' -import { DOCUMENTS_URL } from '@gateway/constants' -import { internal } from '@hapi/boom' -import { logger, IAuthHeader } from '@opencrvs/commons' - -export async function uploadSvg(fileData: string, authHeader: IAuthHeader) { - const url = new URL('upload-svg', DOCUMENTS_URL).toString() - const res = await fetch(url, { - method: 'POST', - headers: { - ...authHeader, - 'Content-Type': 'image/svg+xml' - }, - body: Buffer.from(fileData) - }) - if (!res.ok) { - logger.error(await res.json()) - throw internal() - } - return (await res.json()).refUrl -} diff --git a/packages/workflow/src/documents.ts b/packages/workflow/src/documents.ts index 671c4b444c2..87c1157905b 100644 --- a/packages/workflow/src/documents.ts +++ b/packages/workflow/src/documents.ts @@ -18,36 +18,23 @@ import { } from '@opencrvs/commons/types' import { CertifyInput, IssueInput } from './records/validations' -const fetchDocuments = async ( - suffix: string, +export async function uploadFileToMinio( + fileData: string, authHeader: IAuthHeader, - method = 'GET', - body: string | undefined = undefined -): Promise => { - const result = await fetch(`${DOCUMENTS_URL}${suffix}`, { - method, + svg?: boolean +): Promise { + const suffix = svg ? '/upload-svg' : '/upload' + const request = { + method: 'POST', headers: { ...authHeader, - 'Content-Type': 'application/json' + 'Content-Type': svg ? 'image/svg+xml' : 'application/json' }, - body - }) + body: svg ? fileData : JSON.stringify({ fileData: fileData }) + } + const result = await fetch(`${DOCUMENTS_URL}${suffix}`, request) const res = await result.json() - return res -} - -export async function uploadBase64ToMinio( - fileData: string, - authHeader: IAuthHeader -): Promise { - const docUploadResponse = await fetchDocuments( - '/upload', - authHeader, - 'POST', - JSON.stringify({ fileData: fileData }) - ) - - return docUploadResponse.refUrl + return res.refUrl } export async function uploadCertificateAttachmentsToDocumentsStore< @@ -58,13 +45,14 @@ export async function uploadCertificateAttachmentsToDocumentsStore< certificateDetails.collector.affidavit ) { for (const affidavit of certificateDetails.collector.affidavit) { - affidavit.data = await uploadBase64ToMinio(affidavit.data, authHeader) + affidavit.data = await uploadFileToMinio(affidavit.data, authHeader, true) } } if ('data' in certificateDetails) { - certificateDetails.data = await uploadBase64ToMinio( + certificateDetails.data = await uploadFileToMinio( certificateDetails.data, - authHeader + authHeader, + true ) } return certificateDetails @@ -86,7 +74,7 @@ function uploadOrNormaliseSignatureData( authHeader: IAuthHeader ) { if (isBase64FileString(signature)) { - return uploadBase64ToMinio(signature, authHeader) + return uploadFileToMinio(signature, authHeader) } if (isPresignedUrl(signature)) { @@ -138,7 +126,7 @@ export async function uploadBase64AttachmentsToDocumentsStore( if (record.registration?.attachments) { for (const attachment of record.registration.attachments) { if (attachment.data && isBase64FileString(attachment.data)) { - const fileUri = await uploadBase64ToMinio(attachment.data, authHeader) + const fileUri = await uploadFileToMinio(attachment.data, authHeader) attachment.data = fileUri } } @@ -151,10 +139,7 @@ export async function uploadBase64AttachmentsToDocumentsStore( if (certificate.collector.affidavit) { for (const affidavit of certificate.collector.affidavit) { if (affidavit.data && isBase64FileString(affidavit.data)) { - const fileUri = await uploadBase64ToMinio( - affidavit.data, - authHeader - ) + const fileUri = await uploadFileToMinio(affidavit.data, authHeader) affidavit.data = fileUri } } @@ -162,7 +147,7 @@ export async function uploadBase64AttachmentsToDocumentsStore( if (certificate.collector.photo) { for (const photo of certificate.collector.photo) { if (photo.data && isBase64FileString(photo.data)) { - const fileUri = await uploadBase64ToMinio(photo.data, authHeader) + const fileUri = await uploadFileToMinio(photo.data, authHeader) photo.data = fileUri } } @@ -172,13 +157,13 @@ export async function uploadBase64AttachmentsToDocumentsStore( if (record.registration?.correction?.attachments) { for (const attachment of record.registration.correction.attachments) { if (attachment.data && isBase64FileString(attachment.data)) { - const fileUri = await uploadBase64ToMinio(attachment.data, authHeader) + const fileUri = await uploadFileToMinio(attachment.data, authHeader) attachment.data = fileUri } } } if (record.registration?.correction?.payment?.attachmentData) { - const fileUri = await uploadBase64ToMinio( + const fileUri = await uploadFileToMinio( record.registration.correction.payment.attachmentData, authHeader ) diff --git a/packages/workflow/src/records/handler/correction/request.ts b/packages/workflow/src/records/handler/correction/request.ts index a645336add2..af6e8438857 100644 --- a/packages/workflow/src/records/handler/correction/request.ts +++ b/packages/workflow/src/records/handler/correction/request.ts @@ -12,7 +12,7 @@ import { conflict } from '@hapi/boom' import { getAuthHeader } from '@opencrvs/commons/http' import { CorrectionRequestedRecord } from '@opencrvs/commons/types' -import { uploadBase64ToMinio } from '@workflow/documents' +import { uploadFileToMinio } from '@workflow/documents' import { getLoggedInPractitionerResource, getPractitionerOfficeId @@ -51,7 +51,7 @@ export const requestCorrectionRoute = createRoute({ const paymentAttachmentUrl = correctionDetails.payment?.attachmentData && - (await uploadBase64ToMinio( + (await uploadFileToMinio( correctionDetails.payment.attachmentData, getAuthHeader(request) )) @@ -59,7 +59,7 @@ export const requestCorrectionRoute = createRoute({ const proofOfLegalCorrectionAttachments = await Promise.all( correctionDetails.attachments.map(async (attachment) => ({ type: attachment.type, - url: await uploadBase64ToMinio(attachment.data, getAuthHeader(request)) + url: await uploadFileToMinio(attachment.data, getAuthHeader(request)) })) ) From a10ba8a8b7c16a767f2d3132ee0074aa1be762c9 Mon Sep 17 00:00:00 2001 From: euanmillar Date: Wed, 20 Nov 2024 11:59:16 +0000 Subject: [PATCH 3/7] Fixed test --- packages/workflow/src/records/handler/certify.test.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/workflow/src/records/handler/certify.test.ts b/packages/workflow/src/records/handler/certify.test.ts index 56a9af5e170..0b77f5614b5 100644 --- a/packages/workflow/src/records/handler/certify.test.ts +++ b/packages/workflow/src/records/handler/certify.test.ts @@ -57,7 +57,7 @@ describe('Certify record endpoint', () => { // Upload certificate to minio mswServer.use( - rest.post('http://localhost:9050/upload', (_, res, ctx) => { + rest.post('http://localhost:9050/upload-svg', (_, res, ctx) => { return res( ctx.json({ refUrl: '/ocrvs/6e964d7a-25d0-4524-bdc2-b1f29d1e816c' }) ) @@ -153,7 +153,7 @@ describe('Certify record endpoint', () => { // Upload certificate to minio mswServer.use( - rest.post('http://localhost:9050/upload', (_, res, ctx) => { + rest.post('http://localhost:9050/upload-svg', (_, res, ctx) => { return res( ctx.json({ refUrl: '/ocrvs/6e964d7a-25d0-4524-bdc2-b1f29d1e816c' }) ) From 6c2f5f6976cd5a216f7eb0c202e8853343350319 Mon Sep 17 00:00:00 2001 From: euanmillar Date: Wed, 20 Nov 2024 12:02:00 +0000 Subject: [PATCH 4/7] Revert mistaken removal of response code --- packages/documents/src/features/uploadDocument/handler.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/documents/src/features/uploadDocument/handler.ts b/packages/documents/src/features/uploadDocument/handler.ts index cce0f134b72..fc860f6021f 100644 --- a/packages/documents/src/features/uploadDocument/handler.ts +++ b/packages/documents/src/features/uploadDocument/handler.ts @@ -41,7 +41,9 @@ export async function documentUploadHandler( ...payload.metaData }) - return h.response({ refUrl: `/${MINIO_BUCKET}/${generateFileName}` }) + return h + .response({ refUrl: `/${MINIO_BUCKET}/${generateFileName}` }) + .code(200) } catch (error) { return Promise.reject(new Error(`request failed: ${error.message}`)) } From 790ff53a66a72784f583a278f874912115668823 Mon Sep 17 00:00:00 2001 From: euanmillar Date: Wed, 4 Dec 2024 10:40:55 +0000 Subject: [PATCH 5/7] renamed method --- packages/workflow/src/documents.ts | 34 ++++++++++++++++++++++-------- 1 file changed, 25 insertions(+), 9 deletions(-) diff --git a/packages/workflow/src/documents.ts b/packages/workflow/src/documents.ts index 87c1157905b..e31854b449a 100644 --- a/packages/workflow/src/documents.ts +++ b/packages/workflow/src/documents.ts @@ -20,17 +20,34 @@ import { CertifyInput, IssueInput } from './records/validations' export async function uploadFileToMinio( fileData: string, - authHeader: IAuthHeader, - svg?: boolean + authHeader: IAuthHeader +): Promise { + const suffix = '/upload' + const request = { + method: 'POST', + headers: { + ...authHeader, + 'Content-Type': 'application/json' + }, + body: JSON.stringify({ fileData: fileData }) + } + const result = await fetch(`${DOCUMENTS_URL}${suffix}`, request) + const res = await result.json() + return res.refUrl +} + +export async function uploadSVGToMinio( + fileData: string, + authHeader: IAuthHeader ): Promise { - const suffix = svg ? '/upload-svg' : '/upload' + const suffix = '/upload-svg' const request = { method: 'POST', headers: { ...authHeader, - 'Content-Type': svg ? 'image/svg+xml' : 'application/json' + 'Content-Type': 'image/svg+xml' }, - body: svg ? fileData : JSON.stringify({ fileData: fileData }) + body: fileData } const result = await fetch(`${DOCUMENTS_URL}${suffix}`, request) const res = await result.json() @@ -45,14 +62,13 @@ export async function uploadCertificateAttachmentsToDocumentsStore< certificateDetails.collector.affidavit ) { for (const affidavit of certificateDetails.collector.affidavit) { - affidavit.data = await uploadFileToMinio(affidavit.data, authHeader, true) + affidavit.data = await uploadSVGToMinio(affidavit.data, authHeader) } } if ('data' in certificateDetails) { - certificateDetails.data = await uploadFileToMinio( + certificateDetails.data = await uploadSVGToMinio( certificateDetails.data, - authHeader, - true + authHeader ) } return certificateDetails From 69cd98a02099d789005cbcde7c7785eb0e009e2c Mon Sep 17 00:00:00 2001 From: euanmillar Date: Fri, 6 Dec 2024 15:01:34 +0000 Subject: [PATCH 6/7] Affidavt is not an SVG --- packages/workflow/src/documents.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/workflow/src/documents.ts b/packages/workflow/src/documents.ts index e31854b449a..3af2de96b64 100644 --- a/packages/workflow/src/documents.ts +++ b/packages/workflow/src/documents.ts @@ -62,7 +62,7 @@ export async function uploadCertificateAttachmentsToDocumentsStore< certificateDetails.collector.affidavit ) { for (const affidavit of certificateDetails.collector.affidavit) { - affidavit.data = await uploadSVGToMinio(affidavit.data, authHeader) + affidavit.data = await uploadFileToMinio(affidavit.data, authHeader) } } if ('data' in certificateDetails) { From 1e3724293d1a0e92f342caf2e5086e6c70a60dcc Mon Sep 17 00:00:00 2001 From: Riku Rouvila Date: Mon, 9 Dec 2024 11:56:02 +0200 Subject: [PATCH 7/7] Apply suggestions from code review --- packages/workflow/src/documents.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/workflow/src/documents.ts b/packages/workflow/src/documents.ts index 3af2de96b64..debe9bc2105 100644 --- a/packages/workflow/src/documents.ts +++ b/packages/workflow/src/documents.ts @@ -36,7 +36,7 @@ export async function uploadFileToMinio( return res.refUrl } -export async function uploadSVGToMinio( +async function uploadSVGToMinio( fileData: string, authHeader: IAuthHeader ): Promise {