From be1feefda150bdcf7abc84a4c7f42c80c7a1bf55 Mon Sep 17 00:00:00 2001 From: Bryan Killian Date: Thu, 2 Jan 2020 17:45:59 -0500 Subject: [PATCH 1/2] ensure metadata is correct. if a file does not change, then the putObject call will not run. this forces a copyObject for all files that match the globs in s3Sync.params --- index.js | 80 +++++++++++++++++++++++++++++++++++++++++++++++++++++--- 1 file changed, 77 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 5309c27..af8a391 100644 --- a/index.js +++ b/index.js @@ -5,7 +5,7 @@ const s3 = require('@auth0/s3'); const chalk = require('chalk'); const minimatch = require('minimatch'); const path = require('path'); - +const fs = require('fs'); const messagePrefix = 'S3 Sync: '; class ServerlessS3Sync { @@ -18,7 +18,8 @@ class ServerlessS3Sync { s3sync: { usage: 'Sync directories and S3 prefixes', lifecycleEvents: [ - 'sync' + 'sync', + 'metadata' ] } }; @@ -26,7 +27,8 @@ class ServerlessS3Sync { this.hooks = { 'after:deploy:deploy': () => options.nos3sync?undefined:BbPromise.bind(this).then(this.sync), 'before:remove:remove': () => BbPromise.bind(this).then(this.clear), - 's3sync:sync': () => BbPromise.bind(this).then(this.sync) + 's3sync:sync': () => BbPromise.bind(this).then(this.sync), + 's3sync:metadata': () => BbPromise.bind(this).then(this.syncMetadata) }; } @@ -179,6 +181,78 @@ class ServerlessS3Sync { }); } + syncMetadata() { + const s3Sync = this.serverless.service.custom.s3Sync; + const cli = this.serverless.cli; + if (!Array.isArray(s3Sync)) { + cli.consoleLog(`${messagePrefix}${chalk.red('No configuration found')}`) + return Promise.resolve(); + } + cli.consoleLog(`${messagePrefix}${chalk.yellow('Syncing metadata...')}`); + const servicePath = this.servicePath; + const promises = s3Sync.map( async (s) => { + let bucketPrefix = ''; + if (s.hasOwnProperty('bucketPrefix')) { + bucketPrefix = s.bucketPrefix; + } + let acl = 'private'; + if (s.hasOwnProperty('acl')) { + acl = s.acl; + } + if (!s.bucketName || !s.localDir) { + throw 'Invalid custom.s3Sync'; + } + const localDir = [servicePath, s.localDir].join('/'); + let filesToSync = []; + if(Array.isArray(s.params)) { + s.params.forEach((param) => { + const glob = Object.keys(param)[0]; + let files = this.getLocalFiles(localDir, []); + minimatch.match(files, `${path.resolve(localDir)}/${glob}`, {matchBase: true}).forEach((match) => { + filesToSync.push({name: match, params: this.extractMetaParams(param)}); + }); + }); + } + return filesToSync.forEach((file) => { + return new Promise((resolve) => { + let params = { + CopySource: file.name.replace(localDir, `${s.bucketName}${bucketPrefix == '' ? '' : bucketPrefix}/`), + Key: file.name.replace(localDir, ''), + Bucket: s.bucketName, + Metadata: file.params, + MetadataDirective: 'REPLACE' + }; + const uploader = this.client().copyObject(params); + uploader.on('error', (err) => { + throw err; + }); + uploader.on('end', () => { + resolve('done'); + }); + }); + }); + }); + cli.consoleLog(`${JSON.stringify(promises)}`); + return Promise.all((promises)) + .then(() => { + cli.printDot(); + cli.consoleLog(''); + cli.consoleLog(`${messagePrefix}${chalk.yellow('Synced metadata.')}`); + }); + } + + getLocalFiles(dir, files) { + fs.readdirSync(dir).forEach(file => { + let fullPath = path.join(dir, file); + if (fs.lstatSync(fullPath).isDirectory()) { + this.getLocalFiles(fullPath, files); + } else { + files.push(fullPath); + } + }); + return files; + } + extractMetaParams(config) { const validParams = {}; const keys = Object.keys(config); From aa20fdf7f740b7aef0d24bc10812bfe2829e69e6 Mon Sep 17 00:00:00 2001 From: Bryan Killian Date: Fri, 3 Jan 2020 10:28:16 -0500 Subject: [PATCH 2/2] forgot to put the ACL here. by default we set private, if copy-object is called without acl it will be private. set explicit acl in custom.s3Sync[n] to have you desired acl --- index.js | 1 + 1 file changed, 1 insertion(+) diff --git a/index.js b/index.js index af8a391..f9aa2f7 100644 --- a/index.js +++ b/index.js @@ -220,6 +220,7 @@ class ServerlessS3Sync { Key: file.name.replace(localDir, ''), Bucket: s.bucketName, Metadata: file.params, + ACL: acl, MetadataDirective: 'REPLACE' }; const uploader = this.client().copyObject(params);