diff --git a/lib/plugins/s3/index.js b/lib/plugins/s3/index.js index 605db11969..01671168da 100644 --- a/lib/plugins/s3/index.js +++ b/lib/plugins/s3/index.js @@ -1,7 +1,6 @@ import path from 'node:path'; import { promises as fsPromises } from 'node:fs'; -import pLimit from 'p-limit'; import { getLogger } from '@sitespeed.io/log'; import { S3Client, PutObjectCommand } from '@aws-sdk/client-s3'; @@ -9,6 +8,7 @@ import { SitespeedioPlugin } from '@sitespeed.io/plugin'; import { throwIfMissing } from '../../support/util.js'; import { recursiveReaddir } from '../../support/fileUtil.js'; import { getContentType } from './contentType.js'; +import { runWithConcurrencyLimit } from './limit.js'; const log = getLogger('sitespeedio.plugin.s3'); @@ -86,31 +86,36 @@ export default class S3Plugin extends SitespeedioPlugin { try { const files = await recursiveReaddir(baseDir, true); - const limit = pLimit(s3Options.maxAsyncS3 || 20); - const uploadPromises = files.map(file => - limit(() => - uploadFile( - file, - this.s3Client, - s3Options, - this.storageManager.getStoragePrefix(), - baseDir - ) - ) - ); - await Promise.all(uploadPromises); + const tasks = files.map(file => async () => { + return uploadFile( + file, + this.s3Client, + s3Options, + this.storageManager.getStoragePrefix(), + baseDir + ); + }); + + await runWithConcurrencyLimit(tasks, s3Options.maxAsyncS3 || 20); if (this.options.copyLatestFilesToBase) { const rootPath = path.join(baseDir, '..'); const directoriesAsArray = rootPath.split(path.sep); const rootName = directoriesAsArray.at(-1); const latestFiles = await recursiveReaddir(rootPath, true); - const latestUploadPromises = latestFiles.map(file => - limit(() => - uploadFile(file, this.s3Client, s3Options, rootName, rootPath) - ) + const latestTasks = latestFiles.map(file => async () => { + return uploadFile( + file, + this.s3Client, + s3Options, + rootName, + rootPath + ); + }); + await runWithConcurrencyLimit( + latestTasks, + s3Options.maxAsyncS3 || 20 ); - await Promise.all(latestUploadPromises); } log.info('Finished upload to S3'); diff --git a/lib/plugins/s3/limit.js b/lib/plugins/s3/limit.js new file mode 100644 index 0000000000..2d01b75130 --- /dev/null +++ b/lib/plugins/s3/limit.js @@ -0,0 +1,34 @@ +export async function runWithConcurrencyLimit(tasks, limit) { + const running = new Set(); + + async function runNext() { + if (tasks.length === 0) { + return; + } + + const task = tasks.shift(); + const promise = task() + .catch(error => { + throw error; + }) + .finally(() => { + running.delete(promise); + void runNext(); + }); + + running.add(promise); + if (running.size < limit) { + void runNext(); + } + } + + const starters = []; + for (let index = 0; index < limit && tasks.length > 0; index++) { + starters.push(runNext()); + } + + await Promise.allSettled(starters); + if (running.size > 0) { + await Promise.allSettled(Array.from(running)); + } +} diff --git a/npm-shrinkwrap.json b/npm-shrinkwrap.json index 6a1f4ac15c..044fc3cb4a 100644 --- a/npm-shrinkwrap.json +++ b/npm-shrinkwrap.json @@ -33,7 +33,6 @@ "markdown": "0.5.0", "node-scp": "0.0.23", "ora": "8.0.1", - "p-limit": "6.1.0", "pug": "3.0.3", "simplecrawler": "1.1.9", "tape": "5.8.1", @@ -7768,31 +7767,6 @@ "os-tmpdir": "^1.0.0" } }, - "node_modules/p-limit": { - "version": "6.1.0", - "resolved": "https://registry.npmjs.org/p-limit/-/p-limit-6.1.0.tgz", - "integrity": "sha512-H0jc0q1vOzlEk0TqAKXKZxdl7kX3OFUzCnNVUnq5Pc3DGo0kpeaMuPqxQn235HibwBEb0/pm9dgKTjXy66fBkg==", - "dependencies": { - "yocto-queue": "^1.1.1" - }, - "engines": { - "node": ">=18" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, - "node_modules/p-limit/node_modules/yocto-queue": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/yocto-queue/-/yocto-queue-1.1.1.tgz", - "integrity": "sha512-b4JR1PFR10y1mKjhHY9LaGo6tmrgjit7hxVIeAmyMw3jegXR4dhYqLaQF5zMXZxY7tLpMyJeLjr1C4rLmkVe8g==", - "engines": { - "node": ">=12.20" - }, - "funding": { - "url": "https://github.com/sponsors/sindresorhus" - } - }, "node_modules/p-map": { "version": "7.0.2", "resolved": "https://registry.npmjs.org/p-map/-/p-map-7.0.2.tgz", diff --git a/package.json b/package.json index e6a943ba79..4b941efc94 100644 --- a/package.json +++ b/package.json @@ -104,7 +104,6 @@ "markdown": "0.5.0", "node-scp": "0.0.23", "ora": "8.0.1", - "p-limit": "6.1.0", "pug": "3.0.3", "simplecrawler": "1.1.9", "tape": "5.8.1",