From e33a91d050c41e99b6385ce7e1676005818a611a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Roma=C5=84ski?= Date: Thu, 13 Jan 2022 14:03:07 +0100 Subject: [PATCH 1/3] feat(upload): Add possibility to upload big files --- src/lib/api/upload/upload.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/lib/api/upload/upload.ts b/src/lib/api/upload/upload.ts index 6ae39da9..4f71ee33 100644 --- a/src/lib/api/upload/upload.ts +++ b/src/lib/api/upload/upload.ts @@ -34,7 +34,8 @@ export interface ProgressEvent { } const DEFAULT_PROGRESS_INTERVAL = 1000; - +const BIG_FILE_THRESHOLD = 50 * 1000 * 1000 * 1000 // 50GB +const BIG_FILE_PART_SIZE = 10 * 1024 * 1024; // 10MB const normalizeProgress = (current, last) => { current.totalBytes = Math.max(current.totalBytes, last.totalBytes); current.totalPercent = Math.max(current.totalPercent, last.totalPercent); @@ -206,6 +207,9 @@ export class Upload extends EventEmitter { const f = await getFile(input, this.sanitizerOptions); f.customName = this.overrideFileName; + if (f.size > BIG_FILE_THRESHOLD){ + this.uploader.setPartSize(BIG_FILE_THRESHOLD); + } this.uploader.addFile(f); this.startProgressInterval(); From 731694802da8268ce72cb010d58ac88f31f88b7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Roma=C5=84ski?= Date: Thu, 13 Jan 2022 14:04:14 +0100 Subject: [PATCH 2/3] feat(upload): Correct part size --- src/lib/api/upload/upload.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/lib/api/upload/upload.ts b/src/lib/api/upload/upload.ts index 4f71ee33..7effb34e 100644 --- a/src/lib/api/upload/upload.ts +++ b/src/lib/api/upload/upload.ts @@ -208,7 +208,7 @@ export class Upload extends EventEmitter { const f = await getFile(input, this.sanitizerOptions); f.customName = this.overrideFileName; if (f.size > BIG_FILE_THRESHOLD){ - this.uploader.setPartSize(BIG_FILE_THRESHOLD); + this.uploader.setPartSize(BIG_FILE_PART_SIZE); } this.uploader.addFile(f); From eb849a407da334d970b9a14cf18939812407c798 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Piotr=20Roma=C5=84ski?= Date: Thu, 20 Jan 2022 14:09:43 +0100 Subject: [PATCH 3/3] refactor(upload): Set part size in multiupload --- src/lib/api/upload/upload.ts | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/src/lib/api/upload/upload.ts b/src/lib/api/upload/upload.ts index 7effb34e..7a8c5e0d 100644 --- a/src/lib/api/upload/upload.ts +++ b/src/lib/api/upload/upload.ts @@ -20,6 +20,7 @@ import { Session, Security } from '../../client'; import { S3Uploader } from './uploaders/s3'; import { FilestackError, FilestackErrorType } from './../../../filestack_error'; import { SanitizeOptions } from './../../utils'; +import { File as FsFile } from './file'; import { UploadOptions, StoreUploadOptions } from '../upload/types'; import { getFile, InputFile } from './file_tools'; @@ -196,6 +197,19 @@ export class Upload extends EventEmitter { this.uploader.setUploadTags(tags); } + /** + * Set upload part size + * + * @param {File} file + * @param {S3Uploader} uploader + * @memberof Upload + */ + private setPartSize(uploader : S3Uploader, file : FsFile ){ + if(file.size > BIG_FILE_THRESHOLD){ + uploader.setPartSize(BIG_FILE_PART_SIZE); + } + } + /** * Upload single file * @@ -207,9 +221,8 @@ export class Upload extends EventEmitter { const f = await getFile(input, this.sanitizerOptions); f.customName = this.overrideFileName; - if (f.size > BIG_FILE_THRESHOLD){ - this.uploader.setPartSize(BIG_FILE_PART_SIZE); - } + this.setPartSize(this.uploader, f); + this.uploader.addFile(f); this.startProgressInterval(); @@ -241,6 +254,7 @@ export class Upload extends EventEmitter { const f = await getFile(input[i], this.sanitizerOptions); f.customName = this.overrideFileName; + this.setPartSize(this.uploader, f); this.uploader.addFile(f); }