From dc1e488b50dd7e2deab3e8b28c7d6ece36b90b0e Mon Sep 17 00:00:00 2001 From: Denis DelGrosso <85250797+ddelgrosso1@users.noreply.github.com> Date: Mon, 15 Jul 2024 10:06:36 -0400 Subject: [PATCH 1/2] feat: add function to allow user to set destination in transfer manager (#2497) * feat: add function to allow user to set destination in transfer manager * lint --- src/transfer-manager.ts | 12 +++++++++--- test/transfer-manager.ts | 19 +++++++++++++++++++ 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index d0fe21f75..f253461d9 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -94,6 +94,10 @@ const GCCL_GCS_CMD_FEATURE = { export interface UploadManyFilesOptions { concurrencyLimit?: number; + customDestinationBuilder?( + path: string, + options: UploadManyFilesOptions + ): string; skipIfExists?: boolean; prefix?: string; passthroughOptions?: Omit; @@ -411,6 +415,8 @@ export class TransferManager { * @typedef {object} UploadManyFilesOptions * @property {number} [concurrencyLimit] The number of concurrently executing promises * to use when uploading the files. + * @property {Function} [customDestinationBuilder] A fuction that will take the current path of a local file + * and return a string representing a custom path to be used to upload the file to GCS. * @property {boolean} [skipIfExists] Do not upload the file if it already exists in * the bucket. This will set the precondition ifGenerationMatch = 0. * @property {string} [prefix] A prefix to append to all of the uploaded files. @@ -490,9 +496,9 @@ export class TransferManager { [GCCL_GCS_CMD_KEY]: GCCL_GCS_CMD_FEATURE.UPLOAD_MANY, }; - passThroughOptionsCopy.destination = filePath - .split(path.sep) - .join(path.posix.sep); + passThroughOptionsCopy.destination = options.customDestinationBuilder + ? options.customDestinationBuilder(filePath, options) + : filePath.split(path.sep).join(path.posix.sep); if (options.prefix) { passThroughOptionsCopy.destination = path.posix.join( ...options.prefix.split(path.sep), diff --git a/test/transfer-manager.ts b/test/transfer-manager.ts index 741e0a91c..af5b2d7c2 100644 --- a/test/transfer-manager.ts +++ b/test/transfer-manager.ts @@ -26,6 +26,7 @@ import { MultiPartUploadError, MultiPartUploadHelper, UploadOptions, + UploadManyFilesOptions, TransferManager, Storage, DownloadResponse, @@ -173,6 +174,24 @@ describe('Transfer Manager', () => { await transferManager.uploadManyFiles([filePath]); }); + + it('allows the user to apply a custom destination transformation when supplied a custom function', async () => { + const paths = ['a', 'b', 'foo/bar', 'bar.txt']; + const expected = ['foo/a', 'b/bar', 'foo/foo/bar', 'bar.txt/bar']; + sandbox.stub(bucket, 'upload').callsFake((path, options) => { + const uploadOpts = options as UploadOptions; + assert(expected.includes(uploadOpts.destination as string)); + }); + + let callCount = 0; + const transformationFunc = (path: string) => { + assert.strictEqual(path, paths[callCount]); + return expected[callCount++]; + }; + await transferManager.uploadManyFiles(paths, { + customDestinationBuilder: transformationFunc, + }); + }); }); describe('downloadManyFiles', () => { From ef677607f7ce2bb0424886e42189284a83148027 Mon Sep 17 00:00:00 2001 From: "release-please[bot]" <55107282+release-please[bot]@users.noreply.github.com> Date: Mon, 15 Jul 2024 13:33:23 -0400 Subject: [PATCH 2/2] chore(main): release 7.12.0 (#2499) Co-authored-by: release-please[bot] <55107282+release-please[bot]@users.noreply.github.com> --- CHANGELOG.md | 7 +++++++ package.json | 2 +- samples/package.json | 2 +- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 61d47a7fe..67ac44197 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,13 @@ [1]: https://www.npmjs.com/package/@google-cloud/storage?activeTab=versions +## [7.12.0](https://github.com/googleapis/nodejs-storage/compare/v7.11.3...v7.12.0) (2024-07-15) + + +### Features + +* Add function to allow user to set destination in transfer manager ([#2497](https://github.com/googleapis/nodejs-storage/issues/2497)) ([dc1e488](https://github.com/googleapis/nodejs-storage/commit/dc1e488b50dd7e2deab3e8b28c7d6ece36b90b0e)) + ## [7.11.3](https://github.com/googleapis/nodejs-storage/compare/v7.11.2...v7.11.3) (2024-07-09) diff --git a/package.json b/package.json index 9d9ab33f1..248ca0393 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@google-cloud/storage", "description": "Cloud Storage Client Library for Node.js", - "version": "7.11.3", + "version": "7.12.0", "license": "Apache-2.0", "author": "Google Inc.", "engines": { diff --git a/samples/package.json b/samples/package.json index 82900f131..1ab4f11ec 100644 --- a/samples/package.json +++ b/samples/package.json @@ -17,7 +17,7 @@ }, "dependencies": { "@google-cloud/pubsub": "^4.0.0", - "@google-cloud/storage": "^7.11.3", + "@google-cloud/storage": "^7.12.0", "node-fetch": "^2.6.7", "uuid": "^8.0.0", "yargs": "^16.0.0"