diff --git a/.github/.OwlBot.lock.yaml b/.github/.OwlBot.lock.yaml index 9e90d54bf..24943e116 100644 --- a/.github/.OwlBot.lock.yaml +++ b/.github/.OwlBot.lock.yaml @@ -13,5 +13,5 @@ # limitations under the License. docker: image: gcr.io/cloud-devrel-public-resources/owlbot-nodejs:latest - digest: sha256:d920257482ca1cd72978f29f7d28765a9f8c758c21ee0708234db5cf4c5016c2 -# created: 2024-06-12T16:18:41.688792375Z + digest: sha256:609822e3c09b7a1bd90b99655904609f162cc15acb4704f1edf778284c36f429 +# created: 2024-10-01T19:34:30.797530443Z diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 8d11c3a5e..b0cc3a59f 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -5,8 +5,5 @@ # https://help.github.com/en/github/creating-cloning-and-archiving-repositories/about-code-owners#codeowners-syntax -# The yoshi-nodejs team is the default owner for nodejs repositories. -* @googleapis/yoshi-nodejs @googleapis/cloud-storage-dpe - -# The github automation team is the default owner for the auto-approve file. -.github/auto-approve.yml @googleapis/github-automation +# Unless specified, the jsteam is the default owner for nodejs repositories. +* @googleapis/cloud-storage-dpe @googleapis/jsteam \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md deleted file mode 100644 index 6e9c0dc92..000000000 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ /dev/null @@ -1,38 +0,0 @@ ---- -name: Bug report -about: Create a report to help us improve -labels: 'type: bug, priority: p2' ---- - -Thanks for stopping by to let us know something could be better! - -**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. - -1) Is this a client library issue or a product issue? -This is the client library for . We will only be able to assist with issues that pertain to the behaviors of this library. If the issue you're experiencing is due to the behavior of the product itself, please visit the [ Support page]() to reach the most relevant engineers. - -2) Did someone already solve this? - - Search the issues already opened: https://github.com/googleapis/nodejs-storage/issues - - Search the issues on our "catch-all" repository: https://github.com/googleapis/google-cloud-node - - Search or ask on StackOverflow (engineers monitor these tags): http://stackoverflow.com/questions/tagged/google-cloud-platform+node.js - -3) Do you have a support contract? -Please create an issue in the [support console](https://cloud.google.com/support/) to ensure a timely response. - -If the support paths suggested above still do not result in a resolution, please provide the following details. - -#### Environment details - - - OS: - - Node.js version: - - npm version: - - `@google-cloud/storage` version: - -#### Steps to reproduce - - 1. ? - 2. ? - -Making sure to follow these steps will guarantee the quickest resolution possible. - -Thanks! diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml new file mode 100644 index 000000000..a14a91887 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -0,0 +1,99 @@ +name: Bug Report +description: Create a report to help us improve +labels: + - bug +body: + - type: markdown + attributes: + value: > + **PLEASE READ**: If you have a support contract with Google, please + create an issue in the [support + console](https://cloud.google.com/support/) instead of filing on GitHub. + This will ensure a timely response. Otherwise, please make sure to + follow the steps below. + - type: checkboxes + attributes: + label: Please make sure you have searched for information in the following + guides. + options: + - label: "Search the issues already opened: + https://github.com/GoogleCloudPlatform/google-cloud-node/issues" + required: true + - label: "Search StackOverflow: + http://stackoverflow.com/questions/tagged/google-cloud-platform+nod\ + e.js" + required: true + - label: "Check our Troubleshooting guide: + https://github.com/googleapis/google-cloud-node/blob/main/docs/trou\ + bleshooting.md" + required: true + - label: "Check our FAQ: + https://github.com/googleapis/google-cloud-node/blob/main/docs/faq.\ + md" + required: true + - label: "Check our libraries HOW-TO: + https://github.com/googleapis/gax-nodejs/blob/main/client-libraries\ + .md" + required: true + - label: "Check out our authentication guide: + https://github.com/googleapis/google-auth-library-nodejs" + required: true + - label: "Check out handwritten samples for many of our APIs: + https://github.com/GoogleCloudPlatform/nodejs-docs-samples" + required: true + - type: textarea + attributes: + label: > + A screenshot that you have tested with "Try this API". + description: > + As our client libraries are mostly autogenerated, we kindly request + that you test whether your issue is with the client library, or with the + API itself. To do so, please search for your API + here: https://developers.google.com/apis-explorer and attempt to + reproduce the issue in the given method. Please include a screenshot of + the response in "Try this API". This response should NOT match the current + behavior you are experiencing. If the behavior is the same, it means + that you are likely experiencing a bug with the API itself. In that + case, please submit an issue to the API team, either by submitting an + issue in its issue tracker (https://cloud.google.com/support/docs/issue-trackers), or by + submitting an issue in its linked tracker in the .repo-metadata.json + file https://issuetracker.google.com/savedsearches/559782 + validations: + required: true + - type: input + attributes: + label: > + Link to the code that reproduces this issue. A link to a **public** Github Repository or gist with a minimal + reproduction. + description: > + **Skipping this or providing an invalid link will result in the issue being closed** + validations: + required: true + - type: textarea + attributes: + label: > + A step-by-step description of how to reproduce the issue, based on + the linked reproduction. + description: > + Screenshots can be provided in the issue body below. + placeholder: | + 1. Start the application in development (next dev) + 2. Click X + 3. Y will happen + validations: + required: true + - type: textarea + attributes: + label: A clear and concise description of what the bug is, and what you + expected to happen. + placeholder: Following the steps from the previous section, I expected A to + happen, but I observed B instead + validations: + required: true + + - type: textarea + attributes: + label: A clear and concise description WHY you expect this behavior, i.e., was it a recent change, there is documentation that points to this behavior, etc. ** + placeholder: 'Documentation here(link) states that B should happen instead of A' + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/documentation_request.yml b/.github/ISSUE_TEMPLATE/documentation_request.yml new file mode 100644 index 000000000..d42fde52c --- /dev/null +++ b/.github/ISSUE_TEMPLATE/documentation_request.yml @@ -0,0 +1,53 @@ +name: Documentation Requests +description: Requests for more information +body: + - type: markdown + attributes: + value: > + Please use this issue type to log documentation requests against the library itself. + These requests should involve documentation on Github (`.md` files), and should relate to the library + itself. If you have questions or documentation requests for an API, please + reach out to the API tracker itself. + + Please submit an issue to the API team, either by submitting an + issue in its issue tracker https://cloud.google.com/support/docs/issue-trackers), or by + submitting an issue in its linked tracker in the .repo-metadata.json + file in the API under packages/* ([example](https://issuetracker.google.com/savedsearches/559782)). + You can also submit a request to documentation on cloud.google.com itself with the "Send Feedback" + on the bottom of the page. + + + Please note that documentation requests and questions for specific APIs + will be closed. + - type: checkboxes + attributes: + label: Please make sure you have searched for information in the following + guides. + options: + - label: "Search the issues already opened: + https://github.com/GoogleCloudPlatform/google-cloud-node/issues" + required: true + - label: "Check our Troubleshooting guide: + https://googlecloudplatform.github.io/google-cloud-node/#/docs/guid\ + es/troubleshooting" + required: true + - label: "Check our FAQ: + https://googlecloudplatform.github.io/google-cloud-node/#/docs/guid\ + es/faq" + required: true + - label: "Check our libraries HOW-TO: + https://github.com/googleapis/gax-nodejs/blob/main/client-libraries\ + .md" + required: true + - label: "Check out our authentication guide: + https://github.com/googleapis/google-auth-library-nodejs" + required: true + - label: "Check out handwritten samples for many of our APIs: + https://github.com/GoogleCloudPlatform/nodejs-docs-samples" + required: true + - type: textarea + attributes: + label: > + Documentation Request + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md deleted file mode 100644 index b0327dfa0..000000000 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ /dev/null @@ -1,18 +0,0 @@ ---- -name: Feature request -about: Suggest an idea for this library -labels: 'type: feature request, priority: p3' ---- - -Thanks for stopping by to let us know something could be better! - -**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. - - **Is your feature request related to a problem? Please describe.** -A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - **Describe the solution you'd like** -A clear and concise description of what you want to happen. - **Describe alternatives you've considered** -A clear and concise description of any alternative solutions or features you've considered. - **Additional context** -Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml new file mode 100644 index 000000000..b3f121842 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -0,0 +1,53 @@ +name: Feature Request +description: Suggest an idea for this library +labels: + - feature request +body: + - type: markdown + attributes: + value: > + **PLEASE READ**: If you have a support contract with Google, please + create an issue in the [support + console](https://cloud.google.com/support/) instead of filing on GitHub. + This will ensure a timely response. Otherwise, please make sure to + follow the steps below. + - type: textarea + attributes: + label: > + A screenshot that you have tested with "Try this API". + description: > + As our client libraries are mostly autogenerated, we kindly request + that you test whether your feature request is with the client library, or with the + API itself. To do so, please search for your API + here: https://developers.google.com/apis-explorer and attempt to + reproduce the issue in the given method. Please include a screenshot of + the response in "Try this API". This response should NOT match the current + behavior you are experiencing. If the behavior is the same, it means + that you are likely requesting a feature for the API itself. In that + case, please submit an issue to the API team, either by submitting an + issue in its issue tracker https://cloud.google.com/support/docs/issue-trackers, or by + submitting an issue in its linked tracker in the .repo-metadata.json + file in the API under packages/* ([example](https://issuetracker.google.com/savedsearches/559782)) + + Example of library specific issues would be: retry strategies, authentication questions, or issues with typings. + Examples of API issues would include: expanding method parameter types, adding functionality to an API. + validations: + required: true + - type: textarea + attributes: + label: > + What would you like to see in the library? + description: > + Screenshots can be provided in the issue body below. + placeholder: | + 1. Set up authentication like so + 2. Run the program like so + 3. X would be nice to happen + + - type: textarea + attributes: + label: Describe alternatives you've considered + + - type: textarea + attributes: + label: Additional context/notes \ No newline at end of file diff --git a/.github/ISSUE_TEMPLATE/processs_request.md b/.github/ISSUE_TEMPLATE/processs_request.md new file mode 100644 index 000000000..45682e8f1 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/processs_request.md @@ -0,0 +1,4 @@ +--- +name: Process Request +about: Submit a process request to the library. Process requests are any requests related to library infrastructure, for example CI/CD, publishing, releasing, broken links. +--- diff --git a/.github/ISSUE_TEMPLATE/question.md b/.github/ISSUE_TEMPLATE/question.md deleted file mode 100644 index 973231139..000000000 --- a/.github/ISSUE_TEMPLATE/question.md +++ /dev/null @@ -1,12 +0,0 @@ ---- -name: Question -about: Ask a question -labels: 'type: question, priority: p3' ---- - -Thanks for stopping by to ask us a question! Please make sure to include: -- What you're trying to do -- What code you've already tried -- Any error messages you're getting - -**PLEASE READ**: If you have a support contract with Google, please create an issue in the [support console](https://cloud.google.com/support/) instead of filing on GitHub. This will ensure a timely response. diff --git a/.github/ISSUE_TEMPLATE/questions.md b/.github/ISSUE_TEMPLATE/questions.md new file mode 100644 index 000000000..62c1dd1b9 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/questions.md @@ -0,0 +1,8 @@ +--- +name: Question +about: If you have a question, please use Discussions + +--- + +If you have a general question that goes beyond the library itself, we encourage you to use [Discussions](https://github.com//discussions) +to engage with fellow community members! diff --git a/.github/auto-approve.yml b/.github/auto-approve.yml index ec51b072d..7cba0af63 100644 --- a/.github/auto-approve.yml +++ b/.github/auto-approve.yml @@ -1,4 +1,2 @@ processes: - - "NodeDependency" - - "OwlBotTemplateChangesNode" - - "OwlBotPRsNode" \ No newline at end of file + - "NodeDependency" \ No newline at end of file diff --git a/.github/scripts/close-invalid-link.cjs b/.github/scripts/close-invalid-link.cjs new file mode 100644 index 000000000..d7a3688e7 --- /dev/null +++ b/.github/scripts/close-invalid-link.cjs @@ -0,0 +1,56 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +async function closeIssue(github, owner, repo, number) { + await github.rest.issues.createComment({ + owner: owner, + repo: repo, + issue_number: number, + body: 'Issue was opened with an invalid reproduction link. Please make sure the repository is a valid, publicly-accessible github repository, and make sure the url is complete (example: https://github.com/googleapis/google-cloud-node)' + }); + await github.rest.issues.update({ + owner: owner, + repo: repo, + issue_number: number, + state: 'closed' + }); +} +module.exports = async ({github, context}) => { + const owner = context.repo.owner; + const repo = context.repo.repo; + const number = context.issue.number; + + const issue = await github.rest.issues.get({ + owner: owner, + repo: repo, + issue_number: number, + }); + + const isBugTemplate = issue.data.body.includes('Link to the code that reproduces this issue'); + + if (isBugTemplate) { + console.log(`Issue ${number} is a bug template`) + try { + const link = issue.data.body.split('\n')[18].match(/(https?:\/\/(gist\.)?github.com\/.*)/)[0]; + console.log(`Issue ${number} contains this link: ${link}`) + const isValidLink = (await fetch(link)).ok; + console.log(`Issue ${number} has a ${isValidLink ? 'valid' : 'invalid'} link`) + if (!isValidLink) { + await closeIssue(github, owner, repo, number); + } + } catch (err) { + await closeIssue(github, owner, repo, number); + } + } +}; diff --git a/.github/scripts/close-unresponsive.cjs b/.github/scripts/close-unresponsive.cjs new file mode 100644 index 000000000..142dc1265 --- /dev/null +++ b/.github/scripts/close-unresponsive.cjs @@ -0,0 +1,69 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +function labeledEvent(data) { + return data.event === 'labeled' && data.label.name === 'needs more info'; + } + + const numberOfDaysLimit = 15; + const close_message = `This has been closed since a request for information has \ + not been answered for ${numberOfDaysLimit} days. It can be reopened when the \ + requested information is provided.`; + + module.exports = async ({github, context}) => { + const owner = context.repo.owner; + const repo = context.repo.repo; + + const issues = await github.rest.issues.listForRepo({ + owner: owner, + repo: repo, + labels: 'needs more info', + }); + const numbers = issues.data.map((e) => e.number); + + for (const number of numbers) { + const events = await github.paginate( + github.rest.issues.listEventsForTimeline, + { + owner: owner, + repo: repo, + issue_number: number, + }, + (response) => response.data.filter(labeledEvent) + ); + + const latest_response_label = events[events.length - 1]; + + const created_at = new Date(latest_response_label.created_at); + const now = new Date(); + const diff = now - created_at; + const diffDays = diff / (1000 * 60 * 60 * 24); + + if (diffDays > numberOfDaysLimit) { + await github.rest.issues.update({ + owner: owner, + repo: repo, + issue_number: number, + state: 'closed', + }); + + await github.rest.issues.createComment({ + owner: owner, + repo: repo, + issue_number: number, + body: close_message, + }); + } + } + }; diff --git a/.github/scripts/remove-response-label.cjs b/.github/scripts/remove-response-label.cjs new file mode 100644 index 000000000..887cf349e --- /dev/null +++ b/.github/scripts/remove-response-label.cjs @@ -0,0 +1,33 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +module.exports = async ({ github, context }) => { + const commenter = context.actor; + const issue = await github.rest.issues.get({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + const author = issue.data.user.login; + const labels = issue.data.labels.map((e) => e.name); + + if (author === commenter && labels.includes('needs more info')) { + await github.rest.issues.removeLabel({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + name: 'needs more info', + }); + } + }; diff --git a/.github/workflows/issues-no-repro.yaml b/.github/workflows/issues-no-repro.yaml new file mode 100644 index 000000000..442a46bcc --- /dev/null +++ b/.github/workflows/issues-no-repro.yaml @@ -0,0 +1,18 @@ +name: invalid_link +on: + issues: + types: [opened, reopened] + +jobs: + close: + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + - uses: actions/github-script@v7 + with: + script: | + const script = require('./.github/scripts/close-invalid-link.cjs') + await script({github, context}) diff --git a/.github/workflows/response.yaml b/.github/workflows/response.yaml new file mode 100644 index 000000000..6ed37326f --- /dev/null +++ b/.github/workflows/response.yaml @@ -0,0 +1,35 @@ +name: no_response +on: + schedule: + - cron: '30 1 * * *' # Run every day at 01:30 + workflow_dispatch: + issue_comment: + +jobs: + close: + if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + - uses: actions/github-script@v7 + with: + script: | + const script = require('./.github/scripts/close-unresponsive.cjs') + await script({github, context}) + + remove_label: + if: github.event_name == 'issue_comment' + runs-on: ubuntu-latest + permissions: + issues: write + pull-requests: write + steps: + - uses: actions/checkout@v4 + - uses: actions/github-script@v7 + with: + script: | + const script = require('./.github/scripts/remove-response-label.cjs') + await script({github, context}) diff --git a/.kokoro/continuous/node14/system-test.cfg b/.kokoro/continuous/node14/system-test.cfg index 9ba9ad1c7..c7c9507a8 100644 --- a/.kokoro/continuous/node14/system-test.cfg +++ b/.kokoro/continuous/node14/system-test.cfg @@ -8,5 +8,5 @@ env_vars: { env_vars: { key: "SECRET_MANAGER_KEYS" - value: "long-door-651-kokoro-system-test-service-account" + value: "long-door-651-kokoro-system-test-service-account,client-library-test-universe-domain-credential" } \ No newline at end of file diff --git a/.kokoro/presubmit/node14/system-test.cfg b/.kokoro/presubmit/node14/system-test.cfg index 9ba9ad1c7..c7c9507a8 100644 --- a/.kokoro/presubmit/node14/system-test.cfg +++ b/.kokoro/presubmit/node14/system-test.cfg @@ -8,5 +8,5 @@ env_vars: { env_vars: { key: "SECRET_MANAGER_KEYS" - value: "long-door-651-kokoro-system-test-service-account" + value: "long-door-651-kokoro-system-test-service-account,client-library-test-universe-domain-credential" } \ No newline at end of file diff --git a/.kokoro/release/publish.cfg b/.kokoro/release/publish.cfg index baea5db3c..019cc24a5 100644 --- a/.kokoro/release/publish.cfg +++ b/.kokoro/release/publish.cfg @@ -18,7 +18,7 @@ before_action { env_vars: { key: "SECRET_MANAGER_KEYS" - value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem" + value: "releasetool-publish-reporter-app,releasetool-publish-reporter-googleapis-installation,releasetool-publish-reporter-pem,client-library-test-universe-domain-credential" } # Download trampoline resources. diff --git a/.kokoro/system-test.sh b/.kokoro/system-test.sh index 0b3043d26..e219954ae 100755 --- a/.kokoro/system-test.sh +++ b/.kokoro/system-test.sh @@ -22,6 +22,12 @@ export NPM_CONFIG_PREFIX=${HOME}/.npm-global export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/secret_manager/long-door-651-kokoro-system-test-service-account export GCLOUD_PROJECT=long-door-651 +# For universe domain testing +export TEST_UNIVERSE_DOMAIN_CREDENTIAL=${KOKORO_GFILE_DIR}/secret_manager/client-library-test-universe-domain-credential +export TEST_UNIVERSE_DOMAIN=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-domain) +export TEST_UNIVERSE_PROJECT_ID=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-project-id) +export TEST_UNIVERSE_LOCATION=$(gcloud secrets versions access latest --project cloud-devrel-kokoro-resources --secret=client-library-test-universe-storage-location) + cd $(dirname $0)/.. # Run a pre-test hook, if a pre-system-test.sh is in the project diff --git a/CHANGELOG.md b/CHANGELOG.md index 31e5a5eb0..d9ebb06dc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,16 @@ [1]: https://www.npmjs.com/package/@google-cloud/storage?activeTab=versions +## [7.14.0](https://github.com/googleapis/nodejs-storage/compare/v7.13.0...v7.14.0) (2024-10-29) + + +### Features + +* Add support for restore token ([#2548](https://github.com/googleapis/nodejs-storage/issues/2548)) ([8241e91](https://github.com/googleapis/nodejs-storage/commit/8241e91e78d47b4cdaea2d941f75fd6a4fa29230)) +* Adds integration tests for Universe Domain configuration ([#2538](https://github.com/googleapis/nodejs-storage/issues/2538)) ([53db6ba](https://github.com/googleapis/nodejs-storage/commit/53db6ba7406b99e507cacfa6195cb5a7d308914b)) +* Adds integration tests for Universe Domain configuration with ([53db6ba](https://github.com/googleapis/nodejs-storage/commit/53db6ba7406b99e507cacfa6195cb5a7d308914b)) +* **storage:** Add support for 'skipIfExists' option for downloadMany ([#2526](https://github.com/googleapis/nodejs-storage/issues/2526)) ([729efb2](https://github.com/googleapis/nodejs-storage/commit/729efb213f96b1a406a1caa54870f50e96796639)) + ## [7.13.0](https://github.com/googleapis/nodejs-storage/compare/v7.12.1...v7.13.0) (2024-09-17) diff --git a/README.md b/README.md index 8480241f1..b3dbb15b9 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Google APIs Client Libraries, in [Client Libraries Explained][explained]. 1. [Select or create a Cloud Platform project][projects]. 1. [Enable billing for your project][billing]. 1. [Enable the Google Cloud Storage API][enable_api]. -1. [Set up authentication with a service account][auth] so you can access the +1. [Set up authentication][auth] so you can access the API from your local workstation. ### Installing the client library @@ -288,4 +288,4 @@ See [LICENSE](https://github.com/googleapis/nodejs-storage/blob/main/LICENSE) [projects]: https://console.cloud.google.com/project [billing]: https://support.google.com/cloud/answer/6293499#enable-billing [enable_api]: https://console.cloud.google.com/flows/enableapi?apiid=storage-api.googleapis.com -[auth]: https://cloud.google.com/docs/authentication/getting-started +[auth]: https://cloud.google.com/docs/authentication/external/set-up-adc-local diff --git a/owlbot.py b/owlbot.py index 28ca96604..8e2e3b940 100644 --- a/owlbot.py +++ b/owlbot.py @@ -21,7 +21,16 @@ common_templates = gcp.CommonTemplates() templates = common_templates.node_library(source_location='build/src') -s.copy(templates, excludes=['.jsdoc.js', '.github/release-please.yml', '.github/sync-repo-settings.yaml', '.prettierrc.js', '.mocharc.js']) +s.copy(templates, excludes=['.jsdoc.js', + '.github/release-please.yml', + '.github/sync-repo-settings.yaml', + '.prettierrc.js', + '.mocharc.js', + '.kokoro/continuous/node14/system-test.cfg', + '.kokoro/presubmit/node14/system-test.cfg', + '.kokoro/release/publish.cfg', + '.kokoro/system-test.sh' + ]) # Create .config directory under $HOME to get around permissions issues # with resumable upload. diff --git a/package.json b/package.json index 8c3245cd9..dc5a819b8 100644 --- a/package.json +++ b/package.json @@ -1,7 +1,7 @@ { "name": "@google-cloud/storage", "description": "Cloud Storage Client Library for Node.js", - "version": "7.13.0", + "version": "7.14.0", "license": "Apache-2.0", "author": "Google Inc.", "engines": { @@ -123,7 +123,7 @@ "proxyquire": "^2.1.3", "sinon": "^18.0.0", "nise": "6.0.0", - "path-to-regexp": "6.2.2", + "path-to-regexp": "6.3.0", "tmp": "^0.2.0", "typescript": "^5.1.6", "yargs": "^17.3.1" diff --git a/samples/package.json b/samples/package.json index bd91cd09c..316088f92 100644 --- a/samples/package.json +++ b/samples/package.json @@ -17,7 +17,7 @@ }, "dependencies": { "@google-cloud/pubsub": "^4.0.0", - "@google-cloud/storage": "^7.13.0", + "@google-cloud/storage": "^7.14.0", "node-fetch": "^2.6.7", "uuid": "^8.0.0", "yargs": "^16.0.0" diff --git a/samples/uploadManyFilesWithTransferManager.js b/samples/uploadManyFilesWithTransferManager.js index 241c88968..cc0019f50 100644 --- a/samples/uploadManyFilesWithTransferManager.js +++ b/samples/uploadManyFilesWithTransferManager.js @@ -32,10 +32,10 @@ function main( // The ID of your GCS bucket // const bucketName = 'your-unique-bucket-name'; - // The ID of the first GCS file to download + // The ID of the first GCS file to upload // const firstFilePath = 'your-first-file-name'; - // The ID of the second GCS file to download + // The ID of the second GCS file to upload // const secondFilePath = 'your-second-file-name'; // Imports the Google Cloud client library diff --git a/src/file.ts b/src/file.ts index e808f5c9a..d8ff1a216 100644 --- a/src/file.ts +++ b/src/file.ts @@ -178,6 +178,7 @@ export interface GetFileMetadataCallback { export interface GetFileOptions extends GetConfig { userProject?: string; generation?: number; + restoreToken?: string; softDeleted?: boolean; } @@ -354,6 +355,7 @@ export interface FileOptions { crc32cGenerator?: CRC32CValidatorGenerator; encryptionKey?: string | Buffer; generation?: number | string; + restoreToken?: string; kmsKeyName?: string; preconditionOpts?: PreconditionOptions; userProject?: string; @@ -450,6 +452,7 @@ export interface SetStorageClassCallback { export interface RestoreOptions extends PreconditionOptions { generation: number; + restoreToken?: string; projection?: 'full' | 'noAcl'; } @@ -471,6 +474,7 @@ export interface FileMetadata extends BaseMetadata { eventBasedHold?: boolean | null; readonly eventBasedHoldReleaseTime?: string; generation?: string | number; + restoreToken?: string; hardDeleteTime?: string; kmsKeyName?: string; md5Hash?: string; @@ -547,6 +551,7 @@ class File extends ServiceObject { name: string; generation?: number; + restoreToken?: string; parent!: Bucket; private encryptionKey?: string | Buffer; @@ -844,6 +849,8 @@ class File extends ServiceObject { * @param {string} [options.userProject] The ID of the project which will be * billed for the request. * @param {number} [options.generation] The generation number to get + * @param {string} [options.restoreToken] If this is a soft-deleted object in an HNS-enabled bucket, returns the restore token which will + * be necessary to restore it if there's a name conflict with another object. * @param {boolean} [options.softDeleted] If true, returns the soft-deleted object. Object `generation` is required if `softDeleted` is set to True. * @param {GetFileCallback} [callback] Callback function. @@ -3707,6 +3714,8 @@ class File extends ServiceObject { * @param {string} [userProject] The ID of the project which will be * billed for the request. * @param {number} [generation] If present, selects a specific revision of this object. + * @param {string} [restoreToken] Returns an option that must be specified when getting a soft-deleted object from an HNS-enabled + * bucket that has a naming and generation conflict with another object in the same bucket. * @param {string} [projection] Specifies the set of properties to return. If used, must be 'full' or 'noAcl'. * @param {string | number} [ifGenerationMatch] Request proceeds if the generation of the target resource * matches the value used in the precondition. diff --git a/src/transfer-manager.ts b/src/transfer-manager.ts index f253461d9..234824d77 100644 --- a/src/transfer-manager.ts +++ b/src/transfer-manager.ts @@ -24,7 +24,7 @@ import { } from './file.js'; import pLimit from 'p-limit'; import * as path from 'path'; -import {createReadStream, promises as fsp} from 'fs'; +import {createReadStream, existsSync, promises as fsp} from 'fs'; import {CRC32C} from './crc32c.js'; import {GoogleAuth} from 'google-auth-library'; import {XMLParser, XMLBuilder} from 'fast-xml-parser'; @@ -108,6 +108,7 @@ export interface DownloadManyFilesOptions { prefix?: string; stripPrefix?: string; passthroughOptions?: DownloadOptions; + skipIfExists?: boolean; } export interface DownloadFileInChunksOptions { @@ -524,6 +525,8 @@ export class TransferManager { * @property {string} [stripPrefix] A prefix to remove from all of the downloaded files. * @property {object} [passthroughOptions] {@link DownloadOptions} Options to be passed through * to each individual download operation. + * @property {boolean} [skipIfExists] Do not download the file if it already exists in + * the destination. * */ /** @@ -605,6 +608,12 @@ export class TransferManager { if (options.stripPrefix) { passThroughOptionsCopy.destination = file.name.replace(regex, ''); } + if ( + options.skipIfExists && + existsSync(passThroughOptionsCopy.destination || '') + ) { + continue; + } promises.push( limit(async () => { diff --git a/system-test/storage.ts b/system-test/storage.ts index f192a614c..6c33cf7ac 100644 --- a/system-test/storage.ts +++ b/system-test/storage.ts @@ -792,6 +792,7 @@ describe('storage', function () { describe('soft-delete', () => { let bucket: Bucket; + let hnsBucket: Bucket; const SOFT_DELETE_RETENTION_SECONDS = 7 * 24 * 60 * 60; //7 days in seconds; beforeEach(async () => { @@ -802,11 +803,26 @@ describe('storage', function () { retentionDurationSeconds: SOFT_DELETE_RETENTION_SECONDS, }, }); + + hnsBucket = storage.bucket(generateName()); + await storage.createBucket(hnsBucket.name, { + hierarchicalNamespace: {enabled: true}, + iamConfiguration: { + uniformBucketLevelAccess: { + enabled: true, + }, + }, + softDeletePolicy: { + retentionDurationSeconds: SOFT_DELETE_RETENTION_SECONDS, + }, + }); }); afterEach(async () => { await bucket.deleteFiles({force: true, versions: true}); await bucket.delete(); + await hnsBucket.deleteFiles({force: true, versions: true}); + await hnsBucket.delete(); }); it('should set softDeletePolicy correctly', async () => { @@ -862,6 +878,63 @@ describe('storage', function () { [files] = await bucket.getFiles(); assert.strictEqual(files.length, 1); }); + + it('should LIST soft-deleted files with restore token', async () => { + const f1 = hnsBucket.file('file5a'); + const f2 = hnsBucket.file('file5b'); + await f1.save('file5a'); + await f2.save('file5b'); + await f1.delete(); + await f2.delete(); + const [notSoftDeletedFiles] = await hnsBucket.getFiles(); + assert.strictEqual(notSoftDeletedFiles.length, 0); + const [softDeletedFiles] = await hnsBucket.getFiles({softDeleted: true}); + assert.strictEqual(softDeletedFiles.length, 2); + assert.notStrictEqual( + softDeletedFiles![0].metadata.restoreToken, + undefined + ); + }); + + it('should GET a soft-deleted file with restore token', async () => { + const f1 = hnsBucket.file('file6'); + await f1.save('file6'); + const [metadata] = await f1.getMetadata(); + await f1.delete(); + const [softDeletedFile] = await f1.get({ + softDeleted: true, + generation: parseInt(metadata.generation?.toString() || '0'), + }); + assert(softDeletedFile); + assert.strictEqual( + softDeletedFile.metadata.generation, + metadata.generation + ); + assert.notStrictEqual(softDeletedFile.metadata.restoreToken, undefined); + }); + + it('should restore a soft-deleted file using restoreToken', async () => { + const f1 = hnsBucket.file('file7'); + await f1.save('file7'); + const [metadata] = await f1.getMetadata(); + await f1.delete(); + let [files] = await hnsBucket.getFiles(); + assert.strictEqual(files.length, 0); + const [softDeletedFile] = await f1.get({ + softDeleted: true, + generation: parseInt(metadata.generation?.toString() || '0'), + }); + assert(softDeletedFile); + const restoredFile = await f1.restore({ + generation: parseInt( + softDeletedFile.metadata.generation?.toString() || '0' + ), + restoreToken: softDeletedFile.metadata.restoreToken, + }); + assert(restoredFile); + [files] = await hnsBucket.getFiles(); + assert.strictEqual(files.length, 1); + }); }); describe('dual-region', () => { @@ -3866,6 +3939,68 @@ describe('storage', function () { }); }); + describe('universeDomainTests', () => { + let universeDomainStorage: Storage; + const bucketName = generateName(); + const localFile = fs.readFileSync(FILES.logo.path); + let file: File; + + before(async () => { + const TEST_UNIVERSE_DOMAIN = isNullOrUndefined('TEST_UNIVERSE_DOMAIN'); + const TEST_PROJECT_ID = isNullOrUndefined('TEST_UNIVERSE_PROJECT_ID'); + const TEST_UNIVERSE_LOCATION = isNullOrUndefined( + 'TEST_UNIVERSE_LOCATION' + ); + const CREDENTIAL_PATH = isNullOrUndefined( + 'TEST_UNIVERSE_DOMAIN_CREDENTIAL' + ); + // Create a client with universe domain credentials + universeDomainStorage = new Storage({ + projectId: TEST_PROJECT_ID, + keyFilename: CREDENTIAL_PATH, + universeDomain: TEST_UNIVERSE_DOMAIN, + }); + + const [bucket] = await universeDomainStorage.createBucket(bucketName, { + location: TEST_UNIVERSE_LOCATION, + }); + + file = bucket.file('LogoToSign.jpg'); + fs.createReadStream(FILES.logo.path).pipe(file.createWriteStream()); + }); + + after(async () => { + await deleteFileAsync(file); + await deleteBucketAsync(bucket); + }); + + it('should get bucket', async () => { + const [buckets] = await universeDomainStorage.getBuckets(); + const getBucket = buckets.filter(item => item.name === bucketName); + assert.strictEqual(getBucket[0].name, bucketName); + }); + + it('should get files', async () => { + const fileName = await universeDomainStorage + .bucket(bucketName) + .file(file.name).name; + assert.strictEqual(fileName, file.name); + }); + + it('should create a signed read url', async () => { + const [signedReadUrl] = await file.getSignedUrl({ + version: 'v2', + action: 'read', + expires: Date.now() + 5000, + virtualHostedStyle: true, + }); + + const res = await fetch(signedReadUrl); + const body = await res.text(); + assert.strictEqual(body, localFile.toString()); + }); + }); + async function deleteBucketAsync(bucket: Bucket, options?: {}) { // After files are deleted, eventual consistency may require a bit of a // delay to ensure that the bucket recognizes that the files don't exist @@ -4015,4 +4150,12 @@ describe('storage', function () { function createFileWithContentPromise(content: string) { return bucket.file(`${generateName()}.txt`).save(content); } + + function isNullOrUndefined(envVarName: string) { + const value = process.env[envVarName]; + if (value === undefined || value === null) { + throw new Error(`Please set the ${envVarName} environment variable.`); + } + return value; + } }); diff --git a/test/transfer-manager.ts b/test/transfer-manager.ts index af5b2d7c2..c2d0d750c 100644 --- a/test/transfer-manager.ts +++ b/test/transfer-manager.ts @@ -26,10 +26,10 @@ import { MultiPartUploadError, MultiPartUploadHelper, UploadOptions, - UploadManyFilesOptions, TransferManager, Storage, DownloadResponse, + DownloadManyFilesOptions, } from '../src/index.js'; import assert from 'assert'; import * as path from 'path'; @@ -195,6 +195,10 @@ describe('Transfer Manager', () => { }); describe('downloadManyFiles', () => { + beforeEach(() => { + sandbox.stub(fs, 'existsSync').returns(true); + }); + it('calls download for each provided file', async () => { let count = 0; const firstFile = new File(bucket, 'first.txt'); @@ -276,6 +280,27 @@ describe('Transfer Manager', () => { await transferManager.downloadManyFiles([file], {passthroughOptions}); }); + it('does not download files that already exist locally when skipIfExists is true', async () => { + const firstFile = new File(bucket, 'first.txt'); + sandbox.stub(firstFile, 'download').callsFake(options => { + assert.strictEqual( + (options as DownloadManyFilesOptions).skipIfExists, + 0 + ); + }); + const secondFile = new File(bucket, 'second.txt'); + sandbox.stub(secondFile, 'download').callsFake(options => { + assert.strictEqual( + (options as DownloadManyFilesOptions).skipIfExists, + 0 + ); + }); + + const files = [firstFile, secondFile]; + const options = {skipIfExists: true}; + await transferManager.downloadManyFiles(files, options); + }); + it('does not set the destination when prefix, strip prefix and passthroughOptions.destination are not provided', async () => { const options = {}; const filename = 'first.txt';