From 7605c11cdaf627a9c686c6fa03c105f7c3a84da2 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 9 Jul 2022 16:32:48 +0200 Subject: [PATCH 01/22] bump Node.js to 16 & 18 Signed-off-by: Patrik Oldsberg --- .github/workflows/ci.yml | 6 +++--- .github/workflows/deploy_microsite.yml | 2 +- .github/workflows/deploy_nightly.yml | 2 +- .github/workflows/deploy_packages.yml | 4 ++-- .github/workflows/sync_snyk-github-issues.yml | 2 +- .github/workflows/verify_e2e-linux.yml | 2 +- .github/workflows/verify_e2e-techdocs.yml | 2 +- .github/workflows/verify_e2e-tugboat.yml | 2 +- .github/workflows/verify_e2e-windows.yml | 2 +- .github/workflows/verify_kubernetes.yml | 2 +- .github/workflows/verify_microsite.yml | 2 +- .github/workflows/verify_windows.yml | 2 +- contrib/docker/devops/Dockerfile | 2 +- contrib/docker/frontend-with-nginx/Dockerfile.dockerbuild | 2 +- cypress/tsconfig.json | 4 ++-- docs/features/search/how-to-guides.md | 4 ++-- docs/getting-started/running-backstage-locally.md | 6 +++--- docs/tutorials/quickstart-app-plugin.md | 2 +- package.json | 2 +- packages/cli/config/tsconfig.json | 4 ++-- packages/create-app/templates/default-app/package.json.hbs | 2 +- plugins/scaffolder-backend/scripts/build-nunjucks.js | 2 +- 22 files changed, 30 insertions(+), 30 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 64067188bb1e9..3ab979593c456 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: [14.x, 16.x] + node-version: [16.x, 18.x] env: CI: true @@ -47,7 +47,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: [14.x, 16.x] + node-version: [16.x, 18.x] env: CI: true @@ -121,7 +121,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: [14.x, 16.x] + node-version: [16.x, 18.x] services: postgres13: diff --git a/.github/workflows/deploy_microsite.yml b/.github/workflows/deploy_microsite.yml index af6f2223779a0..1f87c30af2b94 100644 --- a/.github/workflows/deploy_microsite.yml +++ b/.github/workflows/deploy_microsite.yml @@ -10,7 +10,7 @@ jobs: strategy: matrix: - node-version: [14.x] + node-version: [16.x] env: CI: true diff --git a/.github/workflows/deploy_nightly.yml b/.github/workflows/deploy_nightly.yml index 476e5dd10b0ad..74dc60783a76a 100644 --- a/.github/workflows/deploy_nightly.yml +++ b/.github/workflows/deploy_nightly.yml @@ -11,7 +11,7 @@ jobs: strategy: matrix: - node-version: [14.x] + node-version: [16.x] env: CI: true diff --git a/.github/workflows/deploy_packages.yml b/.github/workflows/deploy_packages.yml index f3736d93a25eb..56ff61c75bdd9 100644 --- a/.github/workflows/deploy_packages.yml +++ b/.github/workflows/deploy_packages.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: - node-version: [14.x, 16.x] + node-version: [16.x, 18.x] services: postgres13: @@ -130,7 +130,7 @@ jobs: strategy: matrix: - node-version: [14.x] + node-version: [16.x] env: CI: 'true' diff --git a/.github/workflows/sync_snyk-github-issues.yml b/.github/workflows/sync_snyk-github-issues.yml index dd35ba26f39c6..de05c79300576 100644 --- a/.github/workflows/sync_snyk-github-issues.yml +++ b/.github/workflows/sync_snyk-github-issues.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - node-version: [14.x] + node-version: [16.x] steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/verify_e2e-linux.yml b/.github/workflows/verify_e2e-linux.yml index 3353a84f3bcf7..fefdd1fdd523b 100644 --- a/.github/workflows/verify_e2e-linux.yml +++ b/.github/workflows/verify_e2e-linux.yml @@ -31,7 +31,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest] - node-version: [14.x, 16.x] + node-version: [16.x, 18.x] env: CI: true diff --git a/.github/workflows/verify_e2e-techdocs.yml b/.github/workflows/verify_e2e-techdocs.yml index e1bb2246dc5a2..8d59d1589997e 100644 --- a/.github/workflows/verify_e2e-techdocs.yml +++ b/.github/workflows/verify_e2e-techdocs.yml @@ -18,7 +18,7 @@ jobs: strategy: matrix: - node-version: [14.x, 16.x] + node-version: [16.x, 18.x] env: CI: true diff --git a/.github/workflows/verify_e2e-tugboat.yml b/.github/workflows/verify_e2e-tugboat.yml index 04d9e686e27a7..cb3a463a0d860 100644 --- a/.github/workflows/verify_e2e-tugboat.yml +++ b/.github/workflows/verify_e2e-tugboat.yml @@ -52,7 +52,7 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-node@v3 with: - node-version: '14' + node-version: '16.x' - name: yarn install run: yarn --cwd cypress install diff --git a/.github/workflows/verify_e2e-windows.yml b/.github/workflows/verify_e2e-windows.yml index 4e8025f7d55bd..fc29c1f03bb1a 100644 --- a/.github/workflows/verify_e2e-windows.yml +++ b/.github/workflows/verify_e2e-windows.yml @@ -22,7 +22,7 @@ jobs: strategy: matrix: os: [windows-2019] - node-version: [14.x, 16.x] + node-version: [16.x, 18.x] env: CI: true diff --git a/.github/workflows/verify_kubernetes.yml b/.github/workflows/verify_kubernetes.yml index e64ddec586e51..e8c248699aece 100644 --- a/.github/workflows/verify_kubernetes.yml +++ b/.github/workflows/verify_kubernetes.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - node-version: [14.x, 16.x] + node-version: [16.x, 18.x] env: CI: true diff --git a/.github/workflows/verify_microsite.yml b/.github/workflows/verify_microsite.yml index 50c211609004a..3a24befa84c5e 100644 --- a/.github/workflows/verify_microsite.yml +++ b/.github/workflows/verify_microsite.yml @@ -12,7 +12,7 @@ jobs: strategy: matrix: - node-version: [14.x] + node-version: [16.x] env: CI: true diff --git a/.github/workflows/verify_windows.yml b/.github/workflows/verify_windows.yml index 8bbaa645bf4b3..75a0633cbe352 100644 --- a/.github/workflows/verify_windows.yml +++ b/.github/workflows/verify_windows.yml @@ -13,7 +13,7 @@ jobs: strategy: matrix: - node-version: [14.x, 16.x] + node-version: [16.x, 18.x] env: CI: true diff --git a/contrib/docker/devops/Dockerfile b/contrib/docker/devops/Dockerfile index a7c194618abd1..8703b2ab4858e 100644 --- a/contrib/docker/devops/Dockerfile +++ b/contrib/docker/devops/Dockerfile @@ -1,4 +1,4 @@ -ARG IMAGE_TAG=14-alpine +ARG IMAGE_TAG=16-alpine FROM node:${IMAGE_TAG} diff --git a/contrib/docker/frontend-with-nginx/Dockerfile.dockerbuild b/contrib/docker/frontend-with-nginx/Dockerfile.dockerbuild index 41f931544f1a9..053b4fb4928e6 100644 --- a/contrib/docker/frontend-with-nginx/Dockerfile.dockerbuild +++ b/contrib/docker/frontend-with-nginx/Dockerfile.dockerbuild @@ -35,7 +35,7 @@ -FROM node:14-buster AS build +FROM node:16-buster AS build RUN mkdir /app COPY . /app diff --git a/cypress/tsconfig.json b/cypress/tsconfig.json index d9b4869ecd249..9c4a63b76c52b 100644 --- a/cypress/tsconfig.json +++ b/cypress/tsconfig.json @@ -10,7 +10,7 @@ "incremental": true, "isolatedModules": true, "jsx": "react", - "lib": ["DOM", "DOM.Iterable", "ScriptHost", "ES2020", "ESNext.Promise"], + "lib": ["DOM", "DOM.Iterable", "ScriptHost", "ES2021", "ESNext.Promise"], "module": "ESNext", "moduleResolution": "node", "noEmit": true, @@ -31,7 +31,7 @@ "strictNullChecks": true, "strictPropertyInitialization": true, "stripInternal": true, - "target": "ES2019", + "target": "ES2021", "types": ["node", "cypress"] } } diff --git a/docs/features/search/how-to-guides.md b/docs/features/search/how-to-guides.md index 01d85b8aa260f..5cc2ab5f0696b 100644 --- a/docs/features/search/how-to-guides.md +++ b/docs/features/search/how-to-guides.md @@ -171,6 +171,6 @@ const highlightOverride = { }; ``` -[obj-mode]: https://nodejs.org/docs/latest-v14.x/api/stream.html#stream_object_mode -[read-stream]: https://nodejs.org/docs/latest-v14.x/api/stream.html#stream_readable_streams +[obj-mode]: https://nodejs.org/dist/latest-v16.x/docs/api/stream.html#stream_object_mode +[read-stream]: https://nodejs.org/dist/latest-v16.x/docs/api/stream.html#readable-streams [async-gen]: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of#iterating_over_async_generators diff --git a/docs/getting-started/running-backstage-locally.md b/docs/getting-started/running-backstage-locally.md index 7cc6f6be6a892..3e3e8b0727d43 100644 --- a/docs/getting-started/running-backstage-locally.md +++ b/docs/getting-started/running-backstage-locally.md @@ -21,12 +21,12 @@ This is made easy with a version manager such as # Installing current LTS release nvm install --lts > Installing latest LTS version. -> Downloading and installing node v14.15.1... -> Now using node v14.15.1 (npm v6.14.8) +> Downloading and installing node v16.16.0... +> Now using node v16.16.0 (npm v8.11.0) # Checking your version node --version -> v14.15.1 +> v16.16.0 ``` - Yarn diff --git a/docs/tutorials/quickstart-app-plugin.md b/docs/tutorials/quickstart-app-plugin.md index d1ca61a11fa59..ea0a511c93ae9 100644 --- a/docs/tutorials/quickstart-app-plugin.md +++ b/docs/tutorials/quickstart-app-plugin.md @@ -20,7 +20,7 @@ title: Adding Custom Plugin to Existing Monorepo App > functionality, extend the Sidebar to make our life easy. Finally, we add > custom code to display GitHub repository information. > -> This document assumes you have Node.js 14 active along with Yarn and Python. +> This document assumes you have Node.js 16 active along with Yarn and Python. > Please note, that at the time of this writing, the current version is > 0.1.1-alpha.21. This guide can still be used with future versions, just, > verify as you go. If you run into issues, you can compare your setup with mine diff --git a/package.json b/package.json index 9f17e5ee36847..417fd328c6d92 100644 --- a/package.json +++ b/package.json @@ -2,7 +2,7 @@ "name": "root", "private": true, "engines": { - "node": "14 || 16" + "node": "16 || 18" }, "scripts": { "dev": "concurrently \"yarn start\" \"yarn start-backend\"", diff --git a/packages/cli/config/tsconfig.json b/packages/cli/config/tsconfig.json index 33a8b0a9bd59e..5486f1900f408 100644 --- a/packages/cli/config/tsconfig.json +++ b/packages/cli/config/tsconfig.json @@ -11,7 +11,7 @@ "incremental": true, "isolatedModules": true, "jsx": "react", - "lib": ["DOM", "DOM.Iterable", "ScriptHost", "ES2020"], + "lib": ["DOM", "DOM.Iterable", "ScriptHost", "ES2021"], "module": "ESNext", "moduleResolution": "node", "noEmit": false, @@ -32,7 +32,7 @@ "strictNullChecks": true, "strictPropertyInitialization": true, "stripInternal": true, - "target": "ES2019", + "target": "ES2021", "types": ["node", "jest", "webpack-env"], "useDefineForClassFields": true } diff --git a/packages/create-app/templates/default-app/package.json.hbs b/packages/create-app/templates/default-app/package.json.hbs index c5810d888020b..adcd3a57743a9 100644 --- a/packages/create-app/templates/default-app/package.json.hbs +++ b/packages/create-app/templates/default-app/package.json.hbs @@ -3,7 +3,7 @@ "version": "1.0.0", "private": true, "engines": { - "node": "14 || 16" + "node": "16 || 18" }, "scripts": { "dev": "concurrently \"yarn start\" \"yarn start-backend\"", diff --git a/plugins/scaffolder-backend/scripts/build-nunjucks.js b/plugins/scaffolder-backend/scripts/build-nunjucks.js index b195f96159d47..8b98ac43059a4 100755 --- a/plugins/scaffolder-backend/scripts/build-nunjucks.js +++ b/plugins/scaffolder-backend/scripts/build-nunjucks.js @@ -58,7 +58,7 @@ require('esbuild') bundle: true, format: 'cjs', platform: 'node', - target: 'node14', + target: 'node16', banner: { js: NUNJUCKS_LICENSE }, external: ['fsevents'], outfile: path.resolve(__dirname, '../assets/nunjucks.js.txt'), From 384eaa230720e511706679dde5ed5769aefdfd1e Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Sat, 9 Jul 2022 16:37:54 +0200 Subject: [PATCH 02/22] changests: add changesets for Node.js 18 bump Signed-off-by: Patrik Oldsberg --- .changeset/thirty-deers-float.md | 19 +++++++++++++++++++ .changeset/two-timers-pump.md | 5 +++++ 2 files changed, 24 insertions(+) create mode 100644 .changeset/thirty-deers-float.md create mode 100644 .changeset/two-timers-pump.md diff --git a/.changeset/thirty-deers-float.md b/.changeset/thirty-deers-float.md new file mode 100644 index 0000000000000..198d6f909dbb0 --- /dev/null +++ b/.changeset/thirty-deers-float.md @@ -0,0 +1,19 @@ +--- +'@backstage/create-app': patch +--- + +Switched Node.js version to support version 16 & 18, rather than 14 & 16. To switch the Node.js version in your own project, apply the following change to the root `package.json`: + +```diff + "engines": { +- "node": "14 || 16" ++ "node": "16 || 18" + }, +``` + +As well as the following change to `packages/app/package.json`: + +```diff +- "@types/node": "^14.14.32", ++ "@types/node": "^16.11.26", +``` diff --git a/.changeset/two-timers-pump.md b/.changeset/two-timers-pump.md new file mode 100644 index 0000000000000..e3c1ec2351cc3 --- /dev/null +++ b/.changeset/two-timers-pump.md @@ -0,0 +1,5 @@ +--- +'@backstage/cli': minor +--- + +Switched `tsconfig.json` to target and support `ES2021`, in line with the bump to Node.js 16 & 18. From 1d43bcfa6d4090109bfabef06b49a4069120dd33 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 11 Jul 2022 13:49:25 +0200 Subject: [PATCH 03/22] auth-backend: add script for re-publishing openid-client Co-authored-by: blam Signed-off-by: Patrik Oldsberg --- .../scripts/republish-openid-client.js | 109 ++++++++++++++++++ 1 file changed, 109 insertions(+) create mode 100755 plugins/auth-backend/scripts/republish-openid-client.js diff --git a/plugins/auth-backend/scripts/republish-openid-client.js b/plugins/auth-backend/scripts/republish-openid-client.js new file mode 100755 index 0000000000000..9d4497e5905de --- /dev/null +++ b/plugins/auth-backend/scripts/republish-openid-client.js @@ -0,0 +1,109 @@ +#!/usr/bin/env node +/* eslint-disable import/no-extraneous-dependencies */ +/* + * Copyright 2022 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const fetch = require('node-fetch'); +const zlib = require('zlib'); + +// eslint-disable-next-line import/no-extraneous-dependencies +const libpub = require('libnpmpublish'); +const concatStream = require('concat-stream'); +const tar = require('tar-stream'); + +/* + +This works around an incompatibility in node versioning policies between Backstage and +the openid-client package. This script downloads a target version of the openid-client +package and re-publishes it as a new version of the openid-client-any-engine package. + +Usage: ./republish-openid-client.js + +Environment Variables: + +NPM_TOKEN: The NPM auth token to use for publishing. +NPM_OTP: The NPM OTP (2FA one time password) to use for publishing. + +*/ + +async function main(args) { + const [version] = args; + if (!version) { + throw new Error('No version provided, Usage: $0 '); + } + + const res = await fetch( + `https://registry.npmjs.org/openid-client/-/openid-client-${version}.tgz`, + ); + if (!res.ok) { + throw new Error(`Failed to fetch openid-client: ${res.status}`); + } + + const { data, manifest } = await new Promise((resolve, reject) => { + const extract = tar.extract(); + const pack = tar.pack(); + let foundPackageJson = undefined; + + res.body.pipe(zlib.createGunzip()).pipe(extract).on('error', reject); + + extract.on('entry', (header, stream, callback) => { + if (header.name === 'package/package.json') { + stream.pipe( + concatStream(fileContents => { + const packageJson = JSON.parse(fileContents.toString('utf8')); + packageJson.name = 'openid-client-any-engine'; + packageJson.description = + 'Re-publish of openid-client that allows any Node.js version above 12.19.0'; + packageJson.engines.node = '>=12.19.0'; + + foundPackageJson = packageJson; + + pack.entry( + { name: 'package/package.json' }, + JSON.stringify(packageJson, null, 2), + ); + callback(); + }), + ); + } else { + stream.pipe(pack.entry(header, callback)); + } + }); + + extract.on('finish', () => { + pack.finalize(); + }); + + pack + .pipe(zlib.createGzip()) + .pipe( + concatStream(d => { + resolve({ data: d, manifest: foundPackageJson }); + }), + ) + .on('error', reject); + }); + + await libpub.publish(manifest, data, { + token: process.env.NPM_TOKEN, + otp: process.env.NPM_OTP, + }); +} + +main(process.argv.slice(2)).catch(error => { + console.error(error); + process.exit(1); +}); From 4ca99cc3672f159f1c05a455241969190edd7e21 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Mon, 11 Jul 2022 14:01:10 +0200 Subject: [PATCH 04/22] auth-backend: use openid-client-any-engine Co-authored-by: blam Signed-off-by: Patrik Oldsberg --- plugins/auth-backend/package.json | 2 +- yarn.lock | 18 +++++++++--------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index 4b3d7910e5715..a4033b90b073b 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -60,7 +60,7 @@ "morgan": "^1.10.0", "node-cache": "^5.1.2", "node-fetch": "^2.6.7", - "openid-client": "^5.1.3", + "openid-client": "npm:openid-client-any-engine@^5.1.3", "passport": "^0.6.0", "passport-auth0": "^1.4.3", "passport-bitbucket-oauth2": "^0.1.2", diff --git a/yarn.lock b/yarn.lock index 909d52aaf6886..f8130e8e31867 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4296,7 +4296,7 @@ __metadata: msw: ^0.47.0 node-cache: ^5.1.2 node-fetch: ^2.6.7 - openid-client: ^5.1.3 + openid-client: "npm:openid-client-any-engine@^5.1.3" passport: ^0.6.0 passport-auth0: ^1.4.3 passport-bitbucket-oauth2: ^0.1.2 @@ -31238,27 +31238,27 @@ __metadata: languageName: node linkType: hard -"openid-client@npm:^5.1.3": - version: 5.1.10 - resolution: "openid-client@npm:5.1.10" +"openid-client@npm:^5.1.6": + version: 5.1.9 + resolution: "openid-client@npm:5.1.9" dependencies: jose: ^4.1.4 lru-cache: ^6.0.0 object-hash: ^2.0.1 oidc-token-hash: ^5.0.1 - checksum: 38a4bf08ea4ee4576043968307cf53f0369df224bd025c1bc348b295152df36c47d2a836dfe1505d15c6b79c05f86aadefad798bd3ea3ccbe90834be01f2245c + checksum: 55390a7eceaafdc340a5f2ece576eb4863fcb4cd8840e4a7a1af66bbaf830623b18cf1dd0b8ce472c6f36c6313e78f92db48ca0c10eb8c681a81e5a3d607fc9f languageName: node linkType: hard -"openid-client@npm:^5.1.6": - version: 5.1.9 - resolution: "openid-client@npm:5.1.9" +"openid-client@npm:openid-client-any-engine@^5.1.3": + version: 5.1.8 + resolution: "openid-client-any-engine@npm:5.1.8" dependencies: jose: ^4.1.4 lru-cache: ^6.0.0 object-hash: ^2.0.1 oidc-token-hash: ^5.0.1 - checksum: 55390a7eceaafdc340a5f2ece576eb4863fcb4cd8840e4a7a1af66bbaf830623b18cf1dd0b8ce472c6f36c6313e78f92db48ca0c10eb8c681a81e5a3d607fc9f + checksum: dbd6f54a1d9ec50b82299d117556003670c796db553b35a50c4dfa57824f33721d12b2afc0d92a5dece936ef35d190f6dceab0258c65eb78d045dfb966b64831 languageName: node linkType: hard From 009db8b28e7d7c9b892b4487afc6152758fbf1d8 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 31 Aug 2022 23:24:50 +0200 Subject: [PATCH 05/22] root: add node-gyp dep to fill missing dep of ssh2 Signed-off-by: Patrik Oldsberg --- package.json | 1 + yarn.lock | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 417fd328c6d92..11e92b530bb13 100644 --- a/package.json +++ b/package.json @@ -73,6 +73,7 @@ "husky": "^8.0.0", "lint-staged": "^13.0.0", "minimist": "^1.2.5", + "node-gyp": "^9.1.0", "prettier": "^2.2.1", "semver": "^7.3.2", "shx": "^0.3.2", diff --git a/yarn.lock b/yarn.lock index f8130e8e31867..6ea74fa4613a9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -30594,7 +30594,7 @@ __metadata: languageName: node linkType: hard -"node-gyp@npm:latest": +"node-gyp@npm:^9.1.0, node-gyp@npm:latest": version: 9.1.0 resolution: "node-gyp@npm:9.1.0" dependencies: @@ -35450,6 +35450,7 @@ __metadata: husky: ^8.0.0 lint-staged: ^13.0.0 minimist: ^1.2.5 + node-gyp: ^9.1.0 prettier: ^2.2.1 semver: ^7.3.2 shx: ^0.3.2 From 24b06ff92b39cf468161d4763c52b53729914a79 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Fri, 19 Aug 2022 11:26:57 +0200 Subject: [PATCH 06/22] root: work around openid-client with resolutions Signed-off-by: Patrik Oldsberg --- package.json | 1 + yarn.lock | 12 ------------ 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/package.json b/package.json index 11e92b530bb13..d7936487685f5 100644 --- a/package.json +++ b/package.json @@ -41,6 +41,7 @@ ] }, "resolutions": { + "openid-client": "npm:openid-client-any-engine@^5.1.3", "@types/react": "^17", "@types/react-dom": "^17" }, diff --git a/yarn.lock b/yarn.lock index 6ea74fa4613a9..8a6cfe1be623e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -31238,18 +31238,6 @@ __metadata: languageName: node linkType: hard -"openid-client@npm:^5.1.6": - version: 5.1.9 - resolution: "openid-client@npm:5.1.9" - dependencies: - jose: ^4.1.4 - lru-cache: ^6.0.0 - object-hash: ^2.0.1 - oidc-token-hash: ^5.0.1 - checksum: 55390a7eceaafdc340a5f2ece576eb4863fcb4cd8840e4a7a1af66bbaf830623b18cf1dd0b8ce472c6f36c6313e78f92db48ca0c10eb8c681a81e5a3d607fc9f - languageName: node - linkType: hard - "openid-client@npm:openid-client-any-engine@^5.1.3": version: 5.1.8 resolution: "openid-client-any-engine@npm:5.1.8" From 6fb165fbd402c18c14ad9074f7419881c989c71f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 1 Sep 2022 00:53:53 +0200 Subject: [PATCH 07/22] scripts/check-type-dependencies: work around exports resolution Signed-off-by: Patrik Oldsberg --- scripts/check-type-dependencies.js | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/scripts/check-type-dependencies.js b/scripts/check-type-dependencies.js index ba362bd5ff73d..ebad9b6dfcd97 100755 --- a/scripts/check-type-dependencies.js +++ b/scripts/check-type-dependencies.js @@ -136,10 +136,21 @@ function findTypesPackage(dep, pkg) { return undefined; } catch { try { - // Finally check if it's just a .d.ts file + // Check if it's just a .d.ts file require.resolve(`${dep}.d.ts`, { paths: [pkg.dir] }); return undefined; } catch { + // And finally a naive lookup of the file directly, in case `require.resolve` fails us due to "exports" + if (fs.existsSync(resolvePath(pkg.dir, `node_modules/${dep}.d.ts`))) { + return undefined; + } + if ( + fs.existsSync( + resolvePath(pkg.dir, `../../node_modules/${dep}.d.ts`), + ) + ) { + return undefined; + } throw mkErr('MissingDepError', `No types for ${dep}`, { dep }); } } From cfb30b700c54419c32198882091043b07f5b6f24 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 22 Sep 2022 11:13:32 +0200 Subject: [PATCH 08/22] bump @kubernetes/client-node to 0.17.1 Signed-off-by: Patrik Oldsberg --- .changeset/silent-moles-chew.md | 8 ++ packages/backend-common/package.json | 2 +- plugins/kubernetes-backend/package.json | 2 +- plugins/kubernetes-common/package.json | 2 +- plugins/kubernetes/package.json | 2 +- yarn.lock | 111 +++++------------------- 6 files changed, 33 insertions(+), 94 deletions(-) create mode 100644 .changeset/silent-moles-chew.md diff --git a/.changeset/silent-moles-chew.md b/.changeset/silent-moles-chew.md new file mode 100644 index 0000000000000..09d00ff81bb82 --- /dev/null +++ b/.changeset/silent-moles-chew.md @@ -0,0 +1,8 @@ +--- +'@backstage/backend-common': patch +'@backstage/plugin-kubernetes': patch +'@backstage/plugin-kubernetes-backend': patch +'@backstage/plugin-kubernetes-common': patch +--- + +Bumped `@kubernetes/client-node` to `^0.17.1`. diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 94b2338ab0974..7dde8e6512cb4 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -42,7 +42,7 @@ "@backstage/types": "workspace:^", "@google-cloud/storage": "^6.0.0", "@keyv/redis": "^2.2.3", - "@kubernetes/client-node": "^0.17.0", + "@kubernetes/client-node": "^0.17.1", "@manypkg/get-packages": "^1.1.3", "@octokit/rest": "^19.0.3", "@types/cors": "^2.8.6", diff --git a/plugins/kubernetes-backend/package.json b/plugins/kubernetes-backend/package.json index 4082bd5b3a627..2cef29c4bb866 100644 --- a/plugins/kubernetes-backend/package.json +++ b/plugins/kubernetes-backend/package.json @@ -43,7 +43,7 @@ "@backstage/plugin-auth-node": "workspace:^", "@backstage/plugin-kubernetes-common": "workspace:^", "@google-cloud/container": "^4.0.0", - "@kubernetes/client-node": "^0.17.0", + "@kubernetes/client-node": "^0.17.1", "@types/express": "^4.17.6", "@types/luxon": "^3.0.0", "aws-sdk": "^2.840.0", diff --git a/plugins/kubernetes-common/package.json b/plugins/kubernetes-common/package.json index 48c04fb222bf5..5ef960a396375 100644 --- a/plugins/kubernetes-common/package.json +++ b/plugins/kubernetes-common/package.json @@ -39,7 +39,7 @@ }, "dependencies": { "@backstage/catalog-model": "workspace:^", - "@kubernetes/client-node": "^0.17.0" + "@kubernetes/client-node": "^0.17.1" }, "devDependencies": { "@backstage/cli": "workspace:^" diff --git a/plugins/kubernetes/package.json b/plugins/kubernetes/package.json index ee4da4b55f20d..349642f65abdd 100644 --- a/plugins/kubernetes/package.json +++ b/plugins/kubernetes/package.json @@ -40,7 +40,7 @@ "@backstage/plugin-catalog-react": "workspace:^", "@backstage/plugin-kubernetes-common": "workspace:^", "@backstage/theme": "workspace:^", - "@kubernetes/client-node": "^0.17.0", + "@kubernetes/client-node": "^0.17.1", "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.57", diff --git a/yarn.lock b/yarn.lock index 8a6cfe1be623e..d967f0a87722b 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3148,7 +3148,7 @@ __metadata: "@backstage/types": "workspace:^" "@google-cloud/storage": ^6.0.0 "@keyv/redis": ^2.2.3 - "@kubernetes/client-node": ^0.17.0 + "@kubernetes/client-node": ^0.17.1 "@manypkg/get-packages": ^1.1.3 "@octokit/rest": ^19.0.3 "@types/archiver": ^5.1.0 @@ -6220,7 +6220,7 @@ __metadata: "@backstage/plugin-auth-node": "workspace:^" "@backstage/plugin-kubernetes-common": "workspace:^" "@google-cloud/container": ^4.0.0 - "@kubernetes/client-node": ^0.17.0 + "@kubernetes/client-node": ^0.17.1 "@types/aws4": ^1.5.1 "@types/express": ^4.17.6 "@types/luxon": ^3.0.0 @@ -6249,7 +6249,7 @@ __metadata: dependencies: "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" - "@kubernetes/client-node": ^0.17.0 + "@kubernetes/client-node": ^0.17.1 languageName: unknown linkType: soft @@ -6268,7 +6268,7 @@ __metadata: "@backstage/plugin-kubernetes-common": "workspace:^" "@backstage/test-utils": "workspace:^" "@backstage/theme": "workspace:^" - "@kubernetes/client-node": ^0.17.0 + "@kubernetes/client-node": ^0.17.1 "@material-ui/core": ^4.12.2 "@material-ui/icons": ^4.9.1 "@material-ui/lab": 4.0.0-alpha.57 @@ -10671,17 +10671,10 @@ __metadata: languageName: node linkType: hard -"@kubernetes/client-node@npm:^0.17.0": - version: 0.17.0 - resolution: "@kubernetes/client-node@npm:0.17.0" +"@kubernetes/client-node@npm:^0.17.1": + version: 0.17.1 + resolution: "@kubernetes/client-node@npm:0.17.1" dependencies: - "@types/js-yaml": ^4.0.1 - "@types/node": ^10.12.0 - "@types/request": ^2.47.1 - "@types/stream-buffers": ^3.0.3 - "@types/tar": ^4.0.3 - "@types/underscore": ^1.8.9 - "@types/ws": ^6.0.1 byline: ^5.0.0 execa: 5.0.0 isomorphic-ws: ^4.0.1 @@ -10697,7 +10690,10 @@ __metadata: tslib: ^1.9.3 underscore: ^1.9.1 ws: ^7.3.1 - checksum: 5a6edce96946966d8b61359ab043b42e69d1b31178133bc3bf4b5a5c9191766986e08f85d6ae814b56546ed20389817a79dca5101342543d79d750f30cba21f7 + dependenciesMeta: + openid-client: + optional: true + checksum: 834ab0ca1f8583b06c4102395ad712c115a6f99201f8dd7f0b59c1a0a5da67b006d1fe5857ad909d762b9cf82a75503665df19fdb640bf620f6123ddb36c8872 languageName: node linkType: hard @@ -13451,13 +13447,6 @@ __metadata: languageName: node linkType: hard -"@types/caseless@npm:*": - version: 0.12.2 - resolution: "@types/caseless@npm:0.12.2" - checksum: 430d15911184ad11e0a8aa21d1ec15fcc93b90b63570c37bf16ebd34457482bfc8de3f5eb6771e0ef986ce183270d4297823b0f492c346255967e78f7292388b - languageName: node - linkType: hard - "@types/classnames@npm:^2.2.9": version: 2.3.1 resolution: "@types/classnames@npm:2.3.1" @@ -14120,7 +14109,7 @@ __metadata: languageName: node linkType: hard -"@types/js-yaml@npm:^4.0.0, @types/js-yaml@npm:^4.0.1": +"@types/js-yaml@npm:^4.0.0": version: 4.0.5 resolution: "@types/js-yaml@npm:4.0.5" checksum: 7dcac8c50fec31643cc9d6444b5503239a861414cdfaa7ae9a38bc22597c4d850c4b8cec3d82d73b3fbca408348ce223b0408d598b32e094470dfffc6d486b4d @@ -14331,15 +14320,6 @@ __metadata: languageName: node linkType: hard -"@types/minipass@npm:*": - version: 3.3.5 - resolution: "@types/minipass@npm:3.3.5" - dependencies: - minipass: "*" - checksum: 160f4ae5416697c947e0c0ee1225fe25973acf73a30d01bdf202447f12bd32ba3ea33d2fc4f2517a038dc408022a5aff5f7a1d92da0157a9b04276d1b0209550 - languageName: node - linkType: hard - "@types/mock-fs@npm:^4.10.0, @types/mock-fs@npm:^4.13.0": version: 4.13.1 resolution: "@types/mock-fs@npm:4.13.1" @@ -14394,7 +14374,7 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:*, @types/node@npm:>= 8, @types/node@npm:>=12.12.47, @types/node@npm:>=13.7.0": +"@types/node@npm:*, @types/node@npm:>=12.12.47, @types/node@npm:>=13.7.0": version: 18.11.9 resolution: "@types/node@npm:18.11.9" checksum: cc0aae109e9b7adefc32eecb838d6fad931663bb06484b5e9cbbbf74865c721b03d16fd8d74ad90e31dbe093d956a7c2c306ba5429ba0c00f3f7505103d7a496 @@ -14408,7 +14388,14 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:^10.1.0, @types/node@npm:^10.12.0": +"@types/node@npm:>= 8": + version: 18.7.4 + resolution: "@types/node@npm:18.7.4" + checksum: 051d2147e4d8129fceb63ee9384259b2f224dbc4e4b0c46d96a6b61cbaad4e3fe4060950e7f4fc3d5692b1e6ea47e68ad03b61155754bfa169593747cfe3f8f4 + languageName: node + linkType: hard + +"@types/node@npm:^10.1.0": version: 10.17.60 resolution: "@types/node@npm:10.17.60" checksum: 2cdb3a77d071ba8513e5e8306fa64bf50e3c3302390feeaeff1fd325dd25c8441369715dfc8e3701011a72fed5958c7dfa94eb9239a81b3c286caa4d97db6eef @@ -14753,18 +14740,6 @@ __metadata: languageName: node linkType: hard -"@types/request@npm:^2.47.1": - version: 2.48.5 - resolution: "@types/request@npm:2.48.5" - dependencies: - "@types/caseless": "*" - "@types/node": "*" - "@types/tough-cookie": "*" - form-data: ^2.5.0 - checksum: 02572a0558b9a95ae1a2fa3912b45c4c12f587e25773b2d9bdbf5dfa6f2da5e3787d22256ee0dc4836250f39cf2dcb39fc4ded5bf554c0f948a623c8772e8bd3 - languageName: node - linkType: hard - "@types/resize-observer-browser@npm:^0.1.6": version: 0.1.7 resolution: "@types/resize-observer-browser@npm:0.1.7" @@ -14951,15 +14926,6 @@ __metadata: languageName: node linkType: hard -"@types/stream-buffers@npm:^3.0.3": - version: 3.0.3 - resolution: "@types/stream-buffers@npm:3.0.3" - dependencies: - "@types/node": "*" - checksum: c3456fa2a18f89d31c54fe169621d8288a9735fb7e92aba45786642659d93be26fa0b4caa5c1043b21f4cfaeeefc24d247fa833586a8a0f3aecedf03adb7b5ac - languageName: node - linkType: hard - "@types/styled-jsx@npm:^2.2.8": version: 2.2.8 resolution: "@types/styled-jsx@npm:2.2.8" @@ -15006,16 +14972,6 @@ __metadata: languageName: node linkType: hard -"@types/tar@npm:^4.0.3": - version: 4.0.5 - resolution: "@types/tar@npm:4.0.5" - dependencies: - "@types/minipass": "*" - "@types/node": "*" - checksum: 476d8af8f4cffcd973de026e043271be76171f9cb07bb869ba38a193ce89ee361b59ff8484e28b57216266063d91502c5208d2ab6b976e7370d9bdf4c4dadadc - languageName: node - linkType: hard - "@types/tar@npm:^6.1.1": version: 6.1.3 resolution: "@types/tar@npm:6.1.3" @@ -15085,13 +15041,6 @@ __metadata: languageName: node linkType: hard -"@types/underscore@npm:^1.8.9": - version: 1.10.23 - resolution: "@types/underscore@npm:1.10.23" - checksum: 5e9458888e8c09a0f61f93f5958d4ce57e02e317b21444319f7e4a11796445e58a78d6c743dad17365aa8fb9c31fa23d5f4d30a9047fa2df30d446ccc5e42023 - languageName: node - linkType: hard - "@types/unist@npm:*, @types/unist@npm:^2.0.0": version: 2.0.6 resolution: "@types/unist@npm:2.0.6" @@ -15141,15 +15090,6 @@ __metadata: languageName: node linkType: hard -"@types/ws@npm:^6.0.1": - version: 6.0.4 - resolution: "@types/ws@npm:6.0.4" - dependencies: - "@types/node": "*" - checksum: b2656a76bfad0c17bb1e3fc237ba7122431c1373669977ed8edef45934c82f71c75d8c71f0a576dc6d98b0954fd94cae0166c6b4ccb40f7e0ee29cc92673519c - languageName: node - linkType: hard - "@types/ws@npm:^8.0.0, @types/ws@npm:^8.5.1": version: 8.5.3 resolution: "@types/ws@npm:8.5.3" @@ -30074,15 +30014,6 @@ __metadata: languageName: node linkType: hard -"minipass@npm:*": - version: 3.3.4 - resolution: "minipass@npm:3.3.4" - dependencies: - yallist: ^4.0.0 - checksum: 5d95a7738c54852ba78d484141e850c792e062666a2d0c681a5ac1021275beb7e1acb077e59f9523ff1defb80901aea4e30fac10ded9a20a25d819a42916ef1b - languageName: node - linkType: hard - "minipass@npm:^3.0.0, minipass@npm:^3.1.0, minipass@npm:^3.1.1, minipass@npm:^3.1.3, minipass@npm:^3.1.6": version: 3.1.6 resolution: "minipass@npm:3.1.6" From 7bcb96a6683bde9521d51c23fab26e13a5b9d731 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 31 Aug 2022 23:36:43 +0200 Subject: [PATCH 09/22] workflows: work around missing node-canvas binaries for node 18 Signed-off-by: Patrik Oldsberg --- .github/workflows/ci.yml | 7 +++++++ .github/workflows/deploy_nightly.yml | 2 ++ .github/workflows/deploy_packages.yml | 8 ++++++++ .github/workflows/verify_e2e-linux.yml | 6 ++++++ .github/workflows/verify_e2e-techdocs.yml | 7 +++++++ .github/workflows/verify_e2e-windows.yml | 10 ++++++++++ .github/workflows/verify_kubernetes.yml | 6 ++++++ .github/workflows/verify_windows.yml | 9 +++++++++ 8 files changed, 55 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ab979593c456..1d8040e993586 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,6 +32,13 @@ jobs: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth + # Needed until there are pre-built binaries of node-canvas for Node 18 + - name: node-canvas dependencies + if: matrix.node-version == '18.x' + run: | + sudo apt update + sudo apt install -y build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev + - name: yarn install uses: backstage/actions/yarn-install@v0.5.6 with: diff --git a/.github/workflows/deploy_nightly.yml b/.github/workflows/deploy_nightly.yml index 74dc60783a76a..bb41481cb10ce 100644 --- a/.github/workflows/deploy_nightly.yml +++ b/.github/workflows/deploy_nightly.yml @@ -25,6 +25,8 @@ jobs: with: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth + + # Needed until there are pre-built binaries for Node 18 - name: yarn install uses: backstage/actions/yarn-install@v0.5.6 with: diff --git a/.github/workflows/deploy_packages.yml b/.github/workflows/deploy_packages.yml index 56ff61c75bdd9..6fffe4d60a59f 100644 --- a/.github/workflows/deploy_packages.yml +++ b/.github/workflows/deploy_packages.yml @@ -66,6 +66,14 @@ jobs: with: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth + + # Needed until there are pre-built binaries of node-canvas for Node 18 + - name: node-canvas dependencies + if: matrix.node-version == '18.x' + run: | + sudo apt update + sudo apt install -y build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev + - name: yarn install uses: backstage/actions/yarn-install@v0.5.6 with: diff --git a/.github/workflows/verify_e2e-linux.yml b/.github/workflows/verify_e2e-linux.yml index fefdd1fdd523b..db8f6662fe9ed 100644 --- a/.github/workflows/verify_e2e-linux.yml +++ b/.github/workflows/verify_e2e-linux.yml @@ -51,6 +51,12 @@ jobs: with: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth + # Needed until there are pre-built binaries of node-canvas for Node 18 + - name: node-canvas dependencies + if: matrix.node-version == '18.x' + run: | + sudo apt update + sudo apt install -y build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev - name: yarn install uses: backstage/actions/yarn-install@v0.5.6 with: diff --git a/.github/workflows/verify_e2e-techdocs.yml b/.github/workflows/verify_e2e-techdocs.yml index 8d59d1589997e..0d6836ad3aba8 100644 --- a/.github/workflows/verify_e2e-techdocs.yml +++ b/.github/workflows/verify_e2e-techdocs.yml @@ -28,6 +28,13 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 + # Needed until there are pre-built binaries of node-canvas for Node 18 + - name: node-canvas dependencies + if: matrix.node-version == '18.x' + run: | + sudo apt update + sudo apt install -y build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev + - name: install dependencies run: yarn install --immutable diff --git a/.github/workflows/verify_e2e-windows.yml b/.github/workflows/verify_e2e-windows.yml index fc29c1f03bb1a..9d1ba8a154afb 100644 --- a/.github/workflows/verify_e2e-windows.yml +++ b/.github/workflows/verify_e2e-windows.yml @@ -52,6 +52,16 @@ jobs: uses: microsoft/setup-msbuild@v1.0.3 - name: setup chrome uses: browser-actions/setup-chrome@latest + + # Needed until there are pre-built binaries of node-canvas for Node 18 + - name: node-canvas dependencies + if: matrix.node-version == '18.x' + # From https://github.com/Automattic/node-canvas/blob/master/.github/workflows/ci.yaml + run: | + Invoke-WebRequest "https://ftp-osl.osuosl.org/pub/gnome/binaries/win64/gtk+/2.22/gtk+-bundle_2.22.1-20101229_win64.zip" -OutFile "gtk.zip" + Expand-Archive gtk.zip -DestinationPath "C:\GTK" + Invoke-WebRequest "https://downloads.sourceforge.net/project/libjpeg-turbo/2.0.4/libjpeg-turbo-2.0.4-vc64.exe" -OutFile "libjpeg.exe" -UserAgent NativeHost + .\libjpeg.exe /S - name: yarn install run: yarn install --immutable diff --git a/.github/workflows/verify_kubernetes.yml b/.github/workflows/verify_kubernetes.yml index e8c248699aece..ea5e3ebe4eb87 100644 --- a/.github/workflows/verify_kubernetes.yml +++ b/.github/workflows/verify_kubernetes.yml @@ -27,6 +27,12 @@ jobs: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth + # Needed until there are pre-built binaries of node-canvas for Node 18 + - name: node-canvas dependencies + if: matrix.node-version == '18.x' + run: | + sudo apt update + sudo apt install -y build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev - name: yarn install uses: backstage/actions/yarn-install@v0.5.6 with: diff --git a/.github/workflows/verify_windows.yml b/.github/workflows/verify_windows.yml index 75a0633cbe352..a092c8d405425 100644 --- a/.github/workflows/verify_windows.yml +++ b/.github/workflows/verify_windows.yml @@ -32,6 +32,15 @@ jobs: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth + # Needed until there are pre-built binaries of node-canvas for Node 18 + - name: node-canvas dependencies + if: matrix.node-version == '18.x' + # From https://github.com/Automattic/node-canvas/blob/master/.github/workflows/ci.yaml + run: | + Invoke-WebRequest "https://ftp-osl.osuosl.org/pub/gnome/binaries/win64/gtk+/2.22/gtk+-bundle_2.22.1-20101229_win64.zip" -OutFile "gtk.zip" + Expand-Archive gtk.zip -DestinationPath "C:\GTK" + Invoke-WebRequest "https://downloads.sourceforge.net/project/libjpeg-turbo/2.0.4/libjpeg-turbo-2.0.4-vc64.exe" -OutFile "libjpeg.exe" -UserAgent NativeHost + .\libjpeg.exe /S # Windows file operation slowness means there's no point caching this - name: yarn install run: yarn install --immutable From 21339ea5951c9f8f2b3782e82c8d89fa458470b9 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 22 Sep 2022 14:51:07 +0200 Subject: [PATCH 10/22] backend-common: work around premature close errors Signed-off-by: Patrik Oldsberg --- .changeset/swift-suits-reply.md | 5 ++ .../src/reading/GerritUrlReader.ts | 8 +-- .../src/reading/tree/ReadableArrayResponse.ts | 10 ++- .../src/reading/tree/TarArchiveResponse.ts | 15 ++--- .../src/reading/tree/util.test.ts | 63 +++++++++++++++++++ .../backend-common/src/reading/tree/util.ts | 35 ++++++++--- 6 files changed, 109 insertions(+), 27 deletions(-) create mode 100644 .changeset/swift-suits-reply.md create mode 100644 packages/backend-common/src/reading/tree/util.test.ts diff --git a/.changeset/swift-suits-reply.md b/.changeset/swift-suits-reply.md new file mode 100644 index 0000000000000..f81f39bee285b --- /dev/null +++ b/.changeset/swift-suits-reply.md @@ -0,0 +1,5 @@ +--- +'@backstage/backend-common': patch +--- + +Refactored internal usage of the build-in `pipeline` from `'stream'` to work around `tar` bug in Node 18. diff --git a/packages/backend-common/src/reading/GerritUrlReader.ts b/packages/backend-common/src/reading/GerritUrlReader.ts index 59b33bb2cb609..9ec5e35f65676 100644 --- a/packages/backend-common/src/reading/GerritUrlReader.ts +++ b/packages/backend-common/src/reading/GerritUrlReader.ts @@ -32,8 +32,7 @@ import fetch, { Response } from 'node-fetch'; import os from 'os'; import { join as joinPath } from 'path'; import tar from 'tar'; -import { pipeline as pipelineCb, Readable } from 'stream'; -import { promisify } from 'util'; +import { Readable } from 'stream'; import { ReaderFactory, ReadTreeOptions, @@ -45,8 +44,7 @@ import { UrlReader, } from './types'; import { ScmIntegrations } from '@backstage/integration'; - -const pipeline = promisify(pipelineCb); +import { pipeStream } from './tree/util'; const createTemporaryDirectory = async (workDir: string): Promise => await fs.mkdtemp(joinPath(workDir, '/gerrit-clone-')); @@ -197,7 +195,7 @@ export class GerritUrlReader implements UrlReader { }); const data = await new Promise(async resolve => { - await pipeline( + await pipeStream( tar.create({ cwd: tempDir }, ['']), concatStream(resolve), ); diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts index eabaa3bc56661..da145b0f4bec4 100644 --- a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts @@ -19,17 +19,15 @@ import platformPath, { basename } from 'path'; import getRawBody from 'raw-body'; import fs from 'fs-extra'; -import { promisify } from 'util'; import tar from 'tar'; -import { pipeline as pipelineCb, Readable } from 'stream'; +import { Readable } from 'stream'; import { ReadTreeResponse, ReadTreeResponseFile, ReadTreeResponseDirOptions, FromReadableArrayOptions, } from '../types'; - -const pipeline = promisify(pipelineCb); +import { pipeStream } from './util'; /** * Wraps a array of Readable objects into a tree response reader. @@ -75,7 +73,7 @@ export class ReadableArrayResponse implements ReadTreeResponse { try { const data = await new Promise(async resolve => { - await pipeline( + await pipeStream( tar.create({ cwd: tmpDir }, ['']), concatStream(resolve), ); @@ -95,7 +93,7 @@ export class ReadableArrayResponse implements ReadTreeResponse { for (let i = 0; i < this.stream.length; i++) { if (!this.stream[i].path.endsWith('/')) { - await pipeline( + await pipeStream( this.stream[i].data, fs.createWriteStream( platformPath.join(dir, basename(this.stream[i].path)), diff --git a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts index 493e283c5f7a7..7d0d8f42a2c81 100644 --- a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts @@ -17,21 +17,18 @@ import concatStream from 'concat-stream'; import fs from 'fs-extra'; import platformPath from 'path'; -import { pipeline as pipelineCb, Readable } from 'stream'; +import { Readable } from 'stream'; import tar, { Parse, ParseStream, ReadEntry } from 'tar'; -import { promisify } from 'util'; import { ReadTreeResponse, ReadTreeResponseDirOptions, ReadTreeResponseFile, } from '../types'; -import { stripFirstDirectoryFromPath } from './util'; +import { pipeStream, stripFirstDirectoryFromPath } from './util'; // Tar types for `Parse` is not a proper constructor, but it should be const TarParseStream = Parse as unknown as { new (): ParseStream }; -const pipeline = promisify(pipelineCb); - /** * Wraps a tar archive stream into a tree response reader. */ @@ -99,7 +96,7 @@ export class TarArchiveResponse implements ReadTreeResponse { } const content = new Promise(async resolve => { - await pipeline(entry, concatStream(resolve)); + await pipeStream(entry, concatStream(resolve)); }); files.push({ @@ -110,7 +107,7 @@ export class TarArchiveResponse implements ReadTreeResponse { entry.resume(); }); - await pipeline(this.stream, parser); + await pipeStream(this.stream, parser); return files; } @@ -128,7 +125,7 @@ export class TarArchiveResponse implements ReadTreeResponse { try { const data = await new Promise(async resolve => { - await pipeline( + await pipeStream( tar.create({ cwd: tmpDir }, ['']), concatStream(resolve), ); @@ -152,7 +149,7 @@ export class TarArchiveResponse implements ReadTreeResponse { let filterError: Error | undefined = undefined; - await pipeline( + await pipeStream( this.stream, tar.extract({ strip, diff --git a/packages/backend-common/src/reading/tree/util.test.ts b/packages/backend-common/src/reading/tree/util.test.ts new file mode 100644 index 0000000000000..662ff3f194ae2 --- /dev/null +++ b/packages/backend-common/src/reading/tree/util.test.ts @@ -0,0 +1,63 @@ +/* + * Copyright 2021 The Backstage Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +import { Readable, Writable } from 'stream'; +import { pipeStream, streamToBuffer } from './util'; + +describe('pipeStream', () => { + it('should pipe a stream', async () => { + const from = Readable.from(['hello']); + + let written = ''; + const to = new Writable({ + write(chunk, encoding, callback) { + written = `${encoding}:${chunk}`; + callback(); + }, + }); + + await pipeStream(from, to); + expect(written).toBe('buffer:hello'); + }); + + it('should forward errors', async () => { + const from = new Readable({ + read() { + throw new Error('oh no'); + }, + }); + const to = new Writable(); + + await expect(pipeStream(from, to)).rejects.toThrow('oh no'); + }); +}); + +describe('streamToBuffer', () => { + it('should read a stream', async () => { + await expect(streamToBuffer(Readable.from(['hello']))).resolves.toBe( + 'hello', + ); + }); + + it('should fail on errors', async () => { + const stream = new Readable({ + read() { + throw new Error('oh no'); + }, + }); + await expect(streamToBuffer(stream)).rejects.toThrow('oh no'); + }); +}); diff --git a/packages/backend-common/src/reading/tree/util.ts b/packages/backend-common/src/reading/tree/util.ts index 63192102f4806..a4e9a6686c8f8 100644 --- a/packages/backend-common/src/reading/tree/util.ts +++ b/packages/backend-common/src/reading/tree/util.ts @@ -14,12 +14,9 @@ * limitations under the License. */ -import { Readable, pipeline as pipelineCb } from 'stream'; -import { promisify } from 'util'; +import { Readable, finished } from 'stream'; import concatStream from 'concat-stream'; -const pipeline = promisify(pipelineCb); - // Matches a directory name + one `/` at the start of any string, // containing any character except `/` one or more times, and ending with a `/` // e.g. Will match `dirA/` in `dirA/dirB/file.ext` @@ -29,13 +26,37 @@ export function stripFirstDirectoryFromPath(path: string): string { return path.replace(directoryNameRegex, ''); } +// Custom pipeline implementation, since pipeline doesn't work well with tar on node 18 +// See https://github.com/npm/node-tar/issues/321 +export function pipeStream( + from: NodeJS.ReadableStream, + to: NodeJS.WritableStream, +): Promise { + return new Promise((resolve, reject) => { + from.pipe(to); + finished(from, fromErr => { + if (fromErr) { + reject(fromErr); + } else { + finished(to, toErr => { + if (toErr) { + reject(toErr); + } else { + resolve(); + } + }); + } + }); + }); +} + // Collect the stream into a buffer and return -export const streamToBuffer = (stream: Readable): Promise => { +export function streamToBuffer(stream: Readable): Promise { return new Promise(async (resolve, reject) => { try { - await pipeline(stream, concatStream(resolve)); + await pipeStream(stream, concatStream(resolve)); } catch (ex) { reject(ex); } }); -}; +} From 90e5b42258bbbbe5ac057ecd2d926cdb92295f0f Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 22 Sep 2022 17:44:51 +0200 Subject: [PATCH 11/22] catalog-backend-module-github: work around lack of msw fetch support in Node 18 Signed-off-by: Patrik Oldsberg --- plugins/catalog-backend-module-github/src/lib/github.test.ts | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/plugins/catalog-backend-module-github/src/lib/github.test.ts b/plugins/catalog-backend-module-github/src/lib/github.test.ts index 25878c2005427..0002c4cef0e94 100644 --- a/plugins/catalog-backend-module-github/src/lib/github.test.ts +++ b/plugins/catalog-backend-module-github/src/lib/github.test.ts @@ -25,6 +25,11 @@ import { getOrganizationRepositories, QueryResponse, } from './github'; +import fetch from 'node-fetch'; + +// Workaround for Node.js 18, where native fetch is available, but not yet picked up by msw +// TODO(Rugvip): remove once https://github.com/mswjs/msw/issues/1388 is resolved +(global as any).fetch = fetch; describe('github', () => { const server = setupServer(); From 428e5d55960e694c17ce5be4104d22f08253b8a6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 1 Nov 2022 15:29:59 +0100 Subject: [PATCH 12/22] Revert "root: work around openid-client with resolutions" This reverts commit 59bcd774351a86193eaa2fe8360cfc867a4f16da. Signed-off-by: Patrik Oldsberg --- package.json | 1 - yarn.lock | 12 ++++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index d7936487685f5..11e92b530bb13 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,6 @@ ] }, "resolutions": { - "openid-client": "npm:openid-client-any-engine@^5.1.3", "@types/react": "^17", "@types/react-dom": "^17" }, diff --git a/yarn.lock b/yarn.lock index d967f0a87722b..5e34fdf91ea5e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -31169,6 +31169,18 @@ __metadata: languageName: node linkType: hard +"openid-client@npm:^5.1.6": + version: 5.1.9 + resolution: "openid-client@npm:5.1.9" + dependencies: + jose: ^4.1.4 + lru-cache: ^6.0.0 + object-hash: ^2.0.1 + oidc-token-hash: ^5.0.1 + checksum: 55390a7eceaafdc340a5f2ece576eb4863fcb4cd8840e4a7a1af66bbaf830623b18cf1dd0b8ce472c6f36c6313e78f92db48ca0c10eb8c681a81e5a3d607fc9f + languageName: node + linkType: hard + "openid-client@npm:openid-client-any-engine@^5.1.3": version: 5.1.8 resolution: "openid-client-any-engine@npm:5.1.8" From 40c578eaf62fa1135f9be1c85d84be19be7399dc Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 1 Nov 2022 15:30:02 +0100 Subject: [PATCH 13/22] Revert "auth-backend: use openid-client-any-engine" This reverts commit fabe460cd5b4c3f5e9b75d5cb5635a87440c6465. Signed-off-by: Patrik Oldsberg --- plugins/auth-backend/package.json | 2 +- yarn.lock | 16 ++-------------- 2 files changed, 3 insertions(+), 15 deletions(-) diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index a4033b90b073b..4b3d7910e5715 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -60,7 +60,7 @@ "morgan": "^1.10.0", "node-cache": "^5.1.2", "node-fetch": "^2.6.7", - "openid-client": "npm:openid-client-any-engine@^5.1.3", + "openid-client": "^5.1.3", "passport": "^0.6.0", "passport-auth0": "^1.4.3", "passport-bitbucket-oauth2": "^0.1.2", diff --git a/yarn.lock b/yarn.lock index 5e34fdf91ea5e..286907726c086 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4296,7 +4296,7 @@ __metadata: msw: ^0.47.0 node-cache: ^5.1.2 node-fetch: ^2.6.7 - openid-client: "npm:openid-client-any-engine@^5.1.3" + openid-client: ^5.1.3 passport: ^0.6.0 passport-auth0: ^1.4.3 passport-bitbucket-oauth2: ^0.1.2 @@ -31169,7 +31169,7 @@ __metadata: languageName: node linkType: hard -"openid-client@npm:^5.1.6": +"openid-client@npm:^5.1.3, openid-client@npm:^5.1.6": version: 5.1.9 resolution: "openid-client@npm:5.1.9" dependencies: @@ -31181,18 +31181,6 @@ __metadata: languageName: node linkType: hard -"openid-client@npm:openid-client-any-engine@^5.1.3": - version: 5.1.8 - resolution: "openid-client-any-engine@npm:5.1.8" - dependencies: - jose: ^4.1.4 - lru-cache: ^6.0.0 - object-hash: ^2.0.1 - oidc-token-hash: ^5.0.1 - checksum: dbd6f54a1d9ec50b82299d117556003670c796db553b35a50c4dfa57824f33721d12b2afc0d92a5dece936ef35d190f6dceab0258c65eb78d045dfb966b64831 - languageName: node - linkType: hard - "optionator@npm:^0.8.1": version: 0.8.3 resolution: "optionator@npm:0.8.3" From 442cbba6256302ed7654099a021bf88fd24a3c72 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 1 Nov 2022 15:30:04 +0100 Subject: [PATCH 14/22] Revert "auth-backend: add script for re-publishing openid-client" This reverts commit 6c7a73e597bc53e0bbe93eab2518b5d4e4b26a05. Signed-off-by: Patrik Oldsberg --- .../scripts/republish-openid-client.js | 109 ------------------ 1 file changed, 109 deletions(-) delete mode 100755 plugins/auth-backend/scripts/republish-openid-client.js diff --git a/plugins/auth-backend/scripts/republish-openid-client.js b/plugins/auth-backend/scripts/republish-openid-client.js deleted file mode 100755 index 9d4497e5905de..0000000000000 --- a/plugins/auth-backend/scripts/republish-openid-client.js +++ /dev/null @@ -1,109 +0,0 @@ -#!/usr/bin/env node -/* eslint-disable import/no-extraneous-dependencies */ -/* - * Copyright 2022 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const fetch = require('node-fetch'); -const zlib = require('zlib'); - -// eslint-disable-next-line import/no-extraneous-dependencies -const libpub = require('libnpmpublish'); -const concatStream = require('concat-stream'); -const tar = require('tar-stream'); - -/* - -This works around an incompatibility in node versioning policies between Backstage and -the openid-client package. This script downloads a target version of the openid-client -package and re-publishes it as a new version of the openid-client-any-engine package. - -Usage: ./republish-openid-client.js - -Environment Variables: - -NPM_TOKEN: The NPM auth token to use for publishing. -NPM_OTP: The NPM OTP (2FA one time password) to use for publishing. - -*/ - -async function main(args) { - const [version] = args; - if (!version) { - throw new Error('No version provided, Usage: $0 '); - } - - const res = await fetch( - `https://registry.npmjs.org/openid-client/-/openid-client-${version}.tgz`, - ); - if (!res.ok) { - throw new Error(`Failed to fetch openid-client: ${res.status}`); - } - - const { data, manifest } = await new Promise((resolve, reject) => { - const extract = tar.extract(); - const pack = tar.pack(); - let foundPackageJson = undefined; - - res.body.pipe(zlib.createGunzip()).pipe(extract).on('error', reject); - - extract.on('entry', (header, stream, callback) => { - if (header.name === 'package/package.json') { - stream.pipe( - concatStream(fileContents => { - const packageJson = JSON.parse(fileContents.toString('utf8')); - packageJson.name = 'openid-client-any-engine'; - packageJson.description = - 'Re-publish of openid-client that allows any Node.js version above 12.19.0'; - packageJson.engines.node = '>=12.19.0'; - - foundPackageJson = packageJson; - - pack.entry( - { name: 'package/package.json' }, - JSON.stringify(packageJson, null, 2), - ); - callback(); - }), - ); - } else { - stream.pipe(pack.entry(header, callback)); - } - }); - - extract.on('finish', () => { - pack.finalize(); - }); - - pack - .pipe(zlib.createGzip()) - .pipe( - concatStream(d => { - resolve({ data: d, manifest: foundPackageJson }); - }), - ) - .on('error', reject); - }); - - await libpub.publish(manifest, data, { - token: process.env.NPM_TOKEN, - otp: process.env.NPM_OTP, - }); -} - -main(process.argv.slice(2)).catch(error => { - console.error(error); - process.exit(1); -}); From 2008aec2b69ba005a4b744f25a2143bf15818b2c Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 1 Nov 2022 15:34:34 +0100 Subject: [PATCH 15/22] pin @kubernetes/client-node to working version Signed-off-by: Patrik Oldsberg --- .changeset/silent-moles-chew.md | 2 +- packages/backend-common/package.json | 2 +- plugins/kubernetes-backend/package.json | 2 +- plugins/kubernetes-common/package.json | 2 +- plugins/kubernetes/package.json | 2 +- yarn.lock | 93 +++++++++++++++++++++---- 6 files changed, 85 insertions(+), 18 deletions(-) diff --git a/.changeset/silent-moles-chew.md b/.changeset/silent-moles-chew.md index 09d00ff81bb82..be71272e288be 100644 --- a/.changeset/silent-moles-chew.md +++ b/.changeset/silent-moles-chew.md @@ -5,4 +5,4 @@ '@backstage/plugin-kubernetes-common': patch --- -Bumped `@kubernetes/client-node` to `^0.17.1`. +Pin `@kubernetes/client-node` version to `0.17.0`. diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 7dde8e6512cb4..2847e86504eff 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -42,7 +42,7 @@ "@backstage/types": "workspace:^", "@google-cloud/storage": "^6.0.0", "@keyv/redis": "^2.2.3", - "@kubernetes/client-node": "^0.17.1", + "@kubernetes/client-node": "0.17.0", "@manypkg/get-packages": "^1.1.3", "@octokit/rest": "^19.0.3", "@types/cors": "^2.8.6", diff --git a/plugins/kubernetes-backend/package.json b/plugins/kubernetes-backend/package.json index 2cef29c4bb866..ea6a1146a8284 100644 --- a/plugins/kubernetes-backend/package.json +++ b/plugins/kubernetes-backend/package.json @@ -43,7 +43,7 @@ "@backstage/plugin-auth-node": "workspace:^", "@backstage/plugin-kubernetes-common": "workspace:^", "@google-cloud/container": "^4.0.0", - "@kubernetes/client-node": "^0.17.1", + "@kubernetes/client-node": "0.17.0", "@types/express": "^4.17.6", "@types/luxon": "^3.0.0", "aws-sdk": "^2.840.0", diff --git a/plugins/kubernetes-common/package.json b/plugins/kubernetes-common/package.json index 5ef960a396375..401341a0e8de1 100644 --- a/plugins/kubernetes-common/package.json +++ b/plugins/kubernetes-common/package.json @@ -39,7 +39,7 @@ }, "dependencies": { "@backstage/catalog-model": "workspace:^", - "@kubernetes/client-node": "^0.17.1" + "@kubernetes/client-node": "0.17.0" }, "devDependencies": { "@backstage/cli": "workspace:^" diff --git a/plugins/kubernetes/package.json b/plugins/kubernetes/package.json index 349642f65abdd..0bed1a213b69e 100644 --- a/plugins/kubernetes/package.json +++ b/plugins/kubernetes/package.json @@ -40,7 +40,7 @@ "@backstage/plugin-catalog-react": "workspace:^", "@backstage/plugin-kubernetes-common": "workspace:^", "@backstage/theme": "workspace:^", - "@kubernetes/client-node": "^0.17.1", + "@kubernetes/client-node": "0.17.0", "@material-ui/core": "^4.12.2", "@material-ui/icons": "^4.9.1", "@material-ui/lab": "4.0.0-alpha.57", diff --git a/yarn.lock b/yarn.lock index 286907726c086..91baa366b58d3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3148,7 +3148,7 @@ __metadata: "@backstage/types": "workspace:^" "@google-cloud/storage": ^6.0.0 "@keyv/redis": ^2.2.3 - "@kubernetes/client-node": ^0.17.1 + "@kubernetes/client-node": 0.17.0 "@manypkg/get-packages": ^1.1.3 "@octokit/rest": ^19.0.3 "@types/archiver": ^5.1.0 @@ -6220,7 +6220,7 @@ __metadata: "@backstage/plugin-auth-node": "workspace:^" "@backstage/plugin-kubernetes-common": "workspace:^" "@google-cloud/container": ^4.0.0 - "@kubernetes/client-node": ^0.17.1 + "@kubernetes/client-node": 0.17.0 "@types/aws4": ^1.5.1 "@types/express": ^4.17.6 "@types/luxon": ^3.0.0 @@ -6249,7 +6249,7 @@ __metadata: dependencies: "@backstage/catalog-model": "workspace:^" "@backstage/cli": "workspace:^" - "@kubernetes/client-node": ^0.17.1 + "@kubernetes/client-node": 0.17.0 languageName: unknown linkType: soft @@ -6268,7 +6268,7 @@ __metadata: "@backstage/plugin-kubernetes-common": "workspace:^" "@backstage/test-utils": "workspace:^" "@backstage/theme": "workspace:^" - "@kubernetes/client-node": ^0.17.1 + "@kubernetes/client-node": 0.17.0 "@material-ui/core": ^4.12.2 "@material-ui/icons": ^4.9.1 "@material-ui/lab": 4.0.0-alpha.57 @@ -10671,10 +10671,17 @@ __metadata: languageName: node linkType: hard -"@kubernetes/client-node@npm:^0.17.1": - version: 0.17.1 - resolution: "@kubernetes/client-node@npm:0.17.1" +"@kubernetes/client-node@npm:0.17.0": + version: 0.17.0 + resolution: "@kubernetes/client-node@npm:0.17.0" dependencies: + "@types/js-yaml": ^4.0.1 + "@types/node": ^10.12.0 + "@types/request": ^2.47.1 + "@types/stream-buffers": ^3.0.3 + "@types/tar": ^4.0.3 + "@types/underscore": ^1.8.9 + "@types/ws": ^6.0.1 byline: ^5.0.0 execa: 5.0.0 isomorphic-ws: ^4.0.1 @@ -10690,10 +10697,7 @@ __metadata: tslib: ^1.9.3 underscore: ^1.9.1 ws: ^7.3.1 - dependenciesMeta: - openid-client: - optional: true - checksum: 834ab0ca1f8583b06c4102395ad712c115a6f99201f8dd7f0b59c1a0a5da67b006d1fe5857ad909d762b9cf82a75503665df19fdb640bf620f6123ddb36c8872 + checksum: 5a6edce96946966d8b61359ab043b42e69d1b31178133bc3bf4b5a5c9191766986e08f85d6ae814b56546ed20389817a79dca5101342543d79d750f30cba21f7 languageName: node linkType: hard @@ -13447,6 +13451,13 @@ __metadata: languageName: node linkType: hard +"@types/caseless@npm:*": + version: 0.12.2 + resolution: "@types/caseless@npm:0.12.2" + checksum: 430d15911184ad11e0a8aa21d1ec15fcc93b90b63570c37bf16ebd34457482bfc8de3f5eb6771e0ef986ce183270d4297823b0f492c346255967e78f7292388b + languageName: node + linkType: hard + "@types/classnames@npm:^2.2.9": version: 2.3.1 resolution: "@types/classnames@npm:2.3.1" @@ -14109,7 +14120,7 @@ __metadata: languageName: node linkType: hard -"@types/js-yaml@npm:^4.0.0": +"@types/js-yaml@npm:^4.0.0, @types/js-yaml@npm:^4.0.1": version: 4.0.5 resolution: "@types/js-yaml@npm:4.0.5" checksum: 7dcac8c50fec31643cc9d6444b5503239a861414cdfaa7ae9a38bc22597c4d850c4b8cec3d82d73b3fbca408348ce223b0408d598b32e094470dfffc6d486b4d @@ -14320,6 +14331,15 @@ __metadata: languageName: node linkType: hard +"@types/minipass@npm:*": + version: 3.1.2 + resolution: "@types/minipass@npm:3.1.2" + dependencies: + "@types/node": "*" + checksum: 0d01e11b5b959625385a482ad29ea16352be42506b459555b0f77fd82235e9c540946cc9c05a73fed1ae30b132914baaa4ccf257ed2cad20bc9773f0a06f4bac + languageName: node + linkType: hard + "@types/mock-fs@npm:^4.10.0, @types/mock-fs@npm:^4.13.0": version: 4.13.1 resolution: "@types/mock-fs@npm:4.13.1" @@ -14395,7 +14415,7 @@ __metadata: languageName: node linkType: hard -"@types/node@npm:^10.1.0": +"@types/node@npm:^10.1.0, @types/node@npm:^10.12.0": version: 10.17.60 resolution: "@types/node@npm:10.17.60" checksum: 2cdb3a77d071ba8513e5e8306fa64bf50e3c3302390feeaeff1fd325dd25c8441369715dfc8e3701011a72fed5958c7dfa94eb9239a81b3c286caa4d97db6eef @@ -14740,6 +14760,18 @@ __metadata: languageName: node linkType: hard +"@types/request@npm:^2.47.1": + version: 2.48.8 + resolution: "@types/request@npm:2.48.8" + dependencies: + "@types/caseless": "*" + "@types/node": "*" + "@types/tough-cookie": "*" + form-data: ^2.5.0 + checksum: 0b7754941e08205dce51635d894ec524df276d2b83ca13b9aab723f9281acecf1108841e9554494cb1cb60f6d6ddbb47ebea97392bcf2bf607f035b3a9b4af45 + languageName: node + linkType: hard + "@types/resize-observer-browser@npm:^0.1.6": version: 0.1.7 resolution: "@types/resize-observer-browser@npm:0.1.7" @@ -14926,6 +14958,15 @@ __metadata: languageName: node linkType: hard +"@types/stream-buffers@npm:^3.0.3": + version: 3.0.4 + resolution: "@types/stream-buffers@npm:3.0.4" + dependencies: + "@types/node": "*" + checksum: 5b432b2bf963d612747b79ac317562888236d6a9ea14414fb055c24e7be9643b5e3c7b7470841fa82802aa1c1c0d752a4ba935bbc0cfb12de6b89f7e1dadee92 + languageName: node + linkType: hard + "@types/styled-jsx@npm:^2.2.8": version: 2.2.8 resolution: "@types/styled-jsx@npm:2.2.8" @@ -14972,6 +15013,16 @@ __metadata: languageName: node linkType: hard +"@types/tar@npm:^4.0.3": + version: 4.0.5 + resolution: "@types/tar@npm:4.0.5" + dependencies: + "@types/minipass": "*" + "@types/node": "*" + checksum: 476d8af8f4cffcd973de026e043271be76171f9cb07bb869ba38a193ce89ee361b59ff8484e28b57216266063d91502c5208d2ab6b976e7370d9bdf4c4dadadc + languageName: node + linkType: hard + "@types/tar@npm:^6.1.1": version: 6.1.3 resolution: "@types/tar@npm:6.1.3" @@ -15041,6 +15092,13 @@ __metadata: languageName: node linkType: hard +"@types/underscore@npm:^1.8.9": + version: 1.11.4 + resolution: "@types/underscore@npm:1.11.4" + checksum: db9f8486bc851b732259e51f42d62aad1ae2158be5724612dc125ece5f5d61c51447f9dea28284c2a0f79cb95e788d01cb5ce97709880019213e69fab0dd1696 + languageName: node + linkType: hard + "@types/unist@npm:*, @types/unist@npm:^2.0.0": version: 2.0.6 resolution: "@types/unist@npm:2.0.6" @@ -15090,6 +15148,15 @@ __metadata: languageName: node linkType: hard +"@types/ws@npm:^6.0.1": + version: 6.0.4 + resolution: "@types/ws@npm:6.0.4" + dependencies: + "@types/node": "*" + checksum: b2656a76bfad0c17bb1e3fc237ba7122431c1373669977ed8edef45934c82f71c75d8c71f0a576dc6d98b0954fd94cae0166c6b4ccb40f7e0ee29cc92673519c + languageName: node + linkType: hard + "@types/ws@npm:^8.0.0, @types/ws@npm:^8.5.1": version: 8.5.3 resolution: "@types/ws@npm:8.5.3" From ee35a695dd72439ccb192211f8b145453e795c3d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 1 Nov 2022 15:37:13 +0100 Subject: [PATCH 16/22] auth-backend: bump openid-client to most recent version Signed-off-by: Patrik Oldsberg --- plugins/auth-backend/package.json | 2 +- yarn.lock | 20 ++++++++++---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/plugins/auth-backend/package.json b/plugins/auth-backend/package.json index 4b3d7910e5715..67deb66b334da 100644 --- a/plugins/auth-backend/package.json +++ b/plugins/auth-backend/package.json @@ -60,7 +60,7 @@ "morgan": "^1.10.0", "node-cache": "^5.1.2", "node-fetch": "^2.6.7", - "openid-client": "^5.1.3", + "openid-client": "^5.2.1", "passport": "^0.6.0", "passport-auth0": "^1.4.3", "passport-bitbucket-oauth2": "^0.1.2", diff --git a/yarn.lock b/yarn.lock index 91baa366b58d3..2c90ba57ecc7e 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4296,7 +4296,7 @@ __metadata: msw: ^0.47.0 node-cache: ^5.1.2 node-fetch: ^2.6.7 - openid-client: ^5.1.3 + openid-client: ^5.2.1 passport: ^0.6.0 passport-auth0: ^1.4.3 passport-bitbucket-oauth2: ^0.1.2 @@ -27157,10 +27157,10 @@ __metadata: languageName: node linkType: hard -"jose@npm:^4.1.4": - version: 4.5.0 - resolution: "jose@npm:4.5.0" - checksum: 51142150a9f571ca318842d34ac0daec49fb70bfb6034c68fd46d8c804a8325065876067202310c8ff3cbfc5ca8e44d4712467b4da0d3cadf1e47c96989ca4ca +"jose@npm:^4.10.0": + version: 4.10.4 + resolution: "jose@npm:4.10.4" + checksum: 0e6caaae0b0303534c0ac23711d45eadfbdbff63d9aeed80965c668b5532c254ab25b48afddc3e1ecfcfd36b4275dee41174a097c5a47a25ce04268c78f3c130 languageName: node linkType: hard @@ -31236,15 +31236,15 @@ __metadata: languageName: node linkType: hard -"openid-client@npm:^5.1.3, openid-client@npm:^5.1.6": - version: 5.1.9 - resolution: "openid-client@npm:5.1.9" +"openid-client@npm:^5.1.6, openid-client@npm:^5.2.1": + version: 5.2.1 + resolution: "openid-client@npm:5.2.1" dependencies: - jose: ^4.1.4 + jose: ^4.10.0 lru-cache: ^6.0.0 object-hash: ^2.0.1 oidc-token-hash: ^5.0.1 - checksum: 55390a7eceaafdc340a5f2ece576eb4863fcb4cd8840e4a7a1af66bbaf830623b18cf1dd0b8ce472c6f36c6313e78f92db48ca0c10eb8c681a81e5a3d607fc9f + checksum: b2e9ee8bafb30981fe8eb4446d86578649f05e61d6289abfe79c863578ef8ab24eb4430461c2b8dbb50cbaff89af01e77439df5ebb87c391ed1e336a0e69e590 languageName: node linkType: hard From a8dac60b0813bd0929d06c612e0d4392b21805dc Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 1 Nov 2022 15:37:32 +0100 Subject: [PATCH 17/22] Revert "workflows: work around missing node-canvas binaries for node 18" This reverts commit 0ab270539ae9ba4ec0b34af9624e7d870f4515da. Signed-off-by: Patrik Oldsberg --- .github/workflows/ci.yml | 7 ------- .github/workflows/deploy_nightly.yml | 2 -- .github/workflows/deploy_packages.yml | 8 -------- .github/workflows/verify_e2e-linux.yml | 6 ------ .github/workflows/verify_e2e-techdocs.yml | 7 ------- .github/workflows/verify_e2e-windows.yml | 10 ---------- .github/workflows/verify_kubernetes.yml | 6 ------ .github/workflows/verify_windows.yml | 9 --------- 8 files changed, 55 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1d8040e993586..3ab979593c456 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -32,13 +32,6 @@ jobs: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth - # Needed until there are pre-built binaries of node-canvas for Node 18 - - name: node-canvas dependencies - if: matrix.node-version == '18.x' - run: | - sudo apt update - sudo apt install -y build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev - - name: yarn install uses: backstage/actions/yarn-install@v0.5.6 with: diff --git a/.github/workflows/deploy_nightly.yml b/.github/workflows/deploy_nightly.yml index bb41481cb10ce..74dc60783a76a 100644 --- a/.github/workflows/deploy_nightly.yml +++ b/.github/workflows/deploy_nightly.yml @@ -25,8 +25,6 @@ jobs: with: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth - - # Needed until there are pre-built binaries for Node 18 - name: yarn install uses: backstage/actions/yarn-install@v0.5.6 with: diff --git a/.github/workflows/deploy_packages.yml b/.github/workflows/deploy_packages.yml index 6fffe4d60a59f..56ff61c75bdd9 100644 --- a/.github/workflows/deploy_packages.yml +++ b/.github/workflows/deploy_packages.yml @@ -66,14 +66,6 @@ jobs: with: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth - - # Needed until there are pre-built binaries of node-canvas for Node 18 - - name: node-canvas dependencies - if: matrix.node-version == '18.x' - run: | - sudo apt update - sudo apt install -y build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev - - name: yarn install uses: backstage/actions/yarn-install@v0.5.6 with: diff --git a/.github/workflows/verify_e2e-linux.yml b/.github/workflows/verify_e2e-linux.yml index db8f6662fe9ed..fefdd1fdd523b 100644 --- a/.github/workflows/verify_e2e-linux.yml +++ b/.github/workflows/verify_e2e-linux.yml @@ -51,12 +51,6 @@ jobs: with: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth - # Needed until there are pre-built binaries of node-canvas for Node 18 - - name: node-canvas dependencies - if: matrix.node-version == '18.x' - run: | - sudo apt update - sudo apt install -y build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev - name: yarn install uses: backstage/actions/yarn-install@v0.5.6 with: diff --git a/.github/workflows/verify_e2e-techdocs.yml b/.github/workflows/verify_e2e-techdocs.yml index 0d6836ad3aba8..8d59d1589997e 100644 --- a/.github/workflows/verify_e2e-techdocs.yml +++ b/.github/workflows/verify_e2e-techdocs.yml @@ -28,13 +28,6 @@ jobs: - uses: actions/checkout@v3 - uses: actions/setup-python@v3 - # Needed until there are pre-built binaries of node-canvas for Node 18 - - name: node-canvas dependencies - if: matrix.node-version == '18.x' - run: | - sudo apt update - sudo apt install -y build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev - - name: install dependencies run: yarn install --immutable diff --git a/.github/workflows/verify_e2e-windows.yml b/.github/workflows/verify_e2e-windows.yml index 9d1ba8a154afb..fc29c1f03bb1a 100644 --- a/.github/workflows/verify_e2e-windows.yml +++ b/.github/workflows/verify_e2e-windows.yml @@ -52,16 +52,6 @@ jobs: uses: microsoft/setup-msbuild@v1.0.3 - name: setup chrome uses: browser-actions/setup-chrome@latest - - # Needed until there are pre-built binaries of node-canvas for Node 18 - - name: node-canvas dependencies - if: matrix.node-version == '18.x' - # From https://github.com/Automattic/node-canvas/blob/master/.github/workflows/ci.yaml - run: | - Invoke-WebRequest "https://ftp-osl.osuosl.org/pub/gnome/binaries/win64/gtk+/2.22/gtk+-bundle_2.22.1-20101229_win64.zip" -OutFile "gtk.zip" - Expand-Archive gtk.zip -DestinationPath "C:\GTK" - Invoke-WebRequest "https://downloads.sourceforge.net/project/libjpeg-turbo/2.0.4/libjpeg-turbo-2.0.4-vc64.exe" -OutFile "libjpeg.exe" -UserAgent NativeHost - .\libjpeg.exe /S - name: yarn install run: yarn install --immutable diff --git a/.github/workflows/verify_kubernetes.yml b/.github/workflows/verify_kubernetes.yml index ea5e3ebe4eb87..e8c248699aece 100644 --- a/.github/workflows/verify_kubernetes.yml +++ b/.github/workflows/verify_kubernetes.yml @@ -27,12 +27,6 @@ jobs: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth - # Needed until there are pre-built binaries of node-canvas for Node 18 - - name: node-canvas dependencies - if: matrix.node-version == '18.x' - run: | - sudo apt update - sudo apt install -y build-essential libcairo2-dev libpango1.0-dev libjpeg-dev libgif-dev librsvg2-dev - name: yarn install uses: backstage/actions/yarn-install@v0.5.6 with: diff --git a/.github/workflows/verify_windows.yml b/.github/workflows/verify_windows.yml index a092c8d405425..75a0633cbe352 100644 --- a/.github/workflows/verify_windows.yml +++ b/.github/workflows/verify_windows.yml @@ -32,15 +32,6 @@ jobs: node-version: ${{ matrix.node-version }} registry-url: https://registry.npmjs.org/ # Needed for auth - # Needed until there are pre-built binaries of node-canvas for Node 18 - - name: node-canvas dependencies - if: matrix.node-version == '18.x' - # From https://github.com/Automattic/node-canvas/blob/master/.github/workflows/ci.yaml - run: | - Invoke-WebRequest "https://ftp-osl.osuosl.org/pub/gnome/binaries/win64/gtk+/2.22/gtk+-bundle_2.22.1-20101229_win64.zip" -OutFile "gtk.zip" - Expand-Archive gtk.zip -DestinationPath "C:\GTK" - Invoke-WebRequest "https://downloads.sourceforge.net/project/libjpeg-turbo/2.0.4/libjpeg-turbo-2.0.4-vc64.exe" -OutFile "libjpeg.exe" -UserAgent NativeHost - .\libjpeg.exe /S # Windows file operation slowness means there's no point caching this - name: yarn install run: yarn install --immutable From 93554b047cd34c1d6f44a81856cfe76218118eb6 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 1 Nov 2022 15:39:04 +0100 Subject: [PATCH 18/22] Revert "backend-common: work around premature close errors" This reverts commit b29587c3187fdd5c78341dce5c6f406b631e1f21. Signed-off-by: Patrik Oldsberg --- .changeset/swift-suits-reply.md | 5 -- .../src/reading/GerritUrlReader.ts | 8 ++- .../src/reading/tree/ReadableArrayResponse.ts | 10 +-- .../src/reading/tree/TarArchiveResponse.ts | 15 +++-- .../src/reading/tree/util.test.ts | 63 ------------------- .../backend-common/src/reading/tree/util.ts | 35 +++-------- 6 files changed, 27 insertions(+), 109 deletions(-) delete mode 100644 .changeset/swift-suits-reply.md delete mode 100644 packages/backend-common/src/reading/tree/util.test.ts diff --git a/.changeset/swift-suits-reply.md b/.changeset/swift-suits-reply.md deleted file mode 100644 index f81f39bee285b..0000000000000 --- a/.changeset/swift-suits-reply.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@backstage/backend-common': patch ---- - -Refactored internal usage of the build-in `pipeline` from `'stream'` to work around `tar` bug in Node 18. diff --git a/packages/backend-common/src/reading/GerritUrlReader.ts b/packages/backend-common/src/reading/GerritUrlReader.ts index 9ec5e35f65676..59b33bb2cb609 100644 --- a/packages/backend-common/src/reading/GerritUrlReader.ts +++ b/packages/backend-common/src/reading/GerritUrlReader.ts @@ -32,7 +32,8 @@ import fetch, { Response } from 'node-fetch'; import os from 'os'; import { join as joinPath } from 'path'; import tar from 'tar'; -import { Readable } from 'stream'; +import { pipeline as pipelineCb, Readable } from 'stream'; +import { promisify } from 'util'; import { ReaderFactory, ReadTreeOptions, @@ -44,7 +45,8 @@ import { UrlReader, } from './types'; import { ScmIntegrations } from '@backstage/integration'; -import { pipeStream } from './tree/util'; + +const pipeline = promisify(pipelineCb); const createTemporaryDirectory = async (workDir: string): Promise => await fs.mkdtemp(joinPath(workDir, '/gerrit-clone-')); @@ -195,7 +197,7 @@ export class GerritUrlReader implements UrlReader { }); const data = await new Promise(async resolve => { - await pipeStream( + await pipeline( tar.create({ cwd: tempDir }, ['']), concatStream(resolve), ); diff --git a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts index da145b0f4bec4..eabaa3bc56661 100644 --- a/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts +++ b/packages/backend-common/src/reading/tree/ReadableArrayResponse.ts @@ -19,15 +19,17 @@ import platformPath, { basename } from 'path'; import getRawBody from 'raw-body'; import fs from 'fs-extra'; +import { promisify } from 'util'; import tar from 'tar'; -import { Readable } from 'stream'; +import { pipeline as pipelineCb, Readable } from 'stream'; import { ReadTreeResponse, ReadTreeResponseFile, ReadTreeResponseDirOptions, FromReadableArrayOptions, } from '../types'; -import { pipeStream } from './util'; + +const pipeline = promisify(pipelineCb); /** * Wraps a array of Readable objects into a tree response reader. @@ -73,7 +75,7 @@ export class ReadableArrayResponse implements ReadTreeResponse { try { const data = await new Promise(async resolve => { - await pipeStream( + await pipeline( tar.create({ cwd: tmpDir }, ['']), concatStream(resolve), ); @@ -93,7 +95,7 @@ export class ReadableArrayResponse implements ReadTreeResponse { for (let i = 0; i < this.stream.length; i++) { if (!this.stream[i].path.endsWith('/')) { - await pipeStream( + await pipeline( this.stream[i].data, fs.createWriteStream( platformPath.join(dir, basename(this.stream[i].path)), diff --git a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts index 7d0d8f42a2c81..493e283c5f7a7 100644 --- a/packages/backend-common/src/reading/tree/TarArchiveResponse.ts +++ b/packages/backend-common/src/reading/tree/TarArchiveResponse.ts @@ -17,18 +17,21 @@ import concatStream from 'concat-stream'; import fs from 'fs-extra'; import platformPath from 'path'; -import { Readable } from 'stream'; +import { pipeline as pipelineCb, Readable } from 'stream'; import tar, { Parse, ParseStream, ReadEntry } from 'tar'; +import { promisify } from 'util'; import { ReadTreeResponse, ReadTreeResponseDirOptions, ReadTreeResponseFile, } from '../types'; -import { pipeStream, stripFirstDirectoryFromPath } from './util'; +import { stripFirstDirectoryFromPath } from './util'; // Tar types for `Parse` is not a proper constructor, but it should be const TarParseStream = Parse as unknown as { new (): ParseStream }; +const pipeline = promisify(pipelineCb); + /** * Wraps a tar archive stream into a tree response reader. */ @@ -96,7 +99,7 @@ export class TarArchiveResponse implements ReadTreeResponse { } const content = new Promise(async resolve => { - await pipeStream(entry, concatStream(resolve)); + await pipeline(entry, concatStream(resolve)); }); files.push({ @@ -107,7 +110,7 @@ export class TarArchiveResponse implements ReadTreeResponse { entry.resume(); }); - await pipeStream(this.stream, parser); + await pipeline(this.stream, parser); return files; } @@ -125,7 +128,7 @@ export class TarArchiveResponse implements ReadTreeResponse { try { const data = await new Promise(async resolve => { - await pipeStream( + await pipeline( tar.create({ cwd: tmpDir }, ['']), concatStream(resolve), ); @@ -149,7 +152,7 @@ export class TarArchiveResponse implements ReadTreeResponse { let filterError: Error | undefined = undefined; - await pipeStream( + await pipeline( this.stream, tar.extract({ strip, diff --git a/packages/backend-common/src/reading/tree/util.test.ts b/packages/backend-common/src/reading/tree/util.test.ts deleted file mode 100644 index 662ff3f194ae2..0000000000000 --- a/packages/backend-common/src/reading/tree/util.test.ts +++ /dev/null @@ -1,63 +0,0 @@ -/* - * Copyright 2021 The Backstage Authors - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -import { Readable, Writable } from 'stream'; -import { pipeStream, streamToBuffer } from './util'; - -describe('pipeStream', () => { - it('should pipe a stream', async () => { - const from = Readable.from(['hello']); - - let written = ''; - const to = new Writable({ - write(chunk, encoding, callback) { - written = `${encoding}:${chunk}`; - callback(); - }, - }); - - await pipeStream(from, to); - expect(written).toBe('buffer:hello'); - }); - - it('should forward errors', async () => { - const from = new Readable({ - read() { - throw new Error('oh no'); - }, - }); - const to = new Writable(); - - await expect(pipeStream(from, to)).rejects.toThrow('oh no'); - }); -}); - -describe('streamToBuffer', () => { - it('should read a stream', async () => { - await expect(streamToBuffer(Readable.from(['hello']))).resolves.toBe( - 'hello', - ); - }); - - it('should fail on errors', async () => { - const stream = new Readable({ - read() { - throw new Error('oh no'); - }, - }); - await expect(streamToBuffer(stream)).rejects.toThrow('oh no'); - }); -}); diff --git a/packages/backend-common/src/reading/tree/util.ts b/packages/backend-common/src/reading/tree/util.ts index a4e9a6686c8f8..63192102f4806 100644 --- a/packages/backend-common/src/reading/tree/util.ts +++ b/packages/backend-common/src/reading/tree/util.ts @@ -14,9 +14,12 @@ * limitations under the License. */ -import { Readable, finished } from 'stream'; +import { Readable, pipeline as pipelineCb } from 'stream'; +import { promisify } from 'util'; import concatStream from 'concat-stream'; +const pipeline = promisify(pipelineCb); + // Matches a directory name + one `/` at the start of any string, // containing any character except `/` one or more times, and ending with a `/` // e.g. Will match `dirA/` in `dirA/dirB/file.ext` @@ -26,37 +29,13 @@ export function stripFirstDirectoryFromPath(path: string): string { return path.replace(directoryNameRegex, ''); } -// Custom pipeline implementation, since pipeline doesn't work well with tar on node 18 -// See https://github.com/npm/node-tar/issues/321 -export function pipeStream( - from: NodeJS.ReadableStream, - to: NodeJS.WritableStream, -): Promise { - return new Promise((resolve, reject) => { - from.pipe(to); - finished(from, fromErr => { - if (fromErr) { - reject(fromErr); - } else { - finished(to, toErr => { - if (toErr) { - reject(toErr); - } else { - resolve(); - } - }); - } - }); - }); -} - // Collect the stream into a buffer and return -export function streamToBuffer(stream: Readable): Promise { +export const streamToBuffer = (stream: Readable): Promise => { return new Promise(async (resolve, reject) => { try { - await pipeStream(stream, concatStream(resolve)); + await pipeline(stream, concatStream(resolve)); } catch (ex) { reject(ex); } }); -} +}; From e92aa15f011beb001c645d5df52623979bd953dc Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 1 Nov 2022 15:58:40 +0100 Subject: [PATCH 19/22] bump canvas to 2.10.2 Signed-off-by: Patrik Oldsberg --- .changeset/blue-items-shop.md | 6 ++++++ plugins/cost-insights/package.json | 2 +- plugins/techdocs/package.json | 2 +- yarn.lock | 15 ++++----------- 4 files changed, 12 insertions(+), 13 deletions(-) create mode 100644 .changeset/blue-items-shop.md diff --git a/.changeset/blue-items-shop.md b/.changeset/blue-items-shop.md new file mode 100644 index 0000000000000..fbf22878645c7 --- /dev/null +++ b/.changeset/blue-items-shop.md @@ -0,0 +1,6 @@ +--- +'@backstage/plugin-cost-insights': patch +'@backstage/plugin-techdocs': patch +--- + +Bumped `canvas` dependency to the latest version, which has better Node.js v18 support. diff --git a/plugins/cost-insights/package.json b/plugins/cost-insights/package.json index b5859fa8867db..aac87aa10f7c4 100644 --- a/plugins/cost-insights/package.json +++ b/plugins/cost-insights/package.json @@ -71,7 +71,7 @@ "@types/recharts": "^1.8.14", "@types/regression": "^2.0.0", "@types/yup": "^0.29.13", - "canvas": "^2.6.1", + "canvas": "^2.10.2", "cross-fetch": "^3.1.5", "msw": "^0.47.0" }, diff --git a/plugins/techdocs/package.json b/plugins/techdocs/package.json index cb03cb9e3f836..9eb6f6af3aff5 100644 --- a/plugins/techdocs/package.json +++ b/plugins/techdocs/package.json @@ -76,7 +76,7 @@ "@types/dompurify": "^2.2.2", "@types/event-source-polyfill": "^1.0.0", "@types/node": "^16.11.26", - "canvas": "^2.6.1", + "canvas": "^2.10.2", "cross-fetch": "^3.1.5", "msw": "^0.47.0" }, diff --git a/yarn.lock b/yarn.lock index 2c90ba57ecc7e..8d6464f02652d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5485,7 +5485,7 @@ __metadata: "@types/recharts": ^1.8.14 "@types/regression": ^2.0.0 "@types/yup": ^0.29.13 - canvas: ^2.6.1 + canvas: ^2.10.2 classnames: ^2.2.6 cross-fetch: ^3.1.5 history: ^5.0.0 @@ -7741,7 +7741,7 @@ __metadata: "@types/dompurify": ^2.2.2 "@types/event-source-polyfill": ^1.0.0 "@types/node": ^16.11.26 - canvas: ^2.6.1 + canvas: ^2.10.2 cross-fetch: ^3.1.5 dompurify: ^2.2.9 event-source-polyfill: 1.0.25 @@ -17991,7 +17991,7 @@ __metadata: languageName: node linkType: hard -"canvas@npm:^2.6.1": +"canvas@npm:^2.10.2": version: 2.10.2 resolution: "canvas@npm:2.10.2" dependencies: @@ -27157,14 +27157,7 @@ __metadata: languageName: node linkType: hard -"jose@npm:^4.10.0": - version: 4.10.4 - resolution: "jose@npm:4.10.4" - checksum: 0e6caaae0b0303534c0ac23711d45eadfbdbff63d9aeed80965c668b5532c254ab25b48afddc3e1ecfcfd36b4275dee41174a097c5a47a25ce04268c78f3c130 - languageName: node - linkType: hard - -"jose@npm:^4.6.0": +"jose@npm:^4.10.0, jose@npm:^4.6.0": version: 4.10.4 resolution: "jose@npm:4.10.4" checksum: 0e6caaae0b0303534c0ac23711d45eadfbdbff63d9aeed80965c668b5532c254ab25b48afddc3e1ecfcfd36b4275dee41174a097c5a47a25ce04268c78f3c130 From 88f99b8b1352d924de67b96115f910ca455f9ae0 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Tue, 1 Nov 2022 17:44:20 +0100 Subject: [PATCH 20/22] bump tar to 6.1.12 Signed-off-by: Patrik Oldsberg --- .changeset/eighty-planets-train.md | 6 ++++++ packages/backend-common/package.json | 2 +- packages/cli/package.json | 2 +- yarn.lock | 6 +++--- 4 files changed, 11 insertions(+), 5 deletions(-) create mode 100644 .changeset/eighty-planets-train.md diff --git a/.changeset/eighty-planets-train.md b/.changeset/eighty-planets-train.md new file mode 100644 index 0000000000000..c4d5d57f6c1fc --- /dev/null +++ b/.changeset/eighty-planets-train.md @@ -0,0 +1,6 @@ +--- +'@backstage/backend-common': patch +'@backstage/cli': patch +--- + +Bumped `tar` dependency to `^6.1.12` in order to ensure Node.js v18 compatibility. diff --git a/packages/backend-common/package.json b/packages/backend-common/package.json index 2847e86504eff..28a4f91aa2adf 100644 --- a/packages/backend-common/package.json +++ b/packages/backend-common/package.json @@ -80,7 +80,7 @@ "request": "^2.88.2", "selfsigned": "^2.0.0", "stoppable": "^1.1.0", - "tar": "^6.1.2", + "tar": "^6.1.12", "uuid": "^8.3.2", "winston": "^3.2.1", "yauzl": "^2.10.0", diff --git a/packages/cli/package.json b/packages/cli/package.json index 9eb6bed3d6a04..be46d74a49841 100644 --- a/packages/cli/package.json +++ b/packages/cli/package.json @@ -117,7 +117,7 @@ "style-loader": "^3.3.1", "sucrase": "^3.20.2", "swc-loader": "^0.2.3", - "tar": "^6.1.2", + "tar": "^6.1.12", "terser-webpack-plugin": "^5.1.3", "util": "^0.12.3", "webpack": "^5.70.0", diff --git a/yarn.lock b/yarn.lock index 8d6464f02652d..6e2d6fb8ebb66 100644 --- a/yarn.lock +++ b/yarn.lock @@ -3209,7 +3209,7 @@ __metadata: selfsigned: ^2.0.0 stoppable: ^1.1.0 supertest: ^6.1.3 - tar: ^6.1.2 + tar: ^6.1.12 uuid: ^8.3.2 winston: ^3.2.1 yauzl: ^2.10.0 @@ -3474,7 +3474,7 @@ __metadata: style-loader: ^3.3.1 sucrase: ^3.20.2 swc-loader: ^0.2.3 - tar: ^6.1.2 + tar: ^6.1.12 terser-webpack-plugin: ^5.1.3 ts-node: ^10.0.0 type-fest: ^2.0.0 @@ -37530,7 +37530,7 @@ __metadata: languageName: node linkType: hard -"tar@npm:^6.1.2": +"tar@npm:^6.1.12, tar@npm:^6.1.2": version: 6.1.12 resolution: "tar@npm:6.1.12" dependencies: From f121e3fc6944496705e7dbd519087cbef7bf7e81 Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Wed, 2 Nov 2022 15:08:45 +0100 Subject: [PATCH 21/22] e2e-test: add wait before checking backend availability Signed-off-by: Patrik Oldsberg --- packages/e2e-test/src/commands/run.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/e2e-test/src/commands/run.ts b/packages/e2e-test/src/commands/run.ts index 3f9d28eadd1cf..f32d577cd4e79 100644 --- a/packages/e2e-test/src/commands/run.ts +++ b/packages/e2e-test/src/commands/run.ts @@ -507,6 +507,7 @@ async function testBackendStart(appDir: string, ...args: string[]) { // Skipping the whole block throw new Error(stderr); } + await new Promise(resolve => setTimeout(resolve, 500)); print('Try to fetch entities from the backend'); // Try fetch entities, should be ok From 864c876e57c950a1ccc8eb756dd6111e79e5371d Mon Sep 17 00:00:00 2001 From: Patrik Oldsberg Date: Thu, 3 Nov 2022 10:59:02 +0100 Subject: [PATCH 22/22] create-app: fix default backend listen config and comments Signed-off-by: Patrik Oldsberg --- .changeset/witty-carrots-live.md | 31 +++++++++++++++++++ .../default-app/app-config.production.yaml | 11 ++----- .../templates/default-app/app-config.yaml.hbs | 5 ++- 3 files changed, 36 insertions(+), 11 deletions(-) create mode 100644 .changeset/witty-carrots-live.md diff --git a/.changeset/witty-carrots-live.md b/.changeset/witty-carrots-live.md new file mode 100644 index 0000000000000..4bafd47dfa552 --- /dev/null +++ b/.changeset/witty-carrots-live.md @@ -0,0 +1,31 @@ +--- +'@backstage/create-app': patch +--- + +Fixed incorrect comments in the templated `app-config.yaml` and `app-config.production.yaml`. The `backend.listen` directive is not in fact needed to override the `backend.baseUrl`, the backend listens to all interfaces by default. The configuration has also been updated to listen to all interfaces, rather than just IPv4 ones, as this is required for Node.js v18. The production configuration now also shows the option to specify `backend.listen` as a single string. + +To apply this changes to an existing app, make the following change to `app-config.yaml`: + +```diff +- # Uncomment the following host directive to bind to all IPv4 interfaces and +- # not just the baseUrl hostname. +- # host: 0.0.0.0 ++ # Uncomment the following host directive to bind to specific interfaces ++ # host: 127.0.0.1 +``` + +And the following change to `app-config.production.yaml`: + +```diff +- listen: +- port: 7007 +- # The following host directive binds to all IPv4 interfaces when its value +- # is "0.0.0.0". This is the most permissive setting. The right value depends +- # on your specific deployment. If you remove the host line entirely, the +- # backend will bind on the interface that corresponds to the backend.baseUrl +- # hostname. +- host: 0.0.0.0 ++ # The listener can also be expressed as a single : string. In this case we bind to ++ # all interfaces, the most permissive setting. The right value depends on your specific deployment. ++ listen: ':7007' +``` diff --git a/packages/create-app/templates/default-app/app-config.production.yaml b/packages/create-app/templates/default-app/app-config.production.yaml index 6535d967d54ec..df09dac50a1db 100644 --- a/packages/create-app/templates/default-app/app-config.production.yaml +++ b/packages/create-app/templates/default-app/app-config.production.yaml @@ -9,14 +9,9 @@ backend: # callers. When its value is "http://localhost:7007", it's strictly private # and can't be reached by others. baseUrl: http://localhost:7007 - listen: - port: 7007 - # The following host directive binds to all IPv4 interfaces when its value - # is "0.0.0.0". This is the most permissive setting. The right value depends - # on your specific deployment. If you remove the host line entirely, the - # backend will bind on the interface that corresponds to the backend.baseUrl - # hostname. - host: 0.0.0.0 + # The listener can also be expressed as a single : string. In this case we bind to + # all interfaces, the most permissive setting. The right value depends on your specific deployment. + listen: ':7007' # config options: https://node-postgres.com/api/client database: diff --git a/packages/create-app/templates/default-app/app-config.yaml.hbs b/packages/create-app/templates/default-app/app-config.yaml.hbs index 4a058deefd0a5..1a45d4015bdb7 100644 --- a/packages/create-app/templates/default-app/app-config.yaml.hbs +++ b/packages/create-app/templates/default-app/app-config.yaml.hbs @@ -15,9 +15,8 @@ backend: baseUrl: http://localhost:7007 listen: port: 7007 - # Uncomment the following host directive to bind to all IPv4 interfaces and - # not just the baseUrl hostname. - # host: 0.0.0.0 + # Uncomment the following host directive to bind to specific interfaces + # host: 127.0.0.1 csp: connect-src: ["'self'", 'http:', 'https:'] # Content-Security-Policy directives follow the Helmet format: https://helmetjs.github.io/#reference