From 16be3cf6593737f8b097466d881f835305bf33e5 Mon Sep 17 00:00:00 2001 From: Daniel de Cloet <34800550+Daniel-I-Am@users.noreply.github.com> Date: Mon, 30 Oct 2023 09:59:21 +0100 Subject: [PATCH] feat(terraform): Fetch ziphash for lock refreshes (#25059) Co-authored-by: Rhys Arkins Co-authored-by: Sebastian Poxhofer Co-authored-by: Michael Kriese --- .../terraform-provider/index.spec.ts | 88 +++++++- .../datasource/terraform-provider/index.ts | 38 ++++ .../datasource/terraform-provider/types.ts | 2 + .../releaseBackendGoogle_4_84_0_SHA256SUMS | 3 + .../terraformCloudBackendAzurermVersions.json | 39 ++++ .../terraformCloudBackendGoogleVersions.json | 39 ++++ .../manager/terraform/lockfile/hash.spec.ts | 196 ++++++++++++++++++ .../manager/terraform/lockfile/hash.ts | 18 +- 8 files changed, 416 insertions(+), 7 deletions(-) create mode 100644 lib/modules/manager/terraform/lockfile/__fixtures__/releaseBackendGoogle_4_84_0_SHA256SUMS create mode 100644 lib/modules/manager/terraform/lockfile/__fixtures__/terraformCloudBackendAzurermVersions.json create mode 100644 lib/modules/manager/terraform/lockfile/__fixtures__/terraformCloudBackendGoogleVersions.json diff --git a/lib/modules/datasource/terraform-provider/index.spec.ts b/lib/modules/datasource/terraform-provider/index.spec.ts index 1869bb9041046e..e9108a2b2b9a30 100644 --- a/lib/modules/datasource/terraform-provider/index.spec.ts +++ b/lib/modules/datasource/terraform-provider/index.spec.ts @@ -9,7 +9,7 @@ const hashicorpGoogleBetaReleases = Fixtures.get( 'releaseBackendIndexGoogleBeta.json' ); const serviceDiscoveryResult = Fixtures.get('service-discovery.json'); -const telmateProxmocVersions = Fixtures.get( +const telmateProxmoxVersions = Fixtures.get( 'telmate-proxmox-versions-response.json' ); @@ -282,7 +282,7 @@ describe('modules/datasource/terraform-provider/index', () => { httpMock .scope(primaryUrl) .get('/v1/providers/Telmate/proxmox/versions') - .reply(200, telmateProxmocVersions) + .reply(200, telmateProxmoxVersions) .get('/.well-known/terraform.json') .reply(200, serviceDiscoveryResult); const result = await terraformProviderDatasource.getBuilds( @@ -297,7 +297,7 @@ describe('modules/datasource/terraform-provider/index', () => { httpMock .scope(primaryUrl) .get('/v1/providers/Telmate/proxmox/versions') - .reply(200, telmateProxmocVersions) + .reply(200, telmateProxmoxVersions) .get('/.well-known/terraform.json') .reply(200, serviceDiscoveryResult) .get('/v1/providers/Telmate/proxmox/2.6.1/download/darwin/arm64') @@ -377,7 +377,7 @@ describe('modules/datasource/terraform-provider/index', () => { httpMock .scope(primaryUrl) .get('/v1/providers/Telmate/proxmox/versions') - .reply(200, telmateProxmocVersions) + .reply(200, telmateProxmoxVersions) .get('/.well-known/terraform.json') .reply(200, serviceDiscoveryResult) .get('/v1/providers/Telmate/proxmox/2.6.1/download/darwin/arm64') @@ -411,4 +411,84 @@ describe('modules/datasource/terraform-provider/index', () => { expect(res).toBeNull(); }); }); + + describe('getZipHashes', () => { + it('can fetch zip hashes', async () => { + httpMock + .scope(secondaryUrl) + .get( + '/terraform-provider-azurerm/2.56.0/terraform-provider-azurerm_2.56.0_SHA256SUMS' + ) + .reply( + 200, + '500d4e787bf046bbe64c4853530aff3dfddee2fdbff0087d7b1e7a8c24388628 terraform-provider-azurerm_2.56.0_darwin_amd64.zip\n' + + '766ff42596d643f9945b3aab2e83e306fe77c3020a5196366bbbb77eeea13b71 terraform-provider-azurerm_2.56.0_linux_amd64.zip\n' + + 'fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f terraform-provider-azurerm_2.56.0_manifest.json' + ); + + const res = await terraformProviderDatasource.getZipHashes([ + { + name: 'azurerm', + version: '2.56.0', + os: 'linux', + arch: 'amd64', + filename: 'terraform-provider-azurerm_2.56.0_linux_amd64.zip', + url: 'https://releases.hashicorp.com/terraform-provider-azurerm/2.56.0/terraform-provider-azurerm_2.56.0_linux_amd64.zip', + shasums_url: + 'https://releases.hashicorp.com/terraform-provider-azurerm/2.56.0/terraform-provider-azurerm_2.56.0_SHA256SUMS', + }, + ]); + + expect(res).toMatchObject([ + '500d4e787bf046bbe64c4853530aff3dfddee2fdbff0087d7b1e7a8c24388628', + '766ff42596d643f9945b3aab2e83e306fe77c3020a5196366bbbb77eeea13b71', + 'fbdb892d9822ed0e4cb60f2fedbdbb556e4da0d88d3b942ae963ed6ff091e48f', + ]); + }); + + it('does not fetch anything when there are no builds passed in', async () => { + const res = await terraformProviderDatasource.getZipHashes([]); + + expect(res).toBeEmptyArray(); + }); + + it('does not fetch anything when there is no shasums_url defined', async () => { + const res = await terraformProviderDatasource.getZipHashes([ + { + name: 'azurerm', + version: '2.56.0', + os: 'linux', + arch: 'amd64', + filename: 'terraform-provider-azurerm_2.56.0_linux_amd64.zip', + url: 'https://releases.hashicorp.com/terraform-provider-azurerm/2.56.0/terraform-provider-azurerm_2.56.0_linux_amd64.zip', + }, + ]); + + expect(res).toBeEmptyArray(); + }); + + it('does not hard fail when the ziphashes endpoint is not available', async () => { + httpMock + .scope(secondaryUrl) + .get( + '/terraform-provider-azurerm/2.56.0/terraform-provider-azurerm_2.56.0_SHA256SUMS' + ) + .reply(404); + + const res = await terraformProviderDatasource.getZipHashes([ + { + name: 'azurerm', + version: '2.56.0', + os: 'linux', + arch: 'amd64', + filename: 'terraform-provider-azurerm_2.56.0_linux_amd64.zip', + url: 'https://releases.hashicorp.com/terraform-provider-azurerm/2.56.0/terraform-provider-azurerm_2.56.0_linux_amd64.zip', + shasums_url: + 'https://releases.hashicorp.com/terraform-provider-azurerm/2.56.0/terraform-provider-azurerm_2.56.0_SHA256SUMS', + }, + ]); + + expect(res).toBeUndefined(); + }); + }); }); diff --git a/lib/modules/datasource/terraform-provider/index.ts b/lib/modules/datasource/terraform-provider/index.ts index 1db137f983bf4f..1ebe37c8697bf8 100644 --- a/lib/modules/datasource/terraform-provider/index.ts +++ b/lib/modules/datasource/terraform-provider/index.ts @@ -282,6 +282,44 @@ export class TerraformProviderDatasource extends TerraformDatasource { return filteredResult.length === result.length ? filteredResult : null; } + @cache({ + namespace: `datasource-${TerraformProviderDatasource.id}-zip-hashes`, + key: (registryURL: string, repository: string, version: string) => + `${registryURL}/${repository}/${version}`, + }) + async getZipHashes(builds: TerraformBuild[]): Promise { + if (builds.length === 0) { + return []; + } + + const zipHashUrl = builds[0].shasums_url; + + if (!zipHashUrl) { + return []; + } + + // The hashes are formatted as the result of sha256sum in plain text, each line: \t + let rawHashData: string; + try { + rawHashData = (await this.http.get(zipHashUrl)).body; + } catch (err) { + /* istanbul ignore next */ + if (err instanceof ExternalHostError) { + throw err; + } + logger.debug( + { err, zipHashUrl }, + `Failed to retrieve zip hashes from ${zipHashUrl}` + ); + return undefined; + } + + return rawHashData + .trimEnd() + .split('\n') + .map((line) => line.split(/\s/)[0]); + } + @cache({ namespace: `datasource-${TerraformProviderDatasource.id}-releaseBackendIndex`, key: (backendLookUpName: string, version: string) => diff --git a/lib/modules/datasource/terraform-provider/types.ts b/lib/modules/datasource/terraform-provider/types.ts index a8e80393aec50c..db663294d20fad 100644 --- a/lib/modules/datasource/terraform-provider/types.ts +++ b/lib/modules/datasource/terraform-provider/types.ts @@ -11,6 +11,7 @@ export interface TerraformBuild { arch: string; filename: string; url: string; + shasums_url?: string; } export interface TerraformProvider { @@ -57,4 +58,5 @@ export interface TerraformRegistryBuildResponse { arch: string; filename: string; download_url: string; + shasums_url?: string; } diff --git a/lib/modules/manager/terraform/lockfile/__fixtures__/releaseBackendGoogle_4_84_0_SHA256SUMS b/lib/modules/manager/terraform/lockfile/__fixtures__/releaseBackendGoogle_4_84_0_SHA256SUMS new file mode 100644 index 00000000000000..78a327e74ac2ad --- /dev/null +++ b/lib/modules/manager/terraform/lockfile/__fixtures__/releaseBackendGoogle_4_84_0_SHA256SUMS @@ -0,0 +1,3 @@ +1d47d00730fab764bddb6d548fed7e124739b0bcebb9f3b3c6aa247de55fb804 terraform-provider-google_4.84.0_linux_amd64.zip +29bff92b4375a35a7729248b3bc5db8991ca1b9ba640fc25b13700e12f99c195 terraform-provider-google_4.84.0_darwin_amd64.zip +f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c terraform-provider-google_4.84.0_manifest.json diff --git a/lib/modules/manager/terraform/lockfile/__fixtures__/terraformCloudBackendAzurermVersions.json b/lib/modules/manager/terraform/lockfile/__fixtures__/terraformCloudBackendAzurermVersions.json new file mode 100644 index 00000000000000..cf76e4beba1dd5 --- /dev/null +++ b/lib/modules/manager/terraform/lockfile/__fixtures__/terraformCloudBackendAzurermVersions.json @@ -0,0 +1,39 @@ +{ + "id": "hashicorp/azurerm", + "versions": [ + { + "version": "2.56.0", + "protocols": [ + "5.0" + ], + "platforms": [ + { + "os": "linux", + "arch": "amd64" + }, + { + "os": "darwin", + "arch": "amd64" + } + ] + }, + { + "version": "1.33.0", + "protocols": [ + "4.0", + "5.0" + ], + "platforms": [ + { + "os": "linux", + "arch": "amd64" + }, + { + "os": "darwin", + "arch": "amd64" + } + ] + } + ], + "warnings": null +} diff --git a/lib/modules/manager/terraform/lockfile/__fixtures__/terraformCloudBackendGoogleVersions.json b/lib/modules/manager/terraform/lockfile/__fixtures__/terraformCloudBackendGoogleVersions.json new file mode 100644 index 00000000000000..ef9cad50b07b71 --- /dev/null +++ b/lib/modules/manager/terraform/lockfile/__fixtures__/terraformCloudBackendGoogleVersions.json @@ -0,0 +1,39 @@ +{ + "id": "hashicorp/google", + "versions": [ + { + "version": "4.84.0", + "protocols": [ + "5.0" + ], + "platforms": [ + { + "os": "linux", + "arch": "amd64" + }, + { + "os": "darwin", + "arch": "amd64" + } + ] + }, + { + "version": "1.33.0", + "protocols": [ + "4.0", + "5.0" + ], + "platforms": [ + { + "os": "linux", + "arch": "amd64" + }, + { + "os": "darwin", + "arch": "amd64" + } + ] + } + ], + "warnings": null +} diff --git a/lib/modules/manager/terraform/lockfile/hash.spec.ts b/lib/modules/manager/terraform/lockfile/hash.spec.ts index 149d0e2291899e..79c14232fba2f1 100644 --- a/lib/modules/manager/terraform/lockfile/hash.spec.ts +++ b/lib/modules/manager/terraform/lockfile/hash.spec.ts @@ -8,7 +8,22 @@ import { TerraformProviderDatasource } from '../../../datasource/terraform-provi import { TerraformProviderHash } from './hash'; const releaseBackendUrl = TerraformProviderDatasource.defaultRegistryUrls[1]; +const terraformCloudReleaseBackendUrl = + TerraformProviderDatasource.defaultRegistryUrls[0]; const releaseBackendAzurerm = Fixtures.get('releaseBackendAzurerm_2_56_0.json'); +const releaseBackendGoogleSha256 = Fixtures.get( + 'releaseBackendGoogle_4_84_0_SHA256SUMS' +); +const terraformCloudSDCJson = Fixtures.get( + 'service-discovery.json', + '../../../../modules/datasource/terraform-provider/' +); +const terraformCloudBackendAzurermVersions = Fixtures.get( + 'terraformCloudBackendAzurermVersions.json' +); +const terraformCloudBackendGoogleVersions = Fixtures.get( + 'terraformCloudBackendGoogleVersions.json' +); const log = logger.logger; @@ -125,4 +140,185 @@ describe('modules/manager/terraform/lockfile/hash', () => { 'h1:I2F2atKZqKEOYk1tTLe15Llf9rVqxz48ZL1eZB9g8zM=', ]); }); + + it('full walkthrough on terraform cloud', async () => { + const readStreamLinux = createReadStream( + 'lib/modules/manager/terraform/lockfile/__fixtures__/test.zip' + ); + const readStreamDarwin = createReadStream( + 'lib/modules/manager/terraform/lockfile/__fixtures__/test.zip' + ); + httpMock + .scope(terraformCloudReleaseBackendUrl) + .get('/.well-known/terraform.json') + .reply(200, terraformCloudSDCJson) + .get('/v1/providers/hashicorp/google/versions') + .reply(200, terraformCloudBackendGoogleVersions) + .get('/v1/providers/hashicorp/google/4.84.0/download/linux/amd64') + .reply(200, { + os: 'linux', + arch: 'amd64', + filename: 'terraform-provider-google_4.84.0_linux_amd64.zip', + shasums_url: + 'https://github.com/hashicorp/terraform-provider-google/releases/download/v4.84.0/terraform-provider-google_4.84.0_SHA256SUMS', + download_url: + 'https://github.com/hashicorp/terraform-provider-google/releases/download/v4.84.0/terraform-provider-google_4.84.0_linux_amd64.zip', + }) + .get('/v1/providers/hashicorp/google/4.84.0/download/darwin/amd64') + .reply(200, { + os: 'darwin', + arch: 'amd64', + filename: 'terraform-provider-google_4.84.0_darwin_amd64.zip', + shasums_url: + 'https://github.com/hashicorp/terraform-provider-google/releases/download/v4.84.0/terraform-provider-google_4.84.0_SHA256SUMS', + download_url: + 'https://github.com/hashicorp/terraform-provider-google/releases/download/v4.84.0/terraform-provider-google_4.84.0_darwin_amd64.zip', + }); + + httpMock + .scope('https://github.com') + .get( + '/hashicorp/terraform-provider-google/releases/download/v4.84.0/terraform-provider-google_4.84.0_SHA256SUMS' + ) + .reply(200, releaseBackendGoogleSha256) + .get( + '/hashicorp/terraform-provider-google/releases/download/v4.84.0/terraform-provider-google_4.84.0_linux_amd64.zip' + ) + .reply(200, readStreamLinux) + .get( + '/hashicorp/terraform-provider-google/releases/download/v4.84.0/terraform-provider-google_4.84.0_darwin_amd64.zip' + ) + .reply(200, readStreamDarwin); + + const result = await TerraformProviderHash.createHashes( + 'https://registry.terraform.io', + 'hashicorp/google', + '4.84.0' + ); + expect(log.error.mock.calls).toBeEmptyArray(); + expect(result).toMatchObject([ + 'h1:I2F2atKZqKEOYk1tTLe15Llf9rVqxz48ZL1eZB9g8zM=', + 'h1:I2F2atKZqKEOYk1tTLe15Llf9rVqxz48ZL1eZB9g8zM=', + 'zh:1d47d00730fab764bddb6d548fed7e124739b0bcebb9f3b3c6aa247de55fb804', + 'zh:29bff92b4375a35a7729248b3bc5db8991ca1b9ba640fc25b13700e12f99c195', + // The hash of a terraform-provider-manifest.json file not fetched by getBuilds + 'zh:f569b65999264a9416862bca5cd2a6177d94ccb0424f3a4ef424428912b9cb3c', + ]); + }); + + it('full walkthrough without ziphashes available', async () => { + const readStreamLinux = createReadStream( + 'lib/modules/manager/terraform/lockfile/__fixtures__/test.zip' + ); + const readStreamDarwin = createReadStream( + 'lib/modules/manager/terraform/lockfile/__fixtures__/test.zip' + ); + httpMock + .scope(terraformCloudReleaseBackendUrl) + .get('/.well-known/terraform.json') + .reply(200, terraformCloudSDCJson) + .get('/v1/providers/hashicorp/azurerm/versions') + .reply(200, terraformCloudBackendAzurermVersions) + .get('/v1/providers/hashicorp/azurerm/2.56.0/download/linux/amd64') + .reply(200, { + os: 'linux', + arch: 'amd64', + filename: 'terraform-provider-azurerm_2.56.0_linux_amd64.zip', + download_url: + 'https://github.com/hashicorp/terraform-provider-azurerm/releases/download/v2.56.0/terraform-provider-azurerm_2.56.0_linux_amd64.zip', + }) + .get('/v1/providers/hashicorp/azurerm/2.56.0/download/darwin/amd64') + .reply(200, { + os: 'darwin', + arch: 'amd64', + filename: 'terraform-provider-azurerm_2.56.0_darwin_amd64.zip', + download_url: + 'https://github.com/hashicorp/terraform-provider-azurerm/releases/download/v2.56.0/terraform-provider-azurerm_2.56.0_darwin_amd64.zip', + }); + + httpMock + .scope('https://github.com') + .get( + '/hashicorp/terraform-provider-azurerm/releases/download/v2.56.0/terraform-provider-azurerm_2.56.0_linux_amd64.zip' + ) + .reply(200, readStreamLinux) + .get( + '/hashicorp/terraform-provider-azurerm/releases/download/v2.56.0/terraform-provider-azurerm_2.56.0_darwin_amd64.zip' + ) + .reply(200, readStreamDarwin); + + const result = await TerraformProviderHash.createHashes( + 'https://registry.terraform.io', + 'hashicorp/azurerm', + '2.56.0' + ); + expect(log.error.mock.calls).toBeEmptyArray(); + expect(result).toMatchObject([ + 'h1:I2F2atKZqKEOYk1tTLe15Llf9rVqxz48ZL1eZB9g8zM=', + 'h1:I2F2atKZqKEOYk1tTLe15Llf9rVqxz48ZL1eZB9g8zM=', + ]); + }); + + it('it does not add any ziphashes when the shasums endpoint fails`', async () => { + const readStreamLinux = createReadStream( + 'lib/modules/manager/terraform/lockfile/__fixtures__/test.zip' + ); + const readStreamDarwin = createReadStream( + 'lib/modules/manager/terraform/lockfile/__fixtures__/test.zip' + ); + + httpMock + .scope(terraformCloudReleaseBackendUrl) + .get('/.well-known/terraform.json') + .reply(200, terraformCloudSDCJson) + .get('/v1/providers/hashicorp/azurerm/versions') + .reply(200, terraformCloudBackendAzurermVersions) + .get('/v1/providers/hashicorp/azurerm/2.56.0/download/linux/amd64') + .reply(200, { + os: 'linux', + arch: 'amd64', + filename: 'terraform-provider-azurerm_2.56.0_linux_amd64.zip', + shasums_url: + 'https://github.com/hashicorp/terraform-provider-azurerm/releases/download/v2.56.0/terraform-provider-azurerm_2.56.0_SHA256SUMS', + download_url: + 'https://github.com/hashicorp/terraform-provider-azurerm/releases/download/v2.56.0/terraform-provider-azurerm_2.56.0_linux_amd64.zip', + }) + .get('/v1/providers/hashicorp/azurerm/2.56.0/download/darwin/amd64') + .reply(200, { + os: 'darwin', + arch: 'amd64', + filename: 'terraform-provider-azurerm_2.56.0_darwin_amd64.zip', + shasums_url: + 'https://github.com/hashicorp/terraform-provider-azurerm/releases/download/v2.56.0/terraform-provider-azurerm_2.56.0_SHA256SUMS', + download_url: + 'https://github.com/hashicorp/terraform-provider-azurerm/releases/download/v2.56.0/terraform-provider-azurerm_2.56.0_darwin_amd64.zip', + }); + + httpMock + .scope('https://github.com') + .get( + '/hashicorp/terraform-provider-azurerm/releases/download/v2.56.0/terraform-provider-azurerm_2.56.0_SHA256SUMS' + ) + .replyWithError('endoint failed') + .get( + '/hashicorp/terraform-provider-azurerm/releases/download/v2.56.0/terraform-provider-azurerm_2.56.0_linux_amd64.zip' + ) + .reply(200, readStreamLinux) + .get( + '/hashicorp/terraform-provider-azurerm/releases/download/v2.56.0/terraform-provider-azurerm_2.56.0_darwin_amd64.zip' + ) + .reply(200, readStreamDarwin); + + const result = await TerraformProviderHash.createHashes( + 'https://registry.terraform.io', + 'hashicorp/azurerm', + '2.56.0' + ); + + expect(log.error.mock.calls).toBeEmptyArray(); + expect(result).toMatchObject([ + 'h1:I2F2atKZqKEOYk1tTLe15Llf9rVqxz48ZL1eZB9g8zM=', + 'h1:I2F2atKZqKEOYk1tTLe15Llf9rVqxz48ZL1eZB9g8zM=', + ]); + }); }); diff --git a/lib/modules/manager/terraform/lockfile/hash.ts b/lib/modules/manager/terraform/lockfile/hash.ts index 94d8cfd80c0ded..f9d37eaf3aee92 100644 --- a/lib/modules/manager/terraform/lockfile/hash.ts +++ b/lib/modules/manager/terraform/lockfile/hash.ts @@ -90,7 +90,9 @@ export class TerraformProviderHash { } } - static async calculateHashes(builds: TerraformBuild[]): Promise { + static async calculateHashScheme1Hashes( + builds: TerraformBuild[] + ): Promise { const cacheDir = await ensureCacheDir('./others/terraform'); // for each build download ZIP, extract content and generate hash for all containing files @@ -112,9 +114,19 @@ export class TerraformProviderHash { if (!builds) { return null; } - const hashes = await TerraformProviderHash.calculateHashes(builds); + + const zhHashes = + (await TerraformProviderHash.terraformDatasource.getZipHashes(builds)) ?? + []; + const h1Hashes = await TerraformProviderHash.calculateHashScheme1Hashes( + builds + ); + + const hashes = []; + hashes.push(...h1Hashes.map((hash) => `h1:${hash}`)); + hashes.push(...zhHashes.map((hash) => `zh:${hash}`)); // sorting the hash alphabetically as terraform does this as well - return hashes.sort().map((hash) => `h1:${hash}`); + return hashes.sort(); } }