Skip to content
Permalink
Browse files
Terraform 0.13 Compatibility (#285)
* Ignore terraform providers for docker build - This can get huge otherwise

* Use tfenv for Terraform version management in Dockerfile

* Allow adding source

* Test 0.12 and 0.13

* Update gitignore

* Make docker example work

* Cleanup obsolete modules

* Fix build

* Linter

* Update snapshots

* Drop schema from generated files

We were planning to do this anyway. It became necessary now, since Terraform 0.13
added at least one new field to the schema (`description_kind`). This caused our
snapshot tests to fail when running with different terraform versions

* Adjust integration tests

* adding new jsii image and the using custom terraform binary names

Co-authored-by: Anubhav Mishra <anubhavmishra@me.com>
  • Loading branch information
skorfmann and anubhavmishra committed Aug 8, 2020
1 parent 7dd1da0 commit 59f55661da3abcd6283dc3d8fa6ceab002a6d302
Show file tree
Hide file tree
Showing 30 changed files with 18,118 additions and 20,963 deletions.
@@ -9,4 +9,5 @@

**/cdktf.out
**/.gen
**/dist
**/dist
**/.terraform
@@ -4,6 +4,9 @@ on: [pull_request]
jobs:
build:
runs-on: ubuntu-latest
strategy:
matrix:
terraform: ["0.12.29", 0.13.0-rc1]
container:
image: hashicorp/jsii-terraform

@@ -16,12 +19,17 @@ jobs:
run: |
tools/align-version.sh
yarn build
env:
TERRAFORM_BINARY_NAME: "terraform${{ matrix.terraform }}"
- name: test
run: |
yarn test
env:
TERRAFORM_BINARY_NAME: "terraform${{ matrix.terraform }}"
- name: create bundle
run: yarn package
- name: integration tests
run: yarn integration
env:
TERRAFORM_CLOUD_TOKEN: ${{ secrets.TERRAFORM_CLOUD_TOKEN }}
TERRAFORM_CLOUD_TOKEN: ${{ secrets.TERRAFORM_CLOUD_TOKEN }}
TERRAFORM_BINARY_NAME: "terraform${{ matrix.terraform }}"
@@ -16,4 +16,5 @@ tsconfig.json
**/*.log
**/coverage
**/dist
**/.terraform
**/.terraform
.vscode
@@ -1,7 +1,13 @@
FROM jsii/superchain

COPY tools/install-terraform.sh /tmp/
RUN yum install -y unzip && curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python

RUN yum install -y jq && /tmp/install-terraform.sh && curl https://raw.githubusercontent.com/pypa/pipenv/master/get-pipenv.py | python
ENV DEFAULT_TERRAFORM_VERSION=0.13.0-rc1

RUN rm -f /tmp/install-terraform.sh && yum remove -y jq
# Install Terraform
RUN AVAILABLE_TERRAFORM_VERSIONS="0.12.29 ${DEFAULT_TERRAFORM_VERSION}" && \
for VERSION in ${AVAILABLE_TERRAFORM_VERSIONS}; do curl -LOk https://releases.hashicorp.com/terraform/${VERSION}/terraform_${VERSION}_linux_amd64.zip && \
mkdir -p /usr/local/bin/tf/versions/${VERSION} && \
unzip terraform_${VERSION}_linux_amd64.zip -d /usr/local/bin/tf/versions/${VERSION} && \
ln -s /usr/local/bin/tf/versions/${VERSION}/terraform /usr/local/bin/terraform${VERSION};rm terraform_${VERSION}_linux_amd64.zip;done && \
ln -s /usr/local/bin/tf/versions/${DEFAULT_TERRAFORM_VERSION}/terraform /usr/local/bin/terraform
@@ -1,6 +1,8 @@
{
"language": "python",
"app": "pipenv run ./main.py",
"terraformProviders": ["docker"],
"terraformProviders": [
"terraform-providers/docker@~> 2.0"
],
"codeMakerOutput": "imports"
}
}
@@ -1,15 +1,17 @@
#!/usr/bin/python3 -tt
from constructs import Construct
from cdktf import App, TerraformStack
from imports.docker import Image, Container
from imports.docker import Image, Container, DockerProvider


class MyStack(TerraformStack):
def __init__(self, scope: Construct, ns: str):
super().__init__(scope, ns)


DockerProvider(self, "provider")

docker_image = Image(self, 'nginx-latest', name='nginx:latest', keep_locally=False)

Container(self, 'nginx-cdktf', name='nginx-python-cdktf',
image=docker_image.name, ports=[
{
@@ -3,9 +3,5 @@
"app": "npm run --silent compile && node main.js",
"terraformProviders": [
"aws@~> 2.0"
],
"terraformModules": [
"terraform-aws-modules/eks/aws",
"terraform-aws-modules/vpc/aws"
]
}
@@ -2,6 +2,6 @@
"language": "typescript",
"app": "npm run --silent compile && node main.js",
"terraformProviders": [
"docker"
"terraform-providers/docker@~> 2.0"
]
}
@@ -15,14 +15,16 @@ Steps:

import { Construct } from 'constructs';
import { App, TerraformStack } from 'cdktf';
import { Container, Image } from './.gen/providers/docker';
import { Container, Image, DockerProvider } from './.gen/providers/docker';

class MyStack extends TerraformStack {
public readonly dockerImage: Image

constructor(scope: Construct, name: string) {
super(scope, name);

new DockerProvider(this, 'provider', {})

this.dockerImage = new Image(this, 'nginxImage', {
name : "nginx:latest",
keepLocally : false
@@ -35,7 +37,7 @@ class MyStack extends TerraformStack {
internal: 80,
external: 8000
}]
});
});

}
}
@@ -9,7 +9,9 @@
"watch": "lerna run --parallel --stream --scope cdktf* watch-preserve-output",
"link-packages": "lerna exec --scope cdktf* yarn link",
"integration": "test/run-against-dist test/test-all.sh",
"release-github": "tools/release-github.sh"
"release-github": "tools/release-github.sh",
"build-docker-jsii": "docker build -t hashicorp/jsii-terraform .",
"push-docker-jsii": "docker push hashicorp/jsii-terraform"
},
"workspaces": {
"packages": [
@@ -1,6 +1,8 @@
import * as path from 'path';
import { exec } from '../../../../lib/util'

const terraformBinaryName = process.env.TERRAFORM_BINARY_NAME || 'terraform'

export enum PlannedResourceAction {
CREATE = 'create',
UPDATE = 'update',
@@ -81,7 +83,7 @@ export class Terraform {
}

public async init(): Promise<void> {
await exec('terraform', ['init'], { cwd: this.workdir, env: process.env })
await exec(terraformBinaryName, ['init'], { cwd: this.workdir, env: process.env })
}

public async plan(destroy = false): Promise<TerraformPlan> {
@@ -90,29 +92,29 @@ export class Terraform {
if (destroy) {
options.push('-destroy')
}
await exec('terraform', options, { cwd: this.workdir, env: process.env });
const jsonPlan = await exec('terraform', ['show', '-json', planFile], { cwd: this.workdir, env: process.env });
await exec(terraformBinaryName, options, { cwd: this.workdir, env: process.env });
const jsonPlan = await exec(terraformBinaryName, ['show', '-json', planFile], { cwd: this.workdir, env: process.env });
return new TerraformPlan(planFile, JSON.parse(jsonPlan));
}

public async deploy(planFile: string, stdout: (chunk: Buffer) => any): Promise<void> {
await exec('terraform', ['apply', '-auto-approve', ...this.stateFileOption, planFile], { cwd: this.workdir, env: process.env }, stdout);
await exec(terraformBinaryName, ['apply', '-auto-approve', ...this.stateFileOption, planFile], { cwd: this.workdir, env: process.env }, stdout);
}

public async destroy(stdout: (chunk: Buffer) => any): Promise<void> {
await exec('terraform', ['destroy', '-auto-approve', ...this.stateFileOption], { cwd: this.workdir, env: process.env }, stdout);
await exec(terraformBinaryName, ['destroy', '-auto-approve', ...this.stateFileOption], { cwd: this.workdir, env: process.env }, stdout);
}

public async version(): Promise<string> {
try {
return await exec('terraform', ['-v'], { cwd: this.workdir, env: process.env });
return await exec(terraformBinaryName, ['-v'], { cwd: this.workdir, env: process.env });
} catch {
throw new Error("Terraform CLI not present - Please install a current version https://learn.hashicorp.com/terraform/getting-started/install.html")
}
}

public async output(): Promise<{[key: string]: TerraformOutput}> {
const output = await exec('terraform', ['output', '-json', ...this.stateFileOption], { cwd: this.workdir, env: process.env }); return JSON.parse(output)
const output = await exec(terraformBinaryName, ['output', '-json', ...this.stateFileOption], { cwd: this.workdir, env: process.env }); return JSON.parse(output)
}

private get stateFileOption() {
@@ -87,7 +87,8 @@ export class ResourceEmitter {
this.code.open(`terraformGeneratorMetadata: {`);
this.code.line(`providerName: '${resource.provider}',`);
this.code.line(`providerVersionConstraint: '${resource.providerVersionConstraint}'`);
this.code.close(`}`);
this.code.close(`},`);
this.code.line(`terraformProviderSource: '${resource.terraformProviderSource}'`);
this.code.close(`});`);
}
}
@@ -22,6 +22,7 @@ export class ResourceModel {
public baseName: string;
public provider: string;
public providerVersionConstraint?: string;
public terraformProviderSource?: string;
public fileName: string;
public filePath: string;
public attributes: AttributeModel[];
@@ -4,12 +4,56 @@ import { ResourceModel } from "./models"
import { ResourceParser } from './resource-parser'
import { ResourceEmitter, StructEmitter } from './emitter'

export class TerraformProviderConstraint {
public version: string;
public source?: string;
public name: string
public fqn: string;

constructor(public cdktfConstraint: string) {
const [ fqn, version ] = cdktfConstraint.split('@');
const nameParts = fqn.split('/');
const name = nameParts.pop();
if (!name) { throw new Error(`Provider name should be properly set in ${cdktfConstraint}`) }

this.name = name;
this.source = nameParts.join('/');
this.version = version;
this.fqn = fqn
}

public isMatching(terraformSchemaName: string): boolean {
const elements = terraformSchemaName.split('/')

if (elements.length === 1) {
return this.name === terraformSchemaName
} else {
const [hostname, scope, provider] = elements

if (!hostname || !scope || !provider) {
throw new Error(`can't handle ${terraformSchemaName}`)
}

return this.name === provider;
}
}
}
interface ProviderData {
name: string;
source: string;
version: string;
}

export interface ProviderConstraints {
[fqn: string]: ProviderData;
}

export class TerraformGenerator {
private resourceParser = new ResourceParser();
private resourceEmitter: ResourceEmitter;
private structEmitter: StructEmitter;
constructor(private readonly code: CodeMaker, schema: ProviderSchema, private providerConstraints?: TerraformProviderConstraint[]) {

constructor(private readonly code: CodeMaker, schema: ProviderSchema, private providerConstraints?: { [name: string]: string }) {
this.code.indentation = 2;
this.resourceEmitter = new ResourceEmitter(this.code)
this.structEmitter = new StructEmitter(this.code)
@@ -19,16 +63,19 @@ export class TerraformGenerator {
return;
}

for (const [name, provider] of Object.entries(schema.provider_schemas)) {
this.emitProvider(name, provider);
for (const [fqpn, provider] of Object.entries(schema.provider_schemas)) {
this.emitProvider(fqpn, provider);
}
}

public async save(outdir: string) {
await this.code.save(outdir);
}

private emitProvider(name: string, provider: Provider) {
private emitProvider(fqpn: string, provider: Provider) {
const name = fqpn.split('/').pop()
if (!name) { throw new Error(`can't handle ${fqpn}`) }

const files: string[] = []
for (const [type, resource] of Object.entries(provider.resource_schemas)) {
files.push(this.emitResourceFile(this.resourceParser.parse(name, type, resource, 'resource')));
@@ -41,7 +88,13 @@ export class TerraformGenerator {
if (provider.provider) {
const providerResource = this.resourceParser.parse(name, `provider`, provider.provider, 'provider')
if (this.providerConstraints) {
providerResource.providerVersionConstraint = this.providerConstraints[name]
const constraint = this.providerConstraints.find((p) => (p.isMatching(fqpn)))
if (!constraint) {
console.log({foo: this.providerConstraints, fqpn})
throw new Error(`can't handle ${fqpn}`)
}
providerResource.providerVersionConstraint = constraint.version;
providerResource.terraformProviderSource = constraint.fqn;
}
files.push(this.emitResourceFile(providerResource));
}
@@ -75,9 +128,6 @@ export class TerraformGenerator {
this.code.line(`// ${resource.linkToDocs}`);
this.code.line(`// generated from terraform resource schema`);
this.code.line();
this.code.line('/*');
this.code.line(resource.schemaAsJson);
this.code.line('*/');
resource.importStatements.forEach(statement => this.code.line(statement))
this.code.line();
this.code.line('// Configuration');
@@ -56,18 +56,25 @@ export interface Block {
}

export async function readSchema(providers: string[]): Promise<ProviderSchema> {
const provider: { [name: string]: { version?: string } } = { };
const provider: { [name: string]: {} } = {};
const requiredProviders: { [name: string]: { source?: string; version?: string } } = { };

for (const p of providers) {
const [ name, version ] = p.split('@');
provider[name] = { version };
const [ fqname, version ] = p.split('@');
const name = fqname.split('/').pop()
if (!name) { throw new Error(`Provider name should be properly set in ${p}`) }

provider[name] = {};
requiredProviders[name] = { version, source: fqname };
}
let schema = '';
const workDir = process.cwd()

await withTempDir('fetchSchema', async () => {
const outdir = process.cwd();
const filePath = path.join(outdir, 'providers.tf.json');
await writeFile(filePath, JSON.stringify({ provider }));
// eslint-disable-next-line @typescript-eslint/camelcase
await writeFile(filePath, JSON.stringify({ provider, terraform: { required_providers: requiredProviders }}));

const env = process.env['TF_PLUGIN_CACHE_DIR'] ? process.env : Object.assign({}, process.env, { 'TF_PLUGIN_CACHE_DIR': await cacheDir(workDir) })

@@ -1,5 +1,5 @@
// generates constructs from terraform providers schema
import { TerraformGenerator } from './generator/provider-generator';
import { TerraformGenerator, TerraformProviderConstraint } from './generator/provider-generator';
import { ProviderSchema, readSchema } from './generator/provider-schema';
import { CodeMaker } from 'codemaker';
import { GetBase } from './base'
@@ -19,11 +19,10 @@ export class GetProvider extends GetBase {
return `providers/${name}/index`;
}

private async parseProviders(providers: string[]): Promise<{ [name: string]: string }> {
const provider: { [name: string]: string } = { };
private async parseProviders(providers: string[]): Promise<TerraformProviderConstraint[]> {
const provider: TerraformProviderConstraint[] = [];
for (const p of providers) {
const [ name, version ] = p.split('@');
provider[name] = version;
provider.push(new TerraformProviderConstraint(p))
}
return provider;
}
@@ -10,6 +10,7 @@
"watch": "tsc -w",
"watch-preserve-output": "tsc -w --preserveWatchOutput",
"lint": "eslint . --ext .ts",
"lint:fix": "eslint . --ext .ts --fix",
"test": "yarn lint && jest",
"jest-watch": "jest --watch",
"package": "./package.sh"

0 comments on commit 59f5566

Please sign in to comment.