Module design advice

[ericzbeard]

I'm hoping to get some ideas for how to design modules that can be combined in creative ways.

My use case is AWS CloudFormation templates. I have a high level pattern that simplifies creating a VPC, and a module I can amend that represents a template. Getting the resources into the amended template feels odd, though.

The Pkl template looks like this:

amends "modules/template.pkl"
import "modules/cloudformation.pkl" as cfn
import "modules/patterns/vpc.pkl"

local pub1 = new vpc.Subnet {
    LogicalId = "Pub1"
    IsPublic = true
    Az = cfn.Select(0, cfn.GetAZs("us-east-1")) 
    Cidr = "10.0.0.0/18"
}

local pub2 = new vpc.Subnet {
    LogicalId = "Pub2"
    IsPublic = true
    Az = cfn.Select(1, cfn.GetAZs("us-east-1")) 
    Cidr = "10.0.64.0/18"
}

local priv1 = new vpc.Subnet {
    LogicalId = "Priv1"
    IsPublic = false
    Az = cfn.Select(0, cfn.GetAZs("us-east-1")) 
    Cidr = "10.0.128.0/18"
    PublicNATGateway = pub1.getNATGateway()
}

local priv2 = new vpc.Subnet {
    LogicalId = "Priv2"
    IsPublic = false
    Az = cfn.Select(1, cfn.GetAZs("us-east-1")) 
    Cidr = "10.0.192.0/18"
    PublicNATGateway = pub2.getNATGateway()
}

local myvpc = new vpc.VPC {
    Subnets {
        pub1
        priv1
        pub2
        priv2
    }
}

Resources {
    // Create the VPC
    for (logicalId, resource in myvpc.getResources("MyVPC")) {
        [logicalId] = resource
    }

    // Create other resources inside the VPC...
}

Notice what I'm doing at the end there, iterating through the output of a function to insert the resources. Is there a better, more intuitive way to do this?

Another use case: later in the template, adding a resource that somehow modifies one of those resources that I emitted earlier. For example, let's say I have a pattern that creates a DynamoDB table, and another pattern that creates a Lambda function (along with its role), and later I want to modify the role to allow the function to access the table. I'm pretty sure I will have to end up modifying the pattern code itself to enable this, but it's very hard to anticipate all user needs.

Maybe it would end up looking something like this:

for (logicalId, resource in myvpc.getResources("MyVPC")) {
    [logicalId] = if logicalId == "Foo" (resource) { ... change stuff ... } else resource
}

[bioball]

A pattern that might be useful to you is to define properties that are derived from other properties. You can define higher level knobs, then produce lower level config as your actual output.

For example, our CircleCI stuff is set up this way. We have a high level config that exposes prb, main, release and releaseBuild as top-level builds, and the actual circleCi output is defined in terms of that: https://github.com/apple/pkl-project-commons/blob/main/packages/pkl.impl.circleci/PklCI.pkl#L84

We then set the module's output to be CircleCI's output: https://github.com/apple/pkl-project-commons/blob/main/packages/pkl.impl.circleci/PklCI.pkl#L236

This is nice because it lets us abstract away machinery that is required to set up these workflows. Instead, we only need to think about what job needs to be run when a PRB is triggered, etc. Here's some sample usage: https://github.com/apple/pkl-go/blob/5827572b811253cff17f32a74b828ddc97d42b90/.circleci/config.pkl#L3-L32

You can take this same idea and make it simpler to define a higher level "VPC". I've adapted your VPC example, and changed it to only provide knobs for Name and Subnets, and have resources defined in terms of those knobs.

patterns/VPC.pkl

import "../cloudformation.pkl" as cfn
import "../aws/ec2/vpc.pkl"
import "../aws/ec2/subnet.pkl"
import "../aws/ec2/routetable.pkl"
import "../aws/ec2/route.pkl"
import "../aws/ec2/eip.pkl"
import "../aws/ec2/natgateway.pkl"
import "../aws/ec2/internetgateway.pkl"
import "../aws/ec2/subnetroutetableassociation.pkl"

/// The name of the VPC
Name: String

/// The subnets within this VPC.
Subnets: Listing<Subnet>

/// The CloudFormation resources representing this VPC.
fixed resources: Mapping<String, cfn.Resource> = new {
    [Name] = new vpc.VPC {
        CidrBlock = "10.0.0.0/16"
        EnableDnsHostnames = true
        EnableDnsSupport = true
        InstanceTenancy = "default"
    }
    ["\(Name)Gateway"] = new internetgateway.InternetGateway {}
    for (subnet in Subnets) {
        ...(subnet) { VpcName = Name }.resources
    }
}

class Subnet {
    VpcName: String
    LogicalId: String
    IsPublic: Boolean
    RouteTable: routetable.RouteTable
    DefaultRoute: route.Route
    Az: cfn.RefString
    Cidr: String
    PublicNATGateway: cfn.RefString?

    local vpcId = cfn.Ref(VpcName)

    local gwId = cfn.Ref("\(VpcName)Gateway")

    local privateResources: Mapping<String, cfn.Resource> = new {
        [LogicalId] = new subnet.Subnet {
            CidrBlock = Cidr
            AvailabilityZone = Az
            MapPublicIpOnLaunch = IsPublic
            VpcId = vpcId
        }

        [LogicalId + "RouteTable"] = new routetable.RouteTable {
            VpcId = vpcId
        }

        [LogicalId + "RouteTableAssociation"] = new subnetroutetableassociation.SubnetRouteTableAssociation {
            RouteTableId = cfn.Ref(LogicalId + "RouteTable")
            SubnetId = cfn.Ref(LogicalId)
        }

        [LogicalId + "DefaultRoute"] = new route.Route {
            DestinationCidrBlock = "0.0.0.0/0"
            NatGatewayId = if (IsPublic) null else PublicNATGateway
            GatewayId = if (IsPublic) gwId else null
            RouteTableId = cfn.Ref(LogicalId + "RouteTable")
        }
    }

    fixed resources: Mapping<String, cfn.Resource> = (privateResources) {
        when (IsPublic) {
            [LogicalId + "NATGateway"] = new natgateway.NatGateway {
                AllocationId = cfn.GetAtt(LogicalId + "EIP", "AllocationId")
                SubnetId = cfn.Ref(LogicalId)
            }

            [LogicalId + "EIP"] = new eip.EIP {
                Domain = vpcId
            }
        }
    }

    fixed natGateway: cfn.RefString? = if (IsPublic) cfn.Ref(LogicalId + "NATGateway") else null
}

output {
    value = resources
}

In your usage sites, it's no longer necessary to define Resources, because that is derived for you.

amends "patterns/VPC.pkl"
import "cloudformation.pkl" as cfn

local pub1 = new Subnet {
  LogicalId = "Pub1"
  IsPublic = true
  Az = cfn.Select(0, cfn.GetAZs("us-east-1"))
  Cidr = "10.0.0.0/18"
}

local pub2 = new Subnet {
  LogicalId = "Pub2"
  IsPublic = true
  Az = cfn.Select(1, cfn.GetAZs("us-east-1"))
  Cidr = "10.0.64.0/18"
}

local priv1 = new Subnet {
  LogicalId = "Priv1"
  IsPublic = false
  Az = cfn.Select(0, cfn.GetAZs("us-east-1"))
  Cidr = "10.0.128.0/18"
  PublicNATGateway = pub1.natGateway
}

local priv2 = new Subnet {
  LogicalId = "Priv2"
  IsPublic = false
  Az = cfn.Select(1, cfn.GetAZs("us-east-1"))
  Cidr = "10.0.192.0/18"
  PublicNATGateway = pub2.natGateway
}

Name = "my-vpc"

Subnets {
  pub1
  pub2
  priv1
  priv2
}

Also, you can use this module as a value by importing it. Accessing .resources will give you the underlying cloudformation resources that back this VPC.

Does this help?

Also: I changed some of your functions to be fixed properties. Fixed properties are much like functions in that they represent a value that must be derived in terms of other values, but are a little nicer because their result gets cached. Functions, on the other hand, will be executed every time (Pkl does not memoize).

[ericzbeard]

Thanks! Learning a lot here.

[ericzbeard]

I still need to amend template.pkl, since I need other elements like Parameters. Still less verbose than the original:

Resources {
    ...myvpc.resources
}