diff --git a/pkg/asset/installconfig/aws/route53.go b/pkg/asset/installconfig/aws/route53.go index 0317e9748be..6f7f98b68c2 100644 --- a/pkg/asset/installconfig/aws/route53.go +++ b/pkg/asset/installconfig/aws/route53.go @@ -1,14 +1,19 @@ package aws import ( + "context" "fmt" + "strconv" "strings" + "time" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials/stscreds" "github.com/aws/aws-sdk-go/aws/endpoints" awss "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/route53" + "github.com/sirupsen/logrus" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/validation/field" "github.com/openshift/installer/pkg/types" @@ -16,6 +21,10 @@ import ( //go:generate mockgen -source=./route53.go -destination=mock/awsroute53_generated.go -package=mock +// regions for which ALIAS records are not available +// https://docs.aws.amazon.com/govcloud-us/latest/UserGuide/govcloud-r53.html +var cnameRegions = sets.New[string]("us-gov-west-1", "us-gov-east-1") + // API represents the calls made to the API. type API interface { GetHostedZone(hostedZone string, cfg *aws.Config) (*route53.GetHostedZoneOutput, error) @@ -148,37 +157,227 @@ func GetR53ClientCfg(sess *awss.Session, roleARN string) *aws.Config { } // CreateOrUpdateRecord Creates or Updates the Route53 Record for the cluster endpoint. -func (c *Client) CreateOrUpdateRecord(ic *types.InstallConfig, target string, cfg *aws.Config) error { - zone, err := c.GetBaseDomain(ic.BaseDomain) - if err != nil { - return err +func (c *Client) CreateOrUpdateRecord(ctx context.Context, ic *types.InstallConfig, target string, intTarget string, phzID string) error { + useCNAME := cnameRegions.Has(ic.AWS.Region) + aliasZoneID := hostedZoneIDPerRegionNLBMap[ic.AWS.Region] + + apiName := fmt.Sprintf("api.%s.", ic.ClusterDomain()) + apiIntName := fmt.Sprintf("api-int.%s.", ic.ClusterDomain()) + + // Create api record in public zone + if ic.Publish == types.ExternalPublishingStrategy { + zone, err := c.GetBaseDomain(ic.BaseDomain) + if err != nil { + return err + } + + svc := route53.New(c.ssn) // we dont want to assume role here + if _, err := createRecord(ctx, svc, aws.StringValue(zone.Id), apiName, target, aliasZoneID, useCNAME); err != nil { + return fmt.Errorf("failed to create records for api: %w", err) + } + logrus.Debugln("Created public API record in public zone") + } + + // Create service with assumed role for PHZ + svc := route53.New(c.ssn, GetR53ClientCfg(c.ssn, ic.AWS.HostedZoneRole)) + + // Create api record in private zone + if _, err := createRecord(ctx, svc, phzID, apiName, intTarget, aliasZoneID, useCNAME); err != nil { + return fmt.Errorf("failed to create records for api: %w", err) + } + logrus.Debugln("Created public API record in private zone") + + // Create api-int record in private zone + if _, err := createRecord(ctx, svc, phzID, apiIntName, intTarget, aliasZoneID, useCNAME); err != nil { + return fmt.Errorf("failed to create records for api-int: %w", err) } + logrus.Debugln("Created private API record in private zone") - params := &route53.ChangeResourceRecordSetsInput{ + return nil +} + +func createRecord(ctx context.Context, client *route53.Route53, zoneID, name, dnsName, aliasZoneID string, useCNAME bool) (*route53.ChangeInfo, error) { + recordSet := &route53.ResourceRecordSet{ + Name: aws.String(name), + } + if useCNAME { + recordSet.SetType("CNAME") + recordSet.SetTTL(10) + recordSet.SetResourceRecords([]*route53.ResourceRecord{ + {Value: aws.String(dnsName)}, + }) + } else { + recordSet.SetType("A") + recordSet.SetAliasTarget(&route53.AliasTarget{ + DNSName: aws.String(dnsName), + HostedZoneId: aws.String(aliasZoneID), + EvaluateTargetHealth: aws.Bool(false), + }) + } + input := &route53.ChangeResourceRecordSetsInput{ + HostedZoneId: aws.String(zoneID), ChangeBatch: &route53.ChangeBatch{ - Comment: aws.String(fmt.Sprintf("Creating record for api and api-int in domain %s", ic.ClusterDomain())), - }, - HostedZoneId: zone.Id, - } - for _, prefix := range []string{"api", "api-int"} { - params.ChangeBatch.Changes = append(params.ChangeBatch.Changes, &route53.Change{ - Action: aws.String("UPSERT"), - ResourceRecordSet: &route53.ResourceRecordSet{ - Name: aws.String(fmt.Sprintf("%s.%s.", prefix, ic.ClusterDomain())), - Type: aws.String("A"), - AliasTarget: &route53.AliasTarget{ - DNSName: aws.String(target), - HostedZoneId: aws.String(hostedZoneIDPerRegionNLBMap[ic.AWS.Region]), - EvaluateTargetHealth: aws.Bool(true), + Comment: aws.String(fmt.Sprintf("Creating record %s", name)), + Changes: []*route53.Change{ + { + Action: aws.String("UPSERT"), + ResourceRecordSet: recordSet, }, }, - }) + }, } + res, err := client.ChangeResourceRecordSetsWithContext(ctx, input) + if err != nil { + return nil, err + } + + return res.ChangeInfo, nil +} + +// HostedZoneInput defines the input parameters for hosted zone creation. +type HostedZoneInput struct { + Name string + InfraID string + VpcID string + Region string + Role string + UserTags map[string]string +} + +// CreateHostedZone creates a private hosted zone. +func (c *Client) CreateHostedZone(ctx context.Context, input *HostedZoneInput) (*route53.HostedZone, error) { + cfg := GetR53ClientCfg(c.ssn, input.Role) svc := route53.New(c.ssn, cfg) - if _, err := svc.ChangeResourceRecordSets(params); err != nil { - return fmt.Errorf("failed to create records for api/api-int: %w", err) + + callRef := fmt.Sprintf("%d", time.Now().Unix()) + res, err := svc.CreateHostedZoneWithContext(ctx, &route53.CreateHostedZoneInput{ + CallerReference: aws.String(callRef), + Name: aws.String(input.Name), + HostedZoneConfig: &route53.HostedZoneConfig{ + PrivateZone: aws.Bool(true), + Comment: aws.String("Created by Openshift Installer"), + }, + VPC: &route53.VPC{ + VPCId: aws.String(input.VpcID), + VPCRegion: aws.String(input.Region), + }, + }) + if err != nil { + return nil, fmt.Errorf("error creating private hosted zone: %w", err) } - return nil + + if res == nil { + return nil, fmt.Errorf("error creating private hosted zone: %w", err) + } + + // Tag the hosted zone + tags := mergeTags(input.UserTags, map[string]string{ + "Name": fmt.Sprintf("%s-int", input.InfraID), + }) + _, err = svc.ChangeTagsForResourceWithContext(ctx, &route53.ChangeTagsForResourceInput{ + ResourceType: aws.String("hostedzone"), + ResourceId: res.HostedZone.Id, + AddTags: r53Tags(tags), + }) + if err != nil { + return nil, fmt.Errorf("failed to tag private hosted zone: %w", err) + } + + // Set SOA minimum TTL + recordSet, err := existingRecordSet(ctx, svc, res.HostedZone.Id, input.Name, "SOA") + if err != nil { + return nil, fmt.Errorf("failed to find SOA record set for private zone: %w", err) + } + if len(recordSet.ResourceRecords) == 0 || recordSet.ResourceRecords[0] == nil || recordSet.ResourceRecords[0].Value == nil { + return nil, fmt.Errorf("failed to find SOA record for private zone") + } + record := recordSet.ResourceRecords[0] + fields := strings.Split(aws.StringValue(record.Value), " ") + if len(fields) != 7 { + return nil, fmt.Errorf("SOA record value has %d fields, expected 7", len(fields)) + } + fields[0] = "60" + record.Value = aws.String(strings.Join(fields, " ")) + req, err := svc.ChangeResourceRecordSetsWithContext(ctx, &route53.ChangeResourceRecordSetsInput{ + HostedZoneId: res.HostedZone.Id, + ChangeBatch: &route53.ChangeBatch{ + Changes: []*route53.Change{ + { + Action: aws.String("UPSERT"), + ResourceRecordSet: recordSet, + }, + }, + }, + }) + if err != nil { + return nil, fmt.Errorf("failed to set SOA TTL to minimum: %w", err) + } + + if err = svc.WaitUntilResourceRecordSetsChangedWithContext(ctx, &route53.GetChangeInput{Id: req.ChangeInfo.Id}); err != nil { + return nil, fmt.Errorf("failed to wait for SOA TTL change: %w", err) + } + + return res.HostedZone, nil +} + +func existingRecordSet(ctx context.Context, client *route53.Route53, zoneID *string, recordName string, recordType string) (*route53.ResourceRecordSet, error) { + name := fqdn(strings.ToLower(recordName)) + res, err := client.ListResourceRecordSetsWithContext(ctx, &route53.ListResourceRecordSetsInput{ + HostedZoneId: zoneID, + StartRecordName: aws.String(name), + StartRecordType: aws.String(recordType), + MaxItems: aws.String("1"), + }) + if err != nil { + return nil, fmt.Errorf("failed to list record sets: %w", err) + } + for _, rs := range res.ResourceRecordSets { + resName := strings.ToLower(cleanRecordName(aws.StringValue(rs.Name))) + resType := strings.ToUpper(aws.StringValue(rs.Type)) + if resName == name && resType == recordType { + return rs, nil + } + } + + return nil, fmt.Errorf("not found") +} + +func fqdn(name string) string { + n := len(name) + if n == 0 || name[n-1] == '.' { + return name + } + return name + "." +} + +func cleanRecordName(name string) string { + s, err := strconv.Unquote(`"` + name + `"`) + if err != nil { + return name + } + return s +} + +func mergeTags(lhsTags, rhsTags map[string]string) map[string]string { + merged := make(map[string]string, len(lhsTags)+len(rhsTags)) + for k, v := range lhsTags { + merged[k] = v + } + for k, v := range rhsTags { + merged[k] = v + } + return merged +} + +func r53Tags(tags map[string]string) []*route53.Tag { + rtags := make([]*route53.Tag, 0, len(tags)) + for k, v := range tags { + rtags = append(rtags, &route53.Tag{ + Key: aws.String(k), + Value: aws.String(v), + }) + } + return rtags } // See https://docs.aws.amazon.com/general/latest/gr/elb.html#elb_region diff --git a/pkg/infrastructure/aws/clusterapi/aws.go b/pkg/infrastructure/aws/clusterapi/aws.go index fdbac0b015f..a56b0498970 100644 --- a/pkg/infrastructure/aws/clusterapi/aws.go +++ b/pkg/infrastructure/aws/clusterapi/aws.go @@ -4,12 +4,22 @@ import ( "context" "fmt" + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/ec2" + "github.com/sirupsen/logrus" + capa "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2" + k8sClient "sigs.k8s.io/controller-runtime/pkg/client" + + awsconfig "github.com/openshift/installer/pkg/asset/installconfig/aws" + "github.com/openshift/installer/pkg/asset/manifests/capiutils" "github.com/openshift/installer/pkg/infrastructure/clusterapi" awstypes "github.com/openshift/installer/pkg/types/aws" ) var _ clusterapi.Provider = (*Provider)(nil) var _ clusterapi.PreProvider = (*Provider)(nil) +var _ clusterapi.InfraReadyProvider = (*Provider)(nil) // Provider implements AWS CAPI installation. type Provider struct{} @@ -24,3 +34,115 @@ func (*Provider) PreProvision(ctx context.Context, in clusterapi.PreProvisionInp } return nil } + +// InfraReady creates private hosted zone and DNS records. +func (*Provider) InfraReady(ctx context.Context, in clusterapi.InfraReadyInput) error { + awsCluster := &capa.AWSCluster{} + key := k8sClient.ObjectKey{ + Name: in.InfraID, + Namespace: capiutils.Namespace, + } + if err := in.Client.Get(ctx, key, awsCluster); err != nil { + return fmt.Errorf("failed to get AWSCluster: %w", err) + } + + awsSession, err := in.InstallConfig.AWS.Session(ctx) + if err != nil { + return fmt.Errorf("failed to get aws session: %w", err) + } + + subnetIDs := make([]string, 0, len(awsCluster.Spec.NetworkSpec.Subnets)) + for _, s := range awsCluster.Spec.NetworkSpec.Subnets { + subnetIDs = append(subnetIDs, s.ResourceID) + } + + vpcID := awsCluster.Spec.NetworkSpec.VPC.ID + if len(subnetIDs) > 0 && len(vpcID) == 0 { + // All subnets belong to the same VPC, so we only need one + vpcID, err = getVPCFromSubnets(ctx, awsSession, awsCluster.Spec.Region, subnetIDs[:1]) + if err != nil { + return err + } + } + + tags := map[string]string{ + fmt.Sprintf("kubernetes.io/cluster/%s", in.InfraID): "owned", + } + for k, v := range awsCluster.Spec.AdditionalTags { + tags[k] = v + } + + client := awsconfig.NewClient(awsSession) + + phzID := in.InstallConfig.Config.AWS.HostedZone + if len(phzID) == 0 { + logrus.Infoln("Creating private Hosted Zone") + res, err := client.CreateHostedZone(ctx, &awsconfig.HostedZoneInput{ + InfraID: in.InfraID, + VpcID: vpcID, + Region: awsCluster.Spec.Region, + Name: in.InstallConfig.Config.ClusterDomain(), + Role: in.InstallConfig.Config.AWS.HostedZoneRole, + UserTags: tags, + }) + if err != nil { + return fmt.Errorf("failed to create private hosted zone: %w", err) + } + phzID = aws.StringValue(res.Id) + } + + logrus.Infoln("Creating Route53 records for control plane load balancer") + apiHost := awsCluster.Status.Network.SecondaryAPIServerELB.DNSName + if awsCluster.Status.Network.APIServerELB.Scheme == capa.ELBSchemeInternetFacing { + apiHost = awsCluster.Status.Network.APIServerELB.DNSName + } + apiIntHost := awsCluster.Spec.ControlPlaneEndpoint.Host + err = client.CreateOrUpdateRecord(ctx, in.InstallConfig.Config, apiHost, apiIntHost, phzID) + if err != nil { + return fmt.Errorf("failed to create route53 records: %w", err) + } + + return nil +} + +func getVPCFromSubnets(ctx context.Context, awsSession *session.Session, region string, subnetIDs []string) (string, error) { + var vpcID string + var lastError error + client := ec2.New(awsSession, aws.NewConfig().WithRegion(region)) + err := client.DescribeSubnetsPagesWithContext( + ctx, + &ec2.DescribeSubnetsInput{SubnetIds: aws.StringSlice(subnetIDs)}, + func(results *ec2.DescribeSubnetsOutput, lastPage bool) bool { + for _, subnet := range results.Subnets { + if subnet.SubnetId == nil { + continue + } + if subnet.SubnetArn == nil { + lastError = fmt.Errorf("%s has no ARN", *subnet.SubnetId) + return false + } + if subnet.VpcId == nil { + lastError = fmt.Errorf("%s has no VPC", *subnet.SubnetId) + return false + } + if subnet.AvailabilityZone == nil { + lastError = fmt.Errorf("%s has no availability zone", *subnet.SubnetId) + return false + } + // All subnets belong to the same VPC + vpcID = aws.StringValue(subnet.VpcId) + lastError = nil + return true + } + return !lastPage + }, + ) + if err == nil { + err = lastError + } + if err != nil { + return "", fmt.Errorf("failed to get VPC from subnets: %w", err) + } + + return vpcID, nil +}