Skip to content

Commit

Permalink
azure: Add support for dns=none
Browse files Browse the repository at this point in the history
  • Loading branch information
hakman committed Jul 13, 2023
1 parent 1675f21 commit 77a7a03
Show file tree
Hide file tree
Showing 16 changed files with 389 additions and 115 deletions.
7 changes: 7 additions & 0 deletions cmd/kops-controller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import (
nodeidentityos "k8s.io/kops/pkg/nodeidentity/openstack"
nodeidentityscw "k8s.io/kops/pkg/nodeidentity/scaleway"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
"k8s.io/kops/upup/pkg/fi/cloudup/azure"
"k8s.io/kops/upup/pkg/fi/cloudup/do"
"k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmverifier"
"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
Expand Down Expand Up @@ -153,6 +154,12 @@ func main() {
setupLog.Error(err, "unable to create verifier")
os.Exit(1)
}
} else if opt.Server.Provider.Azure != nil {
verifier, err = azure.NewAzureVerifier(ctx, opt.Server.Provider.Azure)
if err != nil {
setupLog.Error(err, "unable to create verifier")
os.Exit(1)
}
} else {
klog.Fatalf("server cloud provider config not provided")
}
Expand Down
2 changes: 2 additions & 0 deletions cmd/kops-controller/pkg/config/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package config

import (
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
"k8s.io/kops/upup/pkg/fi/cloudup/azure"
"k8s.io/kops/upup/pkg/fi/cloudup/do"
gcetpm "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm"
"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
Expand Down Expand Up @@ -73,6 +74,7 @@ type ServerProviderOptions struct {
OpenStack *openstack.OpenStackVerifierOptions `json:"openstack,omitempty"`
DigitalOcean *do.DigitalOceanVerifierOptions `json:"do,omitempty"`
Scaleway *scaleway.ScalewayVerifierOptions `json:"scaleway,omitempty"`
Azure *azure.AzureVerifierOptions `json:"azure,omitempty"`
}

// DiscoveryOptions configures our support for discovery, particularly gossip DNS (i.e. k8s.local)
Expand Down
7 changes: 7 additions & 0 deletions nodeup/pkg/model/bootstrap_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"k8s.io/kops/pkg/wellknownports"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
"k8s.io/kops/upup/pkg/fi/cloudup/azure"
"k8s.io/kops/upup/pkg/fi/cloudup/do"
"k8s.io/kops/upup/pkg/fi/cloudup/gce/gcediscovery"
"k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmsigner"
Expand Down Expand Up @@ -93,6 +94,12 @@ func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error {
return err
}
authenticator = a
case kops.CloudProviderAzure:
a, err := azure.NewAzureAuthenticator()
if err != nil {
return err
}
authenticator = a

default:
return fmt.Errorf("unsupported cloud provider for authenticator %q", b.CloudProvider())
Expand Down
2 changes: 1 addition & 1 deletion pkg/apis/kops/model/features.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import (

// UseKopsControllerForNodeBootstrap is true if nodeup should use kops-controller for bootstrapping.
func UseKopsControllerForNodeBootstrap(cloudProvider kops.CloudProviderID) bool {
return cloudProvider != kops.CloudProviderAzure
return true
}

// UseChallengeCallback is true if we should use a callback challenge during node provisioning with kops-controller.
Expand Down
10 changes: 0 additions & 10 deletions pkg/apis/kops/validation/validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -521,17 +521,7 @@ func validateTopology(c *kops.Cluster, topology *kops.TopologySpec, fieldPath *f
}

if topology.DNS != "" {
cloud := c.Spec.GetCloudProvider()
allErrs = append(allErrs, IsValidValue(fieldPath.Child("dns", "type"), &topology.DNS, kops.SupportedDnsTypes)...)

if topology.DNS == kops.DNSTypeNone {
switch cloud {
case kops.CloudProviderOpenstack, kops.CloudProviderHetzner, kops.CloudProviderAWS, kops.CloudProviderGCE, kops.CloudProviderDO, kops.CloudProviderScaleway:
// ok
default:
allErrs = append(allErrs, field.Invalid(fieldPath.Child("dns", "type"), topology.DNS, fmt.Sprintf("not supported for %q", c.Spec.GetCloudProvider())))
}
}
}

return allErrs
Expand Down
31 changes: 29 additions & 2 deletions pkg/model/azuremodel/network.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,10 @@ package azuremodel

import (
"fmt"
"strconv"

"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-05-01/network"
"k8s.io/kops/pkg/wellknownports"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/azuretasks"
"k8s.io/utils/net"
Expand Down Expand Up @@ -109,7 +111,7 @@ func (b *NetworkModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
SourceAddressPrefixes: &k8sAccessIPv4,
SourcePortRange: fi.PtrTo("*"),
DestinationAddressPrefix: fi.PtrTo("*"),
DestinationPortRange: fi.PtrTo("443"),
DestinationPortRange: fi.PtrTo(strconv.Itoa(wellknownports.KubeAPIServer)),
})
}
if len(k8sAccessIPv6) > 0 {
Expand All @@ -122,7 +124,32 @@ func (b *NetworkModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
SourceAddressPrefixes: &k8sAccessIPv6,
SourcePortRange: fi.PtrTo("*"),
DestinationAddressPrefix: fi.PtrTo("*"),
DestinationPortRange: fi.PtrTo("443"),
DestinationPortRange: fi.PtrTo(strconv.Itoa(wellknownports.KubeAPIServer)),
})
}
if b.Cluster.UsesNoneDNS() {
// TODO: Limit access to necessary source address prefixes instead of "0.0.0.0/0" and "::/0"
nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
Name: fi.PtrTo("AllowKopsController"),
Priority: fi.PtrTo[int32](210),
Access: network.SecurityRuleAccessAllow,
Direction: network.SecurityRuleDirectionInbound,
Protocol: network.SecurityRuleProtocolTCP,
SourceAddressPrefix: fi.PtrTo("0.0.0.0/0"),
SourcePortRange: fi.PtrTo("*"),
DestinationAddressPrefix: fi.PtrTo("*"),
DestinationPortRange: fi.PtrTo(strconv.Itoa(wellknownports.KopsControllerPort)),
})
nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
Name: fi.PtrTo("AllowKopsController_v6"),
Priority: fi.PtrTo[int32](211),
Access: network.SecurityRuleAccessAllow,
Direction: network.SecurityRuleDirectionInbound,
Protocol: network.SecurityRuleProtocolTCP,
SourceAddressPrefix: fi.PtrTo("::/0"),
SourcePortRange: fi.PtrTo("*"),
DestinationAddressPrefix: fi.PtrTo("*"),
DestinationPortRange: fi.PtrTo(strconv.Itoa(wellknownports.KopsControllerPort)),
})
}
var nodePortAccessIPv4, nodePortAccessIPv6 []string
Expand Down
22 changes: 2 additions & 20 deletions upup/pkg/fi/cloudup/apply_cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -1446,33 +1446,15 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit
}
}

case kops.CloudProviderDO, kops.CloudProviderScaleway:
// Use any IP address that is found (including public ones)
for _, additionalIP := range apiserverAdditionalIPs {
controlPlaneIPs = append(controlPlaneIPs, additionalIP)
}

case kops.CloudProviderGCE:
case kops.CloudProviderDO, kops.CloudProviderScaleway, kops.CloudProviderGCE, kops.CloudProviderAzure:
// Use any IP address that is found (including public ones)
for _, additionalIP := range apiserverAdditionalIPs {
controlPlaneIPs = append(controlPlaneIPs, additionalIP)
}
}

if cluster.UsesNoneDNS() {
switch cluster.Spec.GetCloudProvider() {
case kops.CloudProviderAWS, kops.CloudProviderHetzner, kops.CloudProviderOpenstack:
bootConfig.APIServerIPs = controlPlaneIPs

case kops.CloudProviderDO, kops.CloudProviderScaleway:
bootConfig.APIServerIPs = controlPlaneIPs

case kops.CloudProviderGCE:
bootConfig.APIServerIPs = controlPlaneIPs

default:
return nil, nil, fmt.Errorf("'none' DNS topology is not supported for cloud %q", cluster.Spec.GetCloudProvider())
}
bootConfig.APIServerIPs = controlPlaneIPs
} else {
// If we do have a fixed IP, we use it (on some clouds, initially)
switch cluster.Spec.GetCloudProvider() {
Expand Down
104 changes: 104 additions & 0 deletions upup/pkg/fi/cloudup/azure/authenticator.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,104 @@
/*
Copyright 2022 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package azure

import (
"encoding/json"
"fmt"
"io"
"net/http"
"strings"

"k8s.io/kops/pkg/bootstrap"
)

const AzureAuthenticationTokenPrefix = "x-azure-id "

type azureAuthenticator struct {
}

var _ bootstrap.Authenticator = &azureAuthenticator{}

func NewAzureAuthenticator() (bootstrap.Authenticator, error) {
return &azureAuthenticator{}, nil
}

func (h *azureAuthenticator) CreateToken(body []byte) (string, error) {
m, err := queryInstanceMetadata()
if err != nil {
return "", fmt.Errorf("querying instance metadata: %w", err)
}

// The fully qualified VMSS VM resource ID format is:
// /subscriptions/SUBSCRIPTION_ID/resourceGroups/RESOURCE_GROUP_NAME/providers/Microsoft.Compute/virtualMachineScaleSets/VMSS_NAME/virtualMachines/VMSS_INDEX
r := strings.Split(m.Compute.ResourceID, "/")
if len(r) != 11 || r[7] != "virtualMachineScaleSets" || r[9] != "virtualMachines" {
return "", fmt.Errorf("unexpected resource ID format: %q", m.Compute.ResourceID)
}

vmssName := r[8]
vmssIndex := r[10]

return AzureAuthenticationTokenPrefix + vmssName + " " + vmssIndex, nil
}

type instanceComputeMetadata struct {
ResourceGroupName string `json:"resourceGroupName"`
ResourceID string `json:"resourceId"`
SubscriptionID string `json:"subscriptionId"`
}

type instanceMetadata struct {
Compute *instanceComputeMetadata `json:"compute"`
}

// queryInstanceMetadata queries Azure Instance Metadata Service (IMDS)
// https://learn.microsoft.com/en-us/azure/virtual-machines/instance-metadata-service?tabs=linux
func queryInstanceMetadata() (*instanceMetadata, error) {
transport := &http.Transport{Proxy: nil}

client := http.Client{Transport: transport}

req, err := http.NewRequest("GET", "http://169.254.169.254/metadata/instance", nil)
if err != nil {
return nil, fmt.Errorf("creating a new request: %w", err)
}
req.Header.Add("Metadata", "True")

q := req.URL.Query()
q.Add("format", "json")
q.Add("api-version", "2021-02-01")
req.URL.RawQuery = q.Encode()

resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("sending request to the instance metadata server: %w", err)
}

defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("reading a response from the metadata server: %w", err)
}
metadata := &instanceMetadata{}
err = json.Unmarshal(body, metadata)
if err != nil {
return nil, fmt.Errorf("unmarshalling instance metadata: %w", err)
}

return metadata, nil
}
9 changes: 9 additions & 0 deletions upup/pkg/fi/cloudup/azure/loadbalancer.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
type LoadBalancersClient interface {
CreateOrUpdate(ctx context.Context, resourceGroupName, loadBalancerName string, parameters network.LoadBalancer) error
List(ctx context.Context, resourceGroupName string) ([]network.LoadBalancer, error)
Get(ctx context.Context, resourceGroupName string, loadBalancerName string) (*network.LoadBalancer, error)
Delete(ctx context.Context, resourceGroupName, loadBalancerName string) error
}

Expand All @@ -53,6 +54,14 @@ func (c *loadBalancersClientImpl) List(ctx context.Context, resourceGroupName st
return l, nil
}

func (c *loadBalancersClientImpl) Get(ctx context.Context, resourceGroupName string, loadBalancerName string) (*network.LoadBalancer, error) {
l, err := c.c.Get(ctx, resourceGroupName, loadBalancerName, "frontendIpConfigurations/publicIpAddress")
if err != nil {
return nil, err
}
return &l, nil
}

func (c *loadBalancersClientImpl) Delete(ctx context.Context, resourceGroupName, loadBalancerName string) error {
future, err := c.c.Delete(ctx, resourceGroupName, loadBalancerName)
if err != nil {
Expand Down
Loading

0 comments on commit 77a7a03

Please sign in to comment.