Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

azure: Add support for dns=none #15627

Merged
merged 5 commits into from
Jul 16, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions cmd/kops-controller/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ import (
nodeidentityos "k8s.io/kops/pkg/nodeidentity/openstack"
nodeidentityscw "k8s.io/kops/pkg/nodeidentity/scaleway"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
"k8s.io/kops/upup/pkg/fi/cloudup/azure"
"k8s.io/kops/upup/pkg/fi/cloudup/do"
"k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmverifier"
"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
Expand Down Expand Up @@ -153,6 +154,12 @@ func main() {
setupLog.Error(err, "unable to create verifier")
os.Exit(1)
}
} else if opt.Server.Provider.Azure != nil {
verifier, err = azure.NewAzureVerifier(ctx, opt.Server.Provider.Azure)
if err != nil {
setupLog.Error(err, "unable to create verifier")
os.Exit(1)
}
} else {
klog.Fatalf("server cloud provider config not provided")
}
Expand Down
2 changes: 2 additions & 0 deletions cmd/kops-controller/pkg/config/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@ package config

import (
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
"k8s.io/kops/upup/pkg/fi/cloudup/azure"
"k8s.io/kops/upup/pkg/fi/cloudup/do"
gcetpm "k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm"
"k8s.io/kops/upup/pkg/fi/cloudup/hetzner"
Expand Down Expand Up @@ -73,6 +74,7 @@ type ServerProviderOptions struct {
OpenStack *openstack.OpenStackVerifierOptions `json:"openstack,omitempty"`
DigitalOcean *do.DigitalOceanVerifierOptions `json:"do,omitempty"`
Scaleway *scaleway.ScalewayVerifierOptions `json:"scaleway,omitempty"`
Azure *azure.AzureVerifierOptions `json:"azure,omitempty"`
}

// DiscoveryOptions configures our support for discovery, particularly gossip DNS (i.e. k8s.local)
Expand Down
7 changes: 7 additions & 0 deletions nodeup/pkg/model/bootstrap_client.go
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"k8s.io/kops/pkg/wellknownports"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/awsup"
"k8s.io/kops/upup/pkg/fi/cloudup/azure"
"k8s.io/kops/upup/pkg/fi/cloudup/do"
"k8s.io/kops/upup/pkg/fi/cloudup/gce/gcediscovery"
"k8s.io/kops/upup/pkg/fi/cloudup/gce/tpm/gcetpmsigner"
Expand Down Expand Up @@ -93,6 +94,12 @@ func (b BootstrapClientBuilder) Build(c *fi.NodeupModelBuilderContext) error {
return err
}
authenticator = a
case kops.CloudProviderAzure:
a, err := azure.NewAzureAuthenticator()
if err != nil {
return err
}
authenticator = a

default:
return fmt.Errorf("unsupported cloud provider for authenticator %q", b.CloudProvider())
Expand Down
2 changes: 1 addition & 1 deletion pkg/apis/kops/model/features.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ import (

// UseKopsControllerForNodeBootstrap is true if nodeup should use kops-controller for bootstrapping.
func UseKopsControllerForNodeBootstrap(cloudProvider kops.CloudProviderID) bool {
return cloudProvider != kops.CloudProviderAzure
return true
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

👍 👍 👍

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'll take on removing the dead code this leaves behind.

}

// UseChallengeCallback is true if we should use a callback challenge during node provisioning with kops-controller.
Expand Down
10 changes: 0 additions & 10 deletions pkg/apis/kops/validation/validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -521,17 +521,7 @@ func validateTopology(c *kops.Cluster, topology *kops.TopologySpec, fieldPath *f
}

if topology.DNS != "" {
cloud := c.Spec.GetCloudProvider()
allErrs = append(allErrs, IsValidValue(fieldPath.Child("dns", "type"), &topology.DNS, kops.SupportedDnsTypes)...)

if topology.DNS == kops.DNSTypeNone {
switch cloud {
case kops.CloudProviderOpenstack, kops.CloudProviderHetzner, kops.CloudProviderAWS, kops.CloudProviderGCE, kops.CloudProviderDO, kops.CloudProviderScaleway:
// ok
default:
allErrs = append(allErrs, field.Invalid(fieldPath.Child("dns", "type"), topology.DNS, fmt.Sprintf("not supported for %q", c.Spec.GetCloudProvider())))
}
}
}

return allErrs
Expand Down
46 changes: 44 additions & 2 deletions pkg/model/azuremodel/network.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,11 @@ package azuremodel

import (
"fmt"
"strconv"

"github.com/Azure/azure-sdk-for-go/services/network/mgmt/2022-05-01/network"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/wellknownports"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/azuretasks"
"k8s.io/utils/net"
Expand Down Expand Up @@ -109,7 +112,7 @@ func (b *NetworkModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
SourceAddressPrefixes: &k8sAccessIPv4,
SourcePortRange: fi.PtrTo("*"),
DestinationAddressPrefix: fi.PtrTo("*"),
DestinationPortRange: fi.PtrTo("443"),
DestinationPortRange: fi.PtrTo(strconv.Itoa(wellknownports.KubeAPIServer)),
})
}
if len(k8sAccessIPv6) > 0 {
Expand All @@ -122,9 +125,48 @@ func (b *NetworkModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
SourceAddressPrefixes: &k8sAccessIPv6,
SourcePortRange: fi.PtrTo("*"),
DestinationAddressPrefix: fi.PtrTo("*"),
DestinationPortRange: fi.PtrTo("443"),
DestinationPortRange: fi.PtrTo(strconv.Itoa(wellknownports.KubeAPIServer)),
})
}
if b.Cluster.UsesNoneDNS() {
if b.Cluster.Spec.API.LoadBalancer != nil && b.Cluster.Spec.API.LoadBalancer.Type == kops.LoadBalancerTypeInternal {
nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
Name: fi.PtrTo("AllowKopsController"),
Priority: fi.PtrTo[int32](210),
Access: network.SecurityRuleAccessAllow,
Direction: network.SecurityRuleDirectionInbound,
Protocol: network.SecurityRuleProtocolTCP,
SourceAddressPrefix: fi.PtrTo(b.Cluster.Spec.Networking.NetworkCIDR),
SourcePortRange: fi.PtrTo("*"),
DestinationAddressPrefix: fi.PtrTo("*"),
DestinationPortRange: fi.PtrTo(strconv.Itoa(wellknownports.KopsControllerPort)),
})
} else {
// TODO: Limit access to necessary source address prefixes instead of "0.0.0.0/0" and "::/0"
nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
Name: fi.PtrTo("AllowKopsController"),
Priority: fi.PtrTo[int32](210),
Access: network.SecurityRuleAccessAllow,
Direction: network.SecurityRuleDirectionInbound,
Protocol: network.SecurityRuleProtocolTCP,
SourceAddressPrefix: fi.PtrTo("0.0.0.0/0"),
SourcePortRange: fi.PtrTo("*"),
DestinationAddressPrefix: fi.PtrTo("*"),
DestinationPortRange: fi.PtrTo(strconv.Itoa(wellknownports.KopsControllerPort)),
})
nsgTask.SecurityRules = append(nsgTask.SecurityRules, &azuretasks.NetworkSecurityRule{
Name: fi.PtrTo("AllowKopsController_v6"),
Priority: fi.PtrTo[int32](211),
Access: network.SecurityRuleAccessAllow,
Direction: network.SecurityRuleDirectionInbound,
Protocol: network.SecurityRuleProtocolTCP,
SourceAddressPrefix: fi.PtrTo("::/0"),
SourcePortRange: fi.PtrTo("*"),
DestinationAddressPrefix: fi.PtrTo("*"),
DestinationPortRange: fi.PtrTo(strconv.Itoa(wellknownports.KopsControllerPort)),
})
}
}
var nodePortAccessIPv4, nodePortAccessIPv6 []string
for _, cidr := range b.Cluster.Spec.NodePortAccess {
switch net.IPFamilyOfCIDRString(cidr) {
Expand Down
24 changes: 13 additions & 11 deletions pkg/model/azuremodel/vmscaleset.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,17 +49,19 @@ func (b *VMScaleSetModelBuilder) Build(c *fi.CloudupModelBuilderContext) error {
}
c.AddTask(vmss)

// Create tasks for assigning built-in roles to VM Scale Sets.
// See https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles
// for the ID definitions.
roleDefIDs := map[string]string{
// Owner
"owner": "8e3af657-a8ff-443c-a75c-2fe8c4bcb635",
// Storage Blob Data Contributor
"blob": "ba92f5b4-2d11-453d-a403-e96b0029c9fe",
}
for k, roleDefID := range roleDefIDs {
c.AddTask(b.buildRoleAssignmentTask(vmss, k, roleDefID))
if ig.IsControlPlane() || b.Cluster.UsesLegacyGossip() {
// Create tasks for assigning built-in roles to VM Scale Sets.
// See https://docs.microsoft.com/en-us/azure/role-based-access-control/built-in-roles
// for the ID definitions.
roleDefIDs := map[string]string{
// Owner
"owner": "8e3af657-a8ff-443c-a75c-2fe8c4bcb635",
// Storage Blob Data Contributor
"blob": "ba92f5b4-2d11-453d-a403-e96b0029c9fe",
}
for k, roleDefID := range roleDefIDs {
c.AddTask(b.buildRoleAssignmentTask(vmss, k, roleDefID))
}
}
}

Expand Down
22 changes: 2 additions & 20 deletions upup/pkg/fi/cloudup/apply_cluster.go
Original file line number Diff line number Diff line change
Expand Up @@ -1446,33 +1446,15 @@ func (n *nodeUpConfigBuilder) BuildConfig(ig *kops.InstanceGroup, apiserverAddit
}
}

case kops.CloudProviderDO, kops.CloudProviderScaleway:
// Use any IP address that is found (including public ones)
for _, additionalIP := range apiserverAdditionalIPs {
controlPlaneIPs = append(controlPlaneIPs, additionalIP)
}

case kops.CloudProviderGCE:
case kops.CloudProviderDO, kops.CloudProviderScaleway, kops.CloudProviderGCE, kops.CloudProviderAzure:
// Use any IP address that is found (including public ones)
for _, additionalIP := range apiserverAdditionalIPs {
controlPlaneIPs = append(controlPlaneIPs, additionalIP)
}
}

if cluster.UsesNoneDNS() {
switch cluster.Spec.GetCloudProvider() {
case kops.CloudProviderAWS, kops.CloudProviderHetzner, kops.CloudProviderOpenstack:
bootConfig.APIServerIPs = controlPlaneIPs

case kops.CloudProviderDO, kops.CloudProviderScaleway:
bootConfig.APIServerIPs = controlPlaneIPs

case kops.CloudProviderGCE:
bootConfig.APIServerIPs = controlPlaneIPs

default:
return nil, nil, fmt.Errorf("'none' DNS topology is not supported for cloud %q", cluster.Spec.GetCloudProvider())
}
bootConfig.APIServerIPs = controlPlaneIPs
} else {
// If we do have a fixed IP, we use it (on some clouds, initially)
switch cluster.Spec.GetCloudProvider() {
Expand Down
109 changes: 109 additions & 0 deletions upup/pkg/fi/cloudup/azure/authenticator.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,109 @@
/*
Copyright 2022 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package azure

import (
"encoding/json"
"fmt"
"io"
"net/http"
"strings"

"k8s.io/kops/pkg/bootstrap"
)

const AzureAuthenticationTokenPrefix = "x-azure-id "

type azureAuthenticator struct {
}

var _ bootstrap.Authenticator = &azureAuthenticator{}

func NewAzureAuthenticator() (bootstrap.Authenticator, error) {
return &azureAuthenticator{}, nil
}

func (h *azureAuthenticator) CreateToken(body []byte) (string, error) {
m, err := queryInstanceMetadata()
if err != nil {
return "", fmt.Errorf("querying instance metadata: %w", err)
}

vmId := m.Compute.VMID
if vmId == "" {
return "", fmt.Errorf("missing virtual machine ID")
}

// The fully qualified VMSS VM resource ID format is:
// /subscriptions/SUBSCRIPTION_ID/resourceGroups/RESOURCE_GROUP_NAME/providers/Microsoft.Compute/virtualMachineScaleSets/VMSS_NAME/virtualMachines/VMSS_INDEX
r := strings.Split(m.Compute.ResourceID, "/")
if len(r) != 11 || r[7] != "virtualMachineScaleSets" || r[9] != "virtualMachines" {
return "", fmt.Errorf("unexpected resource ID format: %q", m.Compute.ResourceID)
}
vmssName := r[8]
vmssIndex := r[10]

return AzureAuthenticationTokenPrefix + vmId + " " + vmssName + " " + vmssIndex, nil
}

type instanceComputeMetadata struct {
ResourceGroupName string `json:"resourceGroupName"`
ResourceID string `json:"resourceId"`
SubscriptionID string `json:"subscriptionId"`
VMID string `json:"vmId"`
}

type instanceMetadata struct {
Compute *instanceComputeMetadata `json:"compute"`
}

// queryInstanceMetadata queries Azure Instance Metadata Service (IMDS)
// https://learn.microsoft.com/en-us/azure/virtual-machines/instance-metadata-service?tabs=linux
func queryInstanceMetadata() (*instanceMetadata, error) {
transport := &http.Transport{Proxy: nil}

client := http.Client{Transport: transport}

req, err := http.NewRequest("GET", "http://169.254.169.254/metadata/instance", nil)
if err != nil {
return nil, fmt.Errorf("creating a new request: %w", err)
}
req.Header.Add("Metadata", "True")

q := req.URL.Query()
q.Add("format", "json")
q.Add("api-version", "2021-02-01")
req.URL.RawQuery = q.Encode()

resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("sending request to the instance metadata server: %w", err)
}

defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("reading a response from the metadata server: %w", err)
}
metadata := &instanceMetadata{}
err = json.Unmarshal(body, metadata)
if err != nil {
return nil, fmt.Errorf("unmarshalling instance metadata: %w", err)
}

return metadata, nil
}
9 changes: 9 additions & 0 deletions upup/pkg/fi/cloudup/azure/loadbalancer.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
type LoadBalancersClient interface {
CreateOrUpdate(ctx context.Context, resourceGroupName, loadBalancerName string, parameters network.LoadBalancer) error
List(ctx context.Context, resourceGroupName string) ([]network.LoadBalancer, error)
Get(ctx context.Context, resourceGroupName string, loadBalancerName string) (*network.LoadBalancer, error)
Delete(ctx context.Context, resourceGroupName, loadBalancerName string) error
}

Expand All @@ -53,6 +54,14 @@ func (c *loadBalancersClientImpl) List(ctx context.Context, resourceGroupName st
return l, nil
}

func (c *loadBalancersClientImpl) Get(ctx context.Context, resourceGroupName string, loadBalancerName string) (*network.LoadBalancer, error) {
l, err := c.c.Get(ctx, resourceGroupName, loadBalancerName, "frontendIpConfigurations/publicIpAddress")
if err != nil {
return nil, err
}
return &l, nil
}

func (c *loadBalancersClientImpl) Delete(ctx context.Context, resourceGroupName, loadBalancerName string) error {
future, err := c.c.Delete(ctx, resourceGroupName, loadBalancerName)
if err != nil {
Expand Down
Loading
Loading