Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Automated cherry pick of #14930: no dns for OpenStack #14989

Merged
merged 1 commit into from Jan 12, 2023
Merged

Automated cherry pick of #14930: no dns for OpenStack #14989

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
5 changes: 5 additions & 0 deletions cmd/kops/create_cluster_integration_test.go
View file
Open in desktop
Expand Up @@ -74,6 +74,11 @@ func TestCreateClusterOpenStackOctavia(t *testing.T) {
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/ha_openstack_octavia", "v1alpha2")
}

func TestCreateClusterOpenStackNoDNS(t *testing.T) {
t.Setenv("OS_REGION_NAME", "us-test1")
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/ha_openstack_nodns", "v1alpha2")
}

// TestCreateClusterCilium runs kops with the cilium networking flags
func TestCreateClusterCilium(t *testing.T) {
runCreateClusterIntegrationTest(t, "../../tests/integration/create_cluster/cilium-eni", "v1alpha2")
Expand Down
6 changes: 3 additions & 3 deletions nodeup/pkg/model/etc_hosts.go
View file
Open in desktop
Expand Up @@ -46,15 +46,15 @@ func (b *EtcHostsBuilder) Build(c *fi.NodeupModelBuilderContext) error {
Addresses: []string{"127.0.0.1"},
})
}
} else if b.BootConfig.APIServerIP != "" {
} else if len(b.BootConfig.APIServerIPs) > 0 {
task.Records = append(task.Records, nodetasks.HostRecord{
Hostname: b.Cluster.APIInternalName(),
Addresses: []string{b.BootConfig.APIServerIP},
Addresses: b.BootConfig.APIServerIPs,
})
if b.UseKopsControllerForNodeBootstrap() {
task.Records = append(task.Records, nodetasks.HostRecord{
Hostname: "kops-controller.internal." + b.Cluster.Name,
Addresses: []string{b.BootConfig.APIServerIP},
Addresses: b.BootConfig.APIServerIPs,
})
}
}
Expand Down
4 changes: 2 additions & 2 deletions nodeup/pkg/model/kops_controller.go
View file
Open in desktop
Expand Up @@ -62,8 +62,8 @@ func (b *KopsControllerBuilder) Build(c *fi.NodeupModelBuilderContext) error {
Subject: nodetasks.PKIXName{CommonName: "kops-controller"},
AlternateNames: []string{"kops-controller.internal." + b.Cluster.ObjectMeta.Name},
}
if b.BootConfig.APIServerIP != "" {
issueCert.AlternateNames = append(issueCert.AlternateNames, b.BootConfig.APIServerIP)
if len(b.BootConfig.APIServerIPs) > 0 {
issueCert.AlternateNames = append(issueCert.AlternateNames, b.BootConfig.APIServerIPs...)
}
c.AddTask(issueCert)

Expand Down
2 changes: 1 addition & 1 deletion pkg/apis/kops/validation/validation.go
View file
Open in desktop
Expand Up @@ -430,7 +430,7 @@ func validateTopology(c *kops.Cluster, topology *kops.TopologySpec, fieldPath *f
cloud := c.Spec.GetCloudProvider()
value := string(topology.DNS)
allErrs = append(allErrs, IsValidValue(fieldPath.Child("dns", "type"), &value, kops.SupportedDnsTypes)...)
if value == string(kops.DNSTypeNone) && cloud != kops.CloudProviderHetzner && cloud != kops.CloudProviderAWS && cloud != kops.CloudProviderGCE {
if value == string(kops.DNSTypeNone) && cloud != kops.CloudProviderHetzner && cloud != kops.CloudProviderAWS && cloud != kops.CloudProviderGCE && cloud != kops.CloudProviderOpenstack {
allErrs = append(allErrs, field.Invalid(fieldPath.Child("dns", "type"), &value, fmt.Sprintf("not supported for %q", c.Spec.GetCloudProvider())))
}
}
Expand Down
4 changes: 2 additions & 2 deletions pkg/apis/nodeup/config.go
View file
Open in desktop
Expand Up @@ -90,9 +90,9 @@ type BootConfig struct {
ConfigBase *string `json:",omitempty"`
// ConfigServer holds the configuration for the configuration server.
ConfigServer *ConfigServerOptions `json:",omitempty"`
// APIServerIP is the API server IP address.
// APIServerIPs is the API server IP addresses.
// This field is used for adding an alias for api.internal. in /etc/hosts, when Topology.DNS.Type == DNSTypeNone.
APIServerIP string `json:",omitempty"`
APIServerIPs []string `json:",omitempty"`
// InstanceGroupName is the name of the instance group.
InstanceGroupName string `json:",omitempty"`
// InstanceGroupRole is the instance group role.
Expand Down
7 changes: 6 additions & 1 deletion pkg/model/bootstrapscript.go
View file
Open in desktop
Expand Up @@ -168,8 +168,13 @@ func (b *BootstrapScript) buildEnvironmentVariables(cluster *kops.Cluster) (map[
)
}

// credentials needed always when using swift but when using None dns only in control plane
passEnvs := true
if !strings.HasPrefix(cluster.Spec.ConfigBase, "swift://") && cluster.UsesNoneDNS() && !b.ig.IsControlPlane() {
passEnvs = false
}
// Pass in required credentials when using user-defined swift endpoint
if os.Getenv("OS_AUTH_URL") != "" {
if os.Getenv("OS_AUTH_URL") != "" && passEnvs {
for _, envVar := range osEnvs {
env[envVar] = fmt.Sprintf("'%s'", os.Getenv(envVar))
}
Expand Down
13 changes: 10 additions & 3 deletions pkg/model/openstackmodel/servergroup.go
View file
Open in desktop
Expand Up @@ -26,6 +26,7 @@ import (
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/pkg/model"
"k8s.io/kops/pkg/truncate"
"k8s.io/kops/pkg/wellknownports"
"k8s.io/kops/upup/pkg/fi"
"k8s.io/kops/upup/pkg/fi/cloudup/openstack"
"k8s.io/kops/upup/pkg/fi/cloudup/openstacktasks"
Expand Down Expand Up @@ -166,6 +167,10 @@ func (b *ServerGroupModelBuilder) buildInstances(c *fi.CloudupModelBuilderContex
}
c.AddTask(portTask)

if b.Cluster.UsesNoneDNS() && ig.Spec.Role == kops.InstanceGroupRoleControlPlane {
portTask.ForAPIServer = true
}

metaWithName := make(map[string]string)
for k, v := range igMeta {
metaWithName[k] = v
Expand Down Expand Up @@ -315,8 +320,10 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error
}
c.AddTask(poolTask)

nameForResource := fi.ValueOf(lbTask.Name)
listenerTask := &openstacktasks.LBListener{
Name: lbTask.Name,
Name: fi.PtrTo(nameForResource),
Port: fi.PtrTo(wellknownports.KubeAPIServer),
Lifecycle: b.Lifecycle,
Pool: poolTask,
}
Expand All @@ -334,7 +341,7 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error
c.AddTask(listenerTask)

monitorTask := &openstacktasks.PoolMonitor{
Name: lbTask.Name,
Name: fi.PtrTo(nameForResource),
Pool: poolTask,
Lifecycle: b.Lifecycle,
}
Expand All @@ -350,7 +357,7 @@ func (b *ServerGroupModelBuilder) Build(c *fi.CloudupModelBuilderContext) error
Pool: poolTask,
ServerGroup: mastersg,
InterfaceName: fi.PtrTo(ifName),
ProtocolPort: fi.PtrTo(443),
ProtocolPort: fi.PtrTo(wellknownports.KubeAPIServer),
Lifecycle: b.Lifecycle,
Weight: fi.PtrTo(1),
}
Expand Down
160 changes: 160 additions & 0 deletions pkg/model/openstackmodel/servergroup_test.go
View file
Open in desktop
Expand Up @@ -515,6 +515,166 @@ func getServerGroupModelBuilderTestInput() []serverGroupModelBuilderTestInput {
},
},
},
{
desc: "multizone setup 3 masters 3 nodes without bastion with API loadbalancer dns none",
cluster: &kops.Cluster{
ObjectMeta: metav1.ObjectMeta{
Name: "cluster",
},
Spec: kops.ClusterSpec{
API: kops.APISpec{
LoadBalancer: &kops.LoadBalancerAccessSpec{
Type: kops.LoadBalancerTypePublic,
},
},
CloudProvider: kops.CloudProviderSpec{
Openstack: &kops.OpenstackSpec{
BlockStorage: &kops.OpenstackBlockStorageConfig{
Version: fi.PtrTo("v3"),
IgnoreAZ: fi.PtrTo(false),
CreateStorageClass: fi.PtrTo(false),
CSITopologySupport: fi.PtrTo(true),
},
Loadbalancer: &kops.OpenstackLoadbalancerConfig{
FloatingNetwork: fi.PtrTo("test"),
FloatingSubnet: fi.PtrTo("test-lb-subnet"),
Method: fi.PtrTo("ROUND_ROBIN"),
Provider: fi.PtrTo("amphora"),
UseOctavia: fi.PtrTo(true),
},
Monitor: &kops.OpenstackMonitor{
Delay: fi.PtrTo("1m"),
MaxRetries: fi.PtrTo(3),
Timeout: fi.PtrTo("30s"),
},
Network: &kops.OpenstackNetwork{
AvailabilityZoneHints: []*string{fi.PtrTo("zone-1"), fi.PtrTo("zone-2"), fi.PtrTo("zone-3")},
},
Router: &kops.OpenstackRouter{
DNSServers: fi.PtrTo("8.8.8.8,8.8.4.4"),
ExternalSubnet: fi.PtrTo("test-router-subnet"),
ExternalNetwork: fi.PtrTo("test"),
AvailabilityZoneHints: []*string{fi.PtrTo("ha-zone")},
},
Metadata: &kops.OpenstackMetadata{
ConfigDrive: fi.PtrTo(false),
},
},
},
KubernetesVersion: "1.25.0",
Networking: kops.NetworkingSpec{
Subnets: []kops.ClusterSubnetSpec{
{
Name: "subnet-1",
Zone: "zone-1",
Type: kops.SubnetTypePrivate,
},
{
Name: "subnet-2",
Zone: "zone-2",
Type: kops.SubnetTypePrivate,
},
{
Name: "subnet-3",
Zone: "zone-3",
Type: kops.SubnetTypePrivate,
},
},
Topology: &kops.TopologySpec{
ControlPlane: kops.TopologyPrivate,
DNS: kops.DNSTypeNone,
Nodes: kops.TopologyPrivate,
},
},
},
},
instanceGroups: []*kops.InstanceGroup{
{
ObjectMeta: metav1.ObjectMeta{
Name: "master-a",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleControlPlane,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-1"},
Zones: []string{"zone-1"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "node-a",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleNode,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-1"},
Zones: []string{"zone-1"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "master-b",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleControlPlane,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-2"},
Zones: []string{"zone-2"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "node-b",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleNode,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-2"},
Zones: []string{"zone-2"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "master-c",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleControlPlane,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-3"},
Zones: []string{"zone-3"},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "node-c",
},
Spec: kops.InstanceGroupSpec{
Role: kops.InstanceGroupRoleNode,
Image: "image",
MinSize: i32(1),
MaxSize: i32(1),
MachineType: "blc.1-2",
Subnets: []string{"subnet-3"},
Zones: []string{"zone-3"},
},
},
},
},
{
desc: "multizone setup 3 masters 3 nodes without external router",
cluster: &kops.Cluster{
Expand Down
2 changes: 2 additions & 0 deletions pkg/model/openstackmodel/tests/servergroup/adds-additional-security-groups.yaml
View file
Open in desktop
Expand Up @@ -26,6 +26,7 @@ Name: node-1-cluster
Port:
AdditionalSecurityGroups:
- additional-sg
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
Expand Down Expand Up @@ -192,6 +193,7 @@ PublicACL: null
---
AdditionalSecurityGroups:
- additional-sg
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
Expand Down
2 changes: 2 additions & 0 deletions pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-ClusterSpec.yaml
View file
Open in desktop
Expand Up @@ -26,6 +26,7 @@ Metadata:
Name: node-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
Expand Down Expand Up @@ -190,6 +191,7 @@ Name: nodeupconfig-node
PublicACL: null
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
Expand Down
2 changes: 2 additions & 0 deletions pkg/model/openstackmodel/tests/servergroup/adds-cloud-labels-from-InstanceGroupSpec.yaml
View file
Open in desktop
Expand Up @@ -26,6 +26,7 @@ Metadata:
Name: node-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
Expand Down Expand Up @@ -190,6 +191,7 @@ Name: nodeupconfig-node
PublicACL: null
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
Expand Down
2 changes: 2 additions & 0 deletions ...l/openstackmodel/tests/servergroup/configures-server-group-affinity-with-annotations.yaml
View file
Open in desktop
Expand Up @@ -25,6 +25,7 @@ Metadata:
Name: node-1-cluster
Port:
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
Expand Down Expand Up @@ -189,6 +190,7 @@ Name: nodeupconfig-node
PublicACL: null
---
AdditionalSecurityGroups: null
ForAPIServer: false
ID: null
InstanceGroupName: node
Lifecycle: Sync
Expand Down