Skip to content

Commit

Permalink
Merge pull request #12170 from justinsb/gce_ipv6
Browse files Browse the repository at this point in the history 
Initial IPv6 support for GCE
  • Loading branch information
@k8s-ci-robot
k8s-ci-robot committed Aug 22, 2021
2 parents cf2b0fe + 144c841 commit 20e472e
Show file tree
Hide file tree
Showing 10 changed files with 427 additions and 32 deletions.
7 changes: 2 additions & 5 deletions pkg/model/gcemodel/api_loadbalancer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,16 +84,13 @@ func (b *APILoadBalancerBuilder) Build(c *fi.ModelBuilderContext) error {

// Allow traffic into the API (port 443) from KubernetesAPIAccess CIDRs
{
t := &gcetasks.FirewallRule{
Name: s(b.NameForFirewallRule("https-api")),
b.AddFirewallRulesTasks(c, "https-api", &gcetasks.FirewallRule{
Lifecycle: b.Lifecycle,
Network: b.LinkToNetwork(),
SourceRanges: b.Cluster.Spec.KubernetesAPIAccess,
TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleMaster)},
Allowed: []string{"tcp:443"},
}
c.AddTask(t)
})
}
return nil

}
21 changes: 6 additions & 15 deletions pkg/model/gcemodel/external_access.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,17 +49,15 @@ func (b *ExternalAccessModelBuilder) Build(c *fi.ModelBuilderContext) error {
// But I think we can always add more permissions in this case later, but we can't easily take them away
klog.V(2).Infof("bastion is in use; won't configure SSH access to master / node instances")
} else {
c.AddTask(&gcetasks.FirewallRule{
Name: s(b.SafeObjectName("ssh-external-to-master")),
b.AddFirewallRulesTasks(c, "ssh-external-to-master", &gcetasks.FirewallRule{
Lifecycle: b.Lifecycle,
TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleMaster)},
Allowed: []string{"tcp:22"},
SourceRanges: b.Cluster.Spec.SSHAccess,
Network: b.LinkToNetwork(),
})

c.AddTask(&gcetasks.FirewallRule{
Name: s(b.SafeObjectName("ssh-external-to-node")),
b.AddFirewallRulesTasks(c, "ssh-external-to-node", &gcetasks.FirewallRule{
Lifecycle: b.Lifecycle,
TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleNode)},
Allowed: []string{"tcp:22"},
Expand All @@ -74,9 +72,9 @@ func (b *ExternalAccessModelBuilder) Build(c *fi.ModelBuilderContext) error {
if err != nil {
return err
}

nodePortRangeString := nodePortRange.String()
t := &gcetasks.FirewallRule{
Name: s(b.SafeObjectName("nodeport-external-to-node")),
b.AddFirewallRulesTasks(c, "nodeport-external-to-node", &gcetasks.FirewallRule{
Lifecycle: b.Lifecycle,
TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleNode)},
Allowed: []string{
Expand All @@ -85,13 +83,7 @@ func (b *ExternalAccessModelBuilder) Build(c *fi.ModelBuilderContext) error {
},
SourceRanges: b.Cluster.Spec.NodePortAccess,
Network: b.LinkToNetwork(),
}
if len(t.SourceRanges) == 0 {
// Empty SourceRanges is interpreted as 0.0.0.0/0 if tags are empty, so we set a SourceTag
// This is already covered by the normal node-to-node rules, but avoids opening the NodePort range
t.SourceTags = []string{b.GCETagForRole(kops.InstanceGroupRoleNode)}
}
c.AddTask(t)
})
}

if !b.UseLoadBalancerForAPI() {
Expand All @@ -100,8 +92,7 @@ func (b *ExternalAccessModelBuilder) Build(c *fi.ModelBuilderContext) error {
// We need to open security groups directly to the master nodes (instead of via the ELB)

// HTTPS to the master is allowed (for API access)
c.AddTask(&gcetasks.FirewallRule{
Name: s(b.SafeObjectName("kubernetes-master-https")),
b.AddFirewallRulesTasks(c, "kubernetes-master-https", &gcetasks.FirewallRule{
Lifecycle: b.Lifecycle,
TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleMaster)},
Allowed: []string{"tcp:443"},
Expand Down
51 changes: 43 additions & 8 deletions pkg/model/gcemodel/firewall.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,9 @@ limitations under the License.
package gcemodel

import (
"net"
"strings"

"k8s.io/klog/v2"
"k8s.io/kops/pkg/apis/kops"
"k8s.io/kops/upup/pkg/fi"
Expand Down Expand Up @@ -63,15 +66,13 @@ func (b *FirewallModelBuilder) Build(c *fi.ModelBuilderContext) error {
// The traffic is not recognized if it's on the overlay network?
klog.Warningf("Adding overlay network for X -> node rule - HACK")

t := &gcetasks.FirewallRule{
Name: s(b.SafeObjectName("cidr-to-node")),
b.AddFirewallRulesTasks(c, "cidr-to-node", &gcetasks.FirewallRule{
Lifecycle: b.Lifecycle,
Network: b.LinkToNetwork(),
SourceRanges: []string{b.Cluster.Spec.NonMasqueradeCIDR},
TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleNode)},
Allowed: []string{"tcp", "udp", "icmp", "esp", "ah", "sctp"},
}
c.AddTask(t)
})
}

// Allow full traffic from master -> master
Expand Down Expand Up @@ -116,15 +117,49 @@ func (b *FirewallModelBuilder) Build(c *fi.ModelBuilderContext) error {
if b.Cluster.Spec.NonMasqueradeCIDR != "" {
// The traffic is not recognized if it's on the overlay network?
klog.Warningf("Adding overlay network for X -> master rule - HACK")
t := &gcetasks.FirewallRule{
Name: s(b.SafeObjectName("cidr-to-master")),

b.AddFirewallRulesTasks(c, "cidr-to-master", &gcetasks.FirewallRule{
Lifecycle: b.Lifecycle,
Network: b.LinkToNetwork(),
SourceRanges: []string{b.Cluster.Spec.NonMasqueradeCIDR},
TargetTags: []string{b.GCETagForRole(kops.InstanceGroupRoleMaster)},
Allowed: []string{"tcp:443", "tcp:4194"},
}
c.AddTask(t)
})
}
return nil
}

// AddFirewallRulesTasks creates and adds ipv4 and ipv6 gcetasks.FirewallRule Tasks.
// GCE does not allow us to mix ipv4 and ipv6 in the same firewall rule, so we must create separate rules.
// Furthermore, an empty SourceRange with empty SourceTags is interpreted as allow-everything,
// but we intend for it to block everything; so we can Disabled to achieve the desired blocking.
func (b *GCEModelContext) AddFirewallRulesTasks(c *fi.ModelBuilderContext, name string, rule *gcetasks.FirewallRule) {
var ipv4SourceRanges []string
var ipv6SourceRanges []string
for _, sourceRange := range rule.SourceRanges {
_, cidr, err := net.ParseCIDR(sourceRange)
if err != nil {
klog.Fatalf("failed to parse invalid sourceRange %q", sourceRange)
}

// Split into ipv4s and ipv6s, but treat IPv4-mapped IPv6 addresses as IPv6
if cidr.IP.To4() != nil && !strings.Contains(sourceRange, ":") {
ipv4SourceRanges = append(ipv4SourceRanges, sourceRange)
} else {
ipv6SourceRanges = append(ipv6SourceRanges, sourceRange)

}
}

ipv4 := *rule
ipv4.Name = s(b.NameForFirewallRule(name))
ipv4.SourceRanges = ipv4SourceRanges
ipv4.DisableIfEmptySourceRanges()
c.AddTask(&ipv4)

ipv6 := *rule
ipv6.Name = s(b.NameForFirewallRule(name + "-ipv6"))
ipv6.SourceRanges = ipv6SourceRanges
ipv6.DisableIfEmptySourceRanges()
c.AddTask(&ipv6)
}
3 changes: 2 additions & 1 deletion pkg/resources/gce/gce.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,8 @@ const (
)

// Maximum number of `-` separated tokens in a name
const maxPrefixTokens = 4
// Example: nodeport-external-to-node-ipv6
const maxPrefixTokens = 5

func ListResourcesGCE(gceCloud gce.GCECloud, clusterName string, region string) (map[string]*resources.Resource, error) {
if region == "" {
Expand Down
2 changes: 2 additions & 0 deletions ...ntegration/update_cluster/ha_gce/data/aws_s3_bucket_object_cluster-completed.spec_content
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,6 +153,7 @@ spec:
podManifestPath: /etc/kubernetes/manifests
kubernetesApiAccess:
- 0.0.0.0/0
- ::/0
kubernetesVersion: 1.21.0
masterInternalName: api.internal.ha-gce.example.com
masterKubelet:
Expand Down Expand Up @@ -182,6 +183,7 @@ spec:
serviceClusterIPRange: 100.64.0.0/13
sshAccess:
- 0.0.0.0/0
- ::/0
subnets:
- name: us-test1
region: us-test1
Expand Down
2 changes: 2 additions & 0 deletions tests/integration/update_cluster/ha_gce/in-v1alpha2.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ spec:
anonymousAuth: false
kubernetesApiAccess:
- 0.0.0.0/0
- ::/0
kubernetesVersion: v1.21.0
masterPublicName: api.ha-gce.example.com
networking:
Expand All @@ -43,6 +44,7 @@ spec:
project: testproject
sshAccess:
- 0.0.0.0/0
- ::/0
subnets:
- name: us-test1
region: us-test1
Expand Down
102 changes: 101 additions & 1 deletion tests/integration/update_cluster/ha_gce/kubernetes.tf
View file
Original file line number Diff line number Diff line change
Expand Up @@ -249,12 +249,28 @@ resource "google_compute_firewall" "cidr-to-master-ha-gce-example-com" {
ports = ["4194"]
protocol = "tcp"
}
disabled = false
name = "cidr-to-master-ha-gce-example-com"
network = google_compute_network.default.name
source_ranges = ["100.64.0.0/10"]
target_tags = ["ha-gce-example-com-k8s-io-role-master"]
}

resource "google_compute_firewall" "cidr-to-master-ipv6-ha-gce-example-com" {
allow {
ports = ["443"]
protocol = "tcp"
}
allow {
ports = ["4194"]
protocol = "tcp"
}
disabled = true
name = "cidr-to-master-ipv6-ha-gce-example-com"
network = google_compute_network.default.name
target_tags = ["ha-gce-example-com-k8s-io-role-master"]
}

resource "google_compute_firewall" "cidr-to-node-ha-gce-example-com" {
allow {
protocol = "tcp"
Expand All @@ -274,23 +290,62 @@ resource "google_compute_firewall" "cidr-to-node-ha-gce-example-com" {
allow {
protocol = "sctp"
}
disabled = false
name = "cidr-to-node-ha-gce-example-com"
network = google_compute_network.default.name
source_ranges = ["100.64.0.0/10"]
target_tags = ["ha-gce-example-com-k8s-io-role-node"]
}

resource "google_compute_firewall" "cidr-to-node-ipv6-ha-gce-example-com" {
allow {
protocol = "tcp"
}
allow {
protocol = "udp"
}
allow {
protocol = "icmp"
}
allow {
protocol = "esp"
}
allow {
protocol = "ah"
}
allow {
protocol = "sctp"
}
disabled = true
name = "cidr-to-node-ipv6-ha-gce-example-com"
network = google_compute_network.default.name
target_tags = ["ha-gce-example-com-k8s-io-role-node"]
}

resource "google_compute_firewall" "kubernetes-master-https-ha-gce-example-com" {
allow {
ports = ["443"]
protocol = "tcp"
}
disabled = false
name = "kubernetes-master-https-ha-gce-example-com"
network = google_compute_network.default.name
source_ranges = ["0.0.0.0/0"]
target_tags = ["ha-gce-example-com-k8s-io-role-master"]
}

resource "google_compute_firewall" "kubernetes-master-https-ipv6-ha-gce-example-com" {
allow {
ports = ["443"]
protocol = "tcp"
}
disabled = false
name = "kubernetes-master-https-ipv6-ha-gce-example-com"
network = google_compute_network.default.name
source_ranges = ["::/0"]
target_tags = ["ha-gce-example-com-k8s-io-role-master"]
}

resource "google_compute_firewall" "master-to-master-ha-gce-example-com" {
allow {
protocol = "tcp"
Expand All @@ -310,6 +365,7 @@ resource "google_compute_firewall" "master-to-master-ha-gce-example-com" {
allow {
protocol = "sctp"
}
disabled = false
name = "master-to-master-ha-gce-example-com"
network = google_compute_network.default.name
source_tags = ["ha-gce-example-com-k8s-io-role-master"]
Expand All @@ -335,6 +391,7 @@ resource "google_compute_firewall" "master-to-node-ha-gce-example-com" {
allow {
protocol = "sctp"
}
disabled = false
name = "master-to-node-ha-gce-example-com"
network = google_compute_network.default.name
source_tags = ["ha-gce-example-com-k8s-io-role-master"]
Expand All @@ -350,6 +407,7 @@ resource "google_compute_firewall" "node-to-master-ha-gce-example-com" {
ports = ["4194"]
protocol = "tcp"
}
disabled = false
name = "node-to-master-ha-gce-example-com"
network = google_compute_network.default.name
source_tags = ["ha-gce-example-com-k8s-io-role-node"]
Expand All @@ -375,6 +433,7 @@ resource "google_compute_firewall" "node-to-node-ha-gce-example-com" {
allow {
protocol = "sctp"
}
disabled = false
name = "node-to-node-ha-gce-example-com"
network = google_compute_network.default.name
source_tags = ["ha-gce-example-com-k8s-io-role-node"]
Expand All @@ -390,9 +449,24 @@ resource "google_compute_firewall" "nodeport-external-to-node-ha-gce-example-com
ports = ["30000-32767"]
protocol = "udp"
}
disabled = true
name = "nodeport-external-to-node-ha-gce-example-com"
network = google_compute_network.default.name
source_tags = ["ha-gce-example-com-k8s-io-role-node"]
target_tags = ["ha-gce-example-com-k8s-io-role-node"]
}

resource "google_compute_firewall" "nodeport-external-to-node-ipv6-ha-gce-example-com" {
allow {
ports = ["30000-32767"]
protocol = "tcp"
}
allow {
ports = ["30000-32767"]
protocol = "udp"
}
disabled = true
name = "nodeport-external-to-node-ipv6-ha-gce-example-com"
network = google_compute_network.default.name
target_tags = ["ha-gce-example-com-k8s-io-role-node"]
}

Expand All @@ -401,23 +475,49 @@ resource "google_compute_firewall" "ssh-external-to-master-ha-gce-example-com" {
ports = ["22"]
protocol = "tcp"
}
disabled = false
name = "ssh-external-to-master-ha-gce-example-com"
network = google_compute_network.default.name
source_ranges = ["0.0.0.0/0"]
target_tags = ["ha-gce-example-com-k8s-io-role-master"]
}

resource "google_compute_firewall" "ssh-external-to-master-ipv6-ha-gce-example-com" {
allow {
ports = ["22"]
protocol = "tcp"
}
disabled = false
name = "ssh-external-to-master-ipv6-ha-gce-example-com"
network = google_compute_network.default.name
source_ranges = ["::/0"]
target_tags = ["ha-gce-example-com-k8s-io-role-master"]
}

resource "google_compute_firewall" "ssh-external-to-node-ha-gce-example-com" {
allow {
ports = ["22"]
protocol = "tcp"
}
disabled = false
name = "ssh-external-to-node-ha-gce-example-com"
network = google_compute_network.default.name
source_ranges = ["0.0.0.0/0"]
target_tags = ["ha-gce-example-com-k8s-io-role-node"]
}

resource "google_compute_firewall" "ssh-external-to-node-ipv6-ha-gce-example-com" {
allow {
ports = ["22"]
protocol = "tcp"
}
disabled = false
name = "ssh-external-to-node-ipv6-ha-gce-example-com"
network = google_compute_network.default.name
source_ranges = ["::/0"]
target_tags = ["ha-gce-example-com-k8s-io-role-node"]
}

resource "google_compute_instance_group_manager" "a-master-us-test1-a-ha-gce-example-com" {
base_instance_name = "master-us-test1-a"
name = "a-master-us-test1-a-ha-gce-example-com"
Expand Down
Loading

0 comments on commit 20e472e

Please sign in to comment.