Skip to content

Commit

Permalink
feat(autok3s): support --tls-sans
Browse files Browse the repository at this point in the history 
Signed-off-by: Jason-ZW <zhenyang@rancher.com>
  • Loading branch information
@rancher-sy-bot @Jason-ZW
rancher-sy-bot authored and Jason-ZW committed May 14, 2021
1 parent 71d8fa9 commit d1a5324
Show file tree
Hide file tree
Showing 7 changed files with 176 additions and 34 deletions.
6 changes: 6 additions & 0 deletions pkg/cluster/base.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,12 @@ func (p *ProviderBase) GetClusterOptions() []types.Flag {
V: p.Token,
Usage: "K3s token, if empty will automatically generated, see: https://rancher.com/docs/k3s/latest/en/installation/install-options/server-config/#cluster-options",
},
{
Name: "tls-sans",
P: &p.TLSSans,
V: p.TLSSans,
Usage: "Add additional hostnames or IPv4/IPv6 addresses as Subject Alternative Names on the server TLS cert, e.g.(--tls-sans 192.168.1.10 --tls-sans 192.168.2.10)",
},
{
Name: "master",
P: &p.Master,
Expand Down
52 changes: 41 additions & 11 deletions pkg/cluster/cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ import (
)

var (
initCommand = "curl -sLS %s | %s K3S_TOKEN='%s' INSTALL_K3S_EXEC='server --tls-san %s --node-external-ip %s %s' %s sh -"
initCommand = "curl -sLS %s | %s K3S_TOKEN='%s' INSTALL_K3S_EXEC='server %s --node-external-ip %s %s' %s sh -"
joinCommand = "curl -sLS %s | %s K3S_URL='https://%s:6443' K3S_TOKEN='%s' INSTALL_K3S_EXEC='%s' %s sh -"
getTokenCommand = "sudo cat /var/lib/rancher/k3s/server/node-token"
catCfgCommand = "sudo cat /etc/rancher/k3s/k3s.yaml"
Expand Down Expand Up @@ -73,6 +73,22 @@ func (p *ProviderBase) InitK3sCluster(cluster *types.Cluster) error {
publicIP = cluster.MasterNodes[0].PublicIPAddress[0]
}

// append tls-sans to k3s install script:
// 1. appends from --tls-sans flags.
// 2. appends all master nodes' first public address.
var tlsSans string
p.TLSSans = append(p.TLSSans, publicIP)
for _, master := range cluster.MasterNodes {
if master.PublicIPAddress[0] != "" && master.PublicIPAddress[0] != publicIP {
p.TLSSans = append(p.TLSSans, master.PublicIPAddress[0])
}
}
for _, tlsSan := range p.TLSSans {
tlsSans = tlsSans + fmt.Sprintf(" --tls-san %s", tlsSan)
}
// save p.TlsSans to db.
cluster.TLSSans = p.TLSSans

masterExtraArgs := cluster.MasterExtraArgs
workerExtraArgs := cluster.WorkerExtraArgs

Expand Down Expand Up @@ -103,7 +119,7 @@ func (p *ProviderBase) InitK3sCluster(cluster *types.Cluster) error {
if cluster.DockerScript != "" {
dockerCommand = cluster.DockerScript
}
if err := p.initMaster(k3sScript, k3sMirror, dockerMirror, publicIP, master0ExtraArgs, cluster, cluster.MasterNodes[0]); err != nil {
if err := p.initMaster(k3sScript, k3sMirror, dockerMirror, tlsSans, publicIP, master0ExtraArgs, cluster, cluster.MasterNodes[0]); err != nil {
return err
}
p.Logger.Infof("[%s] successfully created k3s master-%d", p.Provider, 1)
Expand All @@ -119,7 +135,7 @@ func (p *ProviderBase) InitK3sCluster(cluster *types.Cluster) error {
if providerExtraArgs != "" {
masterNExtraArgs += providerExtraArgs
}
if err := p.initAdditionalMaster(k3sScript, k3sMirror, dockerMirror, publicIP, masterNExtraArgs, cluster, master); err != nil {
if err := p.initAdditionalMaster(k3sScript, k3sMirror, dockerMirror, tlsSans, publicIP, masterNExtraArgs, cluster, master); err != nil {
return err
}
p.Logger.Infof("[%s] successfully created k3s master-%d", p.Provider, i+1)
Expand Down Expand Up @@ -233,6 +249,20 @@ func (p *ProviderBase) Join(merged, added *types.Cluster) error {
if merged.DockerScript != "" {
dockerCommand = merged.DockerScript
}

// append tls-sans to k3s install script:
// 1. appends from --tls-sans flags.
// 2. appends all master nodes' first public address.
var tlsSans string
for _, master := range added.MasterNodes {
if master.PublicIPAddress[0] != "" {
merged.TLSSans = append(merged.TLSSans, master.PublicIPAddress[0])
}
}
for _, tlsSan := range merged.TLSSans {
tlsSans = tlsSans + fmt.Sprintf(" --tls-san %s", tlsSan)
}

errChan := make(chan error)
waitGroupDone := make(chan bool)
waitGroup := &sync.WaitGroup{}
Expand All @@ -247,7 +277,7 @@ func (p *ProviderBase) Join(merged, added *types.Cluster) error {
if additionalExtraArgs != "" {
extraArgs += additionalExtraArgs
}
if err := p.joinMaster(k3sScript, k3sMirror, dockerMirror, extraArgs, merged, full); err != nil {
if err := p.joinMaster(k3sScript, k3sMirror, dockerMirror, extraArgs, tlsSans, merged, full); err != nil {
return err
}
p.Logger.Infof("[%s] successfully joined k3s master-%d", merged.Provider, i+1)
Expand Down Expand Up @@ -422,7 +452,7 @@ func (p *ProviderBase) DeployExtraManifest(cluster *types.Cluster, cmds []string
return nil
}

func (p *ProviderBase) initMaster(k3sScript, k3sMirror, dockerMirror, ip, extraArgs string, cluster *types.Cluster, master types.Node) error {
func (p *ProviderBase) initMaster(k3sScript, k3sMirror, dockerMirror, tlsSans, ip, extraArgs string, cluster *types.Cluster, master types.Node) error {
if strings.Contains(extraArgs, "--docker") {
p.Logger.Infof("[cluster] install docker command %s", fmt.Sprintf(dockerCommand, dockerMirror))
if _, err := p.execute(&master, []string{fmt.Sprintf(dockerCommand, dockerMirror)}); err != nil {
Expand All @@ -437,17 +467,17 @@ func (p *ProviderBase) initMaster(k3sScript, k3sMirror, dockerMirror, ip, extraA
}

p.Logger.Infof("[cluster] k3s master command: %s", fmt.Sprintf(initCommand, k3sScript, k3sMirror, cluster.Token,
ip, ip, strings.TrimSpace(extraArgs), genK3sVersion(cluster.K3sVersion, cluster.K3sChannel)))
tlsSans, ip, strings.TrimSpace(extraArgs), genK3sVersion(cluster.K3sVersion, cluster.K3sChannel)))

if _, err := p.execute(&master, []string{fmt.Sprintf(initCommand, k3sScript, k3sMirror,
cluster.Token, ip, ip, strings.TrimSpace(extraArgs), genK3sVersion(cluster.K3sVersion, cluster.K3sChannel))}); err != nil {
cluster.Token, tlsSans, ip, strings.TrimSpace(extraArgs), genK3sVersion(cluster.K3sVersion, cluster.K3sChannel))}); err != nil {
return err
}

return nil
}

func (p *ProviderBase) initAdditionalMaster(k3sScript, k3sMirror, dockerMirror, ip, extraArgs string, cluster *types.Cluster, master types.Node) error {
func (p *ProviderBase) initAdditionalMaster(k3sScript, k3sMirror, dockerMirror, tlsSans, ip, extraArgs string, cluster *types.Cluster, master types.Node) error {
sortedExtraArgs := ""

if strings.Contains(extraArgs, "--docker") {
Expand All @@ -463,7 +493,7 @@ func (p *ProviderBase) initAdditionalMaster(k3sScript, k3sMirror, dockerMirror,
}

if !strings.Contains(extraArgs, "server --server") {
sortedExtraArgs += fmt.Sprintf(" server --server %s --tls-san %s --node-external-ip %s", fmt.Sprintf("https://%s:6443", ip), master.PublicIPAddress[0], master.PublicIPAddress[0])
sortedExtraArgs += fmt.Sprintf(" server --server %s %s --node-external-ip %s", fmt.Sprintf("https://%s:6443", ip), tlsSans, master.PublicIPAddress[0])
}

sortedExtraArgs += " " + extraArgs
Expand Down Expand Up @@ -510,11 +540,11 @@ func (p *ProviderBase) initWorker(wg *sync.WaitGroup, errChan chan error, k3sScr
}

func (p *ProviderBase) joinMaster(k3sScript, k3sMirror, dockerMirror,
extraArgs string, merged *types.Cluster, full types.Node) error {
extraArgs, tlsSans string, merged *types.Cluster, full types.Node) error {
sortedExtraArgs := ""

if !strings.Contains(extraArgs, "server --server") {
sortedExtraArgs += fmt.Sprintf(" server --server %s --tls-san %s --node-external-ip %s", fmt.Sprintf("https://%s:6443", merged.IP), full.PublicIPAddress[0], full.PublicIPAddress[0])
sortedExtraArgs += fmt.Sprintf(" server --server %s %s --node-external-ip %s", fmt.Sprintf("https://%s:6443", merged.IP), tlsSans, full.PublicIPAddress[0])
}

if merged.DataStore != "" {
Expand Down
2 changes: 2 additions & 0 deletions pkg/common/db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ var (
provider TEXT not null,
token TEXT,
ip TEXT,
tls_sans TEXT,
cluster_cidr TEXT,
master_extra_args TEXT,
worker_extra_args TEXT,
Expand Down Expand Up @@ -57,6 +58,7 @@ var (
provider TEXT not null,
token TEXT,
ip TEXT,
tls_sans TEXT,
cluster_cidr TEXT,
master_extra_args TEXT,
worker_extra_args TEXT,
Expand Down
1 change: 1 addition & 0 deletions pkg/common/model.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,7 @@ func toCluster(state *ClusterState) types.Cluster {
Status: state.Status,
},
}

p, err := providers.GetProvider(state.Provider)
if err != nil {
logrus.Errorf("failed to get provider by name %s", state.Provider)
Expand Down
1 change: 1 addition & 0 deletions pkg/types/apis/types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package apis

import (
"github.com/cnrancher/autok3s/pkg/types"

"github.com/rancher/wrangler/pkg/schemas"
)

Expand Down
76 changes: 53 additions & 23 deletions pkg/types/autok3s.go
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,11 @@
package types

import (
"database/sql/driver"
"fmt"
"strings"
)

type AutoK3s struct {
Clusters []Cluster `json:"clusters" yaml:"clusters"`
}
Expand All @@ -13,29 +19,30 @@ type Cluster struct {
}

type Metadata struct {
Name string `json:"name" yaml:"name"`
Provider string `json:"provider" yaml:"provider"`
Master string `json:"master" yaml:"master"`
Worker string `json:"worker" yaml:"worker"`
Token string `json:"token,omitempty" yaml:"token,omitempty"`
IP string `json:"ip,omitempty" yaml:"ip,omitempty"`
ClusterCidr string `json:"cluster-cidr,omitempty" yaml:"cluster-cidr,omitempty"`
MasterExtraArgs string `json:"master-extra-args,omitempty" yaml:"master-extra-args,omitempty"`
WorkerExtraArgs string `json:"worker-extra-args,omitempty" yaml:"worker-extra-args,omitempty"`
Registry string `json:"registry,omitempty" yaml:"registry,omitempty"`
DataStore string `json:"datastore,omitempty" yaml:"datastore,omitempty"`
K3sVersion string `json:"k3s-version,omitempty" yaml:"k3s-version,omitempty"`
K3sChannel string `json:"k3s-channel,omitempty" yaml:"k3s-channel,omitempty"`
InstallScript string `json:"k3s-install-script,omitempty" yaml:"k3s-install-script,omitempty"`
Mirror string `json:"k3s-install-mirror,omitempty" yaml:"k3s-install-mirror,omitempty"`
DockerMirror string `json:"dockerMirror,omitempty" yaml:"dockerMirror,omitempty"`
DockerScript string `json:"dockerScript,omitempty" yaml:"dockerScript,omitempty"`
Network string `json:"network,omitempty" yaml:"network,omitempty"`
UI bool `json:"ui" yaml:"ui"`
Cluster bool `json:"cluster" yaml:"cluster"`
ContextName string `json:"context-name" yaml:"context-name"`
RegistryContent string `json:"registry-content,omitempty" yaml:"registry-content,omitempty"`
Manifests string `json:"manifests,omitempty" yaml:"manifests,omitempty"`
Name string `json:"name" yaml:"name"`
Provider string `json:"provider" yaml:"provider"`
Master string `json:"master" yaml:"master"`
Worker string `json:"worker" yaml:"worker"`
Token string `json:"token,omitempty" yaml:"token,omitempty"`
IP string `json:"ip,omitempty" yaml:"ip,omitempty"`
TLSSans StringArray `json:"tls-sans,omitempty" yaml:"tls-sans,omitempty" gorm:"type:stringArray"`
ClusterCidr string `json:"cluster-cidr,omitempty" yaml:"cluster-cidr,omitempty"`
MasterExtraArgs string `json:"master-extra-args,omitempty" yaml:"master-extra-args,omitempty"`
WorkerExtraArgs string `json:"worker-extra-args,omitempty" yaml:"worker-extra-args,omitempty"`
Registry string `json:"registry,omitempty" yaml:"registry,omitempty"`
DataStore string `json:"datastore,omitempty" yaml:"datastore,omitempty"`
K3sVersion string `json:"k3s-version,omitempty" yaml:"k3s-version,omitempty"`
K3sChannel string `json:"k3s-channel,omitempty" yaml:"k3s-channel,omitempty"`
InstallScript string `json:"k3s-install-script,omitempty" yaml:"k3s-install-script,omitempty"`
Mirror string `json:"k3s-install-mirror,omitempty" yaml:"k3s-install-mirror,omitempty"`
DockerMirror string `json:"dockerMirror,omitempty" yaml:"dockerMirror,omitempty"`
DockerScript string `json:"dockerScript,omitempty" yaml:"dockerScript,omitempty"`
Network string `json:"network,omitempty" yaml:"network,omitempty"`
UI bool `json:"ui" yaml:"ui"`
Cluster bool `json:"cluster" yaml:"cluster"`
ContextName string `json:"context-name" yaml:"context-name"`
RegistryContent string `json:"registry-content,omitempty" yaml:"registry-content,omitempty"`
Manifests string `json:"manifests,omitempty" yaml:"manifests,omitempty"`
}

type Status struct {
Expand Down Expand Up @@ -109,3 +116,26 @@ type ClusterNode struct {
Version string `json:"version,omitempty"`
Master bool `json:"-"`
}

type StringArray []string

func (a *StringArray) Scan(value interface{}) (err error) {
switch v := value.(type) {
case string:
*a = strings.Split(v, ",")
default:
return fmt.Errorf("failed to scan array value %v", value)
}
return nil
}

func (a StringArray) Value() (driver.Value, error) {
if a == nil {
return nil, nil
}
return strings.Join(a, ","), nil
}

func (a StringArray) GormDataType() string {
return "stringArray"
}
72 changes: 72 additions & 0 deletions pkg/utils/flag.go
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
package utils

import (
"bytes"
"encoding/csv"
"strings"

"github.com/cnrancher/autok3s/pkg/types"

"github.com/spf13/cobra"
Expand All @@ -24,6 +28,8 @@ func ConvertFlags(cmd *cobra.Command, fs []types.Flag) *pflag.FlagSet {
pf.StringToStringVar(f.P.(*map[string]string), f.Name, t, f.Usage)
case []string:
pf.StringArrayVar(f.P.(*[]string), f.Name, t, f.Usage)
case types.StringArray:
pf.Var(newStringArrayValue(t, f.P.(*types.StringArray)), f.Name, f.Usage)
default:
continue
}
Expand All @@ -43,6 +49,8 @@ func ConvertFlags(cmd *cobra.Command, fs []types.Flag) *pflag.FlagSet {
pf.StringToStringVarP(f.P.(*map[string]string), f.Name, f.ShortHand, t, f.Usage)
case []string:
pf.StringArrayVarP(f.P.(*[]string), f.Name, f.ShortHand, t, f.Usage)
case types.StringArray:
pf.VarP(newStringArrayValue(t, f.P.(*types.StringArray)), f.Name, f.ShortHand, f.Usage)
default:
continue
}
Expand Down Expand Up @@ -74,3 +82,67 @@ func ValidateRequiredFlags(flags *pflag.FlagSet) {
}
})
}

type stringArrayValue struct {
value *types.StringArray
changed bool
}

func newStringArrayValue(val []string, p *types.StringArray) *stringArrayValue {
ssv := new(stringArrayValue)
ssv.value = p
*ssv.value = val
return ssv
}

func (s *stringArrayValue) Set(val string) error {
if !s.changed {
*s.value = []string{val}
s.changed = true
} else {
*s.value = append(*s.value, val)
}
return nil
}

func (s *stringArrayValue) Append(val string) error {
*s.value = append(*s.value, val)
return nil
}

func (s *stringArrayValue) Replace(val []string) error {
out := make([]string, len(val))
for i, d := range val {
out[i] = d
}
*s.value = out
return nil
}

func (s *stringArrayValue) GetSlice() []string {
out := make([]string, len(*s.value))
for i, d := range *s.value {
out[i] = d
}
return out
}

func (s *stringArrayValue) Type() string {
return "stringArray"
}

func (s *stringArrayValue) String() string {
str, _ := writeAsCSV(*s.value)
return "[" + str + "]"
}

func writeAsCSV(ss []string) (string, error) {
b := &bytes.Buffer{}
w := csv.NewWriter(b)
err := w.Write(ss)
if err != nil {
return "", err
}
w.Flush()
return strings.TrimSuffix(b.String(), "\n"), nil
}

0 comments on commit d1a5324

Please sign in to comment.