Skip to content

Commit

Permalink
baremetal: make provisioning networks more configurable
Browse files Browse the repository at this point in the history
This makes the provisioning network more configurable, by allowing to
specify a specific interface, as well as which network should be used.
This is a prerequisite for enabling IPv6 provisioning, as well as
permitting an end-user to provide their own DHCP service in the
provisioning network.

This also adds a ProvisioningDHCPRange to the baremetal platform, and
makes that information available to the startironic template that runs
on the bootstrap. It provides the ability to disable DHCP and rely on an
external provider within the datacenter, by setting
provisioningDHCPDisabled to 'true'.

This introduces the ability to have platform-specific information
in the templating struct.

Co-authored-by: Ian Main <imain@redhat.com>
Co-authored-by: Stephen Benjamin <stephen@redhat.com>
  • Loading branch information
stbenjam and imain committed Jan 10, 2020
1 parent b02b050 commit 61cf2d8
Show file tree
Hide file tree
Showing 8 changed files with 561 additions and 183 deletions.
Expand Up @@ -11,6 +11,10 @@ COREOS_DOWNLOADER_IMAGE=$(image_for ironic-machine-os-downloader || image_for ir
# This image is templated in via the installer pkg/asset/ignition/bootstrap/bootstrap.go
RHCOS_BOOT_IMAGE_URL="{{.BootImage}}"

# This DHCP range is used by dnsmasq to serve DHCP to the cluster. If empty
# dnsmasq will only serve TFTP and DHCP will be disabled.
DHCP_RANGE="{{.PlatformData.BareMetal.ProvisioningDHCPRange}}"

# First we stop any previously started containers, because ExecStop only runs when the ExecStart process
# e.g this script is still running, but we exit if *any* of the containers exits unexpectedly
for name in ironic-api ironic-conductor ironic-inspector dnsmasq httpd mariadb ipa-downloader coreos-downloader; do
Expand All @@ -19,10 +23,9 @@ for name in ironic-api ironic-conductor ironic-inspector dnsmasq httpd mariadb i
done

# Start the provisioning nic if not already started
# Note removal of the hard-coded subnet tracked via https://github.com/openshift/installer/issues/2091
PROVISIONING_NIC=ens4
if ! nmcli -t device | grep "$PROVISIONING_NIC:ethernet:connected:provisioning"; then
nmcli c add type ethernet ifname $PROVISIONING_NIC con-name provisioning ip4 172.22.0.2/24 gw4 172.22.0.1
nmcli c add type ethernet ifname $PROVISIONING_NIC con-name provisioning ip4 {{.PlatformData.BareMetal.ProvisioningIP}}/{{.PlatformData.BareMetal.ProvisioningCIDR}}
nmcli c up provisioning
fi

Expand Down Expand Up @@ -63,6 +66,7 @@ podman run -d --net host --privileged --name mariadb \

podman run -d --net host --privileged --name dnsmasq \
--env PROVISIONING_INTERFACE=$PROVISIONING_NIC \
--env DHCP_RANGE=$DHCP_RANGE \
-v $IRONIC_SHARED_VOLUME:/shared:z --entrypoint /bin/rundnsmasq ${IRONIC_IMAGE}

podman run -d --net host --privileged --name httpd \
Expand Down
39 changes: 39 additions & 0 deletions pkg/asset/ignition/bootstrap/bootstrap.go
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"io"
"io/ioutil"
"net"
"os"
"path"
"path/filepath"
Expand All @@ -30,6 +31,7 @@ import (
"github.com/openshift/installer/pkg/asset/rhcos"
"github.com/openshift/installer/pkg/asset/tls"
"github.com/openshift/installer/pkg/types"
baremetaltypes "github.com/openshift/installer/pkg/types/baremetal"
)

const (
Expand All @@ -50,6 +52,20 @@ type bootstrapTemplateData struct {
Registries []sysregistriesv2.Registry
BootImage string
ClusterDomain string
PlatformData platformTemplateData
}

// platformTemplateData is the data to use to replace values in bootstrap
// template files that are specific to one platform.
type platformTemplateData struct {
BareMetal *baremetalTemplateData
}

// baremetalTemplateData holds data specific to templates used for the baremetal platform.
type baremetalTemplateData struct {
ProvisioningIP string
ProvisioningCIDR int
ProvisioningDHCPRange string
}

// Bootstrap is an asset that generates the ignition config for bootstrap nodes.
Expand Down Expand Up @@ -223,6 +239,8 @@ func (a *Bootstrap) getTemplateData(installConfig *types.InstallConfig, releaseI
registries = append(registries, registry)
}

platformData := getPlatformData(installConfig)

return &bootstrapTemplateData{
AdditionalTrustBundle: installConfig.AdditionalTrustBundle,
FIPS: installConfig.FIPS,
Expand All @@ -233,9 +251,30 @@ func (a *Bootstrap) getTemplateData(installConfig *types.InstallConfig, releaseI
Registries: registries,
BootImage: string(*rhcosImage),
ClusterDomain: installConfig.ClusterDomain(),
PlatformData: platformData,
}, nil
}

// getPlatformData returns platform-specific data for bootstrap templates.
func getPlatformData(config *types.InstallConfig) (platformData platformTemplateData) {
switch config.Platform.Name() {
case baremetaltypes.Name:
_, provNetwork, _ := net.ParseCIDR(config.Platform.BareMetal.ProvisioningNetworkCIDR)
cidr, _ := provNetwork.Mask.Size()

platformData.BareMetal = &baremetalTemplateData{
ProvisioningIP: config.Platform.BareMetal.BootstrapProvisioningIP,
ProvisioningCIDR: cidr,
}

if !config.Platform.BareMetal.ProvisioningDHCPDisabled {
platformData.BareMetal.ProvisioningDHCPRange = config.Platform.BareMetal.ProvisioningDHCPRange
}
}

return
}

func (a *Bootstrap) addStorageFiles(base string, uri string, templateData *bootstrapTemplateData) (err error) {
file, err := data.Assets.Open(uri)
if err != nil {
Expand Down
57 changes: 45 additions & 12 deletions pkg/types/baremetal/defaults/platform.go
Expand Up @@ -4,34 +4,63 @@ import (
"fmt"
"net"

"github.com/apparentlymart/go-cidr/cidr"
"github.com/openshift/installer/pkg/types"
"github.com/openshift/installer/pkg/types/baremetal"
)

// Defaults for the baremetal platform.
const (
LibvirtURI = "qemu:///system"
BootstrapProvisioningIP = "172.22.0.2"
ClusterProvisioningIP = "172.22.0.3"
ExternalBridge = "baremetal"
ProvisioningBridge = "provisioning"
HardwareProfile = "default"
APIVIP = ""
IngressVIP = ""
LibvirtURI = "qemu:///system"
ProvisioningNetworkCIDR = "172.22.0.0/24"
ExternalBridge = "baremetal"
ProvisioningBridge = "provisioning"
ProvisioningNetworkInterface = "ens3"
HardwareProfile = "default"
APIVIP = ""
IngressVIP = ""
)

// Wrapper for net.LookupHost so we can override in the test
var lookupHost = func(host string) (addrs []string, err error) {
return net.LookupHost(host)
}

// SetPlatformDefaults sets the defaults for the platform.
func SetPlatformDefaults(p *baremetal.Platform, c *types.InstallConfig) {
if p.LibvirtURI == "" {
p.LibvirtURI = LibvirtURI
}

if p.ProvisioningNetworkCIDR == "" {
p.ProvisioningNetworkCIDR = ProvisioningNetworkCIDR
}

_, provNet, _ := net.ParseCIDR(p.ProvisioningNetworkCIDR)

// If the user doesn't provide an explicit DHCP range, and DHCP is not
// disabled, then we set a default value from the 10th to 100th
// address in the network.
if !p.ProvisioningDHCPDisabled && p.ProvisioningDHCPRange == "" {
startIP, _ := cidr.Host(provNet, 10)
endIP, _ := cidr.Host(provNet, 100)
p.ProvisioningDHCPRange = fmt.Sprintf("%s,%s", startIP, endIP)
}

if p.BootstrapProvisioningIP == "" {
p.BootstrapProvisioningIP = BootstrapProvisioningIP
// Default to the second address in provisioning network, e.g 172.22.0.2
ip, err := cidr.Host(provNet, 2)
if err == nil {
p.BootstrapProvisioningIP = ip.String()
}
}

if p.ClusterProvisioningIP == "" {
p.ClusterProvisioningIP = ClusterProvisioningIP
// Default to the third address in provisioning network, e.g 172.22.0.3
ip, err := cidr.Host(provNet, 3)
if err == nil {
p.ClusterProvisioningIP = ip.String()
}
}

if p.ExternalBridge == "" {
Expand All @@ -42,6 +71,10 @@ func SetPlatformDefaults(p *baremetal.Platform, c *types.InstallConfig) {
p.ProvisioningBridge = ProvisioningBridge
}

if p.ProvisioningNetworkInterface == "" {
p.ProvisioningNetworkInterface = ProvisioningNetworkInterface
}

for _, host := range p.Hosts {
if host.HardwareProfile == "" {
host.HardwareProfile = HardwareProfile
Expand All @@ -50,7 +83,7 @@ func SetPlatformDefaults(p *baremetal.Platform, c *types.InstallConfig) {

if p.APIVIP == APIVIP {
// This name should resolve to exactly one address
vip, err := net.LookupHost("api." + c.ClusterDomain())
vip, err := lookupHost("api." + c.ClusterDomain())
if err != nil {
// This will fail validation and abort the install
p.APIVIP = fmt.Sprintf("DNS lookup failure: %s", err.Error())
Expand All @@ -61,7 +94,7 @@ func SetPlatformDefaults(p *baremetal.Platform, c *types.InstallConfig) {

if p.IngressVIP == IngressVIP {
// This name should resolve to exactly one address
vip, err := net.LookupHost("test.apps." + c.ClusterDomain())
vip, err := lookupHost("test.apps." + c.ClusterDomain())
if err != nil {
// This will fail validation and abort the install
p.IngressVIP = fmt.Sprintf("DNS lookup failure: %s", err.Error())
Expand Down
101 changes: 101 additions & 0 deletions pkg/types/baremetal/defaults/platform_test.go
@@ -0,0 +1,101 @@
package defaults

import (
"errors"
"testing"

"github.com/stretchr/testify/assert"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

"github.com/openshift/installer/pkg/types"
"github.com/openshift/installer/pkg/types/baremetal"
)

const testClusterName = "test-cluster"

func TestSetPlatformDefaults(t *testing.T) {
// Stub the call to net.LookupHost
lookupHost = func(host string) (addrs []string, err error) {
if host == "api.test-cluster.test" {
ips := []string{"192.168.111.2"}
return ips, nil
} else if host == "test.apps.test-cluster.test" {
ips := []string{"192.168.111.3"}
return ips, nil
} else {
return nil, errors.New("Unknown Host " + host)
}
}

cases := []struct {
name string
platform *baremetal.Platform
expected *baremetal.Platform
}{
{
name: "default_empty",
platform: &baremetal.Platform{},
expected: &baremetal.Platform{
LibvirtURI: "qemu:///system",
ClusterProvisioningIP: "172.22.0.3",
BootstrapProvisioningIP: "172.22.0.2",
ExternalBridge: "baremetal",
ProvisioningBridge: "provisioning",
APIVIP: "192.168.111.2",
IngressVIP: "192.168.111.3",
ProvisioningNetworkCIDR: "172.22.0.0/24",
ProvisioningDHCPRange: "172.22.0.10,172.22.0.100",
ProvisioningNetworkInterface: "ens3",
},
},
{
name: "alternate_cidr",
platform: &baremetal.Platform{
ProvisioningNetworkCIDR: "172.23.0.0/24",
},
expected: &baremetal.Platform{
LibvirtURI: "qemu:///system",
ClusterProvisioningIP: "172.23.0.3",
BootstrapProvisioningIP: "172.23.0.2",
ExternalBridge: "baremetal",
ProvisioningBridge: "provisioning",
APIVIP: "192.168.111.2",
IngressVIP: "192.168.111.3",
ProvisioningNetworkCIDR: "172.23.0.0/24",
ProvisioningDHCPRange: "172.23.0.10,172.23.0.100",
ProvisioningNetworkInterface: "ens3",
},
},
{
name: "alternate_cidr_dhcp_disabled",
platform: &baremetal.Platform{
ProvisioningNetworkCIDR: "172.23.0.0/24",
ProvisioningDHCPDisabled: true,
},
expected: &baremetal.Platform{
LibvirtURI: "qemu:///system",
ClusterProvisioningIP: "172.23.0.3",
BootstrapProvisioningIP: "172.23.0.2",
ExternalBridge: "baremetal",
ProvisioningBridge: "provisioning",
APIVIP: "192.168.111.2",
IngressVIP: "192.168.111.3",
ProvisioningNetworkCIDR: "172.23.0.0/24",
ProvisioningDHCPDisabled: true,
ProvisioningNetworkInterface: "ens3",
},
},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
ic := &types.InstallConfig{
ObjectMeta: metav1.ObjectMeta{
Name: testClusterName,
},
BaseDomain: "test",
}
SetPlatformDefaults(tc.platform, ic)
assert.Equal(t, tc.expected, tc.platform, "unexpected platform")
})
}
}
23 changes: 22 additions & 1 deletion pkg/types/baremetal/platform.go
Expand Up @@ -40,10 +40,31 @@ type Platform struct {
// +optional
ExternalBridge string `json:"externalBridge,omitempty"`

// Provisioning bridge is used for provisioning nodes.
// Provisioning bridge is used for provisioning nodes, on the host that
// will run the bootstrap VM.
// +optional
ProvisioningBridge string `json:"provisioningBridge,omitempty"`

// ProvisioningNetworkInterface is the name of the network interface on a baremetal server
// that it uses to connect to the provisioning network.
ProvisioningNetworkInterface string `json:"provisioningNetworkInterface"`

// ProvisioningNetworkCIDR defines the network to use for provisioning.
// +optional
ProvisioningNetworkCIDR string `json:"provisioningNetworkCIDR,omitempty"`

// ProvisioningDHCPDisabled is used to disable DHCP services. DHCP must
// be provided by an external service, appropriately configured with next-server
// set to BootstrapProvisioningIP for the control plane, and ClusterProvisioningIP
// for workers.
// +optional
ProvisioningDHCPDisabled bool `json:"provisioningDHCPDisabled,omitempty"`

// ProvisioningDHCPRange is used to provide DHCP services to hosts
// for provisioning.
// +optional
ProvisioningDHCPRange string `json:"provisioningDHCPRange,omitempty"`

// Hosts is the information needed to create the objects in Ironic.
Hosts []*Host `json:"hosts"`

Expand Down

0 comments on commit 61cf2d8

Please sign in to comment.