Merge pull request #1608 from kaccardi/topic/reserve-cnci-net

send cnci refresh command

ciao-controller/client.go

@@ -45,6 +45,7 @@ type controllerClient interface {
 	unMapExternalIP(t types.Tenant, m types.MappedIP) error
 	attachVolume(volID string, instanceID string, nodeID string) error
 	ssntpClient() *ssntp.Client
+	CNCIRefresh(cnciID string, cnciList []payloads.CNCINet) error
 }
 
 type ssntpClient struct {
@@ -777,3 +778,23 @@ func (client *ssntpClient) unMapExternalIP(t types.Tenant, m types.MappedIP) err
 	_, err = client.ssntp.SendCommand(ssntp.ReleasePublicIP, y)
 	return err
 }
+
+func (client *ssntpClient) CNCIRefresh(cnciID string, cnciList []payloads.CNCINet) error {
+	payload := payloads.CommandCNCIRefresh{
+		Command: payloads.CNCIRefreshCommand{
+			CNCIUUID: cnciID,
+			CNCIList: cnciList,
+		},
+	}
+
+	y, err := yaml.Marshal(payload)
+	if err != nil {
+		return err
+	}
+
+	glog.Infof("Refresh CNCI %s: %v\n", cnciID, cnciList)
+	glog.V(1).Info(string(y))
+
+	_, err = client.ssntp.SendCommand(ssntp.RefreshCNCI, y)
+	return err
+}

ciao-controller/client_wrapper_test.go

@@ -22,6 +22,7 @@ import (
 	"time"
 
 	"github.com/ciao-project/ciao/ciao-controller/types"
+	"github.com/ciao-project/ciao/payloads"
 	"github.com/ciao-project/ciao/ssntp"
 )
 
@@ -272,3 +273,7 @@ func (client *ssntpClientWrapper) sendAndDelEventChan(cmd ssntp.Event) {
 func (client *ssntpClientWrapper) RemoveInstance(ID string) {
 	client.realClient.RemoveInstance(ID)
 }
+
+func (client *ssntpClientWrapper) CNCIRefresh(cnciID string, cnciList []payloads.CNCINet) error {
+	return client.realClient.CNCIRefresh(cnciID, cnciList)
+}

ciao-controller/cnci.go

@@ -16,8 +16,11 @@ package main
 
 import (
 	"crypto/rand"
+	"encoding/binary"
 	"encoding/hex"
 	"fmt"
+	"hash/crc32"
+	"net"
 	"sync"
 	"time"
 
@@ -122,6 +125,36 @@ func (c *CNCI) transitionState(to CNCIState) {
 	}
 }
 
+func getTunnelIP(subnet string) net.IP {
+	startTunnelIP := net.ParseIP(cnciNet.String())
+	IP, ipNet, err := net.ParseCIDR(subnet)
+	if err != nil {
+		return nil
+	}
+
+	ones, bits := ipNet.Mask.Size()
+	hostBits := bits - ones
+
+	addr := binary.BigEndian.Uint32(IP.To4())
+	mask := binary.BigEndian.Uint32(ipNet.Mask)
+	start := binary.BigEndian.Uint32(startTunnelIP.To4())
+	subnetNum := addr & mask
+
+	// to calculate the tunnelIP, use the significant subnet
+	// bits only. Since the top 12 bits are always the same,
+	// get rid of them.
+	tunnelNum := (subnetNum & 0x00cfffff) >> uint(hostBits)
+
+	// add one to this value so that we don't allocate host 0
+	tunnelNum++
+
+	tunnelIP := make(net.IP, net.IPv4len)
+	addr = start + uint32(tunnelNum)
+	binary.BigEndian.PutUint32(tunnelIP, addr)
+
+	return tunnelIP
+}
+
 // Active will return true if the CNCI has been launched successfully
 func (c *CNCIManager) Active(ID string) bool {
 	c.cnciLock.RLock()
@@ -223,7 +256,12 @@ func (c *CNCIManager) WaitForActive(subnet string) error {
 
 	// we release the lock before waiting because
 	// we need to be able to read the event channel.
-	return waitForEventTimeout(ch, added, cnciEventTimeout)
+	err = waitForEventTimeout(ch, added, cnciEventTimeout)
+	if err != nil {
+		return err
+	}
+
+	return c.refresh()
 }
 
 // ScheduleRemoveSubnet will kick off a timer to remove a subnet after 5 min.
@@ -294,7 +332,12 @@ func (c *CNCIManager) RemoveSubnet(subnet string) error {
 
 	c.cnciLock.Unlock()
 
-	return waitForEventTimeout(ch, removed, cnciEventTimeout)
+	err = waitForEventTimeout(ch, removed, cnciEventTimeout)
+	if err != nil {
+		return err
+	}
+
+	return c.refresh()
 }
 
 // CNCIRemoved will move the CNCI back to the initial state
@@ -401,6 +444,41 @@ func (c *CNCIManager) waitForActive(subnet string) error {
 	return errors.New("CNCI not active")
 }
 
+func (c *CNCIManager) refresh() error {
+	c.cnciLock.RLock()
+	defer c.cnciLock.RUnlock()
+
+	var cnciList []payloads.CNCINet
+
+	// create a ConcentratorInstanceRefresh struct for each cnci
+	for _, cnci := range c.cncis {
+		tunnelID := crc32.ChecksumIEEE([]byte(c.tenant))
+		tunnelIP := getTunnelIP(cnci.instance.Subnet)
+		if tunnelIP == nil {
+			return errors.New("Unable to derive CNCI tunnel IP")
+		}
+
+		r := payloads.CNCINet{
+			PhysicalIP: cnci.instance.IPAddress,
+			Subnet:     cnci.instance.Subnet,
+			TunnelIP:   tunnelIP.String(),
+			TunnelID:   tunnelID,
+		}
+		cnciList = append(cnciList, r)
+	}
+
+	// send the event to each cnci
+	for _, cnci := range c.cncis {
+		err := c.ctrl.client.CNCIRefresh(cnci.instance.ID, cnciList)
+		if err != nil {
+			// keep going, but log error.
+			glog.Warningf("Unable to send cnci refresh to %s: (%v)", cnci.instance.ID, err)
+		}
+	}
+
+	return nil
+}
+
 // GetInstanceCNCI will return the CNCI Instance for a specific tenant Instance
 func (c *CNCIManager) GetInstanceCNCI(ID string) (*types.Instance, error) {
 	// figure out what subnet we are looking for.

ciao-controller/main.go

@@ -22,6 +22,7 @@ import (
 	"crypto/x509"
 	"flag"
 	"fmt"
+	"net"
 	"net/http"
 	"os"
 	"os/signal"
@@ -56,6 +57,23 @@ type controller struct {
 	httpServers         []*http.Server
 }
 
+type cnciNetFlag string
+
+func (c *cnciNetFlag) String() string {
+	return string(*c)
+}
+
+func (c *cnciNetFlag) Set(val string) error {
+	IP := net.ParseIP(val)
+	if IP == nil {
+		return fmt.Errorf("Unable to parse CNCI network address")
+	}
+
+	*c = cnciNetFlag(IP.String())
+
+	return nil
+}
+
 var cert = flag.String("cert", "", "Client certificate")
 var caCert = flag.String("cacert", "", "CA certificate")
 var serverURL = flag.String("url", "", "Server URL")
@@ -73,6 +91,10 @@ var cephID = flag.String("ceph_id", "", "ceph client id")
 
 var adminSSHKey = ""
 
+// this default allows us to have up to 32K hosts within the upper part
+// of the 192.168.0.0/16 private address space.
+var cnciNet cnciNetFlag = "192.168.128.0"
+
 func init() {
 	flag.Parse()
 
@@ -195,6 +217,14 @@ func main() {
 		clientCertCAPath = clusterConfig.Configure.Controller.ClientAuthCACertPath
 	}
 
+	if clusterConfig.Configure.Controller.CNCINet != "" {
+		err = cnciNet.Set(clusterConfig.Configure.Controller.CNCINet)
+		if err != nil {
+			glog.Fatalf("Invalid CNCI Net cluster configuration: %v", err)
+			return
+		}
+	}
+
 	ctl.ds.GenerateCNCIWorkload(cnciVCPUs, cnciMem, cnciDisk, adminSSHKey)
 
 	database.Logger = gloginterface.CiaoGlogLogger{}

ciao-deploy/cmd/setup.go

@@ -140,6 +140,7 @@ func init() {
 	setupCmd.Flags().StringVar(&clusterConf.AdminSSHKeyPath, "admin-ssh-key", "", "Path to SSH public key for accessing CNCI")
 	setupCmd.Flags().StringVar(&clusterConf.ComputeNet, "compute-net", hostNetwork, "Network range for compute network")
 	setupCmd.Flags().StringVar(&clusterConf.MgmtNet, "mgmt-net", hostNetwork, "Network range for management network")
+	setupCmd.Flags().StringVar(&clusterConf.CNCINet, "cnci-net", "192.168.128.0", "Host start address for CNCI mgmt network - must be at least /18")
 	setupCmd.Flags().StringVar(&clusterConf.ServerIP, "server-ip", hostIP, "IP address nodes can reach this host on")
 	setupCmd.Flags().StringVar(&clusterConf.ServerHostname, "server-hostname", deploy.HostnameWithFallback(), "Name or FQDN that this host can be reached on")
 	setupCmd.Flags().StringVar(&imageCacheDirectory, "image-cache-directory", deploy.DefaultImageCacheDir(), "Directory to use for caching of downloaded images")

ciao-deploy/deploy/setup.go

@@ -44,6 +44,7 @@ type ClusterConfiguration struct {
 	AdminSSHKeyPath   string
 	ComputeNet        string
 	MgmtNet           string
+	CNCINet           string
 	ServerIP          string
 	AuthCACertPath    string
 	AuthAdminCertPath string
@@ -96,6 +97,7 @@ func createConfigurationFile(ctx context.Context, clusterConf *ClusterConfigurat
 	config.Configure.Controller.HTTPSCACert = clusterConf.HTTPSCaCertPath
 	config.Configure.Controller.HTTPSKey = clusterConf.HTTPSCertPath
 	config.Configure.Controller.ClientAuthCACertPath = clusterConf.AuthCACertPath
+	config.Configure.Controller.CNCINet = clusterConf.CNCINet
 
 	config.Configure.Controller.AdminSSHKey = adminSSHKeyData
 

ciao-scheduler/scheduler.go

@@ -546,6 +546,10 @@ func (sched *ssntpSchedulerServer) getCommandConcentratorUUID(command ssntp.Comm
 		var cmd payloads.CommandReleasePublicIP
 		err := yaml.Unmarshal(payload, &cmd)
 		return cmd.ReleaseIP.ConcentratorUUID, err
+	case ssntp.RefreshCNCI:
+		var cmd payloads.CommandCNCIRefresh
+		err := yaml.Unmarshal(payload, &cmd)
+		return cmd.Command.CNCIUUID, err
 	}
 }
 
@@ -811,6 +815,8 @@ func (sched *ssntpSchedulerServer) CommandForward(controllerUUID string, command
 		fallthrough
 	case ssntp.Restore:
 		dest, instanceUUID = sched.fwdCmdToComputeNode(command, payload)
+	case ssntp.RefreshCNCI:
+		fallthrough
 	case ssntp.AssignPublicIP:
 		fallthrough
 	case ssntp.ReleasePublicIP:
@@ -1125,6 +1131,10 @@ func setSSNTPForwardRules(sched *ssntpSchedulerServer) {
 			Operand:        ssntp.ReleasePublicIP,
 			CommandForward: sched,
 		},
+		{ // all RefreshCNCI commands are processed by the Command forwarder
+			Operand:        ssntp.RefreshCNCI,
+			CommandForward: sched,
+		},
 	}
 }
 

configuration/configuration_test.go

@@ -65,6 +65,7 @@ const fullValidConf = `configure:
     cnci_disk: 128
     admin_ssh_key: ""
     client_auth_ca_cert_path: /etc/pki/ciao/auth-CA.pem
+    cnci_net: 10.10.0.0
   launcher:
     compute_net:
     - 192.168.1.0/24
@@ -119,6 +120,7 @@ func fillPayload(conf *payloads.Configure) {
 	conf.Configure.Controller.CNCIMem = 128
 	conf.Configure.Controller.CNCIDisk = 128
 	conf.Configure.Controller.ClientAuthCACertPath = clientAuthCACert
+	conf.Configure.Controller.CNCINet = "10.10.0.0"
 	conf.Configure.Launcher.ComputeNetwork = []string{computeNet}
 	conf.Configure.Launcher.ManagementNetwork = []string{mgmtNet}
 	conf.Configure.Launcher.ChildUser = "ciao"

networking/ciao-cnci-agent/client.go

@@ -169,6 +169,17 @@ func createMandatoryDirs() error {
 	return nil
 }
 
+func processRefreshCNCI(cmd *payloads.CommandCNCIRefresh) {
+	c := &cmd.Command
+	glog.Infof("Processing: CiaoCommandCNCIRefresh %v", c)
+
+	// add call to function to refresh cnci.
+	err := refreshCNCI(c)
+	if err != nil {
+		glog.Errorf("Unable to refresh CNCI list: %v", err)
+	}
+}
+
 func processCommand(client *ssntpConn, cmd *cmdWrapper) {
 
 	switch netCmd := cmd.cmd.(type) {
@@ -232,6 +243,10 @@ func processCommand(client *ssntpConn, cmd *cmdWrapper) {
 			}
 		}(cmd)
 
+	case *payloads.CommandCNCIRefresh:
+
+		go processRefreshCNCI(netCmd)
+
 	case *statusConnected:
 		//Block and send this as it does not make sense to send other events
 		//or process commands when we have not yet registered
@@ -291,6 +306,22 @@ func (client *agentClient) CommandNotify(cmd ssntp.Command, frame *ssntp.Frame)
 			client.cmdCh <- &cmdWrapper{&releaseIP}
 		}(payload)
 
+	case ssntp.RefreshCNCI:
+		glog.Infof("CMD: ssntp.RefreshCNCI %v", len(payload))
+
+		go func(payload []byte) {
+			var refreshCNCI payloads.CommandCNCIRefresh
+
+			err := yaml.Unmarshal(payload, &refreshCNCI)
+			if err != nil {
+				glog.Warning("Error unmarshalling CNCI refresh")
+				return
+			}
+			glog.Infof("CMD: ssntp.RefreshCNCI %v", refreshCNCI)
+
+			client.cmdCh <- &cmdWrapper{&refreshCNCI}
+		}(payload)
+
 	default:
 		glog.Infof("CMD: %s", cmd)
 	}

networking/ciao-cnci-agent/network.go

@@ -409,3 +409,20 @@ func releasePubIP(cmd *payloads.PublicIPCommand) error {
 	err = gFw.PublicIPAccess(libsnnet.FwDisable, prIP, puIP, gCnci.ComputeLink[0].Attrs().Name)
 	return errors.Wrapf(err, "release ip")
 }
+
+func refreshCNCI(cmd *payloads.CNCIRefreshCommand) error {
+	var neighbors []libsnnet.Neighbor
+
+	cncis := cmd.CNCIList
+	for _, c := range cncis {
+		n := libsnnet.Neighbor{
+			PhysicalIP: c.PhysicalIP,
+			Subnet:     c.Subnet,
+			TunnelIP:   c.TunnelIP,
+			TunnelID:   c.TunnelID,
+		}
+		neighbors = append(neighbors, n)
+	}
+
+	return gCnci.UpdateNeighbors(neighbors)
+}