diff --git a/cmd/vm-control/createVm.go b/cmd/vm-control/createVm.go index 2cedd877..c57d6053 100644 --- a/cmd/vm-control/createVm.go +++ b/cmd/vm-control/createVm.go @@ -155,6 +155,7 @@ func createVmOnHypervisor(hypervisor string, logger log.DebugLogger) error { if *imageName != "" { request.ImageName = *imageName request.ImageTimeout = *imageTimeout + request.SkipBootloader = *skipBootloader } else if *imageURL != "" { request.ImageURL = *imageURL } else if *imageFile != "" { diff --git a/cmd/vm-control/main.go b/cmd/vm-control/main.go index 397db340..28684ada 100644 --- a/cmd/vm-control/main.go +++ b/cmd/vm-control/main.go @@ -54,7 +54,9 @@ var ( "Time to wait before timing out on probing VM port") secondarySubnetIDs flagutil.StringList secondaryVolumeSizes flagutil.StringList - subnetId = flag.String("subnetId", "", + skipBootloader = flag.Bool("skipBootloader", false, + "If true, directly boot into the kernel") + subnetId = flag.String("subnetId", "", "Subnet ID to launch VM in") requestIPs flagutil.StringList roundupPower = flag.Uint64("roundupPower", 24, diff --git a/cmd/vm-control/replaceVmImage.go b/cmd/vm-control/replaceVmImage.go index 69409627..ecb3badf 100644 --- a/cmd/vm-control/replaceVmImage.go +++ b/cmd/vm-control/replaceVmImage.go @@ -83,6 +83,7 @@ func replaceVmImageOnHypervisor(hypervisor string, ipAddr net.IP, if *imageName != "" { request.ImageName = *imageName request.ImageTimeout = *imageTimeout + request.SkipBootloader = *skipBootloader } else if *imageURL != "" { request.ImageURL = *imageURL } else if *imageFile != "" { diff --git a/hypervisor/manager/vm.go b/hypervisor/manager/vm.go index 9079691f..74f46e5a 100644 --- a/hypervisor/manager/vm.go +++ b/hypervisor/manager/vm.go @@ -339,6 +339,7 @@ func (m *Manager) createVm(conn *srpc.Conn, decoder srpc.Decoder, return conn.Flush() } + m.Logger.Debugln(1, "CreateVm() starting") var request proto.CreateVmRequest if err := decoder.Decode(&request); err != nil { return err @@ -394,11 +395,17 @@ func (m *Manager) createVm(conn *srpc.Conn, decoder srpc.Decoder, if err != nil { return err } - err = m.writeRaw(vm.VolumeLocations[0].Filename, client, fs, - request.MinimumFreeBytes, request.RoundupPower) + writeRawOptions := util.WriteRawOptions{ + MinimumFreeBytes: request.MinimumFreeBytes, + RootLabel: vm.rootLabel(), + RoundupPower: request.RoundupPower, + } + err = m.writeRaw(vm.VolumeLocations[0], "", client, fs, writeRawOptions, + request.SkipBootloader) if err != nil { return sendError(conn, encoder, err) } + m.Logger.Debugln(1, "finished writing volume") if fi, err := os.Stat(vm.VolumeLocations[0].Filename); err != nil { return sendError(conn, encoder, err) } else { @@ -485,6 +492,7 @@ func (m *Manager) createVm(conn *srpc.Conn, decoder srpc.Decoder, return err } vm = nil // Cancel cleanup. + m.Logger.Debugln(1, "CreateVm() finished") return nil } @@ -775,6 +783,10 @@ func (m *Manager) getVmVolume(conn *srpc.Conn, decoder srpc.Decoder, return encoder.Encode(proto.GetVmVolumeResponse{Error: err.Error()}) } defer vm.mutex.RUnlock() + if request.VolumeIndex == 0 && vm.getActiveKernelPath() != "" { + return encoder.Encode(proto.GetVmVolumeResponse{ + Error: "cannot get root volume with separate kernel"}) + } if request.VolumeIndex >= uint(len(vm.VolumeLocations)) { return encoder.Encode(proto.GetVmVolumeResponse{ Error: "index too large"}) @@ -1373,6 +1385,12 @@ func (m *Manager) replaceVmImage(conn *srpc.Conn, decoder srpc.Decoder, } return sendError(conn, encoder, errors.New("VM is not stopped")) } + initrdFilename := vm.getInitrdPath() + tmpInitrdFilename := initrdFilename + ".new" + defer os.Remove(tmpInitrdFilename) + kernelFilename := vm.getKernelPath() + tmpKernelFilename := kernelFilename + ".new" + defer os.Remove(tmpKernelFilename) tmpRootFilename := vm.VolumeLocations[0].Filename + ".new" defer os.Remove(tmpRootFilename) var newSize uint64 @@ -1394,8 +1412,13 @@ func (m *Manager) replaceVmImage(conn *srpc.Conn, decoder srpc.Decoder, if err != nil { return err } - err = m.writeRaw(tmpRootFilename, client, fs, request.MinimumFreeBytes, - request.RoundupPower) + writeRawOptions := util.WriteRawOptions{ + MinimumFreeBytes: request.MinimumFreeBytes, + RootLabel: vm.rootLabel(), + RoundupPower: request.RoundupPower, + } + err = m.writeRaw(vm.VolumeLocations[0], ".new", client, fs, + writeRawOptions, request.SkipBootloader) if err != nil { return sendError(conn, encoder, err) } @@ -1453,6 +1476,10 @@ func (m *Manager) replaceVmImage(conn *srpc.Conn, decoder srpc.Decoder, os.Rename(oldRootFilename, rootFilename) return sendError(conn, encoder, err) } + os.Rename(initrdFilename, initrdFilename+".old") + os.Rename(tmpInitrdFilename, initrdFilename) + os.Rename(kernelFilename, kernelFilename+".old") + os.Rename(tmpKernelFilename, kernelFilename) if request.ImageName != "" { vm.ImageName = request.ImageName } @@ -1534,6 +1561,10 @@ func (m *Manager) restoreVmImage(ipAddr net.IP, if err := os.Rename(oldRootFilename, rootFilename); err != nil { return err } + initrdFilename := vm.getInitrdPath() + os.Rename(initrdFilename+".old", initrdFilename) + kernelFilename := vm.getKernelPath() + os.Rename(kernelFilename+".old", kernelFilename) vm.Volumes[0].Size = uint64(fi.Size()) vm.writeAndSendInfo() return nil @@ -1681,8 +1712,9 @@ func (m *Manager) unregisterVmMetadataNotifier(ipAddr net.IP, return nil } -func (m *Manager) writeRaw(filename string, client *srpc.Client, - fs *filesystem.FileSystem, minimumFreeBytes, roundupPower uint64) error { +func (m *Manager) writeRaw(volume volumeType, extension string, + client *srpc.Client, fs *filesystem.FileSystem, + writeRawOptions util.WriteRawOptions, skipBootloader bool) error { var objectsGetter objectserver.ObjectsGetter if m.objectCache == nil { objectClient := objclient.AttachObjectClient(client) @@ -1691,9 +1723,47 @@ func (m *Manager) writeRaw(filename string, client *srpc.Client, } else { objectsGetter = m.objectCache } - return util.WriteRaw(fs, objectsGetter, filename, privateFilePerms, - mbr.TABLE_TYPE_MSDOS, minimumFreeBytes, - roundupPower, true, true, m.Logger) + writeRawOptions.AllocateBlocks = true + if skipBootloader { + bootInfo, err := util.GetBootInfo(fs, writeRawOptions.RootLabel, "") + if err != nil { + return err + } + dirent := bootInfo.KernelImageDirent + if dirent == nil { + return errors.New("no kernel image found") + } + inode, ok := dirent.Inode().(*filesystem.RegularInode) + if !ok { + return errors.New("kernel image is not a regular file") + } + inode.Size = 0 + filename := path.Join(volume.DirectoryToCleanup, "kernel"+extension) + _, err = objectserver.LinkObject(filename, objectsGetter, inode.Hash) + if err != nil { + return err + } + dirent = bootInfo.InitrdImageDirent + if dirent != nil { + inode, ok := dirent.Inode().(*filesystem.RegularInode) + if !ok { + return errors.New("initrd image is not a regular file") + } + inode.Size = 0 + filename := path.Join(volume.DirectoryToCleanup, "initrd"+extension) + _, err = objectserver.LinkObject(filename, objectsGetter, + inode.Hash) + if err != nil { + return err + } + } + } else { + writeRawOptions.InstallBootloader = true + } + writeRawOptions.WriteFstab = true + return util.WriteRawWithOptions(fs, objectsGetter, + volume.Filename+extension, privateFilePerms, mbr.TABLE_TYPE_MSDOS, + writeRawOptions, m.Logger) } func (vm *vmInfoType) autoDestroy() { @@ -1860,6 +1930,30 @@ func (vm *vmInfoType) discardSnapshot() error { return nil } +func (vm *vmInfoType) getActiveInitrdPath() string { + initrdPath := vm.getInitrdPath() + if _, err := os.Stat(initrdPath); err == nil { + return initrdPath + } + return "" +} + +func (vm *vmInfoType) getActiveKernelPath() string { + kernelPath := vm.getKernelPath() + if _, err := os.Stat(kernelPath); err == nil { + return kernelPath + } + return "" +} + +func (vm *vmInfoType) getInitrdPath() string { + return path.Join(vm.VolumeLocations[0].DirectoryToCleanup, "initrd") +} + +func (vm *vmInfoType) getKernelPath() string { + return path.Join(vm.VolumeLocations[0].DirectoryToCleanup, "kernel") +} + func (vm *vmInfoType) kill() { vm.mutex.Lock() defer vm.mutex.Unlock() @@ -1941,6 +2035,12 @@ func (vm *vmInfoType) processMonitorResponses(monitorSock net.Conn) { close(vm.commandChannel) } +func (vm *vmInfoType) rootLabel() string { + ipAddr := vm.Address.IpAddress + return fmt.Sprintf("rootfs@%02x%02x%02x%02x", + ipAddr[0], ipAddr[1], ipAddr[2], ipAddr[3]) +} + func (vm *vmInfoType) setState(state proto.State) { vm.State = state if !vm.doNotWriteOrSend { @@ -1980,6 +2080,7 @@ func (vm *vmInfoType) setupVolumes(rootSize uint64, func (vm *vmInfoType) startManaging(dhcpTimeout time.Duration, haveManagerLock bool) (bool, error) { vm.monitorSockname = path.Join(vm.dirname, "monitor.sock") + vm.logger.Debugln(1, "startManaging() starting") switch vm.State { case proto.StateStarting: case proto.StateRunning: @@ -2011,7 +2112,7 @@ func (vm *vmInfoType) startManaging(dhcpTimeout time.Duration, } monitorSock, err := net.Dial("unix", vm.monitorSockname) if err != nil { - vm.logger.Debugf(0, "error connecting to: %s: %s\n", + vm.logger.Debugf(1, "error connecting to: %s: %s\n", vm.monitorSockname, err) if err := vm.startVm(haveManagerLock); err != nil { vm.logger.Println(err) @@ -2136,6 +2237,20 @@ func (vm *vmInfoType) startVm(haveManagerLock bool) error { "-runas", vm.manager.Username, "-qmp", "unix:"+vm.monitorSockname+",server,nowait", "-daemonize") + if kernelPath := vm.getActiveKernelPath(); kernelPath != "" { + cmd.Args = append(cmd.Args, "-kernel", kernelPath) + if initrdPath := vm.getActiveInitrdPath(); initrdPath != "" { + cmd.Args = append(cmd.Args, + "-initrd", initrdPath, + "-append", util.MakeKernelOptions("LABEL="+vm.rootLabel(), + "net.ifnames=0"), + ) + } else { + cmd.Args = append(cmd.Args, + "-append", util.MakeKernelOptions("/dev/vda1", "net.ifnames=0"), + ) + } + } cmd.Args = append(cmd.Args, netOptions...) if vm.manager.ShowVgaConsole { cmd.Args = append(cmd.Args, "-vga", "std") diff --git a/lib/filesystem/util/api.go b/lib/filesystem/util/api.go index b74f493e..29c09c2d 100644 --- a/lib/filesystem/util/api.go +++ b/lib/filesystem/util/api.go @@ -1,8 +1,10 @@ package util import ( + "fmt" "io" "os" + "text/template" "github.com/Symantec/Dominator/lib/filesystem" "github.com/Symantec/Dominator/lib/log" @@ -10,6 +12,16 @@ import ( "github.com/Symantec/Dominator/lib/objectserver" ) +type BootInfoType struct { + BootDirectory *filesystem.DirectoryInode + InitrdImageDirent *filesystem.DirectoryEntry + InitrdImageFile string + KernelImageDirent *filesystem.DirectoryEntry + KernelImageFile string + KernelOptions string + grubTemplate *template.Template +} + type ComputedFile struct { Filename string Source string @@ -28,6 +40,11 @@ func CopyMtimes(source, dest *filesystem.FileSystem) { copyMtimes(source, dest) } +func GetBootInfo(fs *filesystem.FileSystem, rootLabel string, + extraKernelOptions string) (*BootInfoType, error) { + return getBootInfo(fs, rootLabel, extraKernelOptions) +} + func GetUnsupportedExt4fsOptions(fs *filesystem.FileSystem, objectsGetter objectserver.ObjectsGetter) ([]string, error) { return getUnsupportedOptions(fs, objectsGetter) @@ -50,6 +67,11 @@ func MakeExt4fs(deviceName, label string, unsupportedOptions []string, logger) } +func MakeKernelOptions(rootDevice, extraOptions string) string { + return fmt.Sprintf("root=%s ro console=tty0 console=ttyS0,115200n8 %s", + rootDevice, extraOptions) +} + func ReplaceComputedFiles(fs *filesystem.FileSystem, computedFilesData *ComputedFilesData, objectsGetter objectserver.ObjectsGetter) ( @@ -74,11 +96,36 @@ func WriteFstabEntry(writer io.Writer, dumpFrequency, checkOrder) } +type WriteRawOptions struct { + AllocateBlocks bool + DoChroot bool + InstallBootloader bool + MinimumFreeBytes uint64 + RootLabel string + RoundupPower uint64 + WriteFstab bool +} + func WriteRaw(fs *filesystem.FileSystem, objectsGetter objectserver.ObjectsGetter, rawFilename string, perm os.FileMode, tableType mbr.TableType, minFreeSpace uint64, roundupPower uint64, makeBootable, allocateBlocks bool, logger log.DebugLogger) error { return writeRaw(fs, objectsGetter, rawFilename, perm, tableType, - minFreeSpace, roundupPower, makeBootable, allocateBlocks, logger) + WriteRawOptions{ + AllocateBlocks: allocateBlocks, + InstallBootloader: makeBootable, + MinimumFreeBytes: minFreeSpace, + WriteFstab: makeBootable, + RoundupPower: roundupPower, + }, + logger) +} + +func WriteRawWithOptions(fs *filesystem.FileSystem, + objectsGetter objectserver.ObjectsGetter, rawFilename string, + perm os.FileMode, tableType mbr.TableType, options WriteRawOptions, + logger log.DebugLogger) error { + return writeRaw(fs, objectsGetter, rawFilename, perm, tableType, options, + logger) } diff --git a/lib/filesystem/util/writeRaw.go b/lib/filesystem/util/writeRaw.go index a3bd8e3a..68bd0f28 100644 --- a/lib/filesystem/util/writeRaw.go +++ b/lib/filesystem/util/writeRaw.go @@ -36,14 +36,6 @@ var ( defaultMkfsFeatures map[string]struct{} // Key: feature name. ) -type bootInfoType struct { - InitrdImageFile string - KernelImageFile string - KernelOptions string - grubTemplate *template.Template - RootLabel string -} - func checkIfPartition(device string) (bool, error) { if isBlock, err := checkIsBlock(device); err != nil { if !os.IsNotExist(err) { @@ -242,22 +234,24 @@ func lookPath(rootDir, file string) (string, error) { func makeAndWriteRoot(fs *filesystem.FileSystem, objectsGetter objectserver.ObjectsGetter, bootDevice, rootDevice string, - makeBootableFlag bool, logger log.DebugLogger) error { + options WriteRawOptions, logger log.DebugLogger) error { unsupportedOptions, err := getUnsupportedOptions(fs, objectsGetter) if err != nil { return err } - var bootInfo *bootInfoType - label := fmt.Sprintf("rootfs@%x", time.Now().Unix()) - if makeBootableFlag { + var bootInfo *BootInfoType + if options.RootLabel == "" { + options.RootLabel = fmt.Sprintf("rootfs@%x", time.Now().Unix()) + } + if options.InstallBootloader { var err error - bootInfo, err = getBootInfo(fs) + bootInfo, err = getBootInfo(fs, options.RootLabel, "net.ifnames=0") if err != nil { return err } - bootInfo.RootLabel = label } - err = makeExt4fs(rootDevice, label, unsupportedOptions, 8192, logger) + err = makeExt4fs(rootDevice, options.RootLabel, unsupportedOptions, 8192, + logger) if err != nil { return err } @@ -275,22 +269,34 @@ func makeAndWriteRoot(fs *filesystem.FileSystem, if err := Unpack(fs, objectsGetter, mountPoint, logger); err != nil { return err } - if !makeBootableFlag { - return nil + if options.WriteFstab { + err := writeRootFstabEntry(mountPoint, options.RootLabel) + if err != nil { + return err + } } - bootInfo.KernelOptions = "net.ifnames=0" - return bootInfo.makeBootable(bootDevice, mountPoint, false, logger) + if options.InstallBootloader { + err := bootInfo.installBootloader(bootDevice, mountPoint, + options.RootLabel, options.DoChroot, logger) + if err != nil { + return err + } + } + return nil } func makeBootable(fs *filesystem.FileSystem, deviceName, rootLabel, rootDir, kernelOptions string, doChroot bool, logger log.DebugLogger) error { - if bootInfo, err := getBootInfo(fs); err != nil { + if err := writeRootFstabEntry(rootDir, rootLabel); err != nil { + return err + } + if bootInfo, err := getBootInfo(fs, rootLabel, kernelOptions); err != nil { return err } else { bootInfo.KernelOptions = kernelOptions - bootInfo.RootLabel = rootLabel - return bootInfo.makeBootable(deviceName, rootDir, doChroot, logger) + return bootInfo.installBootloader(deviceName, rootDir, rootLabel, + doChroot, logger) } } @@ -348,24 +354,31 @@ func sanitiseInput(ch rune) rune { } } -func getBootInfo(fs *filesystem.FileSystem) (*bootInfoType, error) { +func getBootInfo(fs *filesystem.FileSystem, rootLabel string, + extraKernelOptions string) (*BootInfoType, error) { bootDirectory, err := getBootDirectory(fs) if err != nil { return nil, err } - bootInfo := &bootInfoType{} + bootInfo := &BootInfoType{ + BootDirectory: bootDirectory, + KernelOptions: MakeKernelOptions("LABEL="+rootLabel, + extraKernelOptions), + } for _, dirent := range bootDirectory.EntryList { if strings.HasPrefix(dirent.Name, "initrd.img-") || strings.HasPrefix(dirent.Name, "initramfs-") { if bootInfo.InitrdImageFile != "" { return nil, errors.New("multiple initrd images") } + bootInfo.InitrdImageDirent = dirent bootInfo.InitrdImageFile = "/boot/" + dirent.Name } if strings.HasPrefix(dirent.Name, "vmlinuz-") { if bootInfo.KernelImageFile != "" { return nil, errors.New("multiple kernel images") } + bootInfo.KernelImageDirent = dirent bootInfo.KernelImageFile = "/boot/" + dirent.Name } } @@ -377,8 +390,8 @@ func getBootInfo(fs *filesystem.FileSystem) (*bootInfoType, error) { return bootInfo, nil } -func (bootInfo *bootInfoType) makeBootable(deviceName string, rootDir string, - doChroot bool, logger log.DebugLogger) error { +func (bootInfo *BootInfoType) installBootloader(deviceName string, + rootDir, rootLabel string, doChroot bool, logger log.DebugLogger) error { startTime := time.Now() var bootDir, chrootDir string if doChroot { @@ -415,21 +428,10 @@ func (bootInfo *bootInfoType) makeBootable(deviceName string, rootDir string, if err := bootInfo.writeGrubConfig(grubConfigFile); err != nil { return err } - err = bootInfo.writeGrubTemplate(grubConfigFile + ".template") - if err != nil { - return err - } - file, err := os.Create(filepath.Join(rootDir, "etc", "fstab")) - if err != nil { - return err - } else { - defer file.Close() - return writeFstabEntry(file, "LABEL="+bootInfo.RootLabel, "/", "ext4", - "", 0, 1) - } + return bootInfo.writeGrubTemplate(grubConfigFile + ".template") } -func (bootInfo *bootInfoType) writeGrubConfig(filename string) error { +func (bootInfo *BootInfoType) writeGrubConfig(filename string) error { file, err := os.Create(filename) if err != nil { return fmt.Errorf("error creating GRUB config file: %s", err) @@ -441,7 +443,7 @@ func (bootInfo *bootInfoType) writeGrubConfig(filename string) error { return file.Close() } -func (bootInfo *bootInfoType) writeGrubTemplate(filename string) error { +func (bootInfo *BootInfoType) writeGrubTemplate(filename string) error { file, err := os.Create(filename) if err != nil { return fmt.Errorf("error creating GRUB config file template: %s", err) @@ -466,7 +468,7 @@ func writeFstabEntry(writer io.Writer, func writeToBlock(fs *filesystem.FileSystem, objectsGetter objectserver.ObjectsGetter, bootDevice string, - tableType mbr.TableType, makeBootableFlag bool, + tableType mbr.TableType, options WriteRawOptions, logger log.DebugLogger) error { if err := mbr.WriteDefault(bootDevice, tableType); err != nil { return err @@ -475,14 +477,13 @@ func writeToBlock(fs *filesystem.FileSystem, return err } else { return makeAndWriteRoot(fs, objectsGetter, bootDevice, rootDevice, - makeBootableFlag, logger) + options, logger) } } func writeToFile(fs *filesystem.FileSystem, objectsGetter objectserver.ObjectsGetter, rawFilename string, - perm os.FileMode, tableType mbr.TableType, minFreeSpace uint64, - roundupPower uint64, makeBootableFlag, allocateBlocks bool, + perm os.FileMode, tableType mbr.TableType, options WriteRawOptions, logger log.DebugLogger) error { tmpFilename := rawFilename + "~" if file, err := os.OpenFile(tmpFilename, createFlags, perm); err != nil { @@ -493,19 +494,19 @@ func writeToFile(fs *filesystem.FileSystem, } usageEstimate := fs.EstimateUsage(0) minBytes := usageEstimate + usageEstimate>>3 // 12% extra for good luck. - minBytes += minFreeSpace - if roundupPower < 24 { - roundupPower = 24 // 16 MiB. + minBytes += options.MinimumFreeBytes + if options.RoundupPower < 24 { + options.RoundupPower = 24 // 16 MiB. } - imageUnits := minBytes >> roundupPower - if imageUnits<> options.RoundupPower + if imageUnits<