Skip to content
This repository has been archived by the owner on Apr 29, 2024. It is now read-only.

Commit

Permalink
fix: manage kernel versions explicitly
Browse files Browse the repository at this point in the history
  • Loading branch information
eliobischof committed Oct 13, 2021
1 parent f30909a commit 9d408a0
Show file tree
Hide file tree
Showing 9 changed files with 276 additions and 54 deletions.
5 changes: 5 additions & 0 deletions internal/operator/common/software.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,17 @@ type Software struct {
Hostname Package `yaml:",omitempty"`
Sysctl Package `yaml:",omitempty"`
Health Package `yaml:",omitempty"`
Kernel Package `yaml:",omitempty"`
}

func (s *Software) Merge(sw Software) {

zeroPkg := Package{}

if !sw.Kernel.Equals(zeroPkg) && !s.Kernel.Equals(zeroPkg) {
s.Kernel = sw.Kernel
}

if !sw.Containerruntime.Equals(zeroPkg) {
s.Containerruntime = sw.Containerruntime
}
Expand Down
26 changes: 10 additions & 16 deletions internal/operator/nodeagent/dep/conv/converter.go
Original file line number Diff line number Diff line change
@@ -1,11 +1,9 @@
package conv

import (
"bytes"
"fmt"
"os"
"os/exec"
"strings"

"github.com/caos/orbos/internal/operator/nodeagent/dep/kernel"

"github.com/caos/orbos/internal/operator/nodeagent/dep/health"

Expand Down Expand Up @@ -60,18 +58,9 @@ func (d *dependencies) Init() func() error {
if len(sw) == 0 {
return nil
}
errBuf := new(bytes.Buffer)
defer errBuf.Reset()
cmd := exec.Command("yum", "--assumeyes", "remove", "yum-cron")
cmd.Stderr = errBuf
if d.monitor.IsVerbose() {
fmt.Println(strings.Join(cmd.Args, " "))
cmd.Stdout = os.Stdout
}
if err := cmd.Run(); err != nil {
return fmt.Errorf("removing yum-cron failed with stderr %s: %w", errBuf.String(), err)
}
return nil
return d.pm.Remove(&dep.Software{
Package: "yum-cron",
})
}
}

Expand All @@ -82,6 +71,9 @@ func (d *dependencies) Update() error {
func (d *dependencies) ToDependencies(sw common.Software) []*nodeagent.Dependency {

dependencies := []*nodeagent.Dependency{{
Desired: sw.Kernel,
Installer: kernel.New(d.pm),
}, {
Desired: sw.Sysctl,
Installer: sysctl.New(d.monitor),
}, {
Expand Down Expand Up @@ -128,6 +120,8 @@ func (d *dependencies) ToSoftware(dependencies []*nodeagent.Dependency, pkg func

for _, dependency := range dependencies {
switch i := middleware.Unwrap(dependency.Installer).(type) {
case kernel.Installer:
sw.Kernel = pkg(*dependency)
case sysctl.Installer:
sw.Sysctl = pkg(*dependency)
case health.Installer:
Expand Down
30 changes: 12 additions & 18 deletions internal/operator/nodeagent/dep/cri/ensure.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,31 +3,25 @@ package cri
import (
"bytes"
"fmt"
"os"
"os/exec"
"strings"

"github.com/caos/orbos/internal/operator/nodeagent/dep"
)

func (c *criDep) ensureCentOS(runtime string, version string) error {
errBuf := new(bytes.Buffer)
defer errBuf.Reset()
cmd := exec.Command("yum", "--assumeyes", "remove", "docker",
"docker-client",
"docker-client-latest",
"docker-common",
"docker-latest",
"docker-latest-logrotate",
"docker-logrotate",
"docker-engine")
cmd.Stderr = errBuf
if c.monitor.IsVerbose() {
fmt.Println(strings.Join(cmd.Args, " "))
cmd.Stdout = os.Stdout
}
if err := cmd.Run(); err != nil {
return fmt.Errorf("removing older docker versions failed with stderr %s: %w", errBuf.String(), err)

if err := c.manager.Remove(
&dep.Software{Package: "docker"},
&dep.Software{Package: "docker-client"},
&dep.Software{Package: "docker-client-latest"},
&dep.Software{Package: "docker-common"},
&dep.Software{Package: "docker-latest"},
&dep.Software{Package: "docker-latest-logrotate"},
&dep.Software{Package: "docker-logrotate"},
&dep.Software{Package: "docker-engine"},
); err != nil {
return fmt.Errorf("removing older docker versions failed: %w", err)
}

for _, pkg := range []string{"device-mapper-persistent-data", "lvm2"} {
Expand Down
174 changes: 174 additions & 0 deletions internal/operator/nodeagent/dep/kernel/dep.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,174 @@
package kernel

import (
"fmt"
"io/fs"
"os/exec"
"path/filepath"
"strings"
"time"

"github.com/caos/orbos/internal/operator/common"
"github.com/caos/orbos/internal/operator/nodeagent"
"github.com/caos/orbos/internal/operator/nodeagent/dep"
"github.com/caos/orbos/internal/operator/nodeagent/dep/middleware"
)

var _ Installer = (*kernelDep)(nil)

type Installer interface {
isKernel()
nodeagent.Installer
}

type kernelDep struct {
manager *dep.PackageManager
}

/*
New returns the kernel dependency
Node Agent uninstalls all kernels that don't have a corresponding initramfs file except for the currently
loaded kernel. If the currently loaded kernel doesn't have a corresponding initramfs file, Node Agent panics.
If ORBITER desires a specific kernel version, Node Agent installs and locks it, checks the initramfs file and reboots.
It is in the ORBITERS responsibility to ensure not all nodes are updated and rebooted simultaneously.
*/
func New(manager *dep.PackageManager) *kernelDep {
return &kernelDep{
manager: manager,
}
}

func (kernelDep) isKernel() {}

func (kernelDep) Is(other nodeagent.Installer) bool {
_, ok := middleware.Unwrap(other).(Installer)
return ok
}

func (kernelDep) String() string { return "Kernel" }

func (*kernelDep) Equals(other nodeagent.Installer) bool {
_, ok := other.(*kernelDep)
return ok
}

func (k *kernelDep) Current() (pkg common.Package, err error) {
loaded, corrupted, err := k.kernelVersions()
if err != nil {
return pkg, err
}

pkg.Version = loaded

if len(corrupted) > 0 {
pkg.Config = map[string]string{"corrupted": strings.Join(corrupted, ",")}
}

return pkg, nil
}

func (k *kernelDep) Ensure(remove common.Package, ensure common.Package) error {

corruptedKernels := make([]*dep.Software, 0)
corruptedKernelsStr, ok := remove.Config["corrupted"]
if ok && corruptedKernelsStr != "" {
corruptedKernelsStrs := strings.Split(corruptedKernelsStr, ",")
for i := range corruptedKernelsStrs {
corruptedKernels = append(corruptedKernels, &dep.Software{
Package: "kernel",
Version: corruptedKernelsStrs[i],
})
}
}

if err := k.manager.Remove(corruptedKernels...); err != nil {
return err
}

if remove.Version == ensure.Version || ensure.Version == "" {
return nil
}

if err := k.manager.Install(&dep.Software{
Package: "kernel",
Version: ensure.Version,
}); err != nil {
return err
}

initramfsVersions, err := listInitramfsVersions()
if err != nil {
return err
}

var found bool
for i := range initramfsVersions {
if initramfsVersions[i] == ensure.Version {
fmt.Printf("initramfsVersions[i] == ensure.Version: %s == %s: %t\n", initramfsVersions[i], ensure.Version, initramfsVersions[i] == ensure.Version)
found = true
break
}
}
if !found {
return fmt.Errorf("couldn't find a corresponding initramfs file corresponding kernel version %s. Not rebooting", ensure.Version)
}

out, err := exec.Command("reboot").CombinedOutput()
if err != nil {
return fmt.Errorf("rebooting system failed: %s: %w", string(out), err)
}

// give os signal some time before doing anything
time.Sleep(5 * time.Second)
return nil
}

func (k *kernelDep) kernelVersions() (loadedKernel string, corruptedKernels []string, err error) {

loadedKernelBytes, err := exec.Command("uname", "-r").CombinedOutput()
if err != nil {
return loadedKernel, corruptedKernels, fmt.Errorf("getting loaded kernel failed: %s: %w", loadedKernel, err)
}

loadedKernel = string(loadedKernelBytes)

initramfsVersions, err := listInitramfsVersions()
if err != nil {
return loadedKernel, corruptedKernels, err
}

corruptedKernels = make([]string, 0)
kernels:
for _, installedKernel := range k.manager.CurrentVersions("kernel") {
for i := range initramfsVersions {
if initramfsVersions[i] == installedKernel.Version {
continue kernels
}
}
if installedKernel.Version == loadedKernel {
panic("The actively loaded kernel has no corresponding initramfs file. Pleases fix it manually so the machine survives the next reboot")
}
corruptedKernels = append(corruptedKernels, installedKernel.Version)
}

return loadedKernel, corruptedKernels, nil
}

func listInitramfsVersions() ([]string, error) {
initramfsdir := "/boot/"
var initramfsKernels []string
return initramfsKernels, filepath.WalkDir(initramfsdir, func(path string, d fs.DirEntry, err error) error {
if err != nil {
return err
}
if d.IsDir() {
return filepath.SkipDir
}
if strings.HasPrefix(path, initramfsdir+"initramfs-") && strings.HasSuffix(path, ".img") {
initramfsKernels = append(initramfsKernels, path[len(initramfsdir+"initramfs-"):strings.LastIndex(path, ".img")])
}
return nil
})
}
15 changes: 7 additions & 8 deletions internal/operator/nodeagent/dep/package-manager-install.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,13 +10,13 @@ import (
"github.com/caos/orbos/mntr"
)

func (p *PackageManager) rembasedInstall(installVersion *Software, more ...*Software) error {
func (p *PackageManager) rembasedInstall(install ...*Software) error {

errBuf := new(bytes.Buffer)
defer errBuf.Reset()

installPkgs := make([]string, 0)
for _, sw := range append([]*Software{installVersion}, more...) {
for _, sw := range install {

installedVersion, ok := p.installed[sw.Package]
if ok && (sw.Version == "" || sw.Version == installedVersion) {
Expand Down Expand Up @@ -86,14 +86,14 @@ func rembasedInstallPkg(monitor mntr.Monitor, pkg string) error {
}

// TODO: Use lower level apt instead of apt-get?
func (p *PackageManager) debbasedInstall(installVersion *Software, more ...*Software) error {
func (p *PackageManager) debbasedInstall(install ...*Software) error {

errBuf := new(bytes.Buffer)
defer errBuf.Reset()

pkgs := make([]string, len(more)+1)
pkgs := make([]string, len(install))
hold := make([]string, 0)
for idx, sw := range append([]*Software{installVersion}, more...) {
for idx, sw := range install {
pkgs[idx] = sw.Package
if sw.Version == "" {
continue
Expand Down Expand Up @@ -149,9 +149,8 @@ func (p *PackageManager) debbasedInstall(installVersion *Software, more ...*Soft
errBuf.Reset()

p.monitor.WithFields(map[string]interface{}{
"package": installVersion.Package,
"version": installVersion.Version,
}).Debug("Installed package")
"software": pkg,
}).Debug("Holded package")
}
return nil
}
40 changes: 40 additions & 0 deletions internal/operator/nodeagent/dep/package-manager-remove.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
package dep

import (
"bytes"
"fmt"
"os/exec"
"strings"
)

func (p *PackageManager) rembasedRemove(remove ...*Software) error {

swStrs := make([]string, len(remove))
for i, sw := range remove {
swStrs[i] = sw.Package
if sw.Version != "" {
swStrs[i] = fmt.Sprintf("%s-%s", sw.Package, sw.Version)
}
}

errBuf := new(bytes.Buffer)
defer errBuf.Reset()
outBuf := new(bytes.Buffer)
defer outBuf.Reset()

cmd := exec.Command("yum", append([]string{"--assumeyes", "remove"}, swStrs...)...)
cmd.Stderr = errBuf
cmd.Stdout = outBuf
err := cmd.Run()
errStr := errBuf.String()
outStr := outBuf.String()
p.monitor.WithFields(map[string]interface{}{
"command": fmt.Sprintf("'%s'", strings.Join(cmd.Args, "' '")),
"stdout": outStr,
"stderr": errStr,
}).Debug("Executed yum remove")
if err != nil {
return fmt.Errorf("removing yum packages [%s] failed with stderr %s: %w", strings.Join(swStrs, ", "), errStr, err)
}
return nil
}
Loading

0 comments on commit 9d408a0

Please sign in to comment.