Skip to content

Commit

Permalink
kola: promote concept of sdk.LocalBuild throughout
Browse files Browse the repository at this point in the history 
Instead of keeping separate the build metadata object and the build
directory, let's instead raise the visibility of `sdk.LocalBuild`, which
bundles those two things together already.

What I initially set out to solve was that `cosa kola --build ID` no
longer worked, because we weren't passing `--cosa-build` to kola
anymore. But even if we did, it's silly to have `cosa kola` dig into
build dirs to fetch the path to the `meta.json` when `kola` now knows
how to do that.

Concretely, this changes three things:
1. we add a `--cosa-workdir` to allow specifying a cosa dir other than
   the current working dir.
2. the `--cosa-build` argument is now a build ID, not a path to a
   `meta.json`.
3 `kola.CosaBuild` is now a `LocalBuild` so that we also have access to
   the build directory everywhere.
  • Loading branch information
@jlebon
jlebon committed Mar 25, 2020
1 parent 126727b commit a4e4fb5
Show file tree
Hide file tree
Showing 9 changed files with 72 additions and 74 deletions.
8 changes: 4 additions & 4 deletions mantle/cmd/kola/kola.go
View file
Expand Up @@ -592,15 +592,15 @@ func getParentFcosBuildBase() (string, error) {
// parse commitmeta.json for `fedora-coreos.stream`, then fetch the stream
// metadata for that stream, then fetch the release metadata

if kola.CosaBuild.BuildRef == "" {
if kola.CosaBuild.Meta.BuildRef == "" {
return "", errors.New("no ref in build metadata")
}

stream := filepath.Base(kola.CosaBuild.BuildRef)
stream := filepath.Base(kola.CosaBuild.Meta.BuildRef)

var parentVersion string
if kola.CosaBuild.FedoraCoreOsParentVersion != "" {
parentVersion = kola.CosaBuild.FedoraCoreOsParentVersion
if kola.CosaBuild.Meta.FedoraCoreOsParentVersion != "" {
parentVersion = kola.CosaBuild.Meta.FedoraCoreOsParentVersion
} else {
// ok, we're probably operating on a local dev build since the pipeline
// always injects the parent; just instead fetch the release index
Expand Down
42 changes: 22 additions & 20 deletions mantle/cmd/kola/options.go
View file
Expand Up @@ -21,7 +21,6 @@ import (
"strings"

"github.com/coreos/mantle/auth"
"github.com/coreos/mantle/cosa"
"github.com/coreos/mantle/kola"
"github.com/coreos/mantle/platform"
"github.com/coreos/mantle/sdk"
Expand Down Expand Up @@ -58,6 +57,7 @@ func init() {
sv(&kola.Options.IgnitionVersion, "ignition-version", "", "Ignition version override: v2, v3")
ssv(&kola.BlacklistedTests, "blacklist-test", []string{}, "List of tests to blacklist")
bv(&kola.Options.SSHOnTestFailure, "ssh-on-test-failure", false, "SSH into a machine when tests fail")
sv(&kola.Options.CosaWorkdir, "cosa-workdir", "", "coreos-assembler working directory")
sv(&kola.Options.CosaBuild, "cosa-build", "", "File path for coreos-assembler meta.json")
// rhcos-specific options
sv(&kola.Options.OSContainer, "oscontainer", "", "oscontainer image pullspec for pivot (RHCOS only)")
Expand Down Expand Up @@ -142,8 +142,6 @@ func init() {

// Sync up the command line options if there is dependency
func syncOptionsImpl(useCosa bool) error {
var err error

kola.PacketOptions.Board = kola.QEMUOptions.Board
kola.PacketOptions.GSOptions = &kola.GCEOptions

Expand Down Expand Up @@ -175,28 +173,32 @@ func syncOptionsImpl(useCosa bool) error {
}

foundCosa := false
if kola.Options.CosaBuild == "" {
isroot, err := sdk.IsCosaRoot(".")
// specified --cosa-build? fetch that build
if kola.Options.CosaBuild != "" {
if kola.Options.CosaWorkdir == "" {
kola.Options.CosaWorkdir = "."
}

localbuild, err := sdk.GetLocalBuild(kola.Options.CosaWorkdir, kola.Options.CosaBuild)
if err != nil {
return err
}
if isroot {
localbuild, err := sdk.GetLatestLocalBuild()
if err != nil {
return err
}

kola.Options.CosaBuild = filepath.Join(localbuild.Dir, "meta.json")
kola.CosaBuild = localbuild.Meta
foundCosa = true
}
} else {
kola.CosaBuild, err = cosa.ParseBuild(kola.Options.CosaBuild)
kola.CosaBuild = localbuild
foundCosa = true
// didn't specify --cosa-build, but did specify --cosa-workdir? fetch the
// latest from that workdir
} else if kola.Options.CosaWorkdir != "" {
localbuild, err := sdk.GetLatestLocalBuild(kola.Options.CosaWorkdir)
if err != nil {
return err
}

kola.Options.CosaBuild = localbuild.Meta.BuildID
kola.CosaBuild = localbuild
foundCosa = true
}

if foundCosa && useCosa {
if err := syncCosaOptions(); err != nil {
return err
Expand Down Expand Up @@ -247,19 +249,19 @@ func syncOptions() error {
func syncCosaOptions() error {
switch kolaPlatform {
case "qemu-unpriv", "qemu":
if kola.QEMUOptions.DiskImage == "" && kola.CosaBuild.BuildArtifacts.Qemu != nil {
kola.QEMUOptions.DiskImage = filepath.Join(filepath.Dir(kola.Options.CosaBuild), kola.CosaBuild.BuildArtifacts.Qemu.Path)
if kola.QEMUOptions.DiskImage == "" && kola.CosaBuild.Meta.BuildArtifacts.Qemu != nil {
kola.QEMUOptions.DiskImage = filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.Qemu.Path)
}
}

if kola.Options.IgnitionVersion == "" {
if kola.CosaBuild != nil {
kola.Options.IgnitionVersion = sdk.TargetIgnitionVersion(kola.CosaBuild)
kola.Options.IgnitionVersion = sdk.TargetIgnitionVersion(kola.CosaBuild.Meta)
}
}

if kola.Options.Distribution == "" {
distro, err := sdk.TargetDistro(kola.CosaBuild)
distro, err := sdk.TargetDistro(kola.CosaBuild.Meta)
if err != nil {
return err
}
Expand Down
21 changes: 10 additions & 11 deletions mantle/cmd/kola/testiso.go
View file
Expand Up @@ -86,7 +86,6 @@ func runTestIso(cmd *cobra.Command, args []string) error {
}

baseInst := platform.Install{
CosaBuildDir: kola.Options.CosaBuild,
CosaBuild: kola.CosaBuild,

Console: console,
Expand All @@ -109,13 +108,13 @@ func runTestIso(cmd *cobra.Command, args []string) error {
"-device", "virtio-serial", "-device", "virtserialport,chardev=completion,name=completion",
"-chardev", "file,id=completion,path=" + completionfile}

if kola.CosaBuild.BuildArtifacts.Metal == nil {
return fmt.Errorf("Build %s must have a `metal` artifact", kola.CosaBuild.OstreeVersion)
if kola.CosaBuild.Meta.BuildArtifacts.Metal == nil {
return fmt.Errorf("Build %s must have a `metal` artifact", kola.CosaBuild.Meta.OstreeVersion)
}

ranTest := false

foundLegacy := baseInst.CosaBuild.BuildArtifacts.Kernel != nil
foundLegacy := baseInst.CosaBuild.Meta.BuildArtifacts.Kernel != nil
if foundLegacy {
if legacy {
ranTest = true
Expand All @@ -125,16 +124,16 @@ func runTestIso(cmd *cobra.Command, args []string) error {
if err := testPXE(inst, completionfile); err != nil {
return err
}
fmt.Printf("Successfully tested legacy installer for %s\n", kola.CosaBuild.OstreeVersion)
fmt.Printf("Successfully tested legacy installer for %s\n", kola.CosaBuild.Meta.OstreeVersion)
}
} else if legacy {
return fmt.Errorf("build %s has no legacy installer kernel", kola.CosaBuild.Name)
return fmt.Errorf("build %s has no legacy installer kernel", kola.CosaBuild.Meta.Name)
}

foundLive := kola.CosaBuild.BuildArtifacts.LiveKernel != nil
foundLive := kola.CosaBuild.Meta.BuildArtifacts.LiveKernel != nil
if !nolive {
if !foundLive {
return fmt.Errorf("build %s has no live installer kernel", kola.CosaBuild.Name)
return fmt.Errorf("build %s has no live installer kernel", kola.CosaBuild.Meta.Name)
}
if !nopxe {
ranTest = true
Expand All @@ -143,15 +142,15 @@ func runTestIso(cmd *cobra.Command, args []string) error {
if err := testPXE(instPxe, completionfile); err != nil {
return err
}
fmt.Printf("Successfully tested PXE live installer for %s\n", kola.CosaBuild.OstreeVersion)
fmt.Printf("Successfully tested PXE live installer for %s\n", kola.CosaBuild.Meta.OstreeVersion)
}

ranTest = true
instIso := baseInst // Pretend this is Rust and I wrote .copy()
if err := testLiveIso(instIso, completionfile); err != nil {
return err
}
fmt.Printf("Successfully tested ISO live installer for %s\n", kola.CosaBuild.OstreeVersion)
fmt.Printf("Successfully tested ISO live installer for %s\n", kola.CosaBuild.Meta.OstreeVersion)
}

if !ranTest {
Expand Down Expand Up @@ -179,7 +178,7 @@ func testPXE(inst platform.Install, completionfile string) error {
},
}
var configStr string
if sdk.TargetIgnitionVersion(kola.CosaBuild) == "v2" {
if sdk.TargetIgnitionVersion(kola.CosaBuild.Meta) == "v2" {
ignc2, err := ignconverter.Translate3to2(config)
if err != nil {
return err
Expand Down
4 changes: 2 additions & 2 deletions mantle/kola/harness.go
View file
Expand Up @@ -33,7 +33,6 @@ import (

ignv3 "github.com/coreos/ignition/v2/config/v3_0"
ignv3types "github.com/coreos/ignition/v2/config/v3_0/types"
"github.com/coreos/mantle/cosa"
"github.com/coreos/mantle/harness"
"github.com/coreos/mantle/harness/reporters"
"github.com/coreos/mantle/kola/cluster"
Expand All @@ -55,6 +54,7 @@ import (
"github.com/coreos/mantle/platform/machine/openstack"
"github.com/coreos/mantle/platform/machine/packet"
"github.com/coreos/mantle/platform/machine/unprivqemu"
"github.com/coreos/mantle/sdk"
"github.com/coreos/mantle/system"
"github.com/coreos/mantle/util"
)
Expand All @@ -72,7 +72,7 @@ var (
PacketOptions = packetapi.Options{Options: &Options} // glue to set platform options from main
QEMUOptions = unprivqemu.Options{Options: &Options} // glue to set platform options from main

CosaBuild *cosa.Build // this is the parsed object of --cosa-build
CosaBuild *sdk.LocalBuild // this a parsed cosa build

TestParallelism int //glue var to set test parallelism from main
TAPFile string // if not "", write TAP results here
Expand Down
12 changes: 6 additions & 6 deletions mantle/kola/tests/upgrade/basic.go
View file
Expand Up @@ -120,19 +120,19 @@ func fcosUpgradeBasic(c cluster.TestCluster) {
// optimized for qemu testing locally where this won't leave localhost at
// all. cloud testing should mostly be a pipeline thing, where the infra
// connection should be much faster
ostreeTarPath := filepath.Join(filepath.Dir(kola.Options.CosaBuild), kola.CosaBuild.BuildArtifacts.Ostree.Path)
ostreeTarPath := filepath.Join(kola.CosaBuild.Dir, kola.CosaBuild.Meta.BuildArtifacts.Ostree.Path)
if err := cluster.DropFile(c.Machines(), ostreeTarPath); err != nil {
c.Fatal(err)
}

c.MustSSHf(m, "tar -xf %s -C %s", kola.CosaBuild.BuildArtifacts.Ostree.Path, ostreeRepo)
c.MustSSHf(m, "tar -xf %s -C %s", kola.CosaBuild.Meta.BuildArtifacts.Ostree.Path, ostreeRepo)

graph.seedFromMachine(c, m)
graph.addUpdate(c, m, kola.CosaBuild.OstreeVersion, kola.CosaBuild.OstreeCommit)
graph.addUpdate(c, m, kola.CosaBuild.Meta.OstreeVersion, kola.CosaBuild.Meta.OstreeCommit)
})

c.Run("upgrade-from-previous", func(c cluster.TestCluster) {
waitForUpgradeToVersion(c, m, kola.CosaBuild.OstreeVersion)
waitForUpgradeToVersion(c, m, kola.CosaBuild.Meta.OstreeVersion)
})

// Now, synthesize an update and serve that -- this is similar to
Expand All @@ -141,9 +141,9 @@ func fcosUpgradeBasic(c cluster.TestCluster) {
// Zincati & HTTP). Essentially, this sanity-checks that old starting state
// + new content set can update.
c.Run("upgrade-from-current", func(c cluster.TestCluster) {
newVersion := kola.CosaBuild.OstreeVersion + ".kola"
newVersion := kola.CosaBuild.Meta.OstreeVersion + ".kola"
newCommit := c.MustSSHf(m, "ostree commit --repo %s -b %s --tree ref=%s --add-metadata-string version=%s",
ostreeRepo, kola.CosaBuild.BuildRef, kola.CosaBuild.OstreeCommit, newVersion)
ostreeRepo, kola.CosaBuild.Meta.BuildRef, kola.CosaBuild.Meta.OstreeCommit, newVersion)

graph.addUpdate(c, m, newVersion, string(newCommit))

Expand Down
43 changes: 21 additions & 22 deletions mantle/platform/metal.go
View file
Expand Up @@ -31,8 +31,8 @@ import (
"github.com/pkg/errors"
"github.com/vincent-petithory/dataurl"

"github.com/coreos/mantle/cosa"
"github.com/coreos/mantle/platform/conf"
"github.com/coreos/mantle/sdk"
"github.com/coreos/mantle/system"
"github.com/coreos/mantle/system/exec"
"github.com/coreos/mantle/util"
Expand Down Expand Up @@ -77,8 +77,7 @@ var (
)

type Install struct {
CosaBuildDir string
CosaBuild *cosa.Build
CosaBuild *sdk.LocalBuild

Firmware string
Console bool
Expand All @@ -99,8 +98,8 @@ type InstalledMachine struct {
}

func (inst *Install) PXE(kargs []string, ignition string) (*InstalledMachine, error) {
if inst.CosaBuild.BuildArtifacts.Metal == nil {
return nil, fmt.Errorf("Build %s must have a `metal` artifact", inst.CosaBuild.OstreeVersion)
if inst.CosaBuild.Meta.BuildArtifacts.Metal == nil {
return nil, fmt.Errorf("Build %s must have a `metal` artifact", inst.CosaBuild.Meta.OstreeVersion)
}

inst.kargs = kargs
Expand All @@ -109,23 +108,23 @@ func (inst *Install) PXE(kargs []string, ignition string) (*InstalledMachine, er
var err error
var mach *InstalledMachine
if inst.LegacyInstaller {
if inst.CosaBuild.BuildArtifacts.Kernel == nil {
return nil, fmt.Errorf("build %s has no legacy installer kernel", inst.CosaBuild.OstreeVersion)
if inst.CosaBuild.Meta.BuildArtifacts.Kernel == nil {
return nil, fmt.Errorf("build %s has no legacy installer kernel", inst.CosaBuild.Meta.OstreeVersion)
}
mach, err = inst.runPXE(&kernelSetup{
kernel: inst.CosaBuild.BuildArtifacts.Kernel.Path,
initramfs: inst.CosaBuild.BuildArtifacts.Initramfs.Path,
kernel: inst.CosaBuild.Meta.BuildArtifacts.Kernel.Path,
initramfs: inst.CosaBuild.Meta.BuildArtifacts.Initramfs.Path,
}, true)
if err != nil {
return nil, errors.Wrapf(err, "legacy installer")
}
} else {
if inst.CosaBuild.BuildArtifacts.LiveKernel == nil {
return nil, fmt.Errorf("build %s has no live installer kernel", inst.CosaBuild.Name)
if inst.CosaBuild.Meta.BuildArtifacts.LiveKernel == nil {
return nil, fmt.Errorf("build %s has no live installer kernel", inst.CosaBuild.Meta.Name)
}
mach, err = inst.runPXE(&kernelSetup{
kernel: inst.CosaBuild.BuildArtifacts.LiveKernel.Path,
initramfs: inst.CosaBuild.BuildArtifacts.LiveInitramfs.Path,
kernel: inst.CosaBuild.Meta.BuildArtifacts.LiveKernel.Path,
initramfs: inst.CosaBuild.Meta.BuildArtifacts.LiveInitramfs.Path,
}, false)
if err != nil {
return nil, errors.Wrapf(err, "testing live installer")
Expand Down Expand Up @@ -262,7 +261,7 @@ func (inst *Install) setup(kern *kernelSetup) (*installerRun, error) {
return nil, err
}

builddir := filepath.Dir(inst.CosaBuildDir)
builddir := inst.CosaBuild.Dir
serializedConfig := []byte(inst.ignition)
if err := ioutil.WriteFile(filepath.Join(tftpdir, "config.ign"), serializedConfig, 0644); err != nil {
return nil, err
Expand All @@ -274,7 +273,7 @@ func (inst *Install) setup(kern *kernelSetup) (*installerRun, error) {
}
}

metalimg := inst.CosaBuild.BuildArtifacts.Metal.Path
metalimg := inst.CosaBuild.Meta.BuildArtifacts.Metal.Path
metalname, err := setupMetalImage(builddir, metalimg, tftpdir)
if err != nil {
return nil, errors.Wrapf(err, "setting up metal image")
Expand Down Expand Up @@ -504,11 +503,11 @@ func generatePointerIgnitionString(target string) string {
}

func (inst *Install) InstallViaISOEmbed(kargs []string, liveIgniton, targetIgnition string) (*InstalledMachine, error) {
if inst.CosaBuild.BuildArtifacts.Metal == nil {
return nil, fmt.Errorf("Build %s must have a `metal` artifact", inst.CosaBuild.OstreeVersion)
if inst.CosaBuild.Meta.BuildArtifacts.Metal == nil {
return nil, fmt.Errorf("Build %s must have a `metal` artifact", inst.CosaBuild.Meta.OstreeVersion)
}
if inst.CosaBuild.BuildArtifacts.LiveIso == nil {
return nil, fmt.Errorf("Build %s must have a live ISO", inst.CosaBuild.Name)
if inst.CosaBuild.Meta.BuildArtifacts.LiveIso == nil {
return nil, fmt.Errorf("Build %s must have a live ISO", inst.CosaBuild.Meta.Name)
}

if len(inst.kargs) > 0 {
Expand All @@ -534,9 +533,9 @@ func (inst *Install) InstallViaISOEmbed(kargs []string, liveIgniton, targetIgnit
return nil, err
}

builddir := filepath.Dir(inst.CosaBuildDir)
srcisopath := filepath.Join(builddir, inst.CosaBuild.BuildArtifacts.LiveIso.Path)
metalimg := inst.CosaBuild.BuildArtifacts.Metal.Path
builddir := inst.CosaBuild.Dir
srcisopath := filepath.Join(builddir, inst.CosaBuild.Meta.BuildArtifacts.LiveIso.Path)
metalimg := inst.CosaBuild.Meta.BuildArtifacts.Metal.Path
metalname, err := setupMetalImage(builddir, metalimg, tempdir)
if err != nil {
return nil, errors.Wrapf(err, "setting up metal image")
Expand Down
3 changes: 2 additions & 1 deletion mantle/platform/platform.go
View file
Expand Up @@ -161,7 +161,8 @@ type Options struct {
IgnitionVersion string
SystemdDropins []SystemdDropin

CosaBuild string
CosaWorkdir string
CosaBuild string

NoTestExitError bool

Expand Down
11 changes: 3 additions & 8 deletions mantle/sdk/repo.go
View file
Expand Up @@ -70,16 +70,11 @@ func RequireCosaRoot(root string) error {
return nil
}

func GetLatestLocalBuild() (*LocalBuild, error) {
return GetLocalBuild("latest")
func GetLatestLocalBuild(root string) (*LocalBuild, error) {
return GetLocalBuild(root, "latest")
}

func GetLocalBuild(buildid string) (*LocalBuild, error) {
// Maybe in the future we support an environment variable or CLI switch
root, err := os.Getwd()
if err != nil {
return nil, err
}
func GetLocalBuild(root, buildid string) (*LocalBuild, error) {
if err := RequireCosaRoot(root); err != nil {
return nil, err
}
Expand Down
2 changes: 2 additions & 0 deletions src/cmd-kola
View file
Expand Up @@ -56,6 +56,8 @@ kolaargs = ['kola']
if os.getuid() != 0 and not ('-p' in unknown_args):
kolaargs.extend(['-p', 'qemu-unpriv'])

if args.build is not None:
kolaargs.extend(['--cosa-build', args.build])
outputdir = args.output_dir or default_output_dir
kolaargs.extend(['--output-dir', outputdir])
subargs = args.subargs or [default_cmd]
Expand Down

0 comments on commit a4e4fb5

Please sign in to comment.