Skip to content

Commit

Permalink
chore: clean up the output of the imager
Browse files Browse the repository at this point in the history 
Use `Progress`, and options to pass around the way messages are written.

Fixed some tiny issues in the code, but otherwise no functional changes.

To make colored output work with `docker run`, switched back image
generation to use volume mount for output (old mode is still
functioning, but it's not the default, and it works when docker is not
running on the same host).

Signed-off-by: Andrey Smirnov <andrey.smirnov@talos-systems.com>
  • Loading branch information
@smira
smira committed Aug 7, 2023
1 parent fb536af commit e0f3835
Show file tree
Hide file tree
Showing 34 changed files with 353 additions and 169 deletions.
4 changes: 2 additions & 2 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -300,7 +300,7 @@ image-%: ## Builds the specified image. Valid options are aws, azure, digital-oc
@docker pull $(REGISTRY_AND_USERNAME)/imager:$(IMAGE_TAG)
@for platform in $(subst $(,),$(space),$(PLATFORM)); do \
arch=$$(basename "$${platform}") && \
docker run --rm -v /dev:/dev -v $(PWD)/$(ARTIFACTS):/secureboot:ro --network=host --privileged $(REGISTRY_AND_USERNAME)/imager:$(IMAGE_TAG) $* --arch $$arch --tar-to-stdout $(IMAGER_ARGS) | tar xz -C $(ARTIFACTS) ; \
docker run --rm -t -v /dev:/dev -v $(PWD)/$(ARTIFACTS):/secureboot:ro -v $(PWD)/$(ARTIFACTS):/out --network=host --privileged $(REGISTRY_AND_USERNAME)/imager:$(IMAGE_TAG) $* --arch $$arch $(IMAGER_ARGS) ; \
done

images-essential: image-aws image-gcp image-metal secureboot-installer ## Builds only essential images used in the CI (AWS, GCP, and Metal).
Expand All @@ -309,7 +309,7 @@ images: image-aws image-azure image-digital-ocean image-exoscale image-gcp image

sbc-%: ## Builds the specified SBC image. Valid options are rpi_generic, rock64, bananapi_m64, libretech_all_h3_cc_h5, rockpi_4, rockpi_4c, pine64, jetson_nano and nanopi_r4s (e.g. sbc-rpi_generic)
@docker pull $(REGISTRY_AND_USERNAME)/imager:$(IMAGE_TAG)
@docker run --rm -v /dev:/dev --network=host --privileged $(REGISTRY_AND_USERNAME)/imager:$(IMAGE_TAG) $* --arch arm64 --tar-to-stdout $(IMAGER_ARGS) | tar xz -C $(ARTIFACTS)
@docker run --rm -t -v /dev:/dev -v $(PWD)/$(ARTIFACTS):/out --network=host --privileged $(REGISTRY_AND_USERNAME)/imager:$(IMAGE_TAG) $* --arch arm64 $(IMAGER_ARGS)

sbcs: sbc-rpi_generic sbc-rock64 sbc-bananapi_m64 sbc-libretech_all_h3_cc_h5 sbc-rockpi_4 sbc-rockpi_4c sbc-pine64 sbc-jetson_nano sbc-nanopi_r4s ## Builds all known SBC images (Raspberry Pi 4, Rock64, Banana Pi M64, Radxa ROCK Pi 4, Radxa ROCK Pi 4c, Pine64, Libre Computer Board ALL-H3-CC, Jetson Nano and Nano Pi R4S).

Expand Down
25 changes: 18 additions & 7 deletions cmd/installer/cmd/imager/root.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ package imager

import (
"context"
"fmt"
"os"
"runtime"

Expand All @@ -21,6 +20,7 @@ import (
"github.com/siderolabs/talos/pkg/imager"
"github.com/siderolabs/talos/pkg/imager/profile"
"github.com/siderolabs/talos/pkg/machinery/constants"
"github.com/siderolabs/talos/pkg/reporter"
)

var cmdFlags struct {
Expand All @@ -38,12 +38,19 @@ var cmdFlags struct {

// rootCmd represents the base command when called without any subcommands.
var rootCmd = &cobra.Command{
Use: "imager <profile>|-",
Short: "Generate various boot assets and images.",
Long: ``,
Args: cobra.ExactArgs(1),
Use: "imager <profile>|-",
Short: "Generate various boot assets and images.",
Long: ``,
Args: cobra.ExactArgs(1),
SilenceUsage: true,
RunE: func(cmd *cobra.Command, args []string) error {
return cli.WithContext(context.Background(), func(ctx context.Context) error {
report := reporter.New()
report.Report(reporter.Update{
Message: "assembling the finalized profile...",
Status: reporter.StatusRunning,
})

baseProfile := args[0]

var prof profile.Profile
Expand Down Expand Up @@ -98,7 +105,12 @@ var rootCmd = &cobra.Command{
return err
}

if err = imager.Execute(ctx, cmdFlags.OutputPath); err != nil {
if err = imager.Execute(ctx, cmdFlags.OutputPath, report); err != nil {
report.Report(reporter.Update{
Message: err.Error(),
Status: reporter.StatusError,
})

return err
}

Expand All @@ -115,7 +127,6 @@ var rootCmd = &cobra.Command{
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
if err := rootCmd.Execute(); err != nil {
fmt.Fprintln(os.Stderr, err)
os.Exit(1)
}
}
Expand Down
10 changes: 8 additions & 2 deletions cmd/installer/pkg/install/install.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ type Options struct {
ImageSecureboot bool
Version string
BootAssets bootloaderoptions.BootAssets
Printf func(string, ...any)
}

// Mode is the install mode.
Expand Down Expand Up @@ -107,7 +108,7 @@ func Install(ctx context.Context, p runtime.Platform, mode Mode, opts *Options)
return err
}

log.Printf("installation of %s complete", version.Tag)
i.options.Printf("installation of %s complete", version.Tag)

return nil
}
Expand All @@ -132,6 +133,10 @@ func NewInstaller(ctx context.Context, cmdline *procfs.Cmdline, mode Mode, opts
i.options.Version = version.Tag
}

if i.options.Printf == nil {
i.options.Printf = log.Printf
}

if !i.options.Zero {
i.bootloader, err = bootloader.Probe(ctx, i.options.Disk)
if err != nil && !os.IsNotExist(err) {
Expand Down Expand Up @@ -258,6 +263,7 @@ func (i *Installer) Install(ctx context.Context, mode Mode) (err error) {
Version: i.options.Version,
ImageMode: mode.IsImage(),
BootAssets: i.options.BootAssets,
Printf: i.options.Printf,
}); err != nil {
return err
}
Expand All @@ -270,7 +276,7 @@ func (i *Installer) Install(ctx context.Context, mode Mode) (err error) {
return err
}

log.Printf("installing U-Boot for %q", b.Name())
i.options.Printf("installing U-Boot for %q", b.Name())

if err = b.Install(i.options.Disk); err != nil {
return err
Expand Down
29 changes: 16 additions & 13 deletions cmd/installer/pkg/install/manifest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,6 @@ import (
"context"
"errors"
"fmt"
"log"
"os"
"path/filepath"
"strings"
Expand All @@ -33,6 +32,8 @@ type Manifest struct {
Devices map[string]Device
Targets map[string][]*Target
LegacyBIOSSupport bool

Printf func(string, ...any)
}

// Device represents device options.
Expand All @@ -53,6 +54,8 @@ func NewManifest(mode Mode, uefiOnlyBoot bool, bootLoaderPresent bool, opts *Opt
Devices: map[string]Device{},
Targets: map[string][]*Target{},
LegacyBIOSSupport: opts.LegacyBIOSSupport,

Printf: opts.Printf,
}

if opts.Board != constants.BoardNone {
Expand Down Expand Up @@ -248,7 +251,7 @@ func (m *Manifest) executeOnDevice(device Device, targets []*Target) (err error)
if device.Zero {
if err = partition.Format(device.Device, &partition.FormatOptions{
FileSystemType: partition.FilesystemTypeNone,
}); err != nil {
}, m.Printf); err != nil {
return err
}
}
Expand All @@ -272,7 +275,7 @@ func (m *Manifest) executeOnDevice(device Device, targets []*Target) (err error)
return err
}

log.Printf("creating new partition table on %s", device.Device)
m.Printf("creating new partition table on %s", device.Device)

gptOpts := []gpt.Option{
gpt.WithMarkMBRBootable(m.LegacyBIOSSupport),
Expand All @@ -287,8 +290,8 @@ func (m *Manifest) executeOnDevice(device Device, targets []*Target) (err error)
return err
}

log.Printf("logical/physical block size: %d/%d", pt.Header().LBA.LogicalBlockSize, pt.Header().LBA.PhysicalBlockSize)
log.Printf("minimum/optimal I/O size: %d/%d", pt.Header().LBA.MinimalIOSize, pt.Header().LBA.OptimalIOSize)
m.Printf("logical/physical block size: %d/%d", pt.Header().LBA.LogicalBlockSize, pt.Header().LBA.PhysicalBlockSize)
m.Printf("minimum/optimal I/O size: %d/%d", pt.Header().LBA.MinimalIOSize, pt.Header().LBA.OptimalIOSize)

if err = pt.Write(); err != nil {
return err
Expand All @@ -309,7 +312,7 @@ func (m *Manifest) executeOnDevice(device Device, targets []*Target) (err error)

if !created {
if device.ResetPartitionTable {
log.Printf("resetting partition table on %s", device.Device)
m.Printf("resetting partition table on %s", device.Device)

// TODO: how should it work with zero option above?
if err = bd.Reset(); err != nil {
Expand Down Expand Up @@ -343,7 +346,7 @@ func (m *Manifest) executeOnDevice(device Device, targets []*Target) (err error)
// delete all partitions which are not skipped
for _, part := range pt.Partitions().Items() {
if _, ok := keepPartitions[part.Name]; !ok {
log.Printf("deleting partition %s", part.Name)
m.Printf("deleting partition %s", part.Name)

if err = pt.Delete(part); err != nil {
return err
Expand All @@ -363,7 +366,7 @@ func (m *Manifest) executeOnDevice(device Device, targets []*Target) (err error)
}

for i, target := range targets {
if err = target.partition(pt, i); err != nil {
if err = target.partition(pt, i, m.Printf); err != nil {
return fmt.Errorf("failed to partition device: %w", err)
}
}
Expand All @@ -376,7 +379,7 @@ func (m *Manifest) executeOnDevice(device Device, targets []*Target) (err error)
target := target

err = retry.Constant(time.Minute, retry.WithUnits(100*time.Millisecond)).Retry(func() error {
e := target.Format()
e := target.Format(m.Printf)
if e != nil {
if strings.Contains(e.Error(), "No such file or directory") {
// workaround problem with partition device not being visible immediately after partitioning
Expand Down Expand Up @@ -422,7 +425,7 @@ func (m *Manifest) preserveContents(device Device, targets []*Target) (err error

if bd, err = blockdevice.Open(device.Device); err != nil {
// failed to open the block device, probably it's damaged?
log.Printf("warning: skipping preserve contents on %q as block device failed: %s", device.Device, err)
m.Printf("warning: skipping preserve contents on %q as block device failed: %s", device.Device, err)

return nil
}
Expand All @@ -432,7 +435,7 @@ func (m *Manifest) preserveContents(device Device, targets []*Target) (err error

pt, err := bd.PartitionTable()
if err != nil {
log.Printf("warning: skipping preserve contents on %q as partition table failed: %s", device.Device, err)
m.Printf("warning: skipping preserve contents on %q as partition table failed: %s", device.Device, err)

return nil
}
Expand Down Expand Up @@ -473,13 +476,13 @@ func (m *Manifest) preserveContents(device Device, targets []*Target) (err error
}

if sourcePart == nil {
log.Printf("warning: failed to preserve contents of %q on %q, as source partition wasn't found", target.Label, device.Device)
m.Printf("warning: failed to preserve contents of %q on %q, as source partition wasn't found", target.Label, device.Device)

continue
}

if err = target.SaveContents(device, sourcePart, fileSystemType, fnmatchFilters); err != nil {
log.Printf("warning: failed to preserve contents of %q on %q: %s", target.Label, device.Device, err)
m.Printf("warning: failed to preserve contents of %q on %q: %s", target.Label, device.Device, err)
}
}

Expand Down
37 changes: 21 additions & 16 deletions cmd/installer/pkg/install/manifest_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -229,9 +229,10 @@ func (suite *manifestSuite) TestExecuteManifestClean() {
suite.skipUnderBuildkit()

manifest, err := install.NewManifest(install.ModeInstall, false, false, &install.Options{
Disk: suite.loopbackDevice.Name(),
Force: true,
Board: constants.BoardNone,
Disk: suite.loopbackDevice.Name(),
Force: true,
Board: constants.BoardNone,
Printf: suite.T().Logf,
})
suite.Require().NoError(err)

Expand All @@ -249,9 +250,10 @@ func (suite *manifestSuite) TestExecuteManifestForce() {
suite.skipUnderBuildkit()

manifest, err := install.NewManifest(install.ModeInstall, false, false, &install.Options{
Disk: suite.loopbackDevice.Name(),
Force: true,
Board: constants.BoardNone,
Disk: suite.loopbackDevice.Name(),
Force: true,
Board: constants.BoardNone,
Printf: suite.T().Logf,
})
suite.Require().NoError(err)

Expand All @@ -267,10 +269,11 @@ func (suite *manifestSuite) TestExecuteManifestForce() {
// reinstall

manifest, err = install.NewManifest(install.ModeUpgrade, false, true, &install.Options{
Disk: suite.loopbackDevice.Name(),
Force: true,
Zero: true,
Board: constants.BoardNone,
Disk: suite.loopbackDevice.Name(),
Force: true,
Zero: true,
Board: constants.BoardNone,
Printf: suite.T().Logf,
})
suite.Require().NoError(err)

Expand All @@ -288,9 +291,10 @@ func (suite *manifestSuite) TestExecuteManifestPreserve() {
suite.skipUnderBuildkit()

manifest, err := install.NewManifest(install.ModeInstall, false, false, &install.Options{
Disk: suite.loopbackDevice.Name(),
Force: true,
Board: constants.BoardNone,
Disk: suite.loopbackDevice.Name(),
Force: true,
Board: constants.BoardNone,
Printf: suite.T().Logf,
})
suite.Require().NoError(err)

Expand All @@ -306,9 +310,10 @@ func (suite *manifestSuite) TestExecuteManifestPreserve() {
// reinstall

manifest, err = install.NewManifest(install.ModeUpgrade, false, true, &install.Options{
Disk: suite.loopbackDevice.Name(),
Force: false,
Board: constants.BoardNone,
Disk: suite.loopbackDevice.Name(),
Force: false,
Board: constants.BoardNone,
Printf: suite.T().Logf,
})
suite.Require().NoError(err)

Expand Down
10 changes: 5 additions & 5 deletions cmd/installer/pkg/install/target.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -200,11 +200,11 @@ func (t *Target) Locate(pt *gpt.GPT) (*gpt.Partition, error) {
}

// partition creates a new partition on the specified device.
func (t *Target) partition(pt *gpt.GPT, pos int) (err error) {
func (t *Target) partition(pt *gpt.GPT, pos int, printf func(string, ...any)) (err error) {
if t.Skip {
part := pt.Partitions().FindByName(t.Label)
if part != nil {
log.Printf("skipped %s (%s) size %d blocks", t.PartitionName, t.Label, part.Length())
printf("skipped %s (%s) size %d blocks", t.PartitionName, t.Label, part.Length())

t.PartitionName, err = part.Path()
if err != nil {
Expand All @@ -220,7 +220,7 @@ func (t *Target) partition(pt *gpt.GPT, pos int) (err error) {
PartitionType: t.PartitionType,
Size: t.Size,
LegacyBIOSBootable: t.LegacyBIOSBootable,
})
}, printf)
if err != nil {
return err
}
Expand All @@ -231,12 +231,12 @@ func (t *Target) partition(pt *gpt.GPT, pos int) (err error) {
}

// Format creates a filesystem on the device/partition.
func (t *Target) Format() error {
func (t *Target) Format(printf func(string, ...any)) error {
if t.Skip {
return nil
}

return partition.Format(t.PartitionName, t.FormatOptions)
return partition.Format(t.PartitionName, t.FormatOptions, printf)
}

// GetLabel returns the underlaying partition label.
Expand Down
2 changes: 1 addition & 1 deletion internal/app/machined/pkg/runtime/sequencer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ type ResetOptions interface {
// PartitionTarget provides interface to the disk partition.
type PartitionTarget interface {
fmt.Stringer
Format() error
Format(func(string, ...any)) error
GetLabel() string
}

Expand Down
5 changes: 2 additions & 3 deletions internal/app/machined/pkg/runtime/v1alpha1/bootloader/grub/encode.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ package grub
import (
"bytes"
"io"
"log"
"os"
"path/filepath"
"text/template"
Expand Down Expand Up @@ -43,7 +42,7 @@ menuentry "Reset Talos installation and return to maintenance mode" {
`

// Write the grub configuration to the given file.
func (c *Config) Write(path string) error {
func (c *Config) Write(path string, printf func(string, ...any)) error {
dir := filepath.Dir(path)
if err := os.MkdirAll(dir, os.ModeDir); err != nil {
return err
Expand All @@ -56,7 +55,7 @@ func (c *Config) Write(path string) error {
return err
}

log.Printf("writing %s to disk", path)
printf("writing %s to disk", path)

return os.WriteFile(path, wr.Bytes(), 0o600)
}
Expand Down
Loading

0 comments on commit e0f3835

Please sign in to comment.