From e0dfbb8fba3c50652d0ecbae1db0b0660d0766a6 Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Mon, 22 Jan 2024 17:48:04 +0400 Subject: [PATCH] fix: allow META encoded values to be compressed Fixes #8186 This is planned to be backported to Talos 1.6.3. This allows to pass large META values (YAML for platform network configuration) which might otherwise exceed the limit for kernel command line params. Signed-off-by: Andrey Smirnov --- cmd/installer/pkg/install/meta_value.go | 2 +- pkg/imager/imager.go | 6 +- pkg/imager/quirks/quirks.go | 12 +++ pkg/imager/quirks/quirks_test.go | 28 +++++++ pkg/machinery/meta/meta.go | 56 ++++++++++++- pkg/machinery/meta/meta_test.go | 83 ++++++++++++++++--- .../advanced/metal-network-configuration.md | 2 +- 7 files changed, 173 insertions(+), 16 deletions(-) diff --git a/cmd/installer/pkg/install/meta_value.go b/cmd/installer/pkg/install/meta_value.go index 534b447793..32ef01f756 100644 --- a/cmd/installer/pkg/install/meta_value.go +++ b/cmd/installer/pkg/install/meta_value.go @@ -103,7 +103,7 @@ func (s *MetaValues) GetSlice() []string { // Encode returns the encoded values. func (s *MetaValues) Encode() string { - return s.values.Encode() + return s.values.Encode(false) } // Decode the values from the given string. diff --git a/pkg/imager/imager.go b/pkg/imager/imager.go index 2911c494e9..55024151e3 100644 --- a/pkg/imager/imager.go +++ b/pkg/imager/imager.go @@ -20,6 +20,7 @@ import ( "github.com/siderolabs/talos/internal/pkg/secureboot/uki" "github.com/siderolabs/talos/pkg/imager/extensions" "github.com/siderolabs/talos/pkg/imager/profile" + "github.com/siderolabs/talos/pkg/imager/quirks" "github.com/siderolabs/talos/pkg/imager/utils" "github.com/siderolabs/talos/pkg/machinery/config/merge" "github.com/siderolabs/talos/pkg/machinery/constants" @@ -276,7 +277,10 @@ func (i *Imager) buildCmdline() error { // meta values can be written only to the "image" output if len(i.prof.Customization.MetaContents) > 0 && i.prof.Output.Kind != profile.OutKindImage { // pass META values as kernel talos.environment args which will be passed via the environment to the installer - cmdline.Append(constants.KernelParamEnvironment, constants.MetaValuesEnvVar+"="+i.prof.Customization.MetaContents.Encode()) + cmdline.Append( + constants.KernelParamEnvironment, + constants.MetaValuesEnvVar+"="+i.prof.Customization.MetaContents.Encode(quirks.New(i.prof.Version).SupportsCompressedEncodedMETA()), + ) } // apply customization diff --git a/pkg/imager/quirks/quirks.go b/pkg/imager/quirks/quirks.go index 63768e0008..9d1c407384 100644 --- a/pkg/imager/quirks/quirks.go +++ b/pkg/imager/quirks/quirks.go @@ -33,3 +33,15 @@ func (q Quirks) SupportsResetGRUBOption() bool { return q.v.GTE(minVersionResetOption) } + +var minVersionCompressedMETA = semver.MustParse("1.6.3") + +// SupportsCompressedEncodedMETA returns true if the Talos version supports compressed and encoded META as an environment variable. +func (q Quirks) SupportsCompressedEncodedMETA() bool { + // if the version doesn't parse, we assume it's latest Talos + if q.v == nil { + return true + } + + return q.v.GTE(minVersionCompressedMETA) +} diff --git a/pkg/imager/quirks/quirks_test.go b/pkg/imager/quirks/quirks_test.go index 852325e65c..fbf85a9dc6 100644 --- a/pkg/imager/quirks/quirks_test.go +++ b/pkg/imager/quirks/quirks_test.go @@ -35,3 +35,31 @@ func TestSupportsResetOption(t *testing.T) { }) } } + +func TestSupportsCompressedEncodedMETA(t *testing.T) { + for _, test := range []struct { + version string + + expected bool + }{ + { + version: "1.6.3", + expected: true, + }, + { + version: "1.7.0", + expected: true, + }, + { + expected: true, + }, + { + version: "1.6.2", + expected: false, + }, + } { + t.Run(test.version, func(t *testing.T) { + assert.Equal(t, test.expected, quirks.New(test.version).SupportsCompressedEncodedMETA()) + }) + } +} diff --git a/pkg/machinery/meta/meta.go b/pkg/machinery/meta/meta.go index 4e27db571b..0a802bb814 100644 --- a/pkg/machinery/meta/meta.go +++ b/pkg/machinery/meta/meta.go @@ -6,8 +6,11 @@ package meta import ( + "bytes" + "compress/gzip" "encoding/base64" "fmt" + "io" "strconv" "strings" @@ -49,8 +52,29 @@ type Values []Value // // Each Value is encoded a k=v, split by ';' character. // The result is base64 encoded. -func (v Values) Encode() string { - return base64.StdEncoding.EncodeToString([]byte(strings.Join(xslices.Map(v, Value.String), ";"))) +func (v Values) Encode(allowGzip bool) string { + raw := []byte(strings.Join(xslices.Map(v, Value.String), ";")) + + if allowGzip && len(raw) > 256 { + var buf bytes.Buffer + + gzW, err := gzip.NewWriterLevel(&buf, gzip.BestCompression) + if err != nil { + panic(err) + } + + if _, err := gzW.Write(raw); err != nil { + panic(err) + } + + if err := gzW.Close(); err != nil { + panic(err) + } + + raw = buf.Bytes() + } + + return base64.StdEncoding.EncodeToString(raw) } // DecodeValues parses a string representation of Values for the environment variable. @@ -66,6 +90,25 @@ func DecodeValues(s string) (Values, error) { return nil, nil } + // do un-gzip if needed + if hasGzipMagic(b) { + gzR, err := gzip.NewReader(bytes.NewReader(b)) + if err != nil { + return nil, err + } + + defer gzR.Close() //nolint:errcheck + + b, err = io.ReadAll(gzR) + if err != nil { + return nil, err + } + + if err := gzR.Close(); err != nil { + return nil, err + } + } + parts := strings.Split(string(b), ";") result := make(Values, 0, len(parts)) @@ -82,3 +125,12 @@ func DecodeValues(s string) (Values, error) { return result, nil } + +func hasGzipMagic(b []byte) bool { + if len(b) < 10 { + return false + } + + // See https://en.wikipedia.org/wiki/Gzip#File_format. + return b[0] == 0x1f && b[1] == 0x8b +} diff --git a/pkg/machinery/meta/meta_test.go b/pkg/machinery/meta/meta_test.go index 183ac4c248..c7dbb4ae50 100644 --- a/pkg/machinery/meta/meta_test.go +++ b/pkg/machinery/meta/meta_test.go @@ -5,6 +5,8 @@ package meta_test import ( + "fmt" + "strings" "testing" "github.com/stretchr/testify/assert" @@ -35,15 +37,74 @@ func TestValue(t *testing.T) { func TestEncodeDecodeValues(t *testing.T) { t.Parallel() - values := make(meta.Values, 2) - - require.NoError(t, values[0].Parse("10=foo")) - require.NoError(t, values[1].Parse("0xb=bar")) - - encoded := values.Encode() - - decoded, err := meta.DecodeValues(encoded) - require.NoError(t, err) - - assert.Equal(t, values, decoded) + for _, allowGzip := range []bool{false, true} { + allowGzip := allowGzip + + t.Run(fmt.Sprintf("allowGzip=%v", allowGzip), func(t *testing.T) { + t.Parallel() + + for _, test := range []struct { + name string + + values []string + + expectedEncodedSize int + expectedGzippedSize int + }{ + { + name: "empty", + }, + { + name: "simple", + values: []string{ + "10=foo", + "0xb=bar", + }, + + expectedEncodedSize: 20, + expectedGzippedSize: 20, + }, + { + name: "huge", + values: []string{ + "10=" + strings.Repeat("foobar", 256), + "0xb=" + strings.Repeat("baz", 256), + }, + + expectedEncodedSize: 3084, + expectedGzippedSize: 80, + }, + } { + test := test + + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + values := make(meta.Values, len(test.values)) + + for i, v := range test.values { + require.NoError(t, values[i].Parse(v)) + } + + if len(values) == 0 { + values = nil + } + + encoded := values.Encode(allowGzip) + + switch { + case test.expectedEncodedSize > 0 && !allowGzip: + assert.Equal(t, test.expectedEncodedSize, len(encoded)) + case test.expectedGzippedSize > 0 && allowGzip: + assert.Equal(t, test.expectedGzippedSize, len(encoded)) + } + + decoded, err := meta.DecodeValues(encoded) + require.NoError(t, err) + + assert.Equal(t, values, decoded) + }) + } + }) + } } diff --git a/website/content/v1.7/advanced/metal-network-configuration.md b/website/content/v1.7/advanced/metal-network-configuration.md index db4384c568..bbbab45c7d 100644 --- a/website/content/v1.7/advanced/metal-network-configuration.md +++ b/website/content/v1.7/advanced/metal-network-configuration.md @@ -403,7 +403,7 @@ kernel command line: ... talos.environment=INSTALLER_META_BASE64=MHhhPWZvbw== When PXE booting, the value of `INSTALLER_META_BASE64` should be set manually: ```bash -echo -n "0xa=$(cat network.yaml)" | base64 +echo -n "0xa=$(cat network.yaml)" | gzip -9 | base64 ``` The resulting base64 string should be passed as an environment variable `INSTALLER_META_BASE64` to the initial boot of Talos: `talos.environment=INSTALLER_META_BASE64=`.