diff --git a/contrib/completion/bash b/contrib/completion/bash new file mode 100644 index 00000000..dde7f711 --- /dev/null +++ b/contrib/completion/bash @@ -0,0 +1,14 @@ +_completion_booster() { + # All arguments except the first one + args=("${COMP_WORDS[@]:1:$COMP_CWORD}") + + # Only split on newlines + local IFS=$'\n' + + # Call completion (note that the first element of COMP_WORDS is + # the executable itself) + COMPREPLY=($(GO_FLAGS_COMPLETION=1 ${COMP_WORDS[0]} "${args[@]}")) + return 0 +} + +complete -F _completion_booster booster \ No newline at end of file diff --git a/docs/manpage.md b/docs/manpage.md index 84ce819b..74bac4f2 100644 --- a/docs/manpage.md +++ b/docs/manpage.md @@ -73,15 +73,32 @@ Once you are done modifying your config file and want to regenerate booster imag It is a convenience script that performs the same type of image regeneration as if you installed `booster` with your package manager. ## COMMAND-LINE FLAGS - `booster` command accepts following flags: - * `-config` config file to use. Default value is `/etc/booster.yaml`. - * `-universal` generate a universal image - * `-kernelVersion` use modules for the given kernel version. If the flag is not specified then the current kernel is used (as reported by "uname -r"). - * `-output` output file, by default booster.img used - * `-compression` output file compression. Currently supported compression algorithms are "zstd" (default), "gzip" and "none". - * `-strip` strip ELF files (binaries, shared libraries and kernel modules) before adding it to the image - * `-force` overwrite output file if it exists +### Application Options + +* `-v`, `--verbose` Enable verbose output + +### SUBCOMMANDS + +### build +Build initrd image. Usage: `booster [OPTIONS] build [build-OPTIONS] output` + +* `-f`, `--force` Overwrite existing initrd file. +* `--init-binary` Booster 'init' binary location. +* `--compression` Output file compression. Possible values: _zstd_, _gzip_, _xz_, _lz4_, _none_. +* `--kernel-version` Linux kernel version to generate initramfs for. +* `--config` Configuration file path. +* `--universal` Add wide range of modules/tools to allow this image boot at different machines. +* `--strip` Strip ELF files (binaries, shared libraries and kernel modules) before adding it to the image. + +### cat +Show content of the file inside the image. Usage: `booster [OPTIONS] cat image file-in-image` + +### ls +List content of the image. Usage: `booster [OPTIONS] ls image` + +### unpack +Unpack image. Usage: `booster [OPTIONS] unpack image output-dir` ## BOOT TIME KERNEL PARAMETERS Some parts of booster boot functionality can be modified with kernel boot parameters. These parameters are usually set through bootloader config. Booster boot uses following kernel parameters: @@ -167,19 +184,15 @@ provide additional logs. ## EXAMPLES Create an initramfs file specific for the current kernel/host. The output file is booster.img: - $ booster - -The same as above but output image to /tmp/foobar.img: - - $ booster /tmp/foobar.img + $ booster build booster.img Create an universal image with many modules (such as SATA/TPM/NVME/... drivers) included: - $ booster -universal + $ booster build --universal booster.img Create an initramfs for kernel version 5.4.91-1-lts and copy it to /boot/booster-lts.img: - $ booster -kernelVersion 5.4.91-1-lts -output /boot/booster-lts.img + $ booster build --kernel-kersion 5.4.91-1-lts /boot/booster-lts.img Here is a `systemd-boot` configuration stored at /boot/loader/entries/booster.conf. In this example e122d09e-87a9-4b35-83f7-2592ef40cefa is a UUID for the LUKS partition and 08684949-bcbb-47bb-1c17-089aaa59e17e is a UUID for the encrypted filesystem (e.g. ext4). Please refer to your bootloader documentation for more info about its configuration. diff --git a/generator/config.go b/generator/config.go index a386686e..0ab42ee4 100644 --- a/generator/config.go +++ b/generator/config.go @@ -92,7 +92,7 @@ func readGeneratorConfig(file string) (*generatorConfig, error) { } } } - conf.universal = u.Universal || *universal + conf.universal = u.Universal || opts.BuildCommand.Universal if u.Modules != "" { conf.modules = strings.Split(u.Modules, ",") } @@ -112,17 +112,17 @@ func readGeneratorConfig(file string) (*generatorConfig, error) { } // now check command line flags - conf.output = *outputFile - conf.forceOverwrite = *forceOverwriteFile - conf.initBinary = *initBinary - if *compression != "" { - conf.compression = *compression + conf.output = opts.BuildCommand.Args.Output + conf.forceOverwrite = opts.BuildCommand.Force + conf.initBinary = opts.BuildCommand.InitBinary + if opts.BuildCommand.Compression != "" { + conf.compression = opts.BuildCommand.Compression } if conf.compression == "" { conf.compression = "zstd" } - if *kernelVersion != "" { - conf.kernelVersion = *kernelVersion + if opts.BuildCommand.KernelVersion != "" { + conf.kernelVersion = opts.BuildCommand.KernelVersion } else { ver, err := readKernelVersion() if err != nil { @@ -131,11 +131,11 @@ func readGeneratorConfig(file string) (*generatorConfig, error) { conf.kernelVersion = ver } conf.modulesDir = path.Join("/usr/lib/modules", conf.kernelVersion) - conf.debug = *debugEnabled + conf.debug = opts.Verbose conf.readDeviceAliases = readDeviceAliases conf.readHostModules = readHostModules conf.readModprobeOptions = readModprobeOptions - conf.stripBinaries = u.StripBinaries || *strip + conf.stripBinaries = u.StripBinaries || opts.BuildCommand.Strip conf.enableLVM = u.EnableLVM conf.enableMdraid = u.EnableMdraid conf.mdraidConfigPath = u.MdraidConfigPath diff --git a/generator/filetype.go b/generator/filetype.go new file mode 100644 index 00000000..700c9b70 --- /dev/null +++ b/generator/filetype.go @@ -0,0 +1,68 @@ +package main + +import ( + "bytes" + "io" + "os" +) + +type matcher func(seeker io.ReadSeeker) (bool, error) + +var matchers = map[string]matcher{ + "zstd": matchZstd, + "gzip": matchGzip, + "xz": matchXz, + "lz4": matchLz4, + "cpio": matchCpio, +} + +func matchBytes(f io.ReadSeeker, offset int64, marker []byte) (bool, error) { + if _, err := f.Seek(offset, io.SeekStart); err != nil { + return false, err + } + buff := make([]byte, len(marker)) + if _, err := io.ReadFull(f, buff); err != nil { + return false, nil + } + return bytes.Equal(marker, buff), nil +} + +func matchCpio(f io.ReadSeeker) (bool, error) { + return matchBytes(f, 0, []byte{'0', '7', '0', '7', '0', '1'}) // "new" cpio format +} + +func matchLz4(f io.ReadSeeker) (bool, error) { + return matchBytes(f, 0, []byte{0x02, 0x21, 0x4c, 0x18}) // legacy format used by linux loader +} + +func matchXz(f io.ReadSeeker) (bool, error) { + return matchBytes(f, 0, []byte{0xfd, '7', 'z', 'X', 'Z', 0x00}) +} + +func matchGzip(f io.ReadSeeker) (bool, error) { + return matchBytes(f, 0, []byte{0x1f, 0x8b}) +} + +func matchZstd(f io.ReadSeeker) (bool, error) { + return matchBytes(f, 0, []byte{0x28, 0xb5, 0x2f, 0xfd}) +} + +func filetype(r *os.File) (string, error) { + loc, err := r.Seek(0, io.SeekCurrent) + if err != nil { + return "", err + } + defer r.Seek(loc, io.SeekStart) + + for name, match := range matchers { + ok, err := match(r) + if err != nil { + return "", err + } + if ok { + return name, nil + } + } + + return "", nil +} diff --git a/generator/filetype_test.go b/generator/filetype_test.go new file mode 100644 index 00000000..1729c28b --- /dev/null +++ b/generator/filetype_test.go @@ -0,0 +1,35 @@ +package main + +import ( + "os" + "testing" + + "github.com/cavaliercoder/go-cpio" + "github.com/stretchr/testify/require" +) + +func TestFileType(t *testing.T) { + dir := t.TempDir() + check := func(compression, expectedType string) { + fileName := dir + "/" + compression + img, err := NewImage(fileName, compression, false) + require.NoError(t, err) + + require.NoError(t, img.AppendEntry("foo.txt", cpio.ModeRegular, []byte("hello, world!"))) + require.NoError(t, img.Close()) + + f, err := os.Open(fileName) + require.NoError(t, err) + + kind, err := filetype(f) + require.NoError(t, err) + + require.Equal(t, expectedType, kind) + } + + check("zstd", "zstd") + check("gzip", "gzip") + check("xz", "xz") + check("lz4", "lz4") + check("none", "cpio") +} diff --git a/generator/generator.go b/generator/generator.go index f15f9f01..ab66d220 100644 --- a/generator/generator.go +++ b/generator/generator.go @@ -85,7 +85,7 @@ var defaultModulesList = []string{ func generateInitRamfs(conf *generatorConfig) error { if _, err := os.Stat(conf.output); (err == nil || !os.IsNotExist(err)) && !conf.forceOverwrite { - return fmt.Errorf("File %v exists, please specify -force if you want to overwrite it", conf.output) + return fmt.Errorf("File %v exists, please specify --force if you want to overwrite it", conf.output) } img, err := NewImage(conf.output, conf.compression, conf.stripBinaries) diff --git a/generator/generator_test.go b/generator/generator_test.go index 7b1ae31b..41504163 100644 --- a/generator/generator_test.go +++ b/generator/generator_test.go @@ -530,7 +530,7 @@ func testModprobeOptions(t *testing.T) { } func TestGenerator(t *testing.T) { - *debugEnabled = testing.Verbose() + opts.Verbose = testing.Verbose() prepareAssets(t) diff --git a/generator/go.mod b/generator/go.mod index 4333aa32..bb6dd4c7 100644 --- a/generator/go.mod +++ b/generator/go.mod @@ -4,8 +4,8 @@ go 1.17 require ( github.com/cavaliercoder/go-cpio v0.0.0-20180626203310-925f9528c45e - github.com/frankban/quicktest v1.11.3 // indirect github.com/google/renameio v1.0.1 + github.com/jessevdk/go-flags v1.5.0 github.com/klauspost/compress v1.13.6 github.com/pierrec/lz4 v2.6.1+incompatible github.com/stretchr/testify v1.7.0 @@ -17,5 +17,6 @@ require ( require ( github.com/davecgh/go-spew v1.1.1 // indirect + github.com/frankban/quicktest v1.11.3 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect ) diff --git a/generator/go.sum b/generator/go.sum index 2ec8273e..05b0c634 100644 --- a/generator/go.sum +++ b/generator/go.sum @@ -9,6 +9,8 @@ github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/renameio v1.0.1 h1:Lh/jXZmvZxb0BBeSY5VKEfidcbcbenKjZFzM/q0fSeU= github.com/google/renameio v1.0.1/go.mod h1:t/HQoYBZSsWSNK35C6CO/TpPLDVWvxOHboWUAweKUpk= +github.com/jessevdk/go-flags v1.5.0 h1:1jKYvbxEjfUl0fmqTCOfonvskHHXMjBySTLW4y9LFvc= +github.com/jessevdk/go-flags v1.5.0/go.mod h1:Fw0T6WPc1dYxT4mKEZRfG5kJhaTDP9pj1c2EWnYs/m4= github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= @@ -27,6 +29,7 @@ github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 h1:nIPpBwaJSVYIxUFsDv3M8ofmx9yWTog9BfvIu0q41lo= github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8/go.mod h1:HUYIGzjTL3rfEspMxjDjgmT5uz5wzYJKVo23qUhYTos= +golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359 h1:2B5p2L5IfGiD7+b9BOoRMC6DgObAVZV+Fsp050NqXik= golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= diff --git a/generator/main.go b/generator/main.go index 9f392e1b..e91bbd0e 100644 --- a/generator/main.go +++ b/generator/main.go @@ -1,32 +1,58 @@ package main import ( - "flag" "fmt" "log" "os" "runtime" "runtime/pprof" -) -var ( - outputFile = flag.String("output", "booster.img", "Output initrd file") - forceOverwriteFile = flag.Bool("force", false, "Overwrite existing initrd file") - initBinary = flag.String("initBinary", "/usr/lib/booster/init", "Booster 'init' binary location") - compression = flag.String("compression", "", `Output file compression ("zstd", "gzip", "none")`) - kernelVersion = flag.String("kernelVersion", "", "Linux kernel version to generate initramfs for") - configFile = flag.String("config", "/etc/booster.yaml", "Configuration file path") - debugEnabled = flag.Bool("debug", false, "Enable debug output") - universal = flag.Bool("universal", false, "Add wide range of modules/tools to allow this image boot at different machines") - strip = flag.Bool("strip", false, "Strip ELF binaries before adding it to the image") - pprofcpu = flag.String("pprof.cpu", "", "Write cpu profile to file") - pprofmem = flag.String("pprof.mem", "", "Write memory profile to file") + "github.com/jessevdk/go-flags" ) +var opts struct { + Verbose bool `short:"v" long:"verbose" description:"Enable verbose output"` + Pprofcpu string `long:"pprof.cpu" description:"Write cpu profile to file" hidden:"true"` + Pprofmem string `long:"pprof.mem" description:"Write memory profile to file" hidden:"true"` + + BuildCommand struct { + Force bool `short:"f" long:"force" description:"Overwrite existing initrd file"` + InitBinary string `long:"init-binary" default:"/usr/lib/booster/init" description:"Booster 'init' binary location"` + Compression string `long:"compression" choice:"zstd" choice:"gzip" choice:"xz" choice:"lz4" choice:"none" description:"Output file compression"` + KernelVersion string `long:"kernel-version" description:"Linux kernel version to generate initramfs for"` + ConfigFile string `long:"config" default:"/etc/booster.yaml" description:"Configuration file path"` + Universal bool `long:"universal" description:"Add wide range of modules/tools to allow this image boot at different machines"` + Strip bool `long:"strip" description:"Strip ELF files (binaries, shared libraries and kernel modules) before adding it to the image"` + Args struct { + Output string `positional-arg-name:"output" required:"true"` + } `positional-args:"true"` + } `command:"build" description:"Build initrd image"` + + LsCommand struct { + Args struct { + Image string `positional-arg-name:"image" required:"true"` + } `positional-args:"true"` + } `command:"ls" description:"List content of the image"` + + CatCommand struct { + Args struct { + Image string `positional-arg-name:"image" required:"true"` + File string `positional-arg-name:"file-in-image" required:"true"` + } `positional-args:"true"` + } `command:"cat" description:"Show content of the file inside the image"` + + UnpackCommand struct { + Args struct { + Image string `positional-arg-name:"image" required:"true"` + OutputDir string `positional-arg-name:"output-dir" required:"true"` + } `positional-args:"true"` + } `command:"unpack" description:"Unpack image"` +} + type set map[string]bool func debug(format string, v ...interface{}) { - if *debugEnabled { + if opts.Verbose { fmt.Printf(format+"\n", v...) } } @@ -52,8 +78,8 @@ func saveProfile(profile, path string) error { } func runGenerator() error { - if *pprofcpu != "" { - f, err := os.Create(*pprofcpu) + if opts.Pprofcpu != "" { + f, err := os.Create(opts.Pprofcpu) if err != nil { return err } @@ -67,14 +93,14 @@ func runGenerator() error { increaseOpenFileLimit() - conf, err := readGeneratorConfig(*configFile) + conf, err := readGeneratorConfig(opts.BuildCommand.ConfigFile) if err != nil { return err } err = generateInitRamfs(conf) - if *pprofmem != "" { - if err := saveProfile("allocs", *pprofmem); err != nil { + if opts.Pprofmem != "" { + if err := saveProfile("allocs", opts.Pprofmem); err != nil { fmt.Println(err) } } @@ -82,9 +108,28 @@ func runGenerator() error { } func main() { - flag.Parse() + parser := flags.NewParser(&opts, flags.Default) + _, err := parser.Parse() + if err != nil { + if flagsErr, ok := err.(*flags.Error); ok && flagsErr.Type == flags.ErrHelp { + os.Exit(0) + } else { + os.Exit(1) + } + } - if err := runGenerator(); err != nil { + switch parser.Active.Name { + case "build": + err = runGenerator() + case "cat": + err = runCat() + case "ls": + err = runLs() + case "unpack": + err = runUnpack() + } + + if err != nil { log.Fatal(err) } } diff --git a/generator/unpack.go b/generator/unpack.go new file mode 100644 index 00000000..a76f2714 --- /dev/null +++ b/generator/unpack.go @@ -0,0 +1,148 @@ +package main + +import ( + "compress/gzip" + "fmt" + "io" + "os" + "path/filepath" + + "github.com/cavaliercoder/go-cpio" + "github.com/klauspost/compress/zstd" + "github.com/pierrec/lz4" + "github.com/ulikunitz/xz" +) + +var errStop = fmt.Errorf("Stop Processing") + +type processCpioEntryFn func(header *cpio.Header, reader *cpio.Reader) error + +func processImage(file string, fn processCpioEntryFn) error { + input, err := os.Open(file) + if err != nil { + return err + } + + var img *cpio.Reader + + kind, err := filetype(input) + if err != nil { + return err + } + + switch kind { + case "cpio": + img = cpio.NewReader(input) + case "zstd": + zst, err := zstd.NewReader(input) + if err != nil { + return err + } + defer zst.Close() + img = cpio.NewReader(zst) + case "gzip": + gz, err := gzip.NewReader(input) + if err != nil { + return err + } + defer gz.Close() + img = cpio.NewReader(gz) + case "xz": + conf := xz.ReaderConfig{} + if err := conf.Verify(); err != nil { + return err + } + x, err := conf.NewReader(input) + if err != nil { + return err + } + img = cpio.NewReader(x) + case "lz4": + lz := lz4.NewReaderLegacy(input) + img = cpio.NewReader(lz) + } + + for { + hdr, err := img.Next() + if err == io.EOF { + break + } + if err != nil { + return err + } + + err = fn(hdr, img) + if err == errStop { + break + } + if err != nil { + return err + } + } + + return nil +} + +func runUnpack() error { + dir := opts.UnpackCommand.Args.OutputDir + fn := func(hdr *cpio.Header, r *cpio.Reader) error { + out := filepath.Join(dir, hdr.Name) + if err := os.MkdirAll(filepath.Dir(out), 0755); err != nil { + return err + } + m := hdr.Mode &^ cpio.ModePerm + switch m { + case cpio.ModeDir: + if err := os.Mkdir(out, 0755); err != nil { + return err + } + case cpio.ModeSymlink: + if err := os.Symlink(hdr.Linkname, out); err != nil { + return err + } + case cpio.ModeRegular: + fout, err := os.Open(out) + if err != nil { + return err + } + if _, err := io.Copy(fout, r); err != nil { + return err + } + default: + warning("Unknown mode for file %s: %x", hdr.Name, m) + } + return nil + } + return processImage(opts.UnpackCommand.Args.Image, fn) +} + +func runCat() error { + fn := func(hdr *cpio.Header, r *cpio.Reader) error { + if hdr.Name == opts.CatCommand.Args.File { + if _, err := io.Copy(os.Stdout, r); err != nil { + return err + } + return errStop + } + return nil + } + return processImage(opts.CatCommand.Args.Image, fn) +} + +func runLs() error { + fn := func(hdr *cpio.Header, r *cpio.Reader) error { + m := hdr.Mode &^ cpio.ModePerm + switch m { + case cpio.ModeDir: + fmt.Printf("%s/\n", hdr.Name) + case cpio.ModeSymlink: + fmt.Printf("%s -> %s\n", hdr.Name, hdr.Linkname) + case cpio.ModeRegular: + fmt.Println(hdr.Name) + default: + warning("Unknown mode for file %s: %x", hdr.Name, m) + } + return nil + } + return processImage(opts.LsCommand.Args.Image, fn) +} diff --git a/packaging/arch/PKGBUILD b/packaging/arch/PKGBUILD index 37a6f38d..962d45eb 100644 --- a/packaging/arch/PKGBUILD +++ b/packaging/arch/PKGBUILD @@ -64,4 +64,5 @@ package() { install -Dp -m755 packaging/arch/booster-install "$pkgdir/usr/share/libalpm/scripts/booster-install" install -Dp -m644 packaging/arch/60-booster-remove.hook "$pkgdir/usr/share/libalpm/hooks/60-booster-remove.hook" install -Dp -m755 packaging/arch/booster-remove "$pkgdir/usr/share/libalpm/scripts/booster-remove" + install -Dp -m755 contrib/completion/bash "$pkgdir/usr/share/bash-completion/completions/booster" } diff --git a/packaging/arch/booster-install b/packaging/arch/booster-install index 6f09e137..f0284e42 100755 --- a/packaging/arch/booster-install +++ b/packaging/arch/booster-install @@ -26,7 +26,7 @@ for kernel in "${kernels[@]}"; do fi read -r pkgbase < "${kernel}/pkgbase" - booster -force -output /boot/booster-${pkgbase}.img -kernelVersion ${kernel##/usr/lib/modules/} & + booster build --force --kernel-version ${kernel##/usr/lib/modules/} /boot/booster-${pkgbase}.img & install -Dm644 "${kernel}/vmlinuz" "/boot/vmlinuz-${pkgbase}" done diff --git a/packaging/arch/regenerate_images b/packaging/arch/regenerate_images index 1d01d8e4..1e69539e 100755 --- a/packaging/arch/regenerate_images +++ b/packaging/arch/regenerate_images @@ -10,7 +10,7 @@ for kernel in "${kernels[@]}"; do fi read -r pkgbase < "${kernel}/pkgbase" - booster -force -output /boot/booster-${pkgbase}.img -kernelVersion ${kernel##/usr/lib/modules/} & + booster build --force --kernel-version ${kernel##/usr/lib/modules/} /boot/booster-${pkgbase}.img & install -Dm644 "${kernel}/vmlinuz" "/boot/vmlinuz-${pkgbase}" done diff --git a/tests/integration_test.go b/tests/integration_test.go index b8bf1694..0ffb57b3 100644 --- a/tests/integration_test.go +++ b/tests/integration_test.go @@ -67,7 +67,7 @@ func generateInitRamfs(opts Opts) (string, error) { } defer os.Remove(config) - cmd := exec.Command(binariesDir+"/generator", "-force", "-initBinary", binariesDir+"/init", "-kernelVersion", opts.kernelVersion, "-output", output, "-config", config) + cmd := exec.Command(binariesDir+"/generator", "build", "--force", "--init-binary", binariesDir+"/init", "--kernel-version", opts.kernelVersion, "--config", config, output) if testing.Verbose() { log.Print("Create booster.img with " + cmd.String()) cmd.Stdout = os.Stdout