-
Notifications
You must be signed in to change notification settings - Fork 258
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Use nbdkit for direct stream for the http importer (#1508)
* Add nbdkit with qemu-img Nbdkit can be used to stream directly the content of the image together with qemu-img. It supports different plugins and filters. Signed-off-by: Alice Frosi <afrosi@redhat.com> * Use nbdkit to the http importer Add nbdkit to stream directly the image content avoiding the scratch space for the http importer. Signed-off-by: Alice Frosi <afrosi@redhat.com> * Adjust functional test cases with nbdkit With the introduction of nbdkit, certain error messages do not match anymore, and needed to be adjusted to the new behavior. The scratch space is not needed anymore when the source is http for the majority of the cases. Certain test cases checked the use of the scratch space, and those are not valid anymore. Signed-off-by: Alice Frosi <afrosi@redhat.com>
- Loading branch information
Showing
11 changed files
with
491 additions
and
67 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,168 @@ | ||
| package image | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "github.com/pkg/errors" | ||
| "k8s.io/apimachinery/pkg/api/resource" | ||
| "k8s.io/klog/v2" | ||
| "kubevirt.io/containerized-data-importer/pkg/system" | ||
| "net/url" | ||
| "strings" | ||
| ) | ||
|
|
||
| var ( | ||
| nbdkitExecFunction = system.ExecWithLimits | ||
| ) | ||
|
|
||
| type nbdkitOperations struct { | ||
| nbdkit *Nbdkit | ||
| } | ||
|
|
||
| // NewNbdkitOperations return the implementation of nbdkit of QEMUOperations | ||
| func NewNbdkitOperations(n *Nbdkit) QEMUOperations { | ||
| return &nbdkitOperations{nbdkit: n} | ||
| } | ||
|
|
||
| // NbdkitPlugin represents a plugin for nbdkit | ||
| type NbdkitPlugin string | ||
|
|
||
| // NbdkitFilter represents s filter for nbdkit | ||
| type NbdkitFilter string | ||
|
|
||
| // Nbdkit plugins | ||
| const ( | ||
| NbdkitCurlPlugin NbdkitPlugin = "curl" | ||
| ) | ||
|
|
||
| // Nbdkit filters | ||
| const ( | ||
| NbdkitXzFilter NbdkitFilter = "xz" | ||
| NbdkitTarFilter NbdkitFilter = "tar" | ||
| NbdkitGzipFilter NbdkitFilter = "gzip" | ||
| ) | ||
|
|
||
| // Nbdkit represents struct for an nbdkit instance | ||
| type Nbdkit struct { | ||
| NbdPidFile string | ||
| nbdkitArgs []string | ||
| plugin NbdkitPlugin | ||
| pluginArgs []string | ||
| filters []NbdkitFilter | ||
| source *url.URL | ||
| } | ||
|
|
||
| // NewNbdkit creates a new Nbdkit instance with an nbdkit plugin and pid file | ||
| func NewNbdkit(plugin NbdkitPlugin, nbdkitPidFile string) *Nbdkit { | ||
| return &Nbdkit{ | ||
| NbdPidFile: nbdkitPidFile, | ||
| plugin: plugin, | ||
| } | ||
| } | ||
|
|
||
| // NewNbdkitCurl creates a new Nbdkit instance with the curl plugin | ||
| func NewNbdkitCurl(nbdkitPidFile, certDir string) *Nbdkit { | ||
| var pluginArgs []string | ||
| args := []string{"-r"} | ||
| if certDir != "" { | ||
| pluginArgs = append(pluginArgs, fmt.Sprintf("cainfo=%s/%s", certDir, "tls.crt")) | ||
| } | ||
|
|
||
| return &Nbdkit{ | ||
| NbdPidFile: nbdkitPidFile, | ||
| plugin: NbdkitCurlPlugin, | ||
| nbdkitArgs: args, | ||
| pluginArgs: pluginArgs, | ||
| } | ||
| } | ||
|
|
||
| // AddFilter adds a nbdkit filter if it doesn't already exist | ||
| func (n *Nbdkit) AddFilter(filter NbdkitFilter) { | ||
| for _, f := range n.filters { | ||
| if f == filter { | ||
| return | ||
| } | ||
| } | ||
| n.filters = append(n.filters, filter) | ||
| } | ||
|
|
||
| // Info returns the information about the content provided with the url | ||
| func (n *nbdkitOperations) Info(url *url.URL) (*ImgInfo, error) { | ||
| if len(url.Scheme) <= 0 { | ||
| return Info(url) | ||
| } | ||
| n.nbdkit.source = url | ||
| qemuImgArgs := []string{"--output=json"} | ||
| output, err := n.nbdkit.startNbdkitWithQemuImg("info", qemuImgArgs) | ||
| if err != nil { | ||
| return nil, errors.Errorf("%s, %s", output, err.Error()) | ||
| } | ||
| return checkOutputQemuImgInfo(output, url.String()) | ||
| } | ||
|
|
||
| // Validate validates the url | ||
| func (n *nbdkitOperations) Validate(url *url.URL, availableSize int64, filesystemOverhead float64) error { | ||
| info, err := n.Info(url) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return checkIfURLIsValid(info, availableSize, filesystemOverhead, url.String()) | ||
| } | ||
|
|
||
| // ConvertToRawStream converts the content provided by the url to a raw disk in the dest | ||
| func (n *nbdkitOperations) ConvertToRawStream(url *url.URL, dest string, preallocate bool) error { | ||
| if len(url.Scheme) <= 0 { | ||
| return ConvertToRawStream(url, dest, preallocate) | ||
| } | ||
| n.nbdkit.source = url | ||
| qemuImgArgs := []string{"-p", "-O", "raw", dest, "-t", "none"} | ||
| if preallocate { | ||
| klog.V(1).Info("Added preallocation") | ||
| qemuImgArgs = append(qemuImgArgs, []string{"-o", "preallocation=falloc"}...) | ||
| } | ||
| _, err := n.nbdkit.startNbdkitWithQemuImg("convert", qemuImgArgs) | ||
| return err | ||
| } | ||
|
|
||
| // CreateBlankImage creates empty raw image | ||
| func (n *nbdkitOperations) CreateBlankImage(dest string, size resource.Quantity, preallocate bool) error { | ||
| // Use the default function to create an empty raw image | ||
| return CreateBlankImage(dest, size, preallocate) | ||
| } | ||
|
|
||
| func (n *nbdkitOperations) Resize(image string, size resource.Quantity) error { | ||
| return Resize(image, size) | ||
| } | ||
|
|
||
| func (n *Nbdkit) getSource() string { | ||
| var source string | ||
| switch n.plugin { | ||
| case NbdkitCurlPlugin: | ||
| source = fmt.Sprintf("url=%s", n.source.String()) | ||
| default: | ||
| source = "" | ||
| } | ||
| return source | ||
| } | ||
|
|
||
| func (n *Nbdkit) startNbdkitWithQemuImg(qemuImgCmd string, qemuImgArgs []string) ([]byte, error) { | ||
| argsNbdkit := []string{ | ||
| "--foreground", | ||
| "--readonly", | ||
| "-U", "-", | ||
| "--pidfile", n.NbdPidFile, | ||
| } | ||
| // set filters | ||
| for _, f := range n.filters { | ||
| argsNbdkit = append(argsNbdkit, fmt.Sprintf("--filter=%s", f)) | ||
| } | ||
| // set additional arguments | ||
| for _, a := range n.nbdkitArgs { | ||
| argsNbdkit = append(argsNbdkit, a) | ||
| } | ||
| // append nbdkit plugin arguments | ||
| argsNbdkit = append(argsNbdkit, string(n.plugin), strings.Join(n.pluginArgs, " "), n.getSource()) | ||
| // append qemu-img command | ||
| argsNbdkit = append(argsNbdkit, "--run", fmt.Sprintf("qemu-img %s $nbd %v", qemuImgCmd, strings.Join(qemuImgArgs, " "))) | ||
| klog.V(3).Infof("Start nbdkit with: %v", argsNbdkit) | ||
| return nbdkitExecFunction(nil, reportProgress, "nbdkit", argsNbdkit...) | ||
| } |
Oops, something went wrong.