diff --git a/commands/util.go b/commands/util.go index fbb89f7c820..9ec5498a996 100644 --- a/commands/util.go +++ b/commands/util.go @@ -225,7 +225,7 @@ func driversForNodeGroup(ctx context.Context, dockerCli command.Cli, ng *store.N } } - d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, dockerCli.ConfigFile(), kcc, n.Flags, n.ConfigFile, n.DriverOpts, n.Platforms, contextPathHash) + d, err := driver.GetDriver(ctx, "buildx_buildkit_"+n.Name, f, dockerapi, dockerCli.ConfigFile(), kcc, n.Flags, n.Files, n.DriverOpts, n.Platforms, contextPathHash) if err != nil { di.Err = err return nil @@ -360,7 +360,7 @@ func getDefaultDrivers(ctx context.Context, dockerCli command.Cli, defaultOnly b } } - d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), dockerCli.ConfigFile(), nil, nil, "", nil, nil, contextPathHash) + d, err := driver.GetDriver(ctx, "buildx_buildkit_default", nil, dockerCli.Client(), dockerCli.ConfigFile(), nil, nil, nil, nil, nil, contextPathHash) if err != nil { return nil, err } diff --git a/driver/docker-container/driver.go b/driver/docker-container/driver.go index 80f745ecd09..9ea1a78ae7c 100644 --- a/driver/docker-container/driver.go +++ b/driver/docker-container/driver.go @@ -7,6 +7,8 @@ import ( "io/ioutil" "net" "os" + "path" + "path/filepath" "time" "github.com/docker/buildx/driver" @@ -134,15 +136,8 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error { if err != nil { return err } - if f := d.InitConfig.ConfigFile; f != "" { - configFiles, err := confutil.LoadConfigFiles(f) - if err != nil { - return err - } - defer os.RemoveAll(configFiles) - if err := d.copyToContainer(ctx, configFiles, "/"); err != nil { - return err - } + if err := d.copyToContainer(ctx, d.InitConfig.Files); err != nil { + return err } if err := d.start(ctx, l); err != nil { return err @@ -202,7 +197,14 @@ func (d *Driver) copyLogs(ctx context.Context, l progress.SubLogger) error { return rc.Close() } -func (d *Driver) copyToContainer(ctx context.Context, srcPath string, dstDir string) error { +func (d *Driver) copyToContainer(ctx context.Context, files map[string][]byte) error { + srcPath, err := writeConfigFiles(files) + if err != nil { + return err + } + if srcPath != "" { + defer os.RemoveAll(srcPath) + } srcArchive, err := dockerarchive.TarWithOptions(srcPath, &dockerarchive.TarOptions{ ChownOpts: &idtools.Identity{UID: 0, GID: 0}, }) @@ -210,7 +212,7 @@ func (d *Driver) copyToContainer(ctx context.Context, srcPath string, dstDir str return err } defer srcArchive.Close() - return d.DockerAPI.CopyToContainer(ctx, d.Name, dstDir, srcArchive, dockertypes.CopyToContainerOptions{}) + return d.DockerAPI.CopyToContainer(ctx, d.Name, "/", srcArchive, dockertypes.CopyToContainerOptions{}) } func (d *Driver) exec(ctx context.Context, cmd []string) (string, net.Conn, error) { @@ -383,3 +385,27 @@ func (l *logWriter) Write(dt []byte) (int, error) { l.logger.Log(l.stream, dt) return len(dt), nil } + +func writeConfigFiles(m map[string][]byte) (_ string, err error) { + // Temp dir that will be copied to the container + tmpDir, err := os.MkdirTemp("", "buildkitd-config") + if err != nil { + return "", err + } + defer func() { + if err != nil { + os.RemoveAll(tmpDir) + } + }() + for f, dt := range m { + f = path.Join(confutil.DefaultBuildKitConfigDir, f) + p := filepath.Join(tmpDir, f) + if err := os.MkdirAll(filepath.Dir(p), 0700); err != nil { + return "", err + } + if err := os.WriteFile(p, dt, 0600); err != nil { + return "", err + } + } + return tmpDir, nil +} diff --git a/driver/docker/factory.go b/driver/docker/factory.go index e1d0cae0ddb..c960830ea5e 100644 --- a/driver/docker/factory.go +++ b/driver/docker/factory.go @@ -44,7 +44,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver if cfg.DockerAPI == nil { return nil, errors.Errorf("docker driver requires docker API access") } - if cfg.ConfigFile != "" { + if len(cfg.Files) > 0 { return nil, errors.Errorf("setting config file is not supported for docker driver, use dockerd configuration file") } diff --git a/driver/kubernetes/driver.go b/driver/kubernetes/driver.go index 58daf373600..7e524e4b79e 100644 --- a/driver/kubernetes/driver.go +++ b/driver/kubernetes/driver.go @@ -41,7 +41,7 @@ type Driver struct { factory driver.Factory minReplicas int deployment *appsv1.Deployment - configMap *corev1.ConfigMap + configMaps []*corev1.ConfigMap clientset *kubernetes.Clientset deploymentClient clientappsv1.DeploymentInterface podClient clientcorev1.PodInterface @@ -65,16 +65,16 @@ func (d *Driver) Bootstrap(ctx context.Context, l progress.Logger) error { return errors.Wrapf(err, "error for bootstrap %q", d.deployment.Name) } - if d.configMap != nil { + for _, cfg := range d.configMaps { // create ConfigMap first if exists - _, err = d.configMapClient.Create(ctx, d.configMap, metav1.CreateOptions{}) + _, err = d.configMapClient.Create(ctx, cfg, metav1.CreateOptions{}) if err != nil { if !apierrors.IsAlreadyExists(err) { - return errors.Wrapf(err, "error while calling configMapClient.Create for %q", d.configMap.Name) + return errors.Wrapf(err, "error while calling configMapClient.Create for %q", cfg.Name) } - _, err = d.configMapClient.Update(ctx, d.configMap, metav1.UpdateOptions{}) + _, err = d.configMapClient.Update(ctx, cfg, metav1.UpdateOptions{}) if err != nil { - return errors.Wrapf(err, "error while calling configMapClient.Update for %q", d.configMap.Name) + return errors.Wrapf(err, "error while calling configMapClient.Update for %q", cfg.Name) } } } @@ -171,10 +171,10 @@ func (d *Driver) Rm(ctx context.Context, force bool, rmVolume bool) error { return errors.Wrapf(err, "error while calling deploymentClient.Delete for %q", d.deployment.Name) } } - if d.configMap != nil { - if err := d.configMapClient.Delete(ctx, d.configMap.Name, metav1.DeleteOptions{}); err != nil { + for _, cfg := range d.configMaps { + if err := d.configMapClient.Delete(ctx, cfg.Name, metav1.DeleteOptions{}); err != nil { if !apierrors.IsNotFound(err) { - return errors.Wrapf(err, "error while calling configMapClient.Delete for %q", d.configMap.Name) + return errors.Wrapf(err, "error while calling configMapClient.Delete for %q", cfg.Name) } } } diff --git a/driver/kubernetes/factory.go b/driver/kubernetes/factory.go index 9ddc5e564b6..61985e4b316 100644 --- a/driver/kubernetes/factory.go +++ b/driver/kubernetes/factory.go @@ -2,7 +2,6 @@ package kubernetes import ( "context" - "os" "strconv" "strings" @@ -74,18 +73,11 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver BuildkitFlags: cfg.BuildkitFlags, Rootless: false, Platforms: cfg.Platforms, + ConfigFiles: cfg.Files, } deploymentOpt.Qemu.Image = bkimage.QemuImage - if cfg.ConfigFile != "" { - buildkitConfig, err := os.ReadFile(cfg.ConfigFile) - if err != nil { - return nil, err - } - deploymentOpt.BuildkitConfig = buildkitConfig - } - loadbalance := LoadbalanceSticky for k, v := range cfg.DriverOpts { @@ -147,7 +139,7 @@ func (f *factory) New(ctx context.Context, cfg driver.InitConfig) (driver.Driver } } - d.deployment, d.configMap, err = manifest.NewDeployment(deploymentOpt) + d.deployment, d.configMaps, err = manifest.NewDeployment(deploymentOpt) if err != nil { return nil, err } diff --git a/driver/kubernetes/manifest/manifest.go b/driver/kubernetes/manifest/manifest.go index 3673044a0d0..75268247c4d 100644 --- a/driver/kubernetes/manifest/manifest.go +++ b/driver/kubernetes/manifest/manifest.go @@ -1,6 +1,8 @@ package manifest import ( + "fmt" + "path" "strings" "github.com/docker/buildx/util/platformutil" @@ -25,9 +27,8 @@ type DeploymentOpt struct { } BuildkitFlags []string - // BuildkitConfig - // when not empty, will create configmap with buildkit.toml and mounted - BuildkitConfig []byte + // files mounted at /etc/buildkitd + ConfigFiles map[string][]byte Rootless bool NodeSelector map[string]string @@ -43,7 +44,7 @@ const ( AnnotationPlatform = "buildx.docker.com/platform" ) -func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMap, err error) { +func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c []*corev1.ConfigMap, err error) { labels := map[string]string{ "app": opt.Name, } @@ -103,26 +104,23 @@ func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMa }, }, } - - if len(opt.BuildkitConfig) > 0 { - c = &corev1.ConfigMap{ + for _, cfg := range splitConfigFiles(opt.ConfigFiles) { + cc := &corev1.ConfigMap{ TypeMeta: metav1.TypeMeta{ APIVersion: corev1.SchemeGroupVersion.String(), Kind: "ConfigMap", }, ObjectMeta: metav1.ObjectMeta{ Namespace: opt.Namespace, - Name: opt.Name + "-config", + Name: opt.Name + "-" + cfg.name, Annotations: annotations, }, - Data: map[string]string{ - "buildkitd.toml": string(opt.BuildkitConfig), - }, + Data: cfg.files, } d.Spec.Template.Spec.Containers[0].VolumeMounts = []corev1.VolumeMount{{ - Name: "config", - MountPath: "/etc/buildkit", + Name: cfg.name, + MountPath: path.Join("/etc/buildkit", cfg.path), }} d.Spec.Template.Spec.Volumes = []corev1.Volume{{ @@ -130,11 +128,12 @@ func NewDeployment(opt *DeploymentOpt) (d *appsv1.Deployment, c *corev1.ConfigMa VolumeSource: corev1.VolumeSource{ ConfigMap: &corev1.ConfigMapVolumeSource{ LocalObjectReference: corev1.LocalObjectReference{ - Name: c.Name, + Name: cc.Name, }, }, }, }} + c = append(c, cc) } if opt.Qemu.Install { @@ -208,3 +207,35 @@ func toRootless(d *appsv1.Deployment) error { d.Spec.Template.ObjectMeta.Annotations["container.seccomp.security.alpha.kubernetes.io/"+containerName] = "unconfined" return nil } + +type config struct { + name string + path string + files map[string]string +} + +func splitConfigFiles(m map[string][]byte) []config { + var c []config + idx := map[string]int{} + nameIdx := 0 + for k, v := range m { + dir := path.Dir(k) + i, ok := idx[dir] + if !ok { + idx[dir] = len(c) + i = len(c) + name := "config" + if dir != "." { + nameIdx++ + name = fmt.Sprintf("%s-%d", name, nameIdx) + } + c = append(c, config{ + path: dir, + name: name, + files: map[string]string{}, + }) + } + c[i].files[path.Base(k)] = string(v) + } + return c +} diff --git a/driver/manager.go b/driver/manager.go index 33fd22a8c34..2fe2ffa10b6 100644 --- a/driver/manager.go +++ b/driver/manager.go @@ -53,7 +53,7 @@ type InitConfig struct { DockerAPI dockerclient.APIClient KubeClientConfig KubeClientConfig BuildkitFlags []string - ConfigFile string + Files map[string][]byte DriverOpts map[string]string Auth Auth Platforms []specs.Platform @@ -103,17 +103,17 @@ func GetFactory(name string, instanceRequired bool) Factory { return nil } -func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, config string, do map[string]string, platforms []specs.Platform, contextPathHash string) (Driver, error) { +func GetDriver(ctx context.Context, name string, f Factory, api dockerclient.APIClient, auth Auth, kcc KubeClientConfig, flags []string, files map[string][]byte, do map[string]string, platforms []specs.Platform, contextPathHash string) (Driver, error) { ic := InitConfig{ DockerAPI: api, KubeClientConfig: kcc, Name: name, BuildkitFlags: flags, - ConfigFile: config, DriverOpts: do, Auth: auth, Platforms: platforms, ContextPathHash: contextPathHash, + Files: files, } if f == nil { var err error diff --git a/store/nodegroup.go b/store/nodegroup.go index 5eb7c47cd72..c9f97a67228 100644 --- a/store/nodegroup.go +++ b/store/nodegroup.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/containerd/containerd/platforms" + "github.com/docker/buildx/util/confutil" "github.com/docker/buildx/util/platformutil" specs "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" @@ -21,8 +22,9 @@ type Node struct { Endpoint string Platforms []specs.Platform Flags []string - ConfigFile string DriverOpts map[string]string + + Files map[string][]byte } func (ng *NodeGroup) Leave(name string) error { @@ -88,10 +90,18 @@ func (ng *NodeGroup) Update(name, endpoint string, platforms []string, endpoints Name: name, Endpoint: endpoint, Platforms: pp, - ConfigFile: configFile, Flags: flags, DriverOpts: do, } + + if configFile != "" { + files, err := confutil.LoadConfigFiles(configFile) + if err != nil { + return err + } + n.Files = files + } + ng.Nodes = append(ng.Nodes, n) if err := ng.validateDuplicates(endpoint, len(ng.Nodes)-1); err != nil { diff --git a/util/confutil/container.go b/util/confutil/container.go index 9b9447e01e5..9434b8ff6e7 100644 --- a/util/confutil/container.go +++ b/util/confutil/container.go @@ -1,6 +1,7 @@ package confutil import ( + "bytes" "io" "os" "path" @@ -20,31 +21,20 @@ const ( // LoadConfigFiles creates a temp directory with BuildKit config and // registry certificates ready to be copied to a container. -func LoadConfigFiles(bkconfig string) (string, error) { +func LoadConfigFiles(bkconfig string) (map[string][]byte, error) { if _, err := os.Stat(bkconfig); errors.Is(err, os.ErrNotExist) { - return "", errors.Wrapf(err, "buildkit configuration file not found: %s", bkconfig) + return nil, errors.Wrapf(err, "buildkit configuration file not found: %s", bkconfig) } else if err != nil { - return "", errors.Wrapf(err, "invalid buildkit configuration file: %s", bkconfig) + return nil, errors.Wrapf(err, "invalid buildkit configuration file: %s", bkconfig) } // Load config tree btoml, err := loadConfigTree(bkconfig) if err != nil { - return "", err + return nil, err } - // Temp dir that will be copied to the container - tmpDir, err := os.MkdirTemp("", "buildkitd-config") - if err != nil { - return "", err - } - - // Create BuildKit config folders - tmpBuildKitConfigDir := path.Join(tmpDir, DefaultBuildKitConfigDir) - tmpBuildKitCertsDir := path.Join(tmpBuildKitConfigDir, "certs") - if err := os.MkdirAll(tmpBuildKitCertsDir, 0700); err != nil { - return "", err - } + m := make(map[string][]byte) // Iterate through registry config to copy certs and update // BuildKit config with the underlying certs' path in the container. @@ -70,19 +60,20 @@ func LoadConfigFiles(bkconfig string) (string, error) { if regConf == nil { continue } - regCertsDir := path.Join(tmpBuildKitCertsDir, regName) - if err := os.Mkdir(regCertsDir, 0755); err != nil { - return "", err - } + pfx := path.Join("certs", regName) if regConf.Has("ca") { regCAs := regConf.GetArray("ca").([]string) if len(regCAs) > 0 { var cas []string for _, ca := range regCAs { - cas = append(cas, path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(ca))) - if err := copyfile(ca, path.Join(regCertsDir, path.Base(ca))); err != nil { - return "", err + fp := path.Join(pfx, path.Base(ca)) + cas = append(cas, path.Join(DefaultBuildKitConfigDir, fp)) + + dt, err := readFile(ca) + if err != nil { + return nil, errors.Wrapf(err, "failed to read CA file: %s", ca) } + m[fp] = dt } regConf.Set("ca", cas) } @@ -98,47 +89,44 @@ func LoadConfigFiles(bkconfig string) (string, error) { } key := kp.Get("key").(string) if len(key) > 0 { - kp.Set("key", path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(key))) - if err := copyfile(key, path.Join(regCertsDir, path.Base(key))); err != nil { - return "", err + fp := path.Join(pfx, path.Base(key)) + kp.Set("key", path.Join(DefaultBuildKitConfigDir, fp)) + dt, err := readFile(key) + if err != nil { + return nil, errors.Wrapf(err, "failed to read key file: %s", key) } + m[fp] = dt } cert := kp.Get("cert").(string) if len(cert) > 0 { - kp.Set("cert", path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(cert))) - if err := copyfile(cert, path.Join(regCertsDir, path.Base(cert))); err != nil { - return "", err + fp := path.Join(pfx, path.Base(cert)) + kp.Set("cert", path.Join(DefaultBuildKitConfigDir, fp)) + dt, err := readFile(cert) + if err != nil { + return nil, errors.Wrapf(err, "failed to read cert file: %s", cert) } + m[fp] = dt } } } } } - // Write BuildKit config - bkfile, err := os.OpenFile(path.Join(tmpBuildKitConfigDir, "buildkitd.toml"), os.O_CREATE|os.O_WRONLY, 0600) - if err != nil { - return "", err - } - _, err = btoml.WriteTo(bkfile) + b := bytes.NewBuffer(nil) + _, err = btoml.WriteTo(b) if err != nil { - return "", err + return nil, err } + m["buildkitd.toml"] = b.Bytes() - return tmpDir, nil + return m, nil } -func copyfile(src string, dst string) error { - sf, err := os.Open(src) +func readFile(fp string) ([]byte, error) { + sf, err := os.Open(fp) if err != nil { - return err + return nil, err } defer sf.Close() - df, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0600) - if err != nil { - return err - } - defer df.Close() - _, err = io.Copy(df, sf) - return err + return io.ReadAll(io.LimitReader(sf, 1024*1024)) }