Skip to content

Commit

Permalink
container driver: copy ca and user tls registries certs
Browse files Browse the repository at this point in the history
Signed-off-by: CrazyMax <crazy-max@users.noreply.github.com>
  • Loading branch information
crazy-max committed Oct 27, 2021
1 parent 084b6c0 commit 806ae3d
Show file tree
Hide file tree
Showing 69 changed files with 11,414 additions and 33 deletions.
5 changes: 5 additions & 0 deletions docs/reference/buildx_create.md
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,11 @@ Specifies the configuration file for the buildkitd daemon to use. The configurat
can be overridden by [`--buildkitd-flags`](#buildkitd-flags).
See an [example buildkitd configuration file](https://github.com/moby/buildkit/blob/master/docs/buildkitd.toml.md).

Note that if you create a `docker-container` builder and have specified
certificates for registries in the `buildkitd.toml` configuration, the files
will be copied into the container under `/etc/buildkit/certs` and configuration
will be updated to reflect that.

### <a name="driver"></a> Set the builder driver to use (--driver)

```
Expand Down
74 changes: 41 additions & 33 deletions driver/docker-container/driver.go
Original file line number Diff line number Diff line change
@@ -1,17 +1,18 @@
package docker

import (
"archive/tar"
"bytes"
"context"
"io"
"io/ioutil"
"net"
"os"
"path/filepath"
"time"

"github.com/docker/buildx/driver"
"github.com/docker/buildx/driver/bkimage"
"github.com/docker/buildx/util/confutil"
"github.com/docker/buildx/util/imagetools"
"github.com/docker/buildx/util/progress"
"github.com/docker/docker/api/types"
Expand All @@ -20,6 +21,7 @@ import (
"github.com/docker/docker/api/types/mount"
"github.com/docker/docker/api/types/network"
dockerclient "github.com/docker/docker/client"
dockerarchive "github.com/docker/docker/pkg/archive"
"github.com/docker/docker/pkg/stdcopy"
"github.com/moby/buildkit/client"
"github.com/moby/buildkit/util/tracing/detect"
Expand All @@ -28,12 +30,6 @@ import (

const (
volumeStateSuffix = "_state"

// containerStateDir is the location where buildkitd inside the container
// stores its state. The container driver creates a Linux container, so
// this should match the location for Linux, as defined in:
// https://github.com/moby/buildkit/blob/v0.9.0/util/appdefaults/appdefaults_unix.go#L11-L15
containerBuildKitRootDir = "/var/lib/buildkit"
)

type Driver struct {
Expand Down Expand Up @@ -119,7 +115,7 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
{
Type: mount.TypeVolume,
Source: d.Name + volumeStateSuffix,
Target: containerBuildKitRootDir,
Target: confutil.DefaultBuildKitStateDir,
},
},
}
Expand All @@ -139,11 +135,12 @@ func (d *Driver) create(ctx context.Context, l progress.SubLogger) error {
return err
}
if f := d.InitConfig.ConfigFile; f != "" {
buf, err := readFileToTar(f)
configFiles, err := confutil.LoadConfigFiles(f)
if err != nil {
return err
}
if err := d.DockerAPI.CopyToContainer(ctx, d.Name, "/", buf, dockertypes.CopyToContainerOptions{}); err != nil {
defer os.RemoveAll(configFiles)
if err := d.copyToContainer(ctx, configFiles+"/.", "/"); err != nil {
return err
}
}
Expand Down Expand Up @@ -205,6 +202,40 @@ func (d *Driver) copyLogs(ctx context.Context, l progress.SubLogger) error {
return rc.Close()
}

// copyToContainer is based on the implementation from docker/cli
// https://github.com/docker/cli/blob/master/cli/command/container/cp.go
func (d *Driver) copyToContainer(ctx context.Context, srcPath string, dstDir string) error {
srcPath, err := resolveLocalPath(srcPath)
if err != nil {
return err
}
srcStat, err := os.Lstat(srcPath)
if err != nil {
return err
}
srcInfo := dockerarchive.CopyInfo{
Path: srcPath,
Exists: true,
IsDir: srcStat.IsDir(),
}
srcArchive, err := dockerarchive.TarResource(srcInfo)
if err != nil {
return err
}
defer srcArchive.Close()

return d.DockerAPI.CopyToContainer(ctx, d.Name, dstDir, srcArchive, dockertypes.CopyToContainerOptions{
AllowOverwriteDirWithFile: false,
})
}

func resolveLocalPath(localPath string) (absPath string, err error) {
if absPath, err = filepath.Abs(localPath); err != nil {
return
}
return dockerarchive.PreserveTrailingDotOrSeparator(absPath, localPath, filepath.Separator), nil
}

func (d *Driver) exec(ctx context.Context, cmd []string) (string, net.Conn, error) {
execConfig := types.ExecConfig{
Cmd: cmd,
Expand Down Expand Up @@ -366,29 +397,6 @@ func (d *demux) Read(dt []byte) (int, error) {
return d.Reader.Read(dt)
}

func readFileToTar(fn string) (*bytes.Buffer, error) {
buf := bytes.NewBuffer(nil)
tw := tar.NewWriter(buf)
dt, err := ioutil.ReadFile(fn)
if err != nil {
return nil, err
}
if err := tw.WriteHeader(&tar.Header{
Name: "/etc/buildkit/buildkitd.toml",
Size: int64(len(dt)),
Mode: 0644,
}); err != nil {
return nil, err
}
if _, err := tw.Write(dt); err != nil {
return nil, err
}
if err := tw.Close(); err != nil {
return nil, err
}
return buf, nil
}

type logWriter struct {
logger progress.SubLogger
stream int
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@ require (
github.com/moby/buildkit v0.9.1-0.20211019185819-8778943ac3da
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.2-0.20210819154149-5ad6f50d6283
github.com/pelletier/go-toml v1.9.4
github.com/pkg/errors v0.9.1
github.com/serialx/hashring v0.0.0-20190422032157-8b2912629002
github.com/sirupsen/logrus v1.8.1
Expand Down
25 changes: 25 additions & 0 deletions util/confutil/config.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
package confutil

import (
"os"

"github.com/pelletier/go-toml"
"github.com/pkg/errors"
)

// loadConfigTree loads BuildKit config toml tree
func loadConfigTree(fp string) (*toml.Tree, error) {
f, err := os.Open(fp)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
return nil, errors.Wrapf(err, "failed to load config from %s", fp)
}
defer f.Close()
t, err := toml.LoadReader(f)
if err != nil {
return t, errors.Wrap(err, "failed to parse config")
}
return t, nil
}
144 changes: 144 additions & 0 deletions util/confutil/container.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
package confutil

import (
"io"
"os"
"path"

"github.com/pelletier/go-toml"
"github.com/pkg/errors"
)

const (
// DefaultBuildKitStateDir and DefaultBuildKitConfigDir are the location
// where buildkitd inside the container stores its state. Some drivers
// create a Linux container, so this should match the location for Linux,
// as defined in: https://github.com/moby/buildkit/blob/v0.9.0/util/appdefaults/appdefaults_unix.go#L11-L15
DefaultBuildKitStateDir = "/var/lib/buildkit"
DefaultBuildKitConfigDir = "/etc/buildkit"
)

// LoadConfigFiles creates a temp directory with BuildKit config and
// registry certificates ready to be copied to a container.
func LoadConfigFiles(bkconfig string) (string, error) {
if _, err := os.Stat(bkconfig); errors.Is(err, os.ErrNotExist) {
return "", errors.Wrapf(err, "buildkit configuration file not found: %s", bkconfig)
} else if err != nil {
return "", errors.Wrapf(err, "invalid buildkit configuration file: %s", bkconfig)
}

// Load config tree
btoml, err := loadConfigTree(bkconfig)
if err != nil {
return "", err
}

// Temp dir that will be copied to the container
tmpDir, err := os.MkdirTemp("", "buildkitd-config")
if err != nil {
return "", err
}

// Create BuildKit config folders
tmpBuildKitConfigDir := path.Join(tmpDir, DefaultBuildKitConfigDir)
tmpBuildKitCertsDir := path.Join(tmpBuildKitConfigDir, "certs")
if err := os.MkdirAll(tmpBuildKitCertsDir, 0700); err != nil {
return "", err
}

// Iterate through registry config to copy certs and update
// BuildKit config with the underlying certs' path in the container.
//
// The following BuildKit config:
//
// [registry."myregistry.io"]
// ca=["/etc/config/myca.pem"]
// [[registry."myregistry.io".keypair]]
// key="/etc/config/key.pem"
// cert="/etc/config/cert.pem"
//
// will be translated in the container as:
//
// [registry."myregistry.io"]
// ca=["/etc/buildkit/certs/myregistry.io/myca.pem"]
// [[registry."myregistry.io".keypair]]
// key="/etc/buildkit/certs/myregistry.io/key.pem"
// cert="/etc/buildkit/certs/myregistry.io/cert.pem"
if btoml.Has("registry") {
for regName := range btoml.GetArray("registry").(*toml.Tree).Values() {
regConf := btoml.GetPath([]string{"registry", regName}).(*toml.Tree)
if regConf == nil {
continue
}
regCertsDir := path.Join(tmpBuildKitCertsDir, regName)
if err := os.Mkdir(regCertsDir, 0755); err != nil {
return "", err
}
if regConf.Has("ca") {
regCAs := regConf.GetArray("ca").([]string)
if len(regCAs) > 0 {
var cas []string
for _, ca := range regCAs {
cas = append(cas, path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(ca)))
if err := copyfile(ca, path.Join(regCertsDir, path.Base(ca))); err != nil {
return "", err
}
}
regConf.Set("ca", cas)
}
}
if regConf.Has("keypair") {
regKeyPairs := regConf.GetArray("keypair").([]*toml.Tree)
if len(regKeyPairs) == 0 {
continue
}
for _, kp := range regKeyPairs {
if kp == nil {
continue
}
key := kp.Get("key").(string)
if len(key) > 0 {
kp.Set("key", path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(key)))
if err := copyfile(key, path.Join(regCertsDir, path.Base(key))); err != nil {
return "", err
}
}
cert := kp.Get("cert").(string)
if len(cert) > 0 {
kp.Set("cert", path.Join(DefaultBuildKitConfigDir, "certs", regName, path.Base(cert)))
if err := copyfile(cert, path.Join(regCertsDir, path.Base(cert))); err != nil {
return "", err
}
}
}
}
}
}

// Write BuildKit config
bkfile, err := os.OpenFile(path.Join(tmpBuildKitConfigDir, "buildkitd.toml"), os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
return "", err
}
_, err = btoml.WriteTo(bkfile)
if err != nil {
return "", err
}

return tmpDir, nil
}

func copyfile(src string, dst string) error {
sf, err := os.Open(src)
if err != nil {
return err
}
defer sf.Close()
df, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0600)
if err != nil {
return err
}
defer df.Close()
_, err = io.Copy(df, sf)
return err
}
62 changes: 62 additions & 0 deletions vendor/github.com/containerd/containerd/pkg/userns/userns_linux.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

0 comments on commit 806ae3d

Please sign in to comment.