Skip to content

Commit

Permalink
Run registries for pull-through in pods
Browse files Browse the repository at this point in the history
  • Loading branch information
@dmage
dmage committed Dec 3, 2020
1 parent f47575c commit eb9dace
Show file tree
Hide file tree
Showing 225 changed files with 5,221 additions and 23,308 deletions.
1 change: 1 addition & 0 deletions go.mod
View file
Expand Up @@ -33,6 +33,7 @@ require (
github.com/prometheus/client_golang v1.7.1
github.com/satori/go.uuid v1.2.1-0.20180103174451-36e9d2ebbde5 // indirect
github.com/sirupsen/logrus v1.6.0
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9
google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8 // indirect
gopkg.in/yaml.v2 v2.3.0
k8s.io/api v0.19.0
Expand Down
3 changes: 3 additions & 0 deletions pkg/dockerregistry/server/metrics/metrics.go
View file
Expand Up @@ -78,6 +78,9 @@ func dockerErrorCode(err error) string {
if e, ok := err.(errcode.Error); ok {
return e.ErrorCode().String()
}
if strings.Contains(err.Error(), "no basic auth credentials") {
return "UNAUTHORIZED"
}
return "UNKNOWN"
}

Expand Down
49 changes: 48 additions & 1 deletion pkg/testframework/manifest.go
View file
@@ -1,12 +1,16 @@
package testframework

import (
"context"
"encoding/json"
"fmt"
"net/http"

"github.com/docker/distribution"
"github.com/opencontainers/go-digest"
"github.com/pborman/uuid"

"github.com/openshift/image-registry/pkg/testutil"
)

type Schema2ImageData struct {
Expand All @@ -22,9 +26,21 @@ type Schema2ImageData struct {
}

func NewSchema2ImageData() (Schema2ImageData, error) {
cfg := map[string]interface{}{
"rootfs": map[string]interface{}{
"diff_ids": make([]string, 1),
},
"history": make([]struct{}, 1),
}

configContent, err := json.Marshal(&cfg)
if err != nil {
return Schema2ImageData{}, fmt.Errorf("marshal image config: %w", err)
}

data := Schema2ImageData{
ConfigMediaType: "application/vnd.docker.container.image.v1+json",
Config: []byte("{}"),
Config: []byte(configContent),
LayerMediaType: "application/vnd.docker.image.rootfs.diff.tar.gzip",
Layer: []byte("image-registry-integration-test-" + uuid.NewRandom().String()),
ManifestMediaType: "application/vnd.docker.distribution.manifest.v2+json",
Expand Down Expand Up @@ -104,3 +120,34 @@ func ServeImage(w http.ResponseWriter, r *http.Request, image string, data Schem
}
return true
}

func PushSchema2ImageData(ctx context.Context, repo distribution.Repository, tag string, data Schema2ImageData) (distribution.Manifest, error) {
layerDesc := distribution.Descriptor{
Digest: data.LayerDigest,
Size: int64(len(data.Layer)),
}

if err := testutil.UploadBlob(ctx, repo, layerDesc, data.Layer); err != nil {
return nil, fmt.Errorf("upload layer: %w", err)
}

configDesc := distribution.Descriptor{
Digest: data.ConfigDigest,
Size: int64(len(data.Config)),
}

if err := testutil.UploadBlob(ctx, repo, configDesc, data.Config); err != nil {
return nil, fmt.Errorf("upload image config: %w", err)
}

manifest, _, err := distribution.UnmarshalManifest(data.ManifestMediaType, data.Manifest)
if err != nil {
return manifest, fmt.Errorf("parse manifest: %w", err)
}

if err := testutil.UploadManifest(ctx, repo, tag, manifest); err != nil {
return manifest, fmt.Errorf("upload manifest: %w", err)
}

return manifest, nil
}
194 changes: 4 additions & 190 deletions pkg/testframework/master.go
View file
Expand Up @@ -3,24 +3,14 @@ package testframework
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"io"
"io/ioutil"
"net"
"net/http"
"os"
"path"
"path/filepath"
"strconv"
"testing"
"time"

"github.com/docker/distribution"
"github.com/docker/docker/api/types"
"github.com/docker/docker/api/types/container"
"github.com/docker/docker/client"
"github.com/docker/docker/pkg/archive"
"github.com/pborman/uuid"

corev1 "k8s.io/api/core/v1"
Expand Down Expand Up @@ -92,168 +82,6 @@ func (p *MasterProcess) WaitHealthz(configDir string) error {
return WaitHTTP(rt, fmt.Sprintf("https://%s/healthz", u.Host))
}

type MasterContainer struct {
ID string
Port int
NetworkSettings struct {
IPAddress string
}

KubeConfigPath string
}

func StartMasterContainer(configDir string) (MasterInterface, error) {
cli, err := client.NewEnvClient()
if err != nil {
return nil, err
}

ctx := context.Background()

progress, err := cli.ImagePull(ctx, originImageRef, types.ImagePullOptions{})
if err != nil {
return nil, fmt.Errorf("pull image for master container: %v", err)
}
_, copyErr := io.Copy(ioutil.Discard, progress)
if err := progress.Close(); err != nil {
return nil, fmt.Errorf("close pull progress for master container: %v", err)
}
if copyErr != nil {
return nil, fmt.Errorf("read pull progress for master container: %v", copyErr)
}

resp, err := cli.ContainerCreate(ctx, &container.Config{
Image: originImageRef,
Entrypoint: []string{"/bin/sh"},
Cmd: []string{"-ec", `
openshift start master --write-config=/var/lib/origin/openshift.local.config/master
sed -i'' -e '/- domainName:/d' /var/lib/origin/openshift.local.config/master/master-config.yaml
exec openshift start master --config=/var/lib/origin/openshift.local.config/master/master-config.yaml
`},
}, nil, nil, "")
if err != nil {
return nil, fmt.Errorf("create master container: %v", err)
}

if err := cli.ContainerStart(ctx, resp.ID, types.ContainerStartOptions{}); err != nil {
return nil, fmt.Errorf("start master container: %v", err)
}

c := &MasterContainer{
ID: resp.ID,
Port: 8443,
KubeConfigPath: filepath.Join(configDir, "master", "admin.kubeconfig"),
}

inspectResult, err := cli.ContainerInspect(ctx, resp.ID)
if err != nil {
// TODO(dmage): log error
_ = c.Stop()
return nil, fmt.Errorf("inspect master container: %v", err)
}

c.NetworkSettings.IPAddress = inspectResult.NetworkSettings.IPAddress

if err := WaitTCP(c.NetworkSettings.IPAddress + ":" + strconv.Itoa(c.Port)); err != nil {
_ = c.Stop()
return c, err
}

if err := c.WriteConfigs(configDir); err != nil {
_ = c.Stop()
return c, err
}

if err := c.WaitHealthz(configDir); err != nil {
_ = c.Stop()
return c, err
}

return c, nil
}

func (m *MasterContainer) AdminKubeConfigPath() string {
return m.KubeConfigPath
}

func (c *MasterContainer) WriteConfigs(configDir string) error {
cli, err := client.NewEnvClient()
if err != nil {
return err
}

ctx := context.Background()
// If SRC_PATH does end with `/.`, the content of the source directory is copied.
// https://docs.docker.com/engine/reference/commandline/cp/#extended-description
srcPath := "/var/lib/origin/openshift.local.config/."
dstPath := configDir

content, stat, err := cli.CopyFromContainer(ctx, c.ID, srcPath)
if err != nil {
return fmt.Errorf("get configs from master container: %v", err)
}

srcInfo := archive.CopyInfo{
Path: srcPath,
Exists: true,
IsDir: stat.Mode.IsDir(),
}

if err := archive.CopyTo(content, srcInfo, dstPath); err != nil {
return fmt.Errorf("unpack archive with configs from master container: %v", err)
}

return nil
}

func (c *MasterContainer) WaitHealthz(configDir string) error {
caBundlePath := path.Join(configDir, "master", "ca-bundle.crt")

caBundle, err := ioutil.ReadFile(caBundlePath)
if err != nil {
return fmt.Errorf("unable to read CA bundle: %v", err)
}

rootCAs := x509.NewCertPool()
rootCAs.AppendCertsFromPEM(caBundle)
tlsConfig := &tls.Config{
RootCAs: rootCAs,
}
rt := &http.Transport{
Proxy: http.ProxyFromEnvironment,
DialContext: (&net.Dialer{
Timeout: 30 * time.Second,
KeepAlive: 30 * time.Second,
DualStack: true,
}).DialContext,
MaxIdleConns: 100,
IdleConnTimeout: 90 * time.Second,
TLSHandshakeTimeout: 10 * time.Second,
ExpectContinueTimeout: 1 * time.Second,
TLSClientConfig: tlsConfig,
}

return WaitHTTP(rt, fmt.Sprintf("https://%s:%d/healthz", c.NetworkSettings.IPAddress, c.Port))
}

func (c *MasterContainer) Stop() error {
cli, err := client.NewEnvClient()
if err != nil {
return err
}

ctx := context.Background()

if err := cli.ContainerRemove(ctx, c.ID, types.ContainerRemoveOptions{
RemoveVolumes: true,
Force: true,
}); err != nil {
return fmt.Errorf("remove master container %s: %v", c.ID, err)
}

return nil
}

type User struct {
Name string
Token string
Expand Down Expand Up @@ -285,40 +113,30 @@ func (r *Repository) Transport() http.RoundTripper {

type Master struct {
t *testing.T
tmpDir string
container MasterInterface
adminKubeConfig *rest.Config
namespaces []string
}

func NewMaster(t *testing.T) *Master {
tmpDir, err := ioutil.TempDir("", "image-registry-test-")
if err != nil {
t.Fatalf("failed to create a temporary directory for the master container: %v", err)
}

var container MasterInterface
var err error
if path, ok := os.LookupEnv("TEST_KUBECONFIG"); ok {
container, err = StartMasterProcess(path)
} else if path, ok := os.LookupEnv("KUBECONFIG"); ok {
container, err = StartMasterProcess(path)
} else {
container, err = StartMasterContainer(tmpDir)
t.Fatalf("tests should be run with either TEST_KUBECONFIG or KUBECONFIG")
}
if err != nil {
if removeErr := os.RemoveAll(tmpDir); removeErr != nil {
t.Logf("failed to remove the temporary directory: %v", removeErr)
}
t.Fatal(err)
}

m := &Master{
t: t,
tmpDir: tmpDir,
container: container,
}
if err := m.WaitForRoles(); err != nil {
if removeErr := os.RemoveAll(tmpDir); removeErr != nil {
t.Logf("failed to remove the temporary directory: %v", removeErr)
}
t.Fatal(err)
}
return m
Expand Down Expand Up @@ -363,10 +181,6 @@ func (m *Master) Close() {
if err := m.container.Stop(); err != nil {
m.t.Logf("failed to stop the master container: %v", err)
}

if err := os.RemoveAll(m.tmpDir); err != nil {
m.t.Logf("failed to remove the temporary directory: %v", err)
}
}

func (m *Master) AdminKubeConfig() *rest.Config {
Expand Down

0 comments on commit eb9dace

Please sign in to comment.