Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Extend apiserver testserver #55548

Merged
merged 1 commit into from
Nov 23, 2017
Merged

Extend apiserver testserver #55548

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
29 changes: 2 additions & 27 deletions cmd/kube-apiserver/app/testing/BUILD
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,32 +3,6 @@ package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
"go_test",
)

go_test(
name = "go_default_test",
srcs = ["server_test.go"],
importpath = "k8s.io/kubernetes/cmd/kube-apiserver/app/testing",
library = ":go_default_library",
deps = [
"//vendor/k8s.io/api/admissionregistration/v1alpha1:go_default_library",
"//vendor/k8s.io/api/apps/v1beta1:go_default_library",
"//vendor/k8s.io/api/core/v1:go_default_library",
"//vendor/k8s.io/api/networking/v1:go_default_library",
"//vendor/k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1:go_default_library",
"//vendor/k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/api/errors:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/apis/meta/v1/unstructured:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/runtime/schema:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/apiserver/pkg/features:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature:go_default_library",
"//vendor/k8s.io/apiserver/pkg/util/feature/testing:go_default_library",
"//vendor/k8s.io/client-go/dynamic:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
],
)

go_library(
Expand All @@ -38,9 +12,10 @@ go_library(
deps = [
"//cmd/kube-apiserver/app:go_default_library",
"//cmd/kube-apiserver/app/options:go_default_library",
"//vendor/github.com/spf13/pflag:go_default_library",
"//vendor/k8s.io/apimachinery/pkg/util/wait:go_default_library",
"//vendor/k8s.io/apiserver/pkg/registry/generic/registry:go_default_library",
"//vendor/k8s.io/apiserver/pkg/storage/etcd/testing:go_default_library",
"//vendor/k8s.io/apiserver/pkg/storage/storagebackend:go_default_library",
"//vendor/k8s.io/client-go/kubernetes:go_default_library",
"//vendor/k8s.io/client-go/rest:go_default_library",
],
Expand Down
79 changes: 45 additions & 34 deletions cmd/kube-apiserver/app/testing/testserver.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,13 +21,14 @@ import (
"io/ioutil"
"net"
"os"
"strings"
"testing"
"time"

pflag "github.com/spf13/pflag"

"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/registry/generic/registry"
etcdtesting "k8s.io/apiserver/pkg/storage/etcd/testing"
"k8s.io/apiserver/pkg/storage/storagebackend"
"k8s.io/client-go/kubernetes"
restclient "k8s.io/client-go/rest"
"k8s.io/kubernetes/cmd/kube-apiserver/app"
Expand All @@ -37,15 +38,21 @@ import (
// TearDownFunc is to be called to tear down a test server.
type TearDownFunc func()

// StartTestServer starts a etcd server and kube-apiserver. A rest client config and a tear-down func
// are returned.
// TestServer return values supplied by kube-test-ApiServer
type TestServer struct {
ClientConfig *restclient.Config // Rest client config
ServerOpts *options.ServerRunOptions // ServerOpts
TearDownFn TearDownFunc // TearDown function
TmpDir string // Temp Dir used, by the apiserver
}

// StartTestServer starts a etcd server and kube-apiserver. A rest client config and a tear-down func,
// and location of the tmpdir are returned.
//
// Note: we return a tear-down func instead of a stop channel because the later will leak temporariy
// files that becaues Golang testing's call to os.Exit will not give a stop channel go routine
// enough time to remove temporariy files.
func StartTestServer(t *testing.T) (result *restclient.Config, tearDownForCaller TearDownFunc, err error) {
var tmpDir string
var etcdServer *etcdtesting.EtcdTestServer
func StartTestServer(t *testing.T, customFlags []string, storageConfig *storagebackend.Config) (result TestServer, err error) {

// TODO : Remove TrackStorageCleanup below when PR
// https://github.com/kubernetes/kubernetes/pull/50690
Expand All @@ -56,46 +63,45 @@ func StartTestServer(t *testing.T) (result *restclient.Config, tearDownForCaller
tearDown := func() {
registry.CleanupStorage()
close(stopCh)
if etcdServer != nil {
etcdServer.Terminate(t)
}
if len(tmpDir) != 0 {
os.RemoveAll(tmpDir)
if len(result.TmpDir) != 0 {
os.RemoveAll(result.TmpDir)
}
}
defer func() {
if tearDownForCaller == nil {
if result.TearDownFn == nil {
tearDown()
}
}()

t.Logf("Starting etcd...")
etcdServer, storageConfig := etcdtesting.NewUnsecuredEtcd3TestClientServer(t)

tmpDir, err = ioutil.TempDir("", "kubernetes-kube-apiserver")
result.TmpDir, err = ioutil.TempDir("", "kubernetes-kube-apiserver")
if err != nil {
return nil, nil, fmt.Errorf("failed to create temp dir: %v", err)
return result, fmt.Errorf("failed to create temp dir: %v", err)
}

fs := pflag.NewFlagSet("test", pflag.PanicOnError)

s := options.NewServerRunOptions()
s.AddFlags(fs)

s.InsecureServing.BindPort = 0

s.SecureServing.Listener, s.SecureServing.BindPort, err = createListenerOnFreePort()
if err != nil {
return nil, nil, fmt.Errorf("failed to create listener: %v", err)
return result, fmt.Errorf("failed to create listener: %v", err)
}

s.SecureServing.ServerCert.CertDirectory = tmpDir
s.SecureServing.ServerCert.CertDirectory = result.TmpDir
s.ServiceClusterIPRange.IP = net.IPv4(10, 0, 0, 0)
s.ServiceClusterIPRange.Mask = net.CIDRMask(16, 32)
s.Etcd.StorageConfig = *storageConfig
s.Etcd.DefaultStorageMediaType = "application/json"
s.Admission.PluginNames = strings.Split("Initializers,NamespaceLifecycle,LimitRanger,ServiceAccount,PersistentVolumeLabel,DefaultStorageClass,ResourceQuota,DefaultTolerationSeconds", ",")
s.APIEnablement.RuntimeConfig.Set("api/all=true")

fs.Parse(customFlags)

t.Logf("Starting kube-apiserver on port %d...", s.SecureServing.BindPort)
server, err := app.CreateServerChain(s, stopCh)
if err != nil {
return nil, nil, fmt.Errorf("failed to create server chain: %v", err)
return result, fmt.Errorf("failed to create server chain: %v", err)

}
go func(stopCh <-chan struct{}) {
if err := server.PrepareRun().Run(stopCh); err != nil {
Expand All @@ -104,9 +110,10 @@ func StartTestServer(t *testing.T) (result *restclient.Config, tearDownForCaller
}(stopCh)

t.Logf("Waiting for /healthz to be ok...")

client, err := kubernetes.NewForConfig(server.LoopbackClientConfig)
if err != nil {
return nil, nil, fmt.Errorf("failed to create a client: %v", err)
return result, fmt.Errorf("failed to create a client: %v", err)
}
err = wait.Poll(100*time.Millisecond, 30*time.Second, func() (bool, error) {
result := client.CoreV1().RESTClient().Get().AbsPath("/healthz").Do()
Expand All @@ -118,23 +125,27 @@ func StartTestServer(t *testing.T) (result *restclient.Config, tearDownForCaller
return false, nil
})
if err != nil {
return nil, nil, fmt.Errorf("failed to wait for /healthz to return ok: %v", err)
return result, fmt.Errorf("failed to wait for /healthz to return ok: %v", err)
}

// from here the caller must call tearDown
return server.LoopbackClientConfig, tearDown, nil
result.ClientConfig = server.LoopbackClientConfig
result.ServerOpts = s
result.TearDownFn = tearDown

return result, nil
}

// StartTestServerOrDie calls StartTestServer with up to 5 retries on bind error and dies with
// t.Fatal if it does not succeed.
func StartTestServerOrDie(t *testing.T) (*restclient.Config, TearDownFunc) {
config, td, err := StartTestServer(t)
// StartTestServerOrDie calls StartTestServer t.Fatal if it does not succeed.
func StartTestServerOrDie(t *testing.T, flags []string, storageConfig *storagebackend.Config) *TestServer {

result, err := StartTestServer(t, flags, storageConfig)
if err == nil {
return config, td
return &result
}

t.Fatalf("Failed to launch server: %v", err)
return nil, nil
t.Fatalf("failed to launch server: %v", err)
return nil
}

func createListenerOnFreePort() (net.Listener, int, error) {
Expand Down
4 changes: 2 additions & 2 deletions test/integration/framework/etcd.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,9 +27,9 @@ import (
"path/filepath"
"sync"

"k8s.io/kubernetes/pkg/util/env"

"github.com/golang/glog"

"k8s.io/kubernetes/pkg/util/env"
)

var (
Expand Down
75 changes: 8 additions & 67 deletions test/integration/framework/master_utils.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,14 +17,13 @@ limitations under the License.
package framework

import (
"io/ioutil"
"net"
"net/http"
"net/http/httptest"
"path"
goruntime "runtime"
"strconv"
"sync"
"testing"
"time"

"github.com/go-openapi/spec"
Expand All @@ -38,7 +37,6 @@ import (
extensions "k8s.io/api/extensions/v1beta1"
rbac "k8s.io/api/rbac/v1alpha1"
storage "k8s.io/api/storage/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/wait"
Expand All @@ -62,13 +60,10 @@ import (
"k8s.io/kubernetes/pkg/api/legacyscheme"
"k8s.io/kubernetes/pkg/api/testapi"
"k8s.io/kubernetes/pkg/apis/batch"
api "k8s.io/kubernetes/pkg/apis/core"
policy "k8s.io/kubernetes/pkg/apis/policy/v1beta1"
"k8s.io/kubernetes/pkg/client/clientset_generated/internalclientset"
"k8s.io/kubernetes/pkg/controller"
replicationcontroller "k8s.io/kubernetes/pkg/controller/replication"
"k8s.io/kubernetes/pkg/generated/openapi"
"k8s.io/kubernetes/pkg/kubectl"
kubeletclient "k8s.io/kubernetes/pkg/kubelet/client"
"k8s.io/kubernetes/pkg/master"
"k8s.io/kubernetes/pkg/version"
Expand Down Expand Up @@ -395,67 +390,6 @@ func (m *MasterComponents) Stop(apiServer, rcManager bool) {
}
}

func CreateTestingNamespace(baseName string, apiserver *httptest.Server, t *testing.T) *v1.Namespace {
// TODO: Create a namespace with a given basename.
// Currently we neither create the namespace nor delete all its contents at the end.
// But as long as tests are not using the same namespaces, this should work fine.
return &v1.Namespace{
ObjectMeta: metav1.ObjectMeta{
// TODO: Once we start creating namespaces, switch to GenerateName.
Name: baseName,
},
}
}

func DeleteTestingNamespace(ns *v1.Namespace, apiserver *httptest.Server, t *testing.T) {
// TODO: Remove all resources from a given namespace once we implement CreateTestingNamespace.
}

// RCFromManifest reads a .json file and returns the rc in it.
func RCFromManifest(fileName string) *v1.ReplicationController {
data, err := ioutil.ReadFile(fileName)
if err != nil {
glog.Fatalf("Unexpected error reading rc manifest %v", err)
}
var controller v1.ReplicationController
if err := runtime.DecodeInto(testapi.Default.Codec(), data, &controller); err != nil {
glog.Fatalf("Unexpected error reading rc manifest %v", err)
}
return &controller
}

// StopRC stops the rc via kubectl's stop library
func StopRC(rc *v1.ReplicationController, clientset internalclientset.Interface) error {
reaper, err := kubectl.ReaperFor(api.Kind("ReplicationController"), clientset)
if err != nil || reaper == nil {
return err
}
err = reaper.Stop(rc.Namespace, rc.Name, 0, nil)
if err != nil {
return err
}
return nil
}

// ScaleRC scales the given rc to the given replicas.
func ScaleRC(name, ns string, replicas int32, clientset internalclientset.Interface) (*api.ReplicationController, error) {
scaler, err := kubectl.ScalerFor(api.Kind("ReplicationController"), clientset)
if err != nil {
return nil, err
}
retry := &kubectl.RetryParams{Interval: 50 * time.Millisecond, Timeout: DefaultTimeout}
waitForReplicas := &kubectl.RetryParams{Interval: 50 * time.Millisecond, Timeout: DefaultTimeout}
err = scaler.Scale(ns, name, uint(replicas), nil, retry, waitForReplicas)
if err != nil {
return nil, err
}
scaled, err := clientset.Core().ReplicationControllers(ns).Get(name, metav1.GetOptions{})
if err != nil {
return nil, err
}
return scaled, nil
}

// CloseFunc can be called to cleanup the master
type CloseFunc func()

Expand Down Expand Up @@ -524,3 +458,10 @@ func FindFreeLocalPort() (int, error) {
}
return port, nil
}

// SharedEtcd creates a storage config for a shared etcd instance, with a unique prefix.
func SharedEtcd() *storagebackend.Config {
cfg := storagebackend.NewDefaultConfig(path.Join(uuid.New(), "registry"), nil)
cfg.ServerList = []string{GetEtcdURL()}
return cfg
}