From 17d9da39cc29429559375e5cd7fa0ddf41b67cfb Mon Sep 17 00:00:00 2001 From: Mrproliu Date: Thu, 24 Jun 2021 23:38:10 +0800 Subject: [PATCH 1/3] Upgrade docker compose framework to support native port check --- go.mod | 5 +- go.sum | 8 +- internal/components/setup/compose.go | 155 +++++++++------------------ 3 files changed, 56 insertions(+), 112 deletions(-) diff --git a/go.mod b/go.mod index c1c3c9d..ce51fdd 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,14 @@ module github.com/apache/skywalking-infra-e2e go 1.13 require ( - github.com/docker/docker v20.10.6+incompatible + github.com/docker/docker v20.10.7+incompatible + github.com/docker/go-connections v0.4.0 github.com/google/go-cmp v0.5.4 github.com/gorilla/mux v1.8.0 // indirect github.com/morikuni/aec v1.0.0 // indirect github.com/sirupsen/logrus v1.7.0 github.com/spf13/cobra v1.1.1 - github.com/testcontainers/testcontainers-go v0.11.0 + github.com/testcontainers/testcontainers-go v0.11.1 gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.20.7 k8s.io/apimachinery v0.20.7 diff --git a/go.sum b/go.sum index ef890e0..6257f86 100644 --- a/go.sum +++ b/go.sum @@ -215,8 +215,8 @@ github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TT github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v20.10.6+incompatible h1:oXI3Vas8TI8Eu/EjH4srKHJBVqraSzJybhxY7Om9faQ= -github.com/docker/docker v20.10.6+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.7+incompatible h1:Z6O9Nhsjv+ayUEeI1IojKbYcsGdgYSNqxe1s2MYzUhQ= +github.com/docker/docker v20.10.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= @@ -642,8 +642,8 @@ github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tchap/go-patricia v2.2.6+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= -github.com/testcontainers/testcontainers-go v0.11.0 h1:HO5YOx2DYBHqcg4MzVWPj3FuHAv7USWVu94vCSsgiaM= -github.com/testcontainers/testcontainers-go v0.11.0/go.mod h1:HztBCODzuA+YpMXGK8amjO8j50jz2gcT0BOzSKUiYIs= +github.com/testcontainers/testcontainers-go v0.11.1 h1:FiYsB83LSGbiawoV8TpAZGfcCUbtaeeg1SXqEKUxh08= +github.com/testcontainers/testcontainers-go v0.11.1/go.mod h1:/V0UVq+1e7NWYoqTPog179clf0Qp9TOyp4EcXaEFQz8= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= diff --git a/internal/components/setup/compose.go b/internal/components/setup/compose.go index 31b7aa7..639e1d7 100644 --- a/internal/components/setup/compose.go +++ b/internal/components/setup/compose.go @@ -21,12 +21,15 @@ package setup import ( "context" "fmt" - "net" "os" "regexp" - "syscall" + "strconv" + "strings" "time" + "github.com/docker/go-connections/nat" + "github.com/testcontainers/testcontainers-go/wait" + "github.com/apache/skywalking-infra-e2e/internal/config" "github.com/apache/skywalking-infra-e2e/internal/constant" "github.com/apache/skywalking-infra-e2e/internal/logger" @@ -57,13 +60,8 @@ func ComposeSetup(e2eConfig *config.E2EConfig) error { } identifier := GetIdentity() compose := testcontainers.NewLocalDockerCompose(composeFilePaths, identifier) - execError := compose.WithCommand([]string{"up", "-d"}).Invoke() - if execError.Error != nil { - return execError.Error - } - // record time now - timeNow := time.Now() + // bind wait port timeout := e2eConfig.Setup.Timeout var waitTimeout time.Duration if timeout <= 0 { @@ -71,16 +69,37 @@ func ComposeSetup(e2eConfig *config.E2EConfig) error { } else { waitTimeout = time.Duration(timeout) * time.Second } - logger.Log.Debugf("wait timeout is %d seconds", int(waitTimeout.Seconds())) - - // find exported port and build env + serviceWithPorts := make(map[string][]int) for service, content := range compose.Services { serviceConfig := content.(map[interface{}]interface{}) ports := serviceConfig["ports"] if ports == nil { continue } + serviceWithPorts[service] = []int{} + portList := ports.([]interface{}) + for inx := range portList { + exportPort, err := getExpectPort(portList[inx]) + if err != nil { + return err + } + serviceWithPorts[service] = append(serviceWithPorts[service], exportPort) + + compose.WithExposedService( + service, + exportPort, + wait.NewHostPortStrategy(nat.Port(fmt.Sprintf("%d/tcp", exportPort))).WithStartupTimeout(waitTimeout)) + } + } + + execError := compose.WithCommand([]string{"up", "-d"}).Invoke() + if execError.Error != nil { + return execError.Error + } + + // find exported port and build env + for service, portList := range serviceWithPorts { container, err := findContainer(cli, fmt.Sprintf("%s_%s", identifier, getInstanceName(service))) if err != nil { return err @@ -89,20 +108,10 @@ func ComposeSetup(e2eConfig *config.E2EConfig) error { for inx := range portList { for _, containerPort := range containerPorts { - if int(containerPort.PrivatePort) != portList[inx].(int) { + if int(containerPort.PrivatePort) != portList[inx] { continue } - // calculate max wait time - waitTimeout = NewTimeout(timeNow, waitTimeout) - timeNow = time.Now() - - // wait port and export - err := waitTCPPortStarted(context.Background(), cli, container, int(containerPort.PublicPort), int(containerPort.PrivatePort), waitTimeout) - if err != nil { - return fmt.Errorf("could wait port exported: %s:%d, %v", service, portList[inx], err) - } - // expose env config to env // format: _ envKey := fmt.Sprintf("%s_%d", service, containerPort.PrivatePort) @@ -112,6 +121,7 @@ func ComposeSetup(e2eConfig *config.E2EConfig) error { return fmt.Errorf("could not set env for %s:%d, %v", service, portList[inx], err) } logger.Log.Infof("expose env : %s : %s", envKey, envValue) + break } } } @@ -119,6 +129,23 @@ func ComposeSetup(e2eConfig *config.E2EConfig) error { return nil } +func getExpectPort(portConfig interface{}) (int, error) { + switch portConfig.(type) { + case int: + return portConfig.(int), nil + case string: + portStr := portConfig.(string) + portInfo := strings.Split(portStr, ":") + if len(portInfo) > 1 { + return strconv.Atoi(portInfo[1]) + } else { + return strconv.Atoi(portInfo[1]) + } + default: + return 0, fmt.Errorf("unknown port infomation: %v", portConfig) + } +} + func findContainer(c *client.Client, instanceName string) (*types.Container, error) { f := filters.NewArgs(filters.Arg("name", instanceName)) containerListOptions := types.ContainerListOptions{Filters: f} @@ -133,90 +160,6 @@ func findContainer(c *client.Client, instanceName string) (*types.Container, err return &containers[0], nil } -func waitTCPPortStarted(ctx context.Context, c *client.Client, container *types.Container, publicPort, interPort int, timeout time.Duration) error { - // limit context to startupTimeout - ctx, cancelContext := context.WithTimeout(ctx, timeout) - defer cancelContext() - - var waitInterval = 100 * time.Millisecond - - // external check - dialer := net.Dialer{} - address := net.JoinHostPort("127.0.0.1", fmt.Sprintf("%d", publicPort)) - for { - conn, err := dialer.DialContext(ctx, "tcp", address) - if err != nil { - if v, ok := err.(*net.OpError); ok { - if v2, ok := (v.Err).(*os.SyscallError); ok { - if isConnRefusedErr(v2.Err) { - time.Sleep(waitInterval) - continue - } - } - } - return err - } - conn.Close() - break - } - - // internal check - command := buildInternalCheckCommand(interPort) - for { - if ctx.Err() != nil { - return ctx.Err() - } - response, err := c.ContainerExecCreate(ctx, container.ID, types.ExecConfig{ - Cmd: []string{"/bin/sh", "-c", command}, - AttachStderr: true, - AttachStdout: true, - }) - if err != nil { - return err - } - - err = c.ContainerExecStart(ctx, response.ID, types.ExecStartCheck{ - Detach: false, - }) - if err != nil { - return err - } - - var exitCode int - for { - execResp, err := c.ContainerExecInspect(ctx, response.ID) - if err != nil { - return err - } - - if !execResp.Running { - exitCode = execResp.ExitCode - break - } - - time.Sleep(waitInterval) - } - - if exitCode == 0 { - return nil - } - } -} - -func buildInternalCheckCommand(internalPort int) string { - command := `( - nc -vz -w 1 localhost %d || - cat /proc/net/tcp | awk '{print $2}' | grep -i :%d || - Date: Thu, 24 Jun 2021 23:55:14 +0800 Subject: [PATCH 2/3] Fix issues --- dist/LICENSE | 2 +- internal/components/setup/compose.go | 13 +++++-------- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/dist/LICENSE b/dist/LICENSE index e25fc62..95ac9a7 100644 --- a/dist/LICENSE +++ b/dist/LICENSE @@ -242,7 +242,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. logrus 1.7.0: https://github.com/sirupsen/logrus MIT go-winio v0.4.16: https://github.com/Microsoft/go-winio MIT aec v1.0.0: https://github.com/morikuni/aec MIT - testcontainers-go v0.11.0: https://github.com/testcontainers/testcontainers-go MIT + testcontainers-go v0.11.1: https://github.com/testcontainers/testcontainers-go MIT ======================================================================== BSD licenses diff --git a/internal/components/setup/compose.go b/internal/components/setup/compose.go index 639e1d7..07ec57c 100644 --- a/internal/components/setup/compose.go +++ b/internal/components/setup/compose.go @@ -130,20 +130,17 @@ func ComposeSetup(e2eConfig *config.E2EConfig) error { } func getExpectPort(portConfig interface{}) (int, error) { - switch portConfig.(type) { + switch conf := portConfig.(type) { case int: - return portConfig.(int), nil + return conf, nil case string: - portStr := portConfig.(string) - portInfo := strings.Split(portStr, ":") + portInfo := strings.Split(conf, ":") if len(portInfo) > 1 { return strconv.Atoi(portInfo[1]) - } else { - return strconv.Atoi(portInfo[1]) } - default: - return 0, fmt.Errorf("unknown port infomation: %v", portConfig) + return strconv.Atoi(portInfo[0]) } + return 0, fmt.Errorf("unknown port information: %v", portConfig) } func findContainer(c *client.Client, instanceName string) (*types.Container, error) { From 61c2b6495053f87dac2add449f782f0499237cf2 Mon Sep 17 00:00:00 2001 From: Mrproliu Date: Fri, 25 Jun 2021 00:01:52 +0800 Subject: [PATCH 3/3] Fix license --- dist/LICENSE | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dist/LICENSE b/dist/LICENSE index 95ac9a7..018bc6e 100644 --- a/dist/LICENSE +++ b/dist/LICENSE @@ -225,12 +225,13 @@ The text of each license is the standard Apache 2.0 license. cli-runtime v0.19.0: https://github.com/kubernetes/cli-runtime Apache 2.0 client-go v0.19.0 https://github.com/kubernetes/client-go Apache 2.0 kubectl v0.19.0: https://github.com/kubernetes/kubectl Apache 2.0 - docker v20.10.3: https://github.com/docker/docker Apache 2.0 + docker v20.10.7: https://github.com/docker/docker Apache 2.0 utils v0.0.0-20201110183641-67b214c5f920: https://github.com/kubernetes/utils Apache 2.0 containerd v1.4.3: https://github.com/containerd/containerd Apache 2.0 go-connections v0.4.0: https://github.com/docker/go-connections Apache 2.0 go-units v0.4.0: https://github.com/docker/go-units Apache 2.0 image-spec v1.0.1: https://github.com/opencontainers/image-spec Apache 2.0 + go-connections v0.4.0: https://github.com/opencontainers/image-spec Apache 2.0 ======================================================================== MIT licenses