diff --git a/build/teamcity/cockroach/nightlies/compose.sh b/build/teamcity/cockroach/nightlies/compose.sh index 4d870b916077..290590bbd4e7 100755 --- a/build/teamcity/cockroach/nightlies/compose.sh +++ b/build/teamcity/cockroach/nightlies/compose.sh @@ -6,22 +6,17 @@ dir="$(dirname $(dirname $(dirname $(dirname "${0}"))))" source "$dir/teamcity-support.sh" source "$dir/teamcity-bazel-support.sh" -# The test failures generated by TestComposeCompare are not necessarily -# failures per se. They're cases of behavioral divergences from Postgres. While -# our compatibility guarantees are not 100%, it's better to treat failures as -# information to occasionally review. -export SKIP_LABEL_TEST_FAILURE=1 - tc_start_block "Run compose tests" bazel build //pkg/cmd/bazci --config=ci BAZEL_BIN=$(bazel info bazel-bin --config=ci) BAZCI=$BAZEL_BIN/pkg/cmd/bazci/bazci_/bazci -bazel build //pkg/cmd/cockroach //pkg/compose/compare/compare:compare_test --config=ci --config=crosslinux --config=test +bazel build //pkg/cmd/cockroach //pkg/compose/compare/compare:compare_test //c-deps:libgeos --config=ci --config=crosslinux --config=test CROSSBIN=$(bazel info bazel-bin --config=ci --config=crosslinux --config=test) COCKROACH=$CROSSBIN/pkg/cmd/cockroach/cockroach_/cockroach COMPAREBIN=$CROSSBIN/pkg/compose/compare/compare/compare_test_/compare_test +LIBGEOSDIR=$(bazel info execution_root --config=crosslinux --config=ci --config=test)/external/archived_cdep_libgeos_linux/lib/ ARTIFACTS_DIR=$PWD/artifacts mkdir -p $ARTIFACTS_DIR @@ -35,6 +30,7 @@ $BAZCI --artifacts_dir=$ARTIFACTS_DIR -- \ --test_env=COCKROACH_RUN_COMPOSE=true \ --test_arg -cockroach --test_arg $COCKROACH \ --test_arg -compare --test_arg $COMPAREBIN \ + --test_arg -libgeosdir --test_arg $LIBGEOSDIR \ --test_timeout=1800 || exit_status=$? tc_end_block "Run compose tests" diff --git a/dev b/dev index fe09bb8f21a4..043d36fd002d 100755 --- a/dev +++ b/dev @@ -8,7 +8,7 @@ fi set -euo pipefail # Bump this counter to force rebuilding `dev` on all machines. -DEV_VERSION=94 +DEV_VERSION=95 THIS_DIR=$(cd "$(dirname "$0")" && pwd) BINARY_DIR=$THIS_DIR/bin/dev-versions diff --git a/pkg/cmd/dev/compose.go b/pkg/cmd/dev/compose.go index 01eccb258c89..92abd6f439b5 100644 --- a/pkg/cmd/dev/compose.go +++ b/pkg/cmd/dev/compose.go @@ -12,11 +12,15 @@ package main import ( "fmt" + "os" "path/filepath" + "time" "github.com/spf13/cobra" ) +const eachFlag = "each" + func makeComposeCmd(runE func(cmd *cobra.Command, args []string) error) *cobra.Command { composeCmd := &cobra.Command{ Use: "compose", @@ -30,6 +34,7 @@ func makeComposeCmd(runE func(cmd *cobra.Command, args []string) error) *cobra.C addCommonTestFlags(composeCmd) composeCmd.Flags().Bool(noRebuildCockroachFlag, false, "set if it is unnecessary to rebuild cockroach (artifacts/cockroach must already exist, e.g. after being created by a previous `dev compose` run)") composeCmd.Flags().String(volumeFlag, "bzlhome", "the Docker volume to use as the container home directory (only used for cross builds)") + composeCmd.Flags().Duration(eachFlag, 10*time.Minute, "individual test timeout") return composeCmd } @@ -40,10 +45,11 @@ func (d *dev) compose(cmd *cobra.Command, _ []string) error { noRebuildCockroach = mustGetFlagBool(cmd, noRebuildCockroachFlag) short = mustGetFlagBool(cmd, shortFlag) timeout = mustGetFlagDuration(cmd, timeoutFlag) + each = mustGetFlagDuration(cmd, eachFlag) ) if !noRebuildCockroach { - crossArgs, targets, err := d.getBasicBuildArgs(ctx, []string{"//pkg/cmd/cockroach:cockroach", "//pkg/compose/compare/compare:compare_test"}) + crossArgs, targets, err := d.getBasicBuildArgs(ctx, []string{"//pkg/cmd/cockroach:cockroach", "//pkg/compose/compare/compare:compare_test", "//c-deps:libgeos"}) if err != nil { return err } @@ -58,8 +64,23 @@ func (d *dev) compose(cmd *cobra.Command, _ []string) error { if err != nil { return err } - cockroachBin := filepath.Join(workspace, "artifacts", "cockroach") - compareBin := filepath.Join(workspace, "artifacts", "compare_test") + artifactsDir := filepath.Join(workspace, "artifacts") + cockroachBin := filepath.Join(artifactsDir, "cockroach") + compareBin := filepath.Join(artifactsDir, "compare_test") + + // Copy libgeos files to a temporary directory, since the compose tests expect + // libgeos files to be in their own directory. + libGeosDir, err := os.MkdirTemp("", "lib") + if err != nil { + return err + } + for _, geoLib := range []string{"libgeos.so", "libgeos_c.so"} { + src := filepath.Join(artifactsDir, geoLib) + dst := filepath.Join(libGeosDir, geoLib) + if err := d.os.CopyFile(src, dst); err != nil { + return err + } + } var args []string args = append(args, "test", "//pkg/compose:compose_test") @@ -75,9 +96,15 @@ func (d *dev) compose(cmd *cobra.Command, _ []string) error { if timeout > 0 { args = append(args, fmt.Sprintf("--test_timeout=%d", int(timeout.Seconds()))) } + if each > 0 { + args = append(args, "--test_arg", "-each", "--test_arg", each.String()) + } args = append(args, "--test_arg", "-cockroach", "--test_arg", cockroachBin) + args = append(args, "--test_arg", "-libgeosdir", "--test_arg", libGeosDir) args = append(args, "--test_arg", "-compare", "--test_arg", compareBin) + args = append(args, "--test_arg", "-test.v") + args = append(args, "--test_output", "all") args = append(args, "--test_env", "COCKROACH_DEV_LICENSE") args = append(args, "--test_env", "COCKROACH_RUN_COMPOSE=true") diff --git a/pkg/compose/compare/docker-compose.yml b/pkg/compose/compare/docker-compose.yml index a0567ba6d557..086ef0473418 100644 --- a/pkg/compose/compare/docker-compose.yml +++ b/pkg/compose/compare/docker-compose.yml @@ -7,6 +7,7 @@ services: command: /cockroach/cockroach start-single-node --insecure --listen-addr cockroach1 volumes: - "${COCKROACH_PATH}:/cockroach/cockroach" + - "${LIBGEOS_DIR_PATH}:/cockroach/lib" cockroach2: # We use a docker image mirror to avoid pulling from 3rd party repos, which sometimes have reliability issues. # See https://cockroachlabs.atlassian.net/wiki/spaces/devinf/pages/3462594561/Docker+image+sync for the details. @@ -14,16 +15,19 @@ services: command: /cockroach/cockroach start-single-node --insecure --listen-addr cockroach2 volumes: - "${COCKROACH_PATH}:/cockroach/cockroach" + - "${LIBGEOS_DIR_PATH}:/cockroach/lib" test: # We use a docker image mirror to avoid pulling from 3rd party repos, which sometimes have reliability issues. # See https://cockroachlabs.atlassian.net/wiki/spaces/devinf/pages/3462594561/Docker+image+sync for the details. image: us-east1-docker.pkg.dev/crl-docker-sync/docker-mirror/docker.io/library/ubuntu:xenial-20170214 environment: - COCKROACH_DEV_LICENSE=$COCKROACH_DEV_LICENSE + - COCKROACH_RUN_COMPOSE_COMPARE=${COCKROACH_RUN_COMPOSE_COMPARE} # compare.test is a binary built by the pkg/compose/prepare.sh in non-bazel builds - command: /compare/compare.test -each ${EACH} -test.run ${TESTS} -artifacts ${ARTIFACTS} + command: /compare/compare.test -test.v -each ${EACH} -test.run ${TESTS} -artifacts ${ARTIFACTS} depends_on: - cockroach1 - cockroach2 volumes: - "${COMPARE_DIR_PATH}:/compare" + - "${LIBGEOS_DIR_PATH}:/compare/lib" diff --git a/pkg/compose/compose_test.go b/pkg/compose/compose_test.go index 627df68daaf0..90f4086ec52a 100644 --- a/pkg/compose/compose_test.go +++ b/pkg/compose/compose_test.go @@ -29,11 +29,12 @@ import ( var ( // flagEach controls how long we are going to run each compose test. Ensure bazel BUILD file // of compose tests has a longer timeout. - flagEach = flag.Duration("each", 10*time.Minute, "individual test timeout") - flagTests = flag.String("tests", ".", "tests within docker compose to run") - flagArtifacts = flag.String("artifacts", "", "artifact directory") - flagCockroach = flag.String("cockroach", "", "path to the cockroach executable") - flagCompare = flag.String("compare", "", "path to the compare test (only valid for bazel-driven test)") + flagEach = flag.Duration("each", 10*time.Minute, "individual test timeout") + flagTests = flag.String("tests", ".", "tests within docker compose to run") + flagArtifacts = flag.String("artifacts", "", "artifact directory") + flagCockroach = flag.String("cockroach", "", "path to the cockroach executable") + flagLibGeosDir = flag.String("libgeosdir", "", "path to the libgeos directory (only valid for bazel-driven test)") + flagCompare = flag.String("compare", "", "path to the compare test (only valid for bazel-driven test)") ) func copyBin(src, dst string) error { @@ -55,7 +56,7 @@ func TestComposeCompare(t *testing.T) { if os.Getenv("COCKROACH_RUN_COMPOSE") == "" { skip.IgnoreLint(t, "COCKROACH_RUN_COMPOSE not set") } - var cockroachBin, compareDir, dockerComposeYml string + var cockroachBin, libGeosDir, compareDir, dockerComposeYml string if bazel.BuiltWithBazel() { var err error dockerComposeYml, err = bazel.Runfile("pkg/compose/compare/docker-compose.yml") @@ -65,6 +66,9 @@ func TestComposeCompare(t *testing.T) { if *flagCockroach == "" { t.Fatal("-cockroach not set") } + if *flagLibGeosDir == "" { + t.Fatal("-libgeosdir not set") + } if *flagCompare == "" { t.Fatal("-compare not set") } @@ -75,12 +79,21 @@ func TestComposeCompare(t *testing.T) { composeBinsDir := t.TempDir() compareDir = composeBinsDir cockroachBin = filepath.Join(composeBinsDir, "cockroach") - err = copyBin(*flagCockroach, cockroachBin) - if err != nil { + libGeosDir = filepath.Join(composeBinsDir, "lib") + if err = os.MkdirAll(libGeosDir, 0755); err != nil { t.Fatal(err) } - err = copyBin(*flagCompare, filepath.Join(composeBinsDir, "compare.test")) - if err != nil { + if err := copyBin(*flagCockroach, cockroachBin); err != nil { + t.Fatal(err) + } + for _, geoLib := range []string{"libgeos.so", "libgeos_c.so"} { + src := filepath.Join(*flagLibGeosDir, geoLib) + dst := filepath.Join(libGeosDir, geoLib) + if err := copyBin(src, dst); err != nil { + t.Fatal(err) + } + } + if err = copyBin(*flagCompare, filepath.Join(composeBinsDir, "compare.test")); err != nil { t.Fatal(err) } if *flagArtifacts == "" { @@ -94,6 +107,10 @@ func TestComposeCompare(t *testing.T) { if cockroachBin == "" { cockroachBin = "../../../cockroach-linux-2.6.32-gnu-amd64" } + libGeosDir = *flagLibGeosDir + if libGeosDir == "" { + libGeosDir = "../../../lib" + } compareDir = "./compare" dockerComposeYml = filepath.Join("compare", "docker-compose.yml") if *flagArtifacts == "" { @@ -112,6 +129,7 @@ func TestComposeCompare(t *testing.T) { fmt.Sprintf("EACH=%s", *flagEach), fmt.Sprintf("TESTS=%s", *flagTests), fmt.Sprintf("COCKROACH_PATH=%s", cockroachBin), + fmt.Sprintf("LIBGEOS_DIR_PATH=%s", libGeosDir), fmt.Sprintf("COMPARE_DIR_PATH=%s", compareDir), fmt.Sprintf("ARTIFACTS=%s", *flagArtifacts), fmt.Sprintf("COCKROACH_DEV_LICENSE=%s", envutil.EnvOrDefaultString("COCKROACH_DEV_LICENSE", "")), @@ -119,8 +137,8 @@ func TestComposeCompare(t *testing.T) { "COCKROACH_RUN_COMPOSE_COMPARE=true", } out, err := cmd.CombinedOutput() + t.Log(string(out)) if err != nil { - t.Log(string(out)) t.Fatal(err) } }