ADBDEV-4345: Fix wrong allocation of writer slice in case of modifying shared CTE

arenadata/.env

@@ -0,0 +1 @@
+COMPOSE_HTTP_TIMEOUT=400

arenadata/Dockerfile

@@ -0,0 +1,74 @@
+FROM centos:centos7 as base
+
+# Install some basic utilities and build tools
+RUN yum makecache && yum update -y ca-certificates && \
+    rpm --import https://mirror.yandex.ru/centos/RPM-GPG-KEY-CentOS-7 && \
+    yum -y install epel-release java-1.8.0-openjdk-devel && \
+    yum -y install git iproute net-tools openssh-server rsync sudo time vim wget unzip \
+                   ant-junit autoconf bison cmake3 flex gperf indent jq libtool gcc-c++ \
+                   krb5-server krb5-workstation xerces-c-devel file && \
+    yum clean all
+
+# install all software we need
+RUN yum makecache && \
+    yum -y install python3-devel && \
+    yum -y install apr-devel bzip2-devel expat-devel libcurl-devel && \
+    yum -y install libevent-devel libuuid-devel libxml2-devel libyaml-devel libzstd-devel && \
+    yum -y install openssl-devel pam-devel readline-devel snappy-devel libuv-devel && \
+    yum -y install apache-ivy libicu perl-ExtUtils-Embed perl-Env perl-JSON && \
+    yum -y install perl-IPC-Run perl-Test-Base libxslt-devel openldap-devel && \
+    yum clean all
+
+# setup ssh configuration
+RUN ssh-keygen -t rsa -N "" -f /root/.ssh/id_rsa && \
+    cat /root/.ssh/id_rsa.pub >> /root/.ssh/authorized_keys && \
+    chmod 0600 /root/.ssh/authorized_keys && \
+    echo -e "password\npassword" | passwd 2> /dev/null && \
+    { ssh-keyscan localhost; ssh-keyscan 0.0.0.0; } >> /root/.ssh/known_hosts && \
+    #
+    ssh-keygen -f /etc/ssh/ssh_host_key -N '' -t rsa1 && \
+    ssh-keygen -f /etc/ssh/ssh_host_rsa_key -N '' -t rsa && \
+    ssh-keygen -f /etc/ssh/ssh_host_dsa_key -N '' -t dsa && \
+    sed -i -e 's|Defaults    requiretty|#Defaults    requiretty|' /etc/sudoers && \
+    sed -ri 's/UsePAM yes/UsePAM no/g;s/PasswordAuthentication yes/PasswordAuthentication no/g' /etc/ssh/sshd_config && \
+    sed -ri 's@^HostKey /etc/ssh/ssh_host_ecdsa_key$@#&@;s@^HostKey /etc/ssh/ssh_host_ed25519_key$@#&@' /etc/ssh/sshd_config
+
+# newer version of gcc and run environment for gpdb
+RUN yum -y install centos-release-scl && \
+    yum -y install --nogpgcheck devtoolset-7-gcc devtoolset-7-gcc-c++ \
+    llvm-toolset-7 llvm-toolset-7-llvm-devel && yum clean all && \
+    ln -s /usr/bin/cmake3 /usr/bin/cmake && \
+    ln -s /usr/bin/ctest3 /usr/bin/ctest && \
+    echo -e 'source /opt/rh/devtoolset-7/enable' >> /opt/gcc_env.sh && \
+    echo -e '#!/bin/sh' >> /etc/profile.d/jdk_home.sh && \
+    echo -e 'export JAVA_HOME=/etc/alternatives/java_sdk' >> /etc/profile.d/jdk_home.sh && \
+    echo -e 'export PATH=$JAVA_HOME/bin:$PATH' >> /etc/profile.d/jdk_home.sh
+
+WORKDIR /home/gpadmin
+
+FROM base as build
+
+COPY . gpdb_src
+
+RUN mkdir /home/gpadmin/bin_gpdb
+
+ENV TARGET_OS_VERSION=7
+ENV TARGET_OS=centos
+ENV OUTPUT_ARTIFACT_DIR=bin_gpdb
+ENV SKIP_UNITTESTS=
+# To set default locale for unittests
+ENV LANG=en_US.UTF-8
+ENV CONFIGURE_FLAGS="--enable-debug-extensions --with-gssapi --enable-cassert --enable-debug --enable-depend"
+
+# Compile with running mocking tests
+RUN scl enable devtoolset-7 llvm-toolset-7 'bash /home/gpadmin/gpdb_src/concourse/scripts/compile_gpdb.bash'
+
+FROM base as code
+COPY . gpdb_src
+RUN rm -rf gpdb_src/.git/
+
+FROM base as test
+COPY --from=code /home/gpadmin/gpdb_src gpdb_src
+COPY --from=build /home/gpadmin/bin_gpdb /home/gpadmin/bin_gpdb
+
+RUN echo -e '/opt/rh/llvm-toolset-7/root/usr/lib64/\n' >> /etc/ld.so.conf && ldconfig

arenadata/Dockerfile.linter

@@ -0,0 +1,12 @@
+FROM ubuntu:20.04
+
+RUN apt update && apt install -y git parallel clang-format-11 && \
+    ln -s /usr/bin/clang-format-11 /usr/bin/clang-format
+
+WORKDIR /opt/gpdb_src
+
+COPY . /opt/gpdb_src
+
+ENV CLANG_FORMAT=clang-format-11
+
+ENTRYPOINT src/tools/fmt gen && git diff --exit-code && src/tools/fmt chk

arenadata/docker-compose.yaml

@@ -0,0 +1,52 @@
+---
+
+version: "3"
+services:
+  #image can be either hub.adsw.io/library/gpdb6_regress:${BRANCH_NAME}-x86-64 or
+  #hub.adsw.io/library/gpdb6_regress:${BRANCH_NAME}-ppc64le depending on the arch
+  mdw:
+    image: "${IMAGE}"
+    working_dir: /home/gpadmin
+    hostname: mdw
+    volumes:
+      - "$PWD/ssh_keys/id_rsa:/home/gpadmin/.ssh/id_rsa"
+      - "$PWD/ssh_keys/id_rsa.pub:/home/gpadmin/.ssh/id_rsa.pub"
+      - "$PWD/allure-results:/tmp/allure-results"
+    privileged: true
+    sysctls:
+      kernel.sem: 500 1024000 200 4096
+    entrypoint: >
+      bash -c "set -e && tail -f /home/gpadmin/gpdb_src/README.md"
+  sdw1:
+    image: "${IMAGE}"
+    privileged: true
+    hostname: sdw1
+    volumes:
+      - "$PWD/ssh_keys/id_rsa:/home/gpadmin/.ssh/id_rsa"
+      - "$PWD/ssh_keys/id_rsa.pub:/home/gpadmin/.ssh/id_rsa.pub"
+    sysctls:
+      kernel.sem: 500 1024000 200 4096
+    entrypoint: >
+      bash -c "set -e && tail -f /home/gpadmin/gpdb_src/README.md"
+  sdw2:
+    image: "${IMAGE}"
+    privileged: true
+    hostname: sdw2
+    volumes:
+      - "$PWD/ssh_keys/id_rsa:/home/gpadmin/.ssh/id_rsa"
+      - "$PWD/ssh_keys/id_rsa.pub:/home/gpadmin/.ssh/id_rsa.pub"
+    sysctls:
+      kernel.sem: 500 1024000 200 4096
+    entrypoint: >
+      bash -c "set -e && tail -f /home/gpadmin/gpdb_src/README.md"
+  sdw3:
+    image: "${IMAGE}"
+    privileged: true
+    hostname: sdw3
+    volumes:
+      - "$PWD/ssh_keys/id_rsa:/home/gpadmin/.ssh/id_rsa"
+      - "$PWD/ssh_keys/id_rsa.pub:/home/gpadmin/.ssh/id_rsa.pub"
+    sysctls:
+      kernel.sem: 500 1024000 200 4096
+    entrypoint: >
+      bash -c "set -e && tail -f /home/gpadmin/gpdb_src/README.md"

arenadata/readme.md

@@ -0,0 +1,103 @@
+# How to run tests
+
+## Build docker gpdb image with developer options
+
+Change directory to gpdb sources destination. Make sure that directry doesn't contain binary objects from previous builds. Then run:
+
+```bash
+docker build -t gpdb7_regress:latest -f arenadata/Dockerfile .
+```
+
+CI pushes docker images to the internal registry for each branch. We can pull it with usage of:
+
+* branch name as tag (latest for `adb-7.x` branch)
+* commit hash:
+  ```bash
+  docker pull hub.adsw.io/library/gpdb7_regress:1353d81
+  ```
+
+## Full regression tests suite run
+
+We need to execute [../concourse/scripts/ic_gpdb.bash](../concourse/scripts/ic_gpdb.bash) in container to create demo cluster and run different test suites against it:
+
+```bash
+ docker run --name gpdb7_opt_on --rm -it -e TEST_OS=centos \
+  -e MAKE_TEST_COMMAND=-k PGOPTIONS='-c optimizer=on' installcheck-world \
+  --sysctl kernel.sem=500 1024000 200 4096 gpdb7_regress:latest \
+  scl enable devtoolset-7 llvm-toolset-7 \
+  /home/gpadmin/gpdb_src/concourse/scripts/ic_gpdb.bash
+```
+
+* we need to modify `MAKE_TEST_COMMAND` environment variable to run different suite. e.g. we may run test againt Postgres optimizer or ORCA with altering `PGOPTIONS` environment variable;
+* we need to run container as `--privileged` to run debugger inside it
+* we need to increase semaphore amount to be able to run demo cluster
+* we need running ssh server to be able to run demo cluster
+
+## ORCA linter
+
+```bash
+docker build -t orca-linter:test -f arenadata/Dockerfile.linter .
+docker run --rm -it orca-linter:test
+```
+
+The work directory must be clean to pass this test. Please, stage or even commit your changes.
+
+## ORCA unit test run
+
+```bash
+docker run --rm -it gpdb7_regress:latest bash -c "gpdb_src/concourse/scripts/unit_tests_gporca.bash"
+```
+
+## How to run demo cluster inside docker container manually
+
+1. Build or pull from internal registry (see above) needed image
+1. Start container with
+   ```bash
+   docker run --name gpdb7_demo --rm -it --sysctl 'kernel.sem=500 1024000 200 4096' gpdb7_regress:latest \
+     scl enable devtoolset-7 llvm-toolset-7 bash
+   ```
+1. Run the next commands in container
+   ```bash
+   source gpdb_src/concourse/scripts/common.bash
+   # this command unpack binaries to `/usr/local/greenplum-db-devel/`
+   install_and_configure_gpdb
+   gpdb_src/concourse/scripts/setup_gpadmin_user.bash
+   make_cluster
+   su - gpadmin -c '
+   source /usr/local/greenplum-db-devel/greenplum_path.sh;
+   source gpdb_src/gpAux/gpdemo/gpdemo-env.sh;
+   psql postgres'
+   ```
+
+## Behave test run
+
+Behave tests now can run locally with docker-compose.
+
+Feature files are located in `gpMgmt/test/behave/mgmt_utils`
+Before run tests you need to build a docker-image
+```bash
+docker build -t "hub.adsw.io/library/gpdb7_regress:${BRANCH_NAME}" -f arenadata/Dockerfile .
+```
+
+Command to run features:
+```bash
+# Run all tests
+bash arenadata/scripts/run_behave_tests.bash
+
+# Run specific features
+bash arenadata/scripts/run_behave_tests.bash gpstart gpstop
+```
+
+
+Tests use `allure-behave` package and store allure output files in `allure-results` folder
+**NOTE** that `allure-behave` has too old a version because it is compatible with `python2`.
+Also, the allure report for each failed test has gpdb logs attached files. See `gpMgmt/test/behave_utils/arenadata/formatter.py`
+It required to add `gpMgmt/tests` directory to `PYTHONPATH`.
+
+Greenplum cluster in Docker containers has its own peculiarities in preparing a cluster for tests.
+All tests are run in one way or another on the demo cluster, wherever possible.
+For example, cross_subnet tests or tests with tag `concourse_cluster` currently not worked because of too complex cluster preconditions.
+
+Tests in a docker-compose cluster use the same ssh keys for `gpadmin` user and pre-add the cluster hosts to `.ssh/know_hosts` and `/etc/hosts`.
+
+Docker containers have installed `sigar` libraries. It is required only for `gpperfmon` tests.

arenadata/scripts/behave_gpdb.bash

@@ -0,0 +1,52 @@
+#!/bin/bash -l
+
+set -eox pipefail
+
+CWDIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )/../../concourse/scripts" && pwd )"
+source "${CWDIR}/common.bash"
+
+function gen_env(){
+		cat > /opt/run_test.sh <<-EOF
+		set -ex
+
+		source /usr/local/greenplum-db-devel/greenplum_path.sh
+
+		source gpdb_src/gpAux/gpdemo/gpdemo-env.sh
+
+		if [[ ${FEATURE} == "gpexpand" ]]; then
+			mkdir -p /home/gpadmin/sqldump
+			wget -nv https://rt.adsw.io/artifactory/common/dump.sql.xz -O /home/gpadmin/sqldump/dump.sql.xz
+
+			xz -d /home/gpadmin/sqldump/dump.sql.xz
+		fi
+
+		cd "\${1}/gpdb_src/gpMgmt/"
+		BEHAVE_TAGS="${BEHAVE_TAGS}"
+		BEHAVE_FLAGS="${BEHAVE_FLAGS}"
+		if [ ! -z "\${BEHAVE_TAGS}" ]; then
+				make -f Makefile.behave behave tags=\${BEHAVE_TAGS}
+		else
+				flags="\${BEHAVE_FLAGS}" make -f Makefile.behave behave
+		fi
+	EOF
+
+		chmod a+x /opt/run_test.sh
+}
+
+function _main() {
+
+		if [ -z "${BEHAVE_TAGS}" ] && [ -z "${BEHAVE_FLAGS}" ]; then
+				echo "FATAL: BEHAVE_TAGS or BEHAVE_FLAGS not set"
+				exit 1
+		fi
+
+		# Run inside a subshell so it does not pollute the environment after
+		# sourcing greenplum_path
+		time (make_cluster)
+
+		time gen_env
+
+		time run_test
+}
+
+_main "$@"

arenadata/scripts/run_behave_tests.bash

@@ -0,0 +1,92 @@
+#!/usr/bin/env bash
+set -x -o pipefail
+
+behave_tests_dir="gpMgmt/test/behave/mgmt_utils"
+
+# TODO concourse_cluster tests are not stable
+# clusters="concourse_cluster ~concourse_cluster,demo_cluster"
+
+clusters="~concourse_cluster"
+
+if [ $# -eq 0 ]
+then
+  # TODO cross_subnet and gpssh tests are excluded
+  features=`ls $behave_tests_dir -1 | grep feature | grep -v -E "cross_subnet|gpssh" | sed 's/\.feature$//'`
+else
+  for feature in $@
+  do
+    if [ ! -f "$behave_tests_dir/$feature.feature" ]
+    then
+      echo "Feature '$feature' doesn't exists"
+      exit 1
+    fi
+  done
+  features=$@
+fi
+
+processes=3
+
+rm -rf allure-results
+mkdir allure-results -pm 777
+mkdir ssh_keys -p
+if [ ! -e "ssh_keys/id_rsa" ]
+then
+  ssh-keygen -P "" -f ssh_keys/id_rsa
+fi
+
+run_feature() {
+  local feature=$1
+  local cluster=$2
+  if [ $cluster = "concourse_cluster" ]; then
+    local project="${feature}_concourse"
+  else
+    local project="${feature}_demo"
+  fi
+  echo "Started $feature behave tests on cluster $cluster and project $project"
+  docker-compose -p $project -f arenadata/docker-compose.yaml --env-file arenadata/.env up -d
+  # Prepare ALL containers first
+  local services=$(docker-compose -p $project -f arenadata/docker-compose.yaml config --services | tr '\n' ' ')
+  for service in $services
+  do
+    docker-compose -p $project -f arenadata/docker-compose.yaml exec -T \
+      $service bash -c "mkdir -p /data/gpdata && chmod -R 777 /data &&
+        source gpdb_src/concourse/scripts/common.bash && install_gpdb &&
+       ./gpdb_src/concourse/scripts/setup_gpadmin_user.bash &&
+       ssh-keyscan ${services/$service/} >> /home/gpadmin/.ssh/known_hosts" &
+  done
+  wait
+  docker-compose -p $project -f arenadata/docker-compose.yaml exec -T \
+    -e FEATURE="$feature" -e BEHAVE_FLAGS="--tags $feature --tags=$cluster \
+      -f behave_utils.arenadata.formatter:CustomFormatter \
+      -o non-existed-output \
+      -f allure_behave.formatter:AllureFormatter \
+      -o /tmp/allure-results"  \
+    mdw gpdb_src/arenadata/scripts/behave_gpdb.bash
+  status=$?
+
+  docker-compose -p $project -f arenadata/docker-compose.yaml --env-file arenadata/.env down -v
+
+  if [[ $status > 0 ]]; then echo "Feature $feature failed with exit code $status"; fi
+  exit $status
+}
+
+pids=""
+exits=0
+for feature in $features
+do
+  for cluster in $clusters
+  do
+     run_feature $feature $cluster &
+     pids+="$! "
+     if [[ $(jobs -r -p | wc -l) -ge $processes ]]; then
+        wait -n
+        ((exits += $?))
+     fi
+  done
+done
+for pid in $pids
+  do
+    wait $pid
+    exits=$((exits + $?))
+  done
+if [[ $exits > 0 ]]; then exit 1; fi