Merge branch main into gen4-update

Signed-off-by: Andres Taylor <andres@planetscale.com>
vitessio · Apr 11, 2022 · 9931b09 · 9931b09
2 parents ee7d40a + 8f4c054
commit 9931b09
Show file tree

Hide file tree

Showing 142 changed files with 48,900 additions and 10,498 deletions.
diff --git a/.github/docker/cluster_test_12/Dockerfile b/.github/docker/cluster_test_12/Dockerfile
@@ -0,0 +1,42 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows"
+
+ARG bootstrap_version=5
+ARG image="vitess/bootstrap:${bootstrap_version}-mysql57"
+
+FROM "${image}"
+
+USER root
+
+# Re-copy sources from working tree
+RUN rm -rf /vt/src/vitess.io/vitess/*
+COPY . /vt/src/vitess.io/vitess
+
+# install XtraBackup
+RUN wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+RUN apt-get update
+RUN apt-get install -y gnupg2
+RUN dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+RUN apt-get update
+RUN apt-get install -y percona-xtrabackup-24
+
+# Set the working directory
+WORKDIR /vt/src/vitess.io/vitess
+
+# Fix permissions
+RUN chown -R vitess:vitess /vt
+
+USER vitess
+
+# Set environment variables
+ENV VTROOT /vt/src/vitess.io/vitess
+# Set the vtdataroot such that it uses the volume mount
+ENV VTDATAROOT /vt/vtdataroot
+
+# create the vtdataroot directory
+RUN mkdir -p $VTDATAROOT
+
+# install goimports
+RUN go install golang.org/x/tools/cmd/goimports@latest
+
+# sleep for 50 minutes
+CMD sleep 3000
diff --git a/.github/docker/cluster_test_18/Dockerfile b/.github/docker/cluster_test_18/Dockerfile
@@ -0,0 +1,45 @@
+# DO NOT MODIFY: THIS FILE IS GENERATED USING "make generate_ci_workflows"
+
+ARG bootstrap_version=5
+ARG image="vitess/bootstrap:${bootstrap_version}-mysql57"
+
+FROM "${image}"
+
+USER root
+
+# Re-copy sources from working tree
+RUN rm -rf /vt/src/vitess.io/vitess/*
+COPY . /vt/src/vitess.io/vitess
+
+# install XtraBackup
+RUN wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
+RUN apt-get update
+RUN apt-get install -y gnupg2
+RUN dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
+RUN apt-get update
+RUN apt-get install -y percona-xtrabackup-24
+
+# Set the working directory
+WORKDIR /vt/src/vitess.io/vitess
+
+# Fix permissions
+RUN chown -R vitess:vitess /vt
+
+USER vitess
+
+# Set environment variables
+ENV VTROOT /vt/src/vitess.io/vitess
+# Set the vtdataroot such that it uses the volume mount
+ENV VTDATAROOT /vt/vtdataroot
+
+# create the vtdataroot directory
+RUN mkdir -p $VTDATAROOT
+
+# install goimports
+RUN go install golang.org/x/tools/cmd/goimports@latest
+
+# make tools
+RUN make tools
+
+# sleep for 50 minutes
+CMD sleep 3000
diff --git a/.github/workflows/cluster_endtoend_12.yml b/.github/workflows/cluster_endtoend_12.yml
@@ -6,83 +6,36 @@ concurrency:
   group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (12)')
   cancel-in-progress: true
 
-env:
-  LAUNCHABLE_ORGANIZATION: "vitess"
-  LAUNCHABLE_WORKSPACE: "vitess-app"
-  GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}"
-
 jobs:
   build:
     name: Run endtoend tests on Cluster (12)
-    runs-on: ubuntu-18.04
+    runs-on: self-hosted
 
     steps:
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: 1.18
-
-    - name: Set up python
-      uses: actions/setup-python@v2
-
-    - name: Tune the OS
-      run: |
-        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
-        # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio
-        echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf
-        sudo sysctl -p /etc/sysctl.conf
-
-    - name: Check out code
-      uses: actions/checkout@v2
-
-    - name: Get dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
-        sudo service mysql stop
-        sudo service etcd stop
-        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
-        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
-        go mod download
-
-        # install JUnit report formatter
-        go install github.com/jstemmer/go-junit-report@latest
-
-        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
-        sudo apt-get install -y gnupg2
-        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
-        sudo apt-get update
-        sudo apt-get install percona-xtrabackup-24
-
-    - name: Setup launchable dependencies
-      run: |
-        # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up
-        pip3 install --user launchable~=1.0 > /dev/null
-
-        # verify that launchable setup is all correct.
-        launchable verify || true
+      - name: Check out code
+        uses: actions/checkout@v2
 
-        # Tell Launchable about the build you are producing and testing
-        launchable record build --name "$GITHUB_RUN_ID" --source .
+      - name: Build Docker Image
+        run: docker build -f ./.github/docker/cluster_test_12/Dockerfile -t cluster_test_12:$GITHUB_SHA  .
 
-    - name: Run cluster endtoend test
-      timeout-minutes: 30
-      run: |
-        # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file
-        # which musn't be more than 107 characters long.
-        export VTDATAROOT="/tmp/"
-        source build.env
+      - name: Run test
+        timeout-minutes: 30
+        run: docker run --name "cluster_test_12_$GITHUB_SHA" cluster_test_12:$GITHUB_SHA /bin/bash -c 'source build.env && go run test.go -keep-data=true -docker=false -print-log -follow -shard 12 -- -- --keep-data=true'
 
-        set -x
+      - name: Print Volume Used
+        if: ${{ always() }}
+        run: |
+          docker inspect -f '{{ (index .Mounts 0).Name }}' cluster_test_12_$GITHUB_SHA
 
-        # run the tests however you normally do, then produce a JUnit XML file
-        eatmydata -- go run test.go -docker=false -follow -shard 12 | tee -a output.txt | go-junit-report -set-exit-code > report.xml
+      - name: Cleanup Docker Volume
+        run: |
+          docker rm -v cluster_test_12_$GITHUB_SHA
 
-    - name: Print test output and Record test result in launchable
-      run: |
-        # send recorded tests to launchable
-        launchable record tests --build "$GITHUB_RUN_ID" go-test . || true
+      - name: Cleanup Docker Container
+        if: ${{ always() }}
+        run: |
+          docker rm -f cluster_test_12_$GITHUB_SHA
 
-        # print test output
-        cat output.txt
-      if: always()
+      - name: Cleanup Docker Image
+        run: |
+          docker image rm cluster_test_12:$GITHUB_SHA
diff --git a/.github/workflows/cluster_endtoend_18.yml b/.github/workflows/cluster_endtoend_18.yml
@@ -6,87 +6,36 @@ concurrency:
   group: format('{0}-{1}', ${{ github.ref }}, 'Cluster (18)')
   cancel-in-progress: true
 
-env:
-  LAUNCHABLE_ORGANIZATION: "vitess"
-  LAUNCHABLE_WORKSPACE: "vitess-app"
-  GITHUB_PR_HEAD_SHA: "${{ github.event.pull_request.head.sha }}"
-
 jobs:
   build:
     name: Run endtoend tests on Cluster (18)
-    runs-on: ubuntu-18.04
+    runs-on: self-hosted
 
     steps:
-    - name: Set up Go
-      uses: actions/setup-go@v2
-      with:
-        go-version: 1.18
-
-    - name: Set up python
-      uses: actions/setup-python@v2
-
-    - name: Tune the OS
-      run: |
-        echo '1024 65535' | sudo tee -a /proc/sys/net/ipv4/ip_local_port_range
-        # Increase the asynchronous non-blocking I/O. More information at https://dev.mysql.com/doc/refman/5.7/en/innodb-parameters.html#sysvar_innodb_use_native_aio
-        echo "fs.aio-max-nr = 1048576" | sudo tee -a /etc/sysctl.conf
-        sudo sysctl -p /etc/sysctl.conf
-
-    - name: Check out code
-      uses: actions/checkout@v2
-
-    - name: Get dependencies
-      run: |
-        sudo apt-get update
-        sudo apt-get install -y mysql-server mysql-client make unzip g++ etcd curl git wget eatmydata
-        sudo service mysql stop
-        sudo service etcd stop
-        sudo ln -s /etc/apparmor.d/usr.sbin.mysqld /etc/apparmor.d/disable/
-        sudo apparmor_parser -R /etc/apparmor.d/usr.sbin.mysqld
-        go mod download
-
-        # install JUnit report formatter
-        go install github.com/jstemmer/go-junit-report@latest
-
-        wget https://repo.percona.com/apt/percona-release_latest.$(lsb_release -sc)_all.deb
-        sudo apt-get install -y gnupg2
-        sudo dpkg -i percona-release_latest.$(lsb_release -sc)_all.deb
-        sudo apt-get update
-        sudo apt-get install percona-xtrabackup-24
-
-    - name: Installing zookeeper and consul
-      run: |
-          make tools
-
-    - name: Setup launchable dependencies
-      run: |
-        # Get Launchable CLI installed. If you can, make it a part of the builder image to speed things up
-        pip3 install --user launchable~=1.0 > /dev/null
-
-        # verify that launchable setup is all correct.
-        launchable verify || true
+      - name: Check out code
+        uses: actions/checkout@v2
 
-        # Tell Launchable about the build you are producing and testing
-        launchable record build --name "$GITHUB_RUN_ID" --source .
+      - name: Build Docker Image
+        run: docker build -f ./.github/docker/cluster_test_18/Dockerfile -t cluster_test_18:$GITHUB_SHA  .
 
-    - name: Run cluster endtoend test
-      timeout-minutes: 30
-      run: |
-        # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file
-        # which musn't be more than 107 characters long.
-        export VTDATAROOT="/tmp/"
-        source build.env
+      - name: Run test
+        timeout-minutes: 30
+        run: docker run --name "cluster_test_18_$GITHUB_SHA" cluster_test_18:$GITHUB_SHA /bin/bash -c 'source build.env && go run test.go -keep-data=true -docker=false -print-log -follow -shard 18 -- -- --keep-data=true'
 
-        set -x
+      - name: Print Volume Used
+        if: ${{ always() }}
+        run: |
+          docker inspect -f '{{ (index .Mounts 0).Name }}' cluster_test_18_$GITHUB_SHA
 
-        # run the tests however you normally do, then produce a JUnit XML file
-        eatmydata -- go run test.go -docker=false -follow -shard 18 | tee -a output.txt | go-junit-report -set-exit-code > report.xml
+      - name: Cleanup Docker Volume
+        run: |
+          docker rm -v cluster_test_18_$GITHUB_SHA
 
-    - name: Print test output and Record test result in launchable
-      run: |
-        # send recorded tests to launchable
-        launchable record tests --build "$GITHUB_RUN_ID" go-test . || true
+      - name: Cleanup Docker Container
+        if: ${{ always() }}
+        run: |
+          docker rm -f cluster_test_18_$GITHUB_SHA
 
-        # print test output
-        cat output.txt
-      if: always()
+      - name: Cleanup Docker Image
+        run: |
+          docker image rm cluster_test_18:$GITHUB_SHA
diff --git a/.github/workflows/cluster_endtoend_vtorc_8.0.yml b/.github/workflows/cluster_endtoend_vtorc_8.0.yml
@@ -77,6 +77,9 @@ jobs:
     - name: Run cluster endtoend test
       timeout-minutes: 30
       run: |
+        # We set the VTDATAROOT to the /tmp folder to reduce the file path of mysql.sock file
+        # which musn't be more than 107 characters long.
+        export VTDATAROOT="/tmp/"
         source build.env
 
         set -x

diff --git a/doc/releasenotes/14_0_0_summary.md b/doc/releasenotes/14_0_0_summary.md
@@ -78,6 +78,16 @@ Table lifecycle now supports views. It ensures to not purge rows from views, and
 
 On Mysql `8.0.23` or later, the states `PURGE` and `EVAC` are automatically skipped, thanks to `8.0.23` improvement to `DROP TABLE` speed of operation.
 
+### Tablet throttler
+
+Added `/throttler/throttled-apps` endpoint, which reports back all current throttling instructions. Note, this only reports explicit throttling requests (sych as ones submitted by `/throtler/throttle-app?app=...`). It does not list incidental rejections based on throttle thresholds.
+
+API endpoint `/throttler/throttle-app` now accepts a `ratio` query argument, a floating point in the range `[0..1]`, where:
+
+- `0` means "do not throttle at all"
+- `1` means "always throttle"
+- any numbr in between is allowd. For example, `0.3` means "throttle in 0.3 probability", ie on a per request and based on a dice roll, there's a `30%` change a request is denied. Overall we can expect about `30%` of requests to be denied. Example: `/throttler/throttle-app?app=vreplication&ratio=0.25`
+
 ### Compatibility
 
 #### Join with `USING`

diff --git a/examples/local/scripts/vtadmin-up.sh b/examples/local/scripts/vtadmin-up.sh
@@ -14,6 +14,7 @@ vtadmin \
   --http-tracing \
   --logtostderr \
   --alsologtostderr \
+  --rbac \
   --rbac-config="./vtadmin/rbac.yaml" \
   --cluster "id=local,name=local,discovery=staticfile,discovery-staticfile-path=./vtadmin/discovery.json,tablet-fqdn-tmpl={{ .Tablet.Hostname }}:15{{ .Tablet.Alias.Uid }}" \
   > "${log_dir}/vtadmin-api.out" 2>&1 &

diff --git a/go/cmd/vtadmin/main.go b/go/cmd/vtadmin/main.go
@@ -141,20 +141,29 @@ func run(cmd *cobra.Command, args []string) {
 }
 
 func main() {
+	// Common flags
 	rootCmd.Flags().StringVar(&opts.Addr, "addr", ":15000", "address to serve on")
 	rootCmd.Flags().DurationVar(&opts.CMuxReadTimeout, "lmux-read-timeout", time.Second, "how long to spend connection muxing")
 	rootCmd.Flags().DurationVar(&opts.LameDuckDuration, "lame-duck-duration", time.Second*5, "length of lame duck period at shutdown")
+
+	// Cluster config flags
 	rootCmd.Flags().Var(&clusterConfigs, "cluster", "per-cluster configuration. any values here take precedence over those in -cluster-defaults or -cluster-config")
 	rootCmd.Flags().Var(&clusterFileConfig, "cluster-config", "path to a yaml cluster configuration. see clusters.example.yaml") // (TODO:@amason) provide example config.
 	rootCmd.Flags().Var(&defaultClusterConfig, "cluster-defaults", "default options for all clusters")
 
+	// Tracing flags
 	rootCmd.Flags().AddGoFlag(flag.Lookup("tracer"))                 // defined in go/vt/trace
 	rootCmd.Flags().AddGoFlag(flag.Lookup("tracing-enable-logging")) // defined in go/vt/trace
 	rootCmd.Flags().AddGoFlag(flag.Lookup("tracing-sampling-type"))  // defined in go/vt/trace
 	rootCmd.Flags().AddGoFlag(flag.Lookup("tracing-sampling-rate"))  // defined in go/vt/trace
 	rootCmd.Flags().BoolVar(&opts.EnableTracing, "grpc-tracing", false, "whether to enable tracing on the gRPC server")
 	rootCmd.Flags().BoolVar(&httpOpts.EnableTracing, "http-tracing", false, "whether to enable tracing on the HTTP server")
 
+	// gRPC server flags
+	rootCmd.Flags().BoolVar(&opts.AllowReflection, "grpc-allow-reflection", false, "whether to register the gRPC server for reflection; this is required to use tools like `grpc_cli`")
+	rootCmd.Flags().BoolVar(&opts.EnableChannelz, "grpc-enable-channelz", false, "whether to enable the channelz service on the gRPC server")
+
+	// HTTP server flags
 	rootCmd.Flags().BoolVar(&httpOpts.DisableCompression, "http-no-compress", false, "whether to disable compression of HTTP API responses")
 	rootCmd.Flags().BoolVar(&httpOpts.DisableDebug, "http-no-debug", false, "whether to disable /debug/pprof/* and /debug/env HTTP endpoints")
 	rootCmd.Flags().Var(&debug.OmitEnv, "http-debug-omit-env", "name of an environment variable to omit from /debug/env, if http debug endpoints are enabled. specify multiple times to omit multiple env vars")
@@ -169,7 +178,7 @@ func main() {
 	)
 	rootCmd.Flags().BoolVar(&httpOpts.EnableDynamicClusters, "http-enable-dynamic-clusters", false, "whether to enable dynamic clusters that are set by request header cookies")
 
-	// rbac flags
+	// RBAC flags
 	rootCmd.Flags().StringVar(&rbacConfigPath, "rbac-config", "", "path to an RBAC config file. must be set if passing --rbac")
 	rootCmd.Flags().BoolVar(&enableRBAC, "rbac", false, "whether to enable RBAC. must be set if not passing --rbac")
 	rootCmd.Flags().BoolVar(&disableRBAC, "no-rbac", false, "whether to disable RBAC. must be set if not passing --no-rbac")