From 8c0e4fe5597520349390b62b3e67135a1c9175c8 Mon Sep 17 00:00:00 2001 From: kkkkun Date: Mon, 14 Mar 2022 11:07:57 +0800 Subject: [PATCH 01/30] delete offline defrag in etcdctl --- etcdctl/README.md | 15 +-------------- etcdctl/ctlv3/command/defrag_command.go | 14 -------------- tests/e2e/ctl_v3_defrag_test.go | 3 --- 3 files changed, 1 insertion(+), 31 deletions(-) diff --git a/etcdctl/README.md b/etcdctl/README.md index db273ec838d9..990a3fdcd18f 100644 --- a/etcdctl/README.md +++ b/etcdctl/README.md @@ -913,7 +913,7 @@ If NOSPACE alarm is present: ### DEFRAG [options] -DEFRAG defragments the backend database file for a set of given endpoints while etcd is running, ~~or directly defragments an etcd data directory while etcd is not running~~. When an etcd member reclaims storage space from deleted and compacted keys, the space is kept in a free list and the database file remains the same size. By defragmenting the database, the etcd member releases this free space back to the file system. +DEFRAG defragments the backend database file for a set of given endpoints while etcd is running. When an etcd member reclaims storage space from deleted and compacted keys, the space is kept in a free list and the database file remains the same size. By defragmenting the database, the etcd member releases this free space back to the file system. **Note: to defragment offline (`--data-dir` flag), use: `etcutl defrag` instead** @@ -921,9 +921,6 @@ DEFRAG defragments the backend database file for a set of given endpoints while **Note that defragmentation request does not get replicated over cluster. That is, the request is only applied to the local node. Specify all members in `--endpoints` flag or `--cluster` flag to automatically find all cluster members.** -#### Options - -- data-dir -- Optional. **Deprecated**. If present, defragments a data directory not in use by etcd. To be removed in v3.6. #### Output @@ -946,16 +943,6 @@ Finished defragmenting etcd member[http://127.0.0.1:22379] Finished defragmenting etcd member[http://127.0.0.1:32379] ``` -To defragment a data directory directly, use the `etcdutl` with `--data-dir` flag -(`etcdctl` will remove this flag in v3.6): - -``` bash -# Defragment while etcd is not running -./etcdutl defrag --data-dir default.etcd -# success (exit status 0) -# Error: cannot open database at default.etcd/member/snap/db -``` - #### Remarks DEFRAG returns a zero exit code only if it succeeded defragmenting all given endpoints. diff --git a/etcdctl/ctlv3/command/defrag_command.go b/etcdctl/ctlv3/command/defrag_command.go index 9b4f29a6aa68..5ebf4483d44d 100644 --- a/etcdctl/ctlv3/command/defrag_command.go +++ b/etcdctl/ctlv3/command/defrag_command.go @@ -20,14 +20,9 @@ import ( "time" "github.com/spf13/cobra" - "go.etcd.io/etcd/etcdutl/v3/etcdutl" "go.etcd.io/etcd/pkg/v3/cobrautl" ) -var ( - defragDataDir string -) - // NewDefragCommand returns the cobra command for "Defrag". func NewDefragCommand() *cobra.Command { cmd := &cobra.Command{ @@ -36,19 +31,10 @@ func NewDefragCommand() *cobra.Command { Run: defragCommandFunc, } cmd.PersistentFlags().BoolVar(&epClusterEndpoints, "cluster", false, "use all endpoints from the cluster member list") - cmd.Flags().StringVar(&defragDataDir, "data-dir", "", "Optional. If present, defragments a data directory not in use by etcd.") - cmd.MarkFlagDirname("data-dir") return cmd } func defragCommandFunc(cmd *cobra.Command, args []string) { - if len(defragDataDir) > 0 { - fmt.Fprintf(os.Stderr, "Use `etcdutl defrag` instead. The --data-dir is going to be decomissioned in v3.6.\n\n") - err := etcdutl.DefragData(defragDataDir) - if err != nil { - cobrautl.ExitWithError(cobrautl.ExitError, err) - } - } failures := 0 c := mustClientFromCmd(cmd) diff --git a/tests/e2e/ctl_v3_defrag_test.go b/tests/e2e/ctl_v3_defrag_test.go index 6abf855f49d1..c47ed3f4cdfc 100644 --- a/tests/e2e/ctl_v3_defrag_test.go +++ b/tests/e2e/ctl_v3_defrag_test.go @@ -22,9 +22,6 @@ import ( func TestCtlV3DefragOnline(t *testing.T) { testCtl(t, defragOnlineTest) } -func TestCtlV3DefragOffline(t *testing.T) { - testCtlWithOffline(t, maintenanceInitKeys, defragOfflineTest) -} func TestCtlV3DefragOfflineEtcdutl(t *testing.T) { testCtlWithOffline(t, maintenanceInitKeys, defragOfflineTest, withEtcdutl()) } From 494a4712488b57d1ea238de6c9a3c2524dccfd7e Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Mon, 14 Mar 2022 14:07:50 +0100 Subject: [PATCH 02/30] *: Restore release documentation --- RELEASE.md | 159 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 159 insertions(+) create mode 100644 RELEASE.md diff --git a/RELEASE.md b/RELEASE.md new file mode 100644 index 000000000000..8085f7c5610b --- /dev/null +++ b/RELEASE.md @@ -0,0 +1,159 @@ +# Release + +The guide talks about how to release a new version of etcd. + +The procedure includes some manual steps for sanity checking, but it can probably be further scripted. Please keep this document up-to-date if making changes to the release process. + +## Release management + +etcd community members are assigned to manage the release each etcd major/minor version as well as manage patches +and to each stable release branch. The managers are responsible for communicating the timelines and status of each +release and for ensuring the stability of the release branch. + +| Releases | Manager | +| -------- | ------- | +| 3.1 patch (post 3.1.0) | Joe Betz [@jpbetz](https://github.com/jpbetz) | +| 3.2 patch (post 3.2.0) | Joe Betz [@jpbetz](https://github.com/jpbetz) | +| 3.3 patch (post 3.3.0) | Gyuho Lee [@gyuho](https://github.com/gyuho) | + +## Prepare release + +Set desired version as environment variable for following steps. Here is an example to release 2.3.0: + +``` +export VERSION=v2.3.0 +export PREV_VERSION=v2.2.5 +``` + +All releases version numbers follow the format of [semantic versioning 2.0.0](http://semver.org/). + +### Major, minor version release, or its pre-release + +- Ensure the relevant milestone on GitHub is complete. All referenced issues should be closed, or moved elsewhere. +- Remove this release from [roadmap](https://github.com/etcd-io/etcd/blob/master/ROADMAP.md), if necessary. +- Ensure the latest upgrade documentation is available. +- Bump [hardcoded MinClusterVerion in the repository](https://github.com/etcd-io/etcd/blob/v3.4.15/version/version.go#L29), if necessary. +- Add feature capability maps for the new version, if necessary. + +### Patch version release + +- To request a backport, devlopers submit cherrypick PRs targeting the release branch. The commits should not include merge commits. The commits should be restricted to bug fixes and security patches. +- The cherrypick PRs should target the appropriate release branch (`base:release--`). `hack/patch/cherrypick.sh` may be used to automatically generate cherrypick PRs. +- The release patch manager reviews the cherrypick PRs. Please discuss carefully what is backported to the patch release. Each patch release should be strictly better than it's predecessor. +- The release patch manager will cherry-pick these commits starting from the oldest one into stable branch. + +## Write release note + +- Write introduction for the new release. For example, what major bug we fix, what new features we introduce or what performance improvement we make. +- Put `[GH XXXX]` at the head of change line to reference Pull Request that introduces the change. Moreover, add a link on it to jump to the Pull Request. +- Find PRs with `release-note` label and explain them in `NEWS` file, as a straightforward summary of changes for end-users. + +## Tag version + +- Bump [hardcoded Version in the repository](https://github.com/etcd-io/etcd/blob/v3.4.15/version/version.go#L30) to the latest version `${VERSION}`. +- Ensure all tests on CI system are passed. +- Manually check etcd is buildable in Linux, Darwin and Windows. +- Manually check upgrade etcd cluster of previous minor version works well. +- Manually check new features work well. +- Add a signed tag through `git tag -s ${VERSION}`. +- Sanity check tag correctness through `git show tags/$VERSION`. +- Push the tag to GitHub through `git push origin tags/$VERSION`. This assumes `origin` corresponds to "https://github.com/etcd-io/etcd\". + +## Build release binaries and images + +- Ensure `docker` is available. + +Run release script in root directory: + +``` +TAG=gcr.io/etcd-development/etcd ./scripts/release.sh ${VERSION} +``` + +It generates all release binaries and images under directory ./release. + +## Sign binaries, images, and source code + +etcd project key must be used to sign the generated binaries and images.`$SUBKEYID` is the key ID of etcd project Yubikey. Connect the key and run `gpg2 --card-status` to get the ID. + +The following commands are used for public release sign: + +``` +cd release +for i in etcd-*{.zip,.tar.gz}; do gpg2 --default-key $SUBKEYID --armor --output ${i}.asc --detach-sign ${i}; done +for i in etcd-*{.zip,.tar.gz}; do gpg2 --verify ${i}.asc ${i}; done + +# sign zipped source code files +wget https://github.com/etcd-io/etcd/archive/${VERSION}.zip +gpg2 --armor --default-key $SUBKEYID --output ${VERSION}.zip.asc --detach-sign ${VERSION}.zip +gpg2 --verify ${VERSION}.zip.asc ${VERSION}.zip + +wget https://github.com/etcd-io/etcd/archive/${VERSION}.tar.gz +gpg2 --armor --default-key $SUBKEYID --output ${VERSION}.tar.gz.asc --detach-sign ${VERSION}.tar.gz +gpg2 --verify ${VERSION}.tar.gz.asc ${VERSION}.tar.gz +``` + +The public key for GPG signing can be found at [CoreOS Application Signing Key](https://coreos.com/security/app-signing-key) + + +## Publish release page in GitHub + +- Set release title as the version name. +- Follow the format of previous release pages. +- Attach the generated binaries and signatures. +- Select whether it is a pre-release. +- Publish the release! + +## Publish docker image in gcr.io + +- Push docker image: + +``` +gcloud docker -- login -u _json_key -p "$(cat /etc/gcp-key-etcd.json)" https://gcr.io + +for TARGET_ARCH in "-arm64" "-ppc64le" "-s390x" ""; do + gcloud docker -- push gcr.io/etcd-development/etcd:${VERSION}${TARGET_ARCH} +done +``` + +- Add `latest` tag to the new image on [gcr.io](https://console.cloud.google.com/gcr/images/etcd-development/GLOBAL/etcd?project=etcd-development&authuser=1) if this is a stable release. + +## Publish docker image in Quay.io + +- Build docker images with quay.io: + +``` +for TARGET_ARCH in "amd64" "arm64" "ppc64le" "s390x"; do + TAG=quay.io/coreos/etcd GOARCH=${TARGET_ARCH} \ + BINARYDIR=release/etcd-${VERSION}-linux-${TARGET_ARCH} \ + BUILDDIR=release \ + ./scripts/build-docker ${VERSION} +done +``` + +- Push docker image: + +``` +docker login quay.io + +for TARGET_ARCH in "-arm64" "-ppc64le" "-s390x" ""; do + docker push quay.io/coreos/etcd:${VERSION}${TARGET_ARCH} +done +``` + +- Add `latest` tag to the new image on [quay.io](https://quay.io/repository/coreos/etcd?tag=latest&tab=tags) if this is a stable release. + +## Announce to the etcd-dev Googlegroup + +- Follow the format of [previous release emails](https://groups.google.com/forum/#!forum/etcd-dev). +- Make sure to include a list of authors that contributed since the previous release - something like the following might be handy: + +``` +git log ...${PREV_VERSION} --pretty=format:"%an" | sort | uniq | tr '\n' ',' | sed -e 's#,#, #g' -e 's#, $##' +``` + +- Send email to etcd-dev@googlegroups.com + +## Post release + +- Create new stable branch through `git push origin ${VERSION_MAJOR}.${VERSION_MINOR}` if this is a major stable release. This assumes `origin` corresponds to "https://github.com/etcd-io/etcd\". +- Bump [hardcoded Version in the repository](https://github.com/etcd-io/etcd/blob/v3.4.15/version/version.go#L30) to the version `${VERSION}+git`. \ No newline at end of file From a1f24e9afc07bd8790cb6368fb2910ef6228e6f4 Mon Sep 17 00:00:00 2001 From: kkkkun Date: Tue, 15 Mar 2022 22:50:56 +0800 Subject: [PATCH 03/30] tests: Migrate endpoint tests to common framework --- tests/common/endpoint_test.go | 58 ++++++++++++++++++ tests/e2e/ctl_v3_auth_test.go | 11 +++- tests/e2e/ctl_v3_endpoint_test.go | 93 ----------------------------- tests/framework/e2e/etcdctl.go | 67 +++++++++++++++++++++ tests/framework/integration.go | 41 +++++++++++++ tests/framework/interface.go | 4 ++ tests/integration/v3_health_test.go | 39 ------------ 7 files changed, 180 insertions(+), 133 deletions(-) create mode 100644 tests/common/endpoint_test.go delete mode 100644 tests/e2e/ctl_v3_endpoint_test.go delete mode 100644 tests/integration/v3_health_test.go diff --git a/tests/common/endpoint_test.go b/tests/common/endpoint_test.go new file mode 100644 index 000000000000..371e85fbbb84 --- /dev/null +++ b/tests/common/endpoint_test.go @@ -0,0 +1,58 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "testing" + "time" + + "go.etcd.io/etcd/tests/v3/framework/config" + "go.etcd.io/etcd/tests/v3/framework/testutils" +) + +func TestEndpointStatus(t *testing.T) { + testRunner.BeforeTest(t) + clus := testRunner.NewCluster(t, config.ClusterConfig{ClusterSize: 3}) + defer clus.Close() + testutils.ExecuteWithTimeout(t, 10*time.Second, func() { + _, err := clus.Client().Status() + if err != nil { + t.Fatalf("get endpoint status error: %v", err) + } + }) +} + +func TestEndpointHashKV(t *testing.T) { + testRunner.BeforeTest(t) + clus := testRunner.NewCluster(t, config.ClusterConfig{ClusterSize: 3}) + defer clus.Close() + testutils.ExecuteWithTimeout(t, 10*time.Second, func() { + _, err := clus.Client().HashKV(0) + if err != nil { + t.Fatalf("get endpoint hashkv error: %v", err) + } + }) +} + +func TestEndpointHealth(t *testing.T) { + testRunner.BeforeTest(t) + clus := testRunner.NewCluster(t, config.ClusterConfig{ClusterSize: 3}) + defer clus.Close() + testutils.ExecuteWithTimeout(t, 10*time.Second, func() { + if err := clus.Client().Health(); err != nil { + t.Fatalf("get endpoint health error: %v", err) + } + }) +} diff --git a/tests/e2e/ctl_v3_auth_test.go b/tests/e2e/ctl_v3_auth_test.go index 4d376f64c8c6..fd15a9e617ec 100644 --- a/tests/e2e/ctl_v3_auth_test.go +++ b/tests/e2e/ctl_v3_auth_test.go @@ -22,7 +22,7 @@ import ( "testing" "time" - "go.etcd.io/etcd/client/v3" + clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/e2e" ) @@ -1256,3 +1256,12 @@ func authTestRevisionConsistency(cx ctlCtx) { cx.t.Fatalf("auth revison shouldn't change when restarting etcd, expected: %d, got: %d", oldAuthRevision, newAuthRevision) } } + +func ctlV3EndpointHealth(cx ctlCtx) error { + cmdArgs := append(cx.PrefixArgs(), "endpoint", "health") + lines := make([]string, cx.epc.Cfg.ClusterSize) + for i := range lines { + lines[i] = "is healthy" + } + return e2e.SpawnWithExpects(cmdArgs, cx.envMap, lines...) +} diff --git a/tests/e2e/ctl_v3_endpoint_test.go b/tests/e2e/ctl_v3_endpoint_test.go deleted file mode 100644 index 8e364e8b4b81..000000000000 --- a/tests/e2e/ctl_v3_endpoint_test.go +++ /dev/null @@ -1,93 +0,0 @@ -// Copyright 2016 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package e2e - -import ( - "context" - "fmt" - "net/url" - "testing" - "time" - - "go.etcd.io/etcd/client/v3" - "go.etcd.io/etcd/tests/v3/framework/e2e" -) - -func TestCtlV3EndpointHealth(t *testing.T) { testCtl(t, endpointHealthTest, withQuorum()) } -func TestCtlV3EndpointStatus(t *testing.T) { testCtl(t, endpointStatusTest, withQuorum()) } -func TestCtlV3EndpointHashKV(t *testing.T) { testCtl(t, endpointHashKVTest, withQuorum()) } - -func endpointHealthTest(cx ctlCtx) { - if err := ctlV3EndpointHealth(cx); err != nil { - cx.t.Fatalf("endpointStatusTest ctlV3EndpointHealth error (%v)", err) - } -} - -func ctlV3EndpointHealth(cx ctlCtx) error { - cmdArgs := append(cx.PrefixArgs(), "endpoint", "health") - lines := make([]string, cx.epc.Cfg.ClusterSize) - for i := range lines { - lines[i] = "is healthy" - } - return e2e.SpawnWithExpects(cmdArgs, cx.envMap, lines...) -} - -func endpointStatusTest(cx ctlCtx) { - if err := ctlV3EndpointStatus(cx); err != nil { - cx.t.Fatalf("endpointStatusTest ctlV3EndpointStatus error (%v)", err) - } -} - -func ctlV3EndpointStatus(cx ctlCtx) error { - cmdArgs := append(cx.PrefixArgs(), "endpoint", "status") - var eps []string - for _, ep := range cx.epc.EndpointsV3() { - u, _ := url.Parse(ep) - eps = append(eps, u.Host) - } - return e2e.SpawnWithExpects(cmdArgs, cx.envMap, eps...) -} - -func endpointHashKVTest(cx ctlCtx) { - if err := ctlV3EndpointHashKV(cx); err != nil { - cx.t.Fatalf("endpointHashKVTest ctlV3EndpointHashKV error (%v)", err) - } -} - -func ctlV3EndpointHashKV(cx ctlCtx) error { - eps := cx.epc.EndpointsV3() - - // get latest hash to compare - cli, err := clientv3.New(clientv3.Config{ - Endpoints: eps, - DialTimeout: 3 * time.Second, - }) - if err != nil { - cx.t.Fatal(err) - } - defer cli.Close() - hresp, err := cli.HashKV(context.TODO(), eps[0], 0) - if err != nil { - cx.t.Fatal(err) - } - - cmdArgs := append(cx.PrefixArgs(), "endpoint", "hashkv") - var ss []string - for _, ep := range cx.epc.EndpointsV3() { - u, _ := url.Parse(ep) - ss = append(ss, fmt.Sprintf("%s, %d", u.Host, hresp.Hash)) - } - return e2e.SpawnWithExpects(cmdArgs, cx.envMap, ss...) -} diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index f807c7fe67d1..18c436748264 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -176,3 +176,70 @@ func (ctl *EtcdctlV3) Compact(rev int64, o config.CompactOption) (*clientv3.Comp return nil, SpawnWithExpect(args, fmt.Sprintf("compacted revision %v", rev)) } + +func (ctl *EtcdctlV3) Status() ([]*clientv3.StatusResponse, error) { + args := ctl.cmdArgs() + args = append(args, "endpoint", "status", "-w", "json") + args = append(args, "--endpoints", strings.Join(ctl.endpoints, ",")) + cmd, err := SpawnCmd(args, nil) + if err != nil { + return nil, err + } + + var epStatus []*struct { + Endpoint string + Status *clientv3.StatusResponse + } + line, err := cmd.Expect("header") + if err != nil { + return nil, err + } + err = json.Unmarshal([]byte(line), &epStatus) + if err != nil { + return nil, err + } + resp := make([]*clientv3.StatusResponse, len(epStatus)) + for _, e := range epStatus { + resp = append(resp, e.Status) + } + return resp, err +} + +func (ctl *EtcdctlV3) HashKV(rev int64) ([]*clientv3.HashKVResponse, error) { + args := ctl.cmdArgs() + args = append(args, "endpoint", "hashkv", "-w", "json") + args = append(args, "--endpoints", strings.Join(ctl.endpoints, ",")) + args = append(args, "--rev", fmt.Sprint(rev)) + cmd, err := SpawnCmd(args, nil) + if err != nil { + return nil, err + } + var epHashKVs []*struct { + Endpoint string + HashKV *clientv3.HashKVResponse + } + line, err := cmd.Expect("header") + if err != nil { + return nil, err + } + err = json.Unmarshal([]byte(line), &epHashKVs) + if err != nil { + return nil, err + } + resp := make([]*clientv3.HashKVResponse, len(epHashKVs)) + for _, e := range epHashKVs { + resp = append(resp, e.HashKV) + } + return resp, err +} + +func (ctl *EtcdctlV3) Health() error { + args := ctl.cmdArgs() + args = append(args, "endpoint", "health") + lines := make([]string, len(ctl.endpoints)) + for i := range lines { + lines[i] = "is healthy" + } + return SpawnWithExpects(args, map[string]string{}, lines...) + +} diff --git a/tests/framework/integration.go b/tests/framework/integration.go index 18aabed5a334..73e15f130ff0 100644 --- a/tests/framework/integration.go +++ b/tests/framework/integration.go @@ -19,9 +19,12 @@ import ( "fmt" "testing" + healthpb "google.golang.org/grpc/health/grpc_health_v1" + "go.etcd.io/etcd/client/pkg/v3/testutil" "go.etcd.io/etcd/client/pkg/v3/transport" clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/integration" "go.uber.org/zap" @@ -155,3 +158,41 @@ func (c integrationClient) Compact(rev int64, o config.CompactOption) (*clientv3 } return c.Client.Compact(ctx, rev, clientOpts...) } + +func (c integrationClient) Status() ([]*clientv3.StatusResponse, error) { + endpoints := c.Client.Endpoints() + var resp []*clientv3.StatusResponse + for _, ep := range endpoints { + status, err := c.Client.Status(context.Background(), ep) + if err != nil { + return nil, err + } + resp = append(resp, status) + } + return resp, nil +} + +func (c integrationClient) HashKV(rev int64) ([]*clientv3.HashKVResponse, error) { + endpoints := c.Client.Endpoints() + var resp []*clientv3.HashKVResponse + for _, ep := range endpoints { + hashKV, err := c.Client.HashKV(context.Background(), ep, rev) + if err != nil { + return nil, err + } + resp = append(resp, hashKV) + } + return resp, nil +} + +func (c integrationClient) Health() error { + cli := healthpb.NewHealthClient(c.Client.ActiveConnection()) + resp, err := cli.Check(context.TODO(), &healthpb.HealthCheckRequest{}) + if err != nil { + return err + } + if resp.Status != healthpb.HealthCheckResponse_SERVING { + return fmt.Errorf("status expected %s, got %s", healthpb.HealthCheckResponse_SERVING, resp.Status) + } + return nil +} diff --git a/tests/framework/interface.go b/tests/framework/interface.go index 5c1ac01145a6..f6982f321a89 100644 --- a/tests/framework/interface.go +++ b/tests/framework/interface.go @@ -37,4 +37,8 @@ type Client interface { Get(key string, opts config.GetOptions) (*clientv3.GetResponse, error) Delete(key string, opts config.DeleteOptions) (*clientv3.DeleteResponse, error) Compact(rev int64, opts config.CompactOption) (*clientv3.CompactResponse, error) + + Status() ([]*clientv3.StatusResponse, error) + HashKV(rev int64) ([]*clientv3.HashKVResponse, error) + Health() error } diff --git a/tests/integration/v3_health_test.go b/tests/integration/v3_health_test.go deleted file mode 100644 index 98b64c63fe05..000000000000 --- a/tests/integration/v3_health_test.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2017 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package integration - -import ( - "context" - "testing" - - "go.etcd.io/etcd/tests/v3/framework/integration" - healthpb "google.golang.org/grpc/health/grpc_health_v1" -) - -func TestHealthCheck(t *testing.T) { - integration.BeforeTest(t) - - clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) - defer clus.Terminate(t) - - cli := healthpb.NewHealthClient(clus.RandClient().ActiveConnection()) - resp, err := cli.Check(context.TODO(), &healthpb.HealthCheckRequest{}) - if err != nil { - t.Fatal(err) - } - if resp.Status != healthpb.HealthCheckResponse_SERVING { - t.Fatalf("status expected %s, got %s", healthpb.HealthCheckResponse_SERVING, resp.Status) - } -} From b50239f9a46a11ec6f4d070e305bf7c2b3e49f50 Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Tue, 15 Mar 2022 16:38:59 +0000 Subject: [PATCH 04/30] integration/client/watch: remove duplicate setup This test was being skipped as it has a duplicate call to integration2.BeginTest. This duplicate call registered a second handler for leak detection which failed and skipped the test as the wrapper (runWatchTest) has already started a cluster. --- tests/integration/clientv3/watch_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/integration/clientv3/watch_test.go b/tests/integration/clientv3/watch_test.go index 1d472ecc55ca..f464ec9bb964 100644 --- a/tests/integration/clientv3/watch_test.go +++ b/tests/integration/clientv3/watch_test.go @@ -299,8 +299,6 @@ func TestWatchCancelRunning(t *testing.T) { } func testWatchCancelRunning(t *testing.T, wctx *watchctx) { - integration2.BeforeTest(t) - ctx, cancel := context.WithCancel(context.Background()) if wctx.ch = wctx.w.Watch(ctx, "a"); wctx.ch == nil { t.Fatalf("expected non-nil watcher channel") From 3bbbef54c17b54186ea2467317d4e60eeec6e785 Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Tue, 15 Mar 2022 17:42:47 +0000 Subject: [PATCH 05/30] integration/clientv3/exp/recipes: Fix lock test TestMutexTryLock(SingleNode|MultiNode) were being skipped as they had duplicate calls to integration2.BeginTest. These duplicate calls registered a second handler for leak detection which failed and skipped the test as the wrappers had already started a cluster. part of #13698 --- .../integration/clientv3/experimental/recipes/v3_lock_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/integration/clientv3/experimental/recipes/v3_lock_test.go b/tests/integration/clientv3/experimental/recipes/v3_lock_test.go index ee130679bd02..ab7e7ff9e9cd 100644 --- a/tests/integration/clientv3/experimental/recipes/v3_lock_test.go +++ b/tests/integration/clientv3/experimental/recipes/v3_lock_test.go @@ -21,7 +21,7 @@ import ( "time" "go.etcd.io/etcd/api/v3/mvccpb" - "go.etcd.io/etcd/client/v3" + clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/client/v3/concurrency" recipe "go.etcd.io/etcd/client/v3/experimental/recipes" integration2 "go.etcd.io/etcd/tests/v3/framework/integration" @@ -113,8 +113,6 @@ func TestMutexTryLockMultiNode(t *testing.T) { } func testMutexTryLock(t *testing.T, lockers int, chooseClient func() *clientv3.Client) { - integration2.BeforeTest(t) - lockedC := make(chan *concurrency.Mutex) notlockedC := make(chan *concurrency.Mutex) stopC := make(chan struct{}) From a045e4bbfc26f45e8cc64663a5a82cbc1b1a2fe2 Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Tue, 15 Mar 2022 11:06:57 +0000 Subject: [PATCH 06/30] expect_test: Look up binaries from the path Not all systems include binaries in the same location. On my (NixOS, so albeit a little weird) system these binaries exist in very different locations. This test switches to looking up the paths from the users PATH or skips the test if they do not exist to improve the `make test` experience on such systems. --- pkg/expect/expect_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkg/expect/expect_test.go b/pkg/expect/expect_test.go index c5ed18ec60bf..f091f1377de6 100644 --- a/pkg/expect/expect_test.go +++ b/pkg/expect/expect_test.go @@ -23,7 +23,7 @@ import ( ) func TestExpectFunc(t *testing.T) { - ep, err := NewExpect("/bin/echo", "hello world") + ep, err := NewExpect("echo", "hello world") if err != nil { t.Fatal(err) } @@ -41,7 +41,7 @@ func TestExpectFunc(t *testing.T) { } func TestEcho(t *testing.T) { - ep, err := NewExpect("/bin/echo", "hello world") + ep, err := NewExpect("echo", "hello world") if err != nil { t.Fatal(err) } @@ -62,7 +62,7 @@ func TestEcho(t *testing.T) { } func TestLineCount(t *testing.T) { - ep, err := NewExpect("/usr/bin/printf", "1\n2\n3") + ep, err := NewExpect("printf", "1\n2\n3") if err != nil { t.Fatal(err) } @@ -83,7 +83,7 @@ func TestLineCount(t *testing.T) { } func TestSend(t *testing.T) { - ep, err := NewExpect("/usr/bin/tr", "a", "b") + ep, err := NewExpect("tr", "a", "b") if err != nil { t.Fatal(err) } @@ -99,7 +99,7 @@ func TestSend(t *testing.T) { } func TestSignal(t *testing.T) { - ep, err := NewExpect("/bin/sleep", "100") + ep, err := NewExpect("sleep", "100") if err != nil { t.Fatal(err) } From edce939f6e8b414df4170577ab7a3cd243d8c2e9 Mon Sep 17 00:00:00 2001 From: ahrtr Date: Thu, 10 Mar 2022 14:37:45 +0800 Subject: [PATCH 07/30] add one more field storageVersion into StatusResponse When performing the downgrade operation, users can confirm whether each member is ready to be downgraded using the field 'storageVersion'. If it's equal to the 'target version' in the downgrade command, then it's ready to be downgraded; otherwise, the etcd member is still in progress of processing the db file. --- CHANGELOG/CHANGELOG-3.6.md | 1 + .../apispec/swagger/rpc.swagger.json | 4 + api/etcdserverpb/rpc.pb.go | 607 ++++++++++-------- api/etcdserverpb/rpc.proto | 2 + etcdctl/ctlv3/command/printer.go | 3 +- etcdctl/ctlv3/command/printer_fields.go | 1 + scripts/etcd_version_annotations.txt | 1 + server/etcdserver/adapters.go | 6 +- server/etcdserver/api/v3rpc/maintenance.go | 7 +- server/etcdserver/server.go | 8 +- 10 files changed, 355 insertions(+), 285 deletions(-) diff --git a/CHANGELOG/CHANGELOG-3.6.md b/CHANGELOG/CHANGELOG-3.6.md index 06c5c6d6200b..f33b697bf5a3 100644 --- a/CHANGELOG/CHANGELOG-3.6.md +++ b/CHANGELOG/CHANGELOG-3.6.md @@ -22,6 +22,7 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v3.6.0). - Add command to generate [shell completion](https://github.com/etcd-io/etcd/pull/13133). - When print endpoint status, [show db size in use](https://github.com/etcd-io/etcd/pull/13639) - [Always print the raft_term in decimal](https://github.com/etcd-io/etcd/pull/13711) when displaying member list in json. +- [Add one more field `storageVersion`](https://github.com/etcd-io/etcd/pull/13773) into the response of command `etcdctl endpoint status`. ### etcdutl v3 diff --git a/Documentation/dev-guide/apispec/swagger/rpc.swagger.json b/Documentation/dev-guide/apispec/swagger/rpc.swagger.json index 07e4b33f52f9..35cc83fbddcb 100644 --- a/Documentation/dev-guide/apispec/swagger/rpc.swagger.json +++ b/Documentation/dev-guide/apispec/swagger/rpc.swagger.json @@ -2771,6 +2771,10 @@ "type": "string", "format": "uint64" }, + "storageVersion": { + "description": "storageVersion is the version of the db file.", + "type": "string" + }, "version": { "description": "version is the cluster protocol version used by the responding member.", "type": "string" diff --git a/api/etcdserverpb/rpc.pb.go b/api/etcdserverpb/rpc.pb.go index 46a8889d1220..52c53f8afcb1 100644 --- a/api/etcdserverpb/rpc.pb.go +++ b/api/etcdserverpb/rpc.pb.go @@ -4235,7 +4235,9 @@ type StatusResponse struct { // dbSizeInUse is the size of the backend database logically in use, in bytes, of the responding member. DbSizeInUse int64 `protobuf:"varint,9,opt,name=dbSizeInUse,proto3" json:"dbSizeInUse,omitempty"` // isLearner indicates if the member is raft learner. - IsLearner bool `protobuf:"varint,10,opt,name=isLearner,proto3" json:"isLearner,omitempty"` + IsLearner bool `protobuf:"varint,10,opt,name=isLearner,proto3" json:"isLearner,omitempty"` + // storageVersion is the version of the db file. + StorageVersion string `protobuf:"bytes,11,opt,name=storageVersion,proto3" json:"storageVersion,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -4344,6 +4346,13 @@ func (m *StatusResponse) GetIsLearner() bool { return false } +func (m *StatusResponse) GetStorageVersion() string { + if m != nil { + return m.StorageVersion + } + return "" +} + type AuthEnableRequest struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -6166,282 +6175,283 @@ func init() { func init() { proto.RegisterFile("rpc.proto", fileDescriptor_77a6da22d6a3feb1) } var fileDescriptor_77a6da22d6a3feb1 = []byte{ - // 4393 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x5c, 0x4f, 0x6f, 0x24, 0x49, - 0x56, 0x77, 0xd6, 0x5f, 0xd7, 0xab, 0x72, 0xb9, 0x1c, 0xed, 0xee, 0xa9, 0xae, 0xe9, 0x76, 0x7b, - 0xb2, 0xa7, 0x67, 0x3c, 0x9e, 0x19, 0xbb, 0xdb, 0x76, 0xef, 0x40, 0xa3, 0x19, 0xb6, 0xda, 0xae, - 0x69, 0x9b, 0x76, 0xdb, 0xde, 0x74, 0x75, 0xcf, 0xce, 0x20, 0xad, 0x49, 0x57, 0x45, 0xdb, 0xb9, - 0xae, 0xca, 0xac, 0xcd, 0xcc, 0x72, 0xdb, 0xcb, 0x61, 0x97, 0x85, 0x65, 0xb5, 0x20, 0xad, 0xc4, - 0x22, 0xa1, 0x15, 0x82, 0x0b, 0x42, 0x82, 0x03, 0x20, 0x38, 0x70, 0x40, 0x1c, 0x38, 0xc0, 0x01, - 0x0e, 0x48, 0x48, 0x7c, 0x01, 0x18, 0xf6, 0xc4, 0x57, 0x40, 0x42, 0xab, 0xf8, 0x97, 0x11, 0x99, - 0x19, 0x65, 0x7b, 0xd6, 0x1e, 0xed, 0x65, 0xba, 0x32, 0xe2, 0xc5, 0xfb, 0xbd, 0x78, 0x2f, 0xe2, - 0xbd, 0x88, 0xf7, 0xc2, 0x03, 0x25, 0x7f, 0xd0, 0x59, 0x18, 0xf8, 0x5e, 0xe8, 0xa1, 0x0a, 0x0e, - 0x3b, 0xdd, 0x00, 0xfb, 0xc7, 0xd8, 0x1f, 0xec, 0x37, 0xa6, 0x0f, 0xbc, 0x03, 0x8f, 0x76, 0x2c, - 0x92, 0x5f, 0x8c, 0xa6, 0x51, 0x27, 0x34, 0x8b, 0xf6, 0xc0, 0x59, 0xec, 0x1f, 0x77, 0x3a, 0x83, - 0xfd, 0xc5, 0xa3, 0x63, 0xde, 0xd3, 0x88, 0x7a, 0xec, 0x61, 0x78, 0x38, 0xd8, 0xa7, 0xff, 0xf0, - 0xbe, 0xd9, 0xa8, 0xef, 0x18, 0xfb, 0x81, 0xe3, 0xb9, 0x83, 0x7d, 0xf1, 0x8b, 0x53, 0xdc, 0x3a, - 0xf0, 0xbc, 0x83, 0x1e, 0x66, 0xe3, 0x5d, 0xd7, 0x0b, 0xed, 0xd0, 0xf1, 0xdc, 0x80, 0xf5, 0x9a, - 0x3f, 0x32, 0xa0, 0x6a, 0xe1, 0x60, 0xe0, 0xb9, 0x01, 0x5e, 0xc7, 0x76, 0x17, 0xfb, 0xe8, 0x36, - 0x40, 0xa7, 0x37, 0x0c, 0x42, 0xec, 0xef, 0x39, 0xdd, 0xba, 0x31, 0x6b, 0xcc, 0xe5, 0xac, 0x12, - 0x6f, 0xd9, 0xe8, 0xa2, 0xd7, 0xa1, 0xd4, 0xc7, 0xfd, 0x7d, 0xd6, 0x9b, 0xa1, 0xbd, 0xe3, 0xac, - 0x61, 0xa3, 0x8b, 0x1a, 0x30, 0xee, 0xe3, 0x63, 0x87, 0xc0, 0xd7, 0xb3, 0xb3, 0xc6, 0x5c, 0xd6, - 0x8a, 0xbe, 0xc9, 0x40, 0xdf, 0x7e, 0x19, 0xee, 0x85, 0xd8, 0xef, 0xd7, 0x73, 0x6c, 0x20, 0x69, - 0x68, 0x63, 0xbf, 0xff, 0xa8, 0xf8, 0xbd, 0xbf, 0xaf, 0x67, 0x97, 0x17, 0xee, 0x9b, 0xff, 0x9c, - 0x87, 0x8a, 0x65, 0xbb, 0x07, 0xd8, 0xc2, 0xdf, 0x1a, 0xe2, 0x20, 0x44, 0x35, 0xc8, 0x1e, 0xe1, - 0x53, 0x2a, 0x47, 0xc5, 0x22, 0x3f, 0x19, 0x23, 0xf7, 0x00, 0xef, 0x61, 0x97, 0x49, 0x50, 0x21, - 0x8c, 0xdc, 0x03, 0xdc, 0x72, 0xbb, 0x68, 0x1a, 0xf2, 0x3d, 0xa7, 0xef, 0x84, 0x1c, 0x9e, 0x7d, - 0xc4, 0xe4, 0xca, 0x25, 0xe4, 0x5a, 0x05, 0x08, 0x3c, 0x3f, 0xdc, 0xf3, 0xfc, 0x2e, 0xf6, 0xeb, - 0xf9, 0x59, 0x63, 0xae, 0xba, 0xf4, 0xe6, 0x82, 0x6a, 0xb1, 0x05, 0x55, 0xa0, 0x85, 0x5d, 0xcf, - 0x0f, 0xb7, 0x09, 0xad, 0x55, 0x0a, 0xc4, 0x4f, 0xf4, 0x31, 0x94, 0x29, 0x93, 0xd0, 0xf6, 0x0f, - 0x70, 0x58, 0x2f, 0x50, 0x2e, 0xf7, 0xce, 0xe1, 0xd2, 0xa6, 0xc4, 0x16, 0x85, 0x67, 0xbf, 0x91, - 0x09, 0x95, 0x00, 0xfb, 0x8e, 0xdd, 0x73, 0xbe, 0x6d, 0xef, 0xf7, 0x70, 0xbd, 0x38, 0x6b, 0xcc, - 0x8d, 0x5b, 0xb1, 0x36, 0x32, 0xff, 0x23, 0x7c, 0x1a, 0xec, 0x79, 0x6e, 0xef, 0xb4, 0x3e, 0x4e, - 0x09, 0xc6, 0x49, 0xc3, 0xb6, 0xdb, 0x3b, 0xa5, 0xd6, 0xf3, 0x86, 0x6e, 0xc8, 0x7a, 0x4b, 0xb4, - 0xb7, 0x44, 0x5b, 0x68, 0xf7, 0x03, 0xa8, 0xf5, 0x1d, 0x77, 0xaf, 0xef, 0x75, 0xf7, 0x22, 0x85, - 0x00, 0x51, 0xc8, 0xe3, 0xe2, 0xef, 0x51, 0x0b, 0x3c, 0xb0, 0xaa, 0x7d, 0xc7, 0x7d, 0xe6, 0x75, - 0x2d, 0xa1, 0x1f, 0x32, 0xc4, 0x3e, 0x89, 0x0f, 0x29, 0x27, 0x87, 0xd8, 0x27, 0xea, 0x90, 0x0f, - 0xe0, 0x1a, 0x41, 0xe9, 0xf8, 0xd8, 0x0e, 0xb1, 0x1c, 0x55, 0x89, 0x8f, 0x9a, 0xea, 0x3b, 0xee, - 0x2a, 0x25, 0x89, 0x0d, 0xb4, 0x4f, 0x52, 0x03, 0x27, 0x92, 0x03, 0xed, 0x93, 0xf8, 0x40, 0xf3, - 0x03, 0x28, 0x45, 0x76, 0x41, 0xe3, 0x90, 0xdb, 0xda, 0xde, 0x6a, 0xd5, 0xc6, 0x10, 0x40, 0xa1, - 0xb9, 0xbb, 0xda, 0xda, 0x5a, 0xab, 0x19, 0xa8, 0x0c, 0xc5, 0xb5, 0x16, 0xfb, 0xc8, 0x34, 0x8a, - 0x3f, 0xe6, 0xeb, 0xed, 0x29, 0x80, 0x34, 0x05, 0x2a, 0x42, 0xf6, 0x69, 0xeb, 0xd3, 0xda, 0x18, - 0x21, 0x7e, 0xd1, 0xb2, 0x76, 0x37, 0xb6, 0xb7, 0x6a, 0x06, 0xe1, 0xb2, 0x6a, 0xb5, 0x9a, 0xed, - 0x56, 0x2d, 0x43, 0x28, 0x9e, 0x6d, 0xaf, 0xd5, 0xb2, 0xa8, 0x04, 0xf9, 0x17, 0xcd, 0xcd, 0xe7, - 0xad, 0x5a, 0x2e, 0x62, 0x26, 0x57, 0xf1, 0x9f, 0x18, 0x30, 0xc1, 0xcd, 0xcd, 0xf6, 0x16, 0x5a, - 0x81, 0xc2, 0x21, 0xdd, 0x5f, 0x74, 0x25, 0x97, 0x97, 0x6e, 0x25, 0xd6, 0x46, 0x6c, 0x0f, 0x5a, - 0x9c, 0x16, 0x99, 0x90, 0x3d, 0x3a, 0x0e, 0xea, 0x99, 0xd9, 0xec, 0x5c, 0x79, 0xa9, 0xb6, 0xc0, - 0x3c, 0xc3, 0xc2, 0x53, 0x7c, 0xfa, 0xc2, 0xee, 0x0d, 0xb1, 0x45, 0x3a, 0x11, 0x82, 0x5c, 0xdf, - 0xf3, 0x31, 0x5d, 0xf0, 0xe3, 0x16, 0xfd, 0x4d, 0x76, 0x01, 0xb5, 0x39, 0x5f, 0xec, 0xec, 0x43, - 0x8a, 0xf7, 0xef, 0x06, 0xc0, 0xce, 0x30, 0x1c, 0xbd, 0xc5, 0xa6, 0x21, 0x7f, 0x4c, 0x10, 0xf8, - 0xf6, 0x62, 0x1f, 0x74, 0x6f, 0x61, 0x3b, 0xc0, 0xd1, 0xde, 0x22, 0x1f, 0x68, 0x16, 0x8a, 0x03, - 0x1f, 0x1f, 0xef, 0x1d, 0x1d, 0x53, 0xb4, 0x71, 0x69, 0xa7, 0x02, 0x69, 0x7f, 0x7a, 0x8c, 0xe6, - 0xa1, 0xe2, 0x1c, 0xb8, 0x9e, 0x8f, 0xf7, 0x18, 0xd3, 0xbc, 0x4a, 0xb6, 0x64, 0x95, 0x59, 0x27, - 0x9d, 0x92, 0x42, 0xcb, 0xa0, 0x0a, 0x5a, 0xda, 0x4d, 0xd2, 0x27, 0xe7, 0xf3, 0x5d, 0x03, 0xca, - 0x74, 0x3e, 0x97, 0x52, 0xf6, 0x92, 0x9c, 0x48, 0x86, 0x0e, 0x4b, 0x29, 0x3c, 0x35, 0x35, 0x29, - 0x82, 0x0b, 0x68, 0x0d, 0xf7, 0x70, 0x88, 0x2f, 0xe3, 0xbc, 0x14, 0x55, 0x66, 0xb5, 0xaa, 0x94, - 0x78, 0x7f, 0x6e, 0xc0, 0xb5, 0x18, 0xe0, 0xa5, 0xa6, 0x5e, 0x87, 0x62, 0x97, 0x32, 0x63, 0x32, - 0x65, 0x2d, 0xf1, 0x89, 0x56, 0x60, 0x9c, 0x8b, 0x14, 0xd4, 0xb3, 0xfa, 0x65, 0x28, 0xa5, 0x2c, - 0x32, 0x29, 0x03, 0x29, 0xe6, 0x3f, 0x66, 0xa0, 0xc4, 0x95, 0xb1, 0x3d, 0x40, 0x4d, 0x98, 0xf0, - 0xd9, 0xc7, 0x1e, 0x9d, 0x33, 0x97, 0xb1, 0x31, 0xda, 0x4f, 0xae, 0x8f, 0x59, 0x15, 0x3e, 0x84, - 0x36, 0xa3, 0x5f, 0x81, 0xb2, 0x60, 0x31, 0x18, 0x86, 0xdc, 0x50, 0xf5, 0x38, 0x03, 0xb9, 0xb4, - 0xd7, 0xc7, 0x2c, 0xe0, 0xe4, 0x3b, 0xc3, 0x10, 0xb5, 0x61, 0x5a, 0x0c, 0x66, 0xf3, 0xe3, 0x62, - 0x64, 0x29, 0x97, 0xd9, 0x38, 0x97, 0xb4, 0x39, 0xd7, 0xc7, 0x2c, 0xc4, 0xc7, 0x2b, 0x9d, 0x68, - 0x4d, 0x8a, 0x14, 0x9e, 0xb0, 0xf8, 0x92, 0x12, 0xa9, 0x7d, 0xe2, 0x72, 0x26, 0x42, 0x5b, 0xcb, - 0x8a, 0x6c, 0xed, 0x13, 0x37, 0x52, 0xd9, 0xe3, 0x12, 0x14, 0x79, 0xb3, 0xf9, 0x6f, 0x19, 0x00, - 0x61, 0xb1, 0xed, 0x01, 0x5a, 0x83, 0xaa, 0xcf, 0xbf, 0x62, 0xfa, 0x7b, 0x5d, 0xab, 0x3f, 0x6e, - 0xe8, 0x31, 0x6b, 0x42, 0x0c, 0x62, 0xe2, 0x7e, 0x04, 0x95, 0x88, 0x8b, 0x54, 0xe1, 0x4d, 0x8d, - 0x0a, 0x23, 0x0e, 0x65, 0x31, 0x80, 0x28, 0xf1, 0x13, 0xb8, 0x1e, 0x8d, 0xd7, 0x68, 0xf1, 0x8d, - 0x33, 0xb4, 0x18, 0x31, 0xbc, 0x26, 0x38, 0xa8, 0x7a, 0x7c, 0xa2, 0x08, 0x26, 0x15, 0x79, 0x53, - 0xa3, 0x48, 0x46, 0xa4, 0x6a, 0x32, 0x92, 0x30, 0xa6, 0x4a, 0x20, 0x61, 0x9f, 0xb5, 0x9b, 0x7f, - 0x99, 0x83, 0xe2, 0xaa, 0xd7, 0x1f, 0xd8, 0x3e, 0x59, 0x44, 0x05, 0x1f, 0x07, 0xc3, 0x5e, 0x48, - 0x15, 0x58, 0x5d, 0xba, 0x1b, 0xc7, 0xe0, 0x64, 0xe2, 0x5f, 0x8b, 0x92, 0x5a, 0x7c, 0x08, 0x19, - 0xcc, 0xa3, 0x7c, 0xe6, 0x02, 0x83, 0x79, 0x8c, 0xe7, 0x43, 0x84, 0x43, 0xc8, 0x4a, 0x87, 0xd0, - 0x80, 0x22, 0x3f, 0xb0, 0x31, 0x67, 0xbd, 0x3e, 0x66, 0x89, 0x06, 0xf4, 0x0e, 0x4c, 0x26, 0x43, - 0x61, 0x9e, 0xd3, 0x54, 0x3b, 0xf1, 0xc8, 0x79, 0x17, 0x2a, 0xb1, 0x08, 0x5d, 0xe0, 0x74, 0xe5, - 0xbe, 0x12, 0x97, 0x6f, 0x08, 0xb7, 0x4e, 0x8e, 0x15, 0x95, 0xf5, 0x31, 0xe1, 0xd8, 0xef, 0x08, - 0xc7, 0x3e, 0xae, 0x06, 0x5a, 0xa2, 0x57, 0xee, 0xe3, 0xdf, 0x54, 0xbd, 0xd6, 0x57, 0xc9, 0xe0, - 0x88, 0x48, 0xba, 0x2f, 0xd3, 0x82, 0x89, 0x98, 0xca, 0x48, 0x8c, 0x6c, 0x7d, 0xed, 0x79, 0x73, - 0x93, 0x05, 0xd4, 0x27, 0x34, 0x86, 0x5a, 0x35, 0x83, 0x04, 0xe8, 0xcd, 0xd6, 0xee, 0x6e, 0x2d, - 0x83, 0x6e, 0x40, 0x69, 0x6b, 0xbb, 0xbd, 0xc7, 0xa8, 0xb2, 0x8d, 0xe2, 0x1f, 0x33, 0x4f, 0x22, - 0xe3, 0xf3, 0xa7, 0x11, 0x4f, 0x1e, 0xa2, 0x95, 0xc8, 0x3c, 0xa6, 0x44, 0x66, 0x43, 0x44, 0xe6, - 0x8c, 0x8c, 0xcc, 0x59, 0x84, 0x20, 0xbf, 0xd9, 0x6a, 0xee, 0xd2, 0x20, 0xcd, 0x58, 0x2f, 0xa7, - 0xa3, 0xf5, 0xe3, 0x2a, 0x54, 0x98, 0x79, 0xf6, 0x86, 0x2e, 0x39, 0x4c, 0xfc, 0x95, 0x01, 0x20, - 0x37, 0x2c, 0x5a, 0x84, 0x62, 0x87, 0x89, 0x50, 0x37, 0xa8, 0x07, 0xbc, 0xae, 0xb5, 0xb8, 0x25, - 0xa8, 0xd0, 0x03, 0x28, 0x06, 0xc3, 0x4e, 0x07, 0x07, 0x22, 0x72, 0xbf, 0x96, 0x74, 0xc2, 0xdc, - 0x21, 0x5a, 0x82, 0x8e, 0x0c, 0x79, 0x69, 0x3b, 0xbd, 0x21, 0x8d, 0xe3, 0x67, 0x0f, 0xe1, 0x74, - 0xd2, 0xc7, 0xfe, 0x99, 0x01, 0x65, 0x65, 0x5b, 0xfc, 0x9c, 0x21, 0xe0, 0x16, 0x94, 0xa8, 0x30, - 0xb8, 0xcb, 0x83, 0xc0, 0xb8, 0x25, 0x1b, 0xd0, 0x57, 0xa0, 0x24, 0x76, 0x92, 0x88, 0x03, 0x75, - 0x3d, 0xdb, 0xed, 0x81, 0x25, 0x49, 0xa5, 0x90, 0x6d, 0x98, 0xa2, 0x7a, 0xea, 0x90, 0xdb, 0x87, - 0xd0, 0xac, 0x7a, 0x2c, 0x37, 0x12, 0xc7, 0xf2, 0x06, 0x8c, 0x0f, 0x0e, 0x4f, 0x03, 0xa7, 0x63, - 0xf7, 0xb8, 0x38, 0xd1, 0xb7, 0xe4, 0xba, 0x0b, 0x48, 0xe5, 0x7a, 0x19, 0x05, 0x48, 0xa6, 0x37, - 0xa0, 0xbc, 0x6e, 0x07, 0x87, 0x5c, 0x48, 0xd9, 0xbe, 0x02, 0x13, 0xa4, 0xfd, 0xe9, 0x8b, 0x0b, - 0x88, 0x2f, 0x46, 0x2d, 0xd3, 0x1b, 0x96, 0x18, 0x76, 0x29, 0x03, 0x21, 0xc8, 0x1d, 0xda, 0xc1, - 0x21, 0x55, 0xc6, 0x84, 0x45, 0x7f, 0xa3, 0x77, 0xa0, 0xd6, 0x61, 0xf3, 0xdf, 0x4b, 0xdc, 0xbb, - 0x26, 0x79, 0xbb, 0x95, 0x12, 0xc8, 0x86, 0x0a, 0x9b, 0xde, 0x55, 0x4b, 0x23, 0x35, 0xd5, 0x80, - 0xc9, 0x5d, 0xd7, 0x1e, 0x04, 0x87, 0x5e, 0x98, 0xd0, 0xe2, 0xb2, 0xf9, 0x77, 0x06, 0xd4, 0x64, - 0xe7, 0xa5, 0x64, 0x78, 0x1b, 0x26, 0x7d, 0xdc, 0xb7, 0x1d, 0xd7, 0x71, 0x0f, 0xf6, 0xf6, 0x4f, - 0x43, 0x1c, 0xf0, 0x0b, 0x69, 0x35, 0x6a, 0x7e, 0x4c, 0x5a, 0x89, 0xb0, 0xfb, 0x3d, 0x6f, 0x9f, - 0xbb, 0x5d, 0xfa, 0x1b, 0xbd, 0x11, 0xf7, 0xbb, 0x25, 0xe1, 0xd0, 0xbe, 0x12, 0xb9, 0x5f, 0x29, - 0xf3, 0x4f, 0x32, 0x50, 0xf9, 0xc4, 0x0e, 0x3b, 0x62, 0x4d, 0xa0, 0x0d, 0xa8, 0x46, 0x8e, 0x99, - 0xb6, 0x70, 0xb9, 0x13, 0x47, 0x08, 0x3a, 0x46, 0xdc, 0x54, 0xc4, 0x11, 0x62, 0xa2, 0xa3, 0x36, - 0x50, 0x56, 0xb6, 0xdb, 0xc1, 0xbd, 0x88, 0x55, 0x66, 0x34, 0x2b, 0x4a, 0xa8, 0xb2, 0x52, 0x1b, - 0xd0, 0xd7, 0xa1, 0x36, 0xf0, 0xbd, 0x03, 0x1f, 0x07, 0x41, 0xc4, 0x8c, 0x05, 0x65, 0x53, 0xc3, - 0x6c, 0x87, 0x93, 0x26, 0xce, 0x25, 0x2b, 0xeb, 0x63, 0xd6, 0xe4, 0x20, 0xde, 0x27, 0x5d, 0xe5, - 0xa4, 0x3c, 0xc1, 0x31, 0x5f, 0xf9, 0x83, 0x2c, 0xa0, 0xf4, 0x34, 0xbf, 0xe8, 0xc1, 0xf7, 0x1e, - 0x54, 0x83, 0xd0, 0xf6, 0x53, 0xab, 0x78, 0x82, 0xb6, 0x46, 0xf1, 0xeb, 0x6d, 0x88, 0x24, 0xdb, - 0x73, 0xbd, 0xd0, 0x79, 0x79, 0xca, 0xae, 0x1c, 0x56, 0x55, 0x34, 0x6f, 0xd1, 0x56, 0xb4, 0x05, - 0xc5, 0x97, 0x4e, 0x2f, 0xc4, 0x7e, 0x50, 0xcf, 0xcf, 0x66, 0xe7, 0xaa, 0x4b, 0xef, 0x9e, 0x67, - 0x98, 0x85, 0x8f, 0x29, 0x7d, 0xfb, 0x74, 0xa0, 0x9e, 0x67, 0x39, 0x13, 0xf5, 0x60, 0x5e, 0xd0, - 0xdf, 0x71, 0x4c, 0x18, 0x7f, 0x45, 0x98, 0xee, 0x39, 0x5d, 0x1a, 0x5d, 0xa3, 0x28, 0xba, 0x62, - 0x15, 0x69, 0xc7, 0x46, 0x17, 0xdd, 0x85, 0xf1, 0x97, 0xbe, 0x7d, 0xd0, 0xc7, 0x6e, 0xc8, 0xee, - 0xed, 0x92, 0x26, 0xea, 0x30, 0x17, 0x00, 0xa4, 0x28, 0x24, 0x96, 0x6d, 0x6d, 0xef, 0x3c, 0x6f, - 0xd7, 0xc6, 0x50, 0x05, 0xc6, 0xb7, 0xb6, 0xd7, 0x5a, 0x9b, 0x2d, 0x12, 0xed, 0x44, 0x14, 0x7b, - 0x20, 0x37, 0x5d, 0x53, 0x18, 0x22, 0xb6, 0x26, 0x54, 0xb9, 0x8c, 0xf8, 0x35, 0x5a, 0xc8, 0x25, - 0x58, 0x3c, 0x30, 0xef, 0xc0, 0xb4, 0x6e, 0x69, 0x08, 0x82, 0x15, 0xf3, 0x5f, 0x32, 0x30, 0xc1, - 0x37, 0xc2, 0xa5, 0x76, 0xee, 0x4d, 0x45, 0x2a, 0x7e, 0xe1, 0x10, 0x4a, 0xaa, 0x43, 0x91, 0x6d, - 0x90, 0x2e, 0xbf, 0xd1, 0x8a, 0x4f, 0xe2, 0x6e, 0xd9, 0x7a, 0xc7, 0x5d, 0x6e, 0xf6, 0xe8, 0x5b, - 0xeb, 0x08, 0xf3, 0x5a, 0x47, 0x88, 0xde, 0x83, 0x89, 0x68, 0xc3, 0xd9, 0x01, 0x3f, 0x2a, 0x95, - 0xa4, 0x29, 0x2a, 0x62, 0x53, 0x91, 0xce, 0x98, 0xcd, 0x8a, 0x23, 0x6c, 0x86, 0xee, 0x41, 0x01, - 0x1f, 0x63, 0x37, 0x0c, 0xea, 0x65, 0x1a, 0x1a, 0x27, 0xc4, 0x15, 0xa9, 0x45, 0x5a, 0x2d, 0xde, - 0x29, 0x4d, 0xf5, 0x11, 0x4c, 0xd1, 0x1b, 0xec, 0x13, 0xdf, 0x76, 0xd5, 0x5b, 0x78, 0xbb, 0xbd, - 0xc9, 0x03, 0x09, 0xf9, 0x89, 0xaa, 0x90, 0xd9, 0x58, 0xe3, 0xfa, 0xc9, 0x6c, 0xac, 0xc9, 0xf1, - 0xbf, 0x6f, 0x00, 0x52, 0x19, 0x5c, 0xca, 0x16, 0x09, 0x14, 0x21, 0x47, 0x56, 0xca, 0x31, 0x0d, - 0x79, 0xec, 0xfb, 0x9e, 0xcf, 0x1c, 0xa5, 0xc5, 0x3e, 0xa4, 0x34, 0xef, 0x73, 0x61, 0x2c, 0x7c, - 0xec, 0x1d, 0x45, 0x1e, 0x80, 0xb1, 0x35, 0xd2, 0xc2, 0xb7, 0xe1, 0x5a, 0x8c, 0xfc, 0x6a, 0x82, - 0xf6, 0x36, 0x4c, 0x52, 0xae, 0xab, 0x87, 0xb8, 0x73, 0x34, 0xf0, 0x1c, 0x37, 0x25, 0x01, 0xba, - 0x4b, 0x7c, 0x97, 0x08, 0x17, 0x64, 0x8a, 0x6c, 0xce, 0x95, 0xa8, 0xb1, 0xdd, 0xde, 0x94, 0x4b, - 0x7d, 0x1f, 0x6e, 0x24, 0x18, 0x8a, 0x99, 0xfd, 0x2a, 0x94, 0x3b, 0x51, 0x63, 0xc0, 0xcf, 0x84, - 0xb7, 0xe3, 0xe2, 0x26, 0x87, 0xaa, 0x23, 0x24, 0xc6, 0xd7, 0xe1, 0xb5, 0x14, 0xc6, 0x55, 0xa8, - 0x63, 0xc5, 0xbc, 0x0f, 0xd7, 0x29, 0xe7, 0xa7, 0x18, 0x0f, 0x9a, 0x3d, 0xe7, 0xf8, 0x7c, 0xb3, - 0x9c, 0xf2, 0xf9, 0x2a, 0x23, 0xbe, 0xdc, 0x65, 0x25, 0xa1, 0x5b, 0x1c, 0xba, 0xed, 0xf4, 0x71, - 0xdb, 0xdb, 0x1c, 0x2d, 0x2d, 0x09, 0xe4, 0x47, 0xf8, 0x34, 0xe0, 0x07, 0x42, 0xfa, 0x5b, 0x7a, - 0xaf, 0xbf, 0x31, 0xb8, 0x3a, 0x55, 0x3e, 0x5f, 0xf2, 0xd6, 0x98, 0x01, 0x38, 0x20, 0x7b, 0x10, - 0x77, 0x49, 0x07, 0xcb, 0xb6, 0x29, 0x2d, 0x91, 0xc0, 0x24, 0x0a, 0x55, 0x92, 0x02, 0xdf, 0xe6, - 0x1b, 0x87, 0xfe, 0x27, 0x48, 0x9d, 0x94, 0xde, 0x82, 0x32, 0xed, 0xd9, 0x0d, 0xed, 0x70, 0x18, - 0x8c, 0xb2, 0xdc, 0xb2, 0xf9, 0x03, 0x83, 0xef, 0x28, 0xc1, 0xe7, 0x52, 0x73, 0x7e, 0x00, 0x05, - 0x7a, 0xe7, 0x13, 0x77, 0x97, 0x9b, 0x9a, 0x85, 0xcd, 0x24, 0xb2, 0x38, 0xa1, 0x72, 0x4e, 0x32, - 0xa0, 0xf0, 0x8c, 0xd6, 0x02, 0x14, 0x69, 0x73, 0xc2, 0x72, 0xae, 0xdd, 0x67, 0x09, 0xc5, 0x92, - 0x45, 0x7f, 0xd3, 0x23, 0x3e, 0xc6, 0xfe, 0x73, 0x6b, 0x93, 0xdd, 0x29, 0x4a, 0x56, 0xf4, 0x4d, - 0x14, 0xdb, 0xe9, 0x39, 0xd8, 0x0d, 0x69, 0x6f, 0x8e, 0xf6, 0x2a, 0x2d, 0xe8, 0x1e, 0x94, 0x9c, - 0x60, 0x13, 0xdb, 0xbe, 0xcb, 0x93, 0xf6, 0x8a, 0x63, 0x96, 0x3d, 0x72, 0x8d, 0x7d, 0x03, 0x6a, - 0x4c, 0xb2, 0x66, 0xb7, 0xab, 0x9c, 0xdf, 0x23, 0x7c, 0x23, 0x81, 0x1f, 0xe3, 0x9f, 0x39, 0x9f, - 0xff, 0xdf, 0x1a, 0x30, 0xa5, 0x00, 0x5c, 0xca, 0x04, 0xef, 0x41, 0x81, 0x55, 0x54, 0xf8, 0x51, - 0x70, 0x3a, 0x3e, 0x8a, 0xc1, 0x58, 0x9c, 0x06, 0x2d, 0x40, 0x91, 0xfd, 0x12, 0x17, 0x33, 0x3d, - 0xb9, 0x20, 0x92, 0x22, 0x2f, 0xc0, 0x35, 0xde, 0x87, 0xfb, 0x9e, 0x6e, 0xcf, 0xe5, 0xe2, 0x1e, - 0xe2, 0xfb, 0x06, 0x4c, 0xc7, 0x07, 0x5c, 0x6a, 0x96, 0x8a, 0xdc, 0x99, 0x2f, 0x24, 0xf7, 0xaf, - 0x09, 0xb9, 0x9f, 0x0f, 0xba, 0xca, 0x91, 0x33, 0xb9, 0xe2, 0x54, 0xeb, 0x66, 0xe2, 0xd6, 0x95, - 0xbc, 0x7e, 0x14, 0xcd, 0x49, 0x30, 0xbb, 0xd4, 0x9c, 0x3e, 0xb8, 0xd0, 0x9c, 0x94, 0x23, 0x58, - 0x6a, 0x72, 0x1b, 0x62, 0x19, 0x6d, 0x3a, 0x41, 0x14, 0x71, 0xde, 0x85, 0x4a, 0xcf, 0x71, 0xb1, - 0xed, 0xf3, 0xaa, 0x90, 0xa1, 0xae, 0xc7, 0x87, 0x56, 0xac, 0x53, 0xb2, 0xfa, 0x6d, 0x03, 0x90, - 0xca, 0xeb, 0x17, 0x63, 0xad, 0x45, 0xa1, 0xe0, 0x1d, 0xdf, 0xeb, 0x7b, 0xe1, 0x79, 0xcb, 0x6c, - 0xc5, 0xfc, 0x5d, 0x03, 0xae, 0x27, 0x46, 0xfc, 0x22, 0x24, 0x5f, 0x31, 0x6f, 0xc1, 0xd4, 0x1a, - 0x16, 0x67, 0xbc, 0x54, 0x36, 0x60, 0x17, 0x90, 0xda, 0x7b, 0x35, 0xa7, 0x98, 0x5f, 0x82, 0xa9, - 0x67, 0xde, 0x31, 0x71, 0xe4, 0xa4, 0x5b, 0xba, 0x29, 0x96, 0x9e, 0x8a, 0xf4, 0x15, 0x7d, 0x4b, - 0xd7, 0xbb, 0x0b, 0x48, 0x1d, 0x79, 0x15, 0xe2, 0x2c, 0x9b, 0xff, 0x6d, 0x40, 0xa5, 0xd9, 0xb3, - 0xfd, 0xbe, 0x10, 0xe5, 0x23, 0x28, 0xb0, 0x5c, 0x0b, 0x4f, 0x9c, 0xbe, 0x15, 0xe7, 0xa7, 0xd2, - 0xb2, 0x8f, 0x26, 0xcb, 0xcc, 0xf0, 0x51, 0x64, 0x2a, 0xbc, 0x56, 0xbc, 0x96, 0xa8, 0x1d, 0xaf, - 0xa1, 0xf7, 0x21, 0x6f, 0x93, 0x21, 0x34, 0xbc, 0x56, 0x93, 0x09, 0x30, 0xca, 0x8d, 0x5c, 0x89, - 0x2c, 0x46, 0x65, 0x7e, 0x08, 0x65, 0x05, 0x01, 0x15, 0x21, 0xfb, 0xa4, 0xc5, 0xaf, 0x49, 0xcd, - 0xd5, 0xf6, 0xc6, 0x0b, 0x96, 0x14, 0xac, 0x02, 0xac, 0xb5, 0xa2, 0xef, 0x8c, 0xa6, 0x54, 0x67, - 0x73, 0x3e, 0x3c, 0x6e, 0xa9, 0x12, 0x1a, 0xa3, 0x24, 0xcc, 0x5c, 0x44, 0x42, 0x09, 0xf1, 0x5b, - 0x06, 0x4c, 0x70, 0xd5, 0x5c, 0x36, 0x34, 0x53, 0xce, 0x23, 0x42, 0xb3, 0x32, 0x0d, 0x8b, 0x13, - 0x4a, 0x19, 0xfe, 0xc9, 0x80, 0xda, 0x9a, 0xf7, 0xca, 0x3d, 0xf0, 0xed, 0x6e, 0xb4, 0x07, 0x3f, - 0x4e, 0x98, 0x73, 0x21, 0x91, 0xbb, 0x4f, 0xd0, 0xcb, 0x86, 0x84, 0x59, 0xeb, 0x32, 0x97, 0xc2, - 0xe2, 0xbb, 0xf8, 0x34, 0xbf, 0x0a, 0x93, 0x89, 0x41, 0xc4, 0x40, 0x2f, 0x9a, 0x9b, 0x1b, 0x6b, - 0xc4, 0x20, 0x34, 0x83, 0xdb, 0xda, 0x6a, 0x3e, 0xde, 0x6c, 0xf1, 0x3a, 0x6b, 0x73, 0x6b, 0xb5, - 0xb5, 0x29, 0x0d, 0xf5, 0x50, 0xcc, 0xe0, 0xa1, 0xd9, 0x83, 0x29, 0x45, 0xa0, 0xcb, 0x96, 0xbb, - 0xf4, 0xf2, 0x4a, 0xb4, 0x3a, 0x4c, 0xf0, 0x53, 0x4e, 0x72, 0xe3, 0xff, 0x5f, 0x06, 0xaa, 0xa2, - 0xeb, 0xcb, 0x91, 0x02, 0xdd, 0x80, 0x42, 0x77, 0x7f, 0xd7, 0xf9, 0xb6, 0xa8, 0xb4, 0xf2, 0x2f, - 0xd2, 0xde, 0x63, 0x38, 0xec, 0xfd, 0x04, 0xff, 0x42, 0xb7, 0xd8, 0xd3, 0x8a, 0x0d, 0xb7, 0x8b, - 0x4f, 0xe8, 0x61, 0x28, 0x67, 0xc9, 0x06, 0x9a, 0xa6, 0xe4, 0xef, 0x2c, 0xe8, 0x5d, 0x57, 0x79, - 0x77, 0x81, 0x96, 0xa1, 0x46, 0x7e, 0x37, 0x07, 0x83, 0x9e, 0x83, 0xbb, 0x8c, 0x01, 0xb9, 0xe6, - 0xe6, 0xe4, 0x69, 0x27, 0x45, 0x80, 0xee, 0x40, 0x81, 0x5e, 0x01, 0x83, 0xfa, 0x38, 0x89, 0xab, - 0x92, 0x94, 0x37, 0xa3, 0x77, 0xa0, 0xcc, 0x24, 0xde, 0x70, 0x9f, 0x07, 0x98, 0xbe, 0x42, 0x50, - 0xf2, 0x21, 0x6a, 0x5f, 0xfc, 0x9c, 0x05, 0xe7, 0x9f, 0xb3, 0x6e, 0xc1, 0x54, 0x73, 0x18, 0x1e, - 0xb6, 0x5c, 0x12, 0xeb, 0x52, 0xb6, 0xb9, 0x0d, 0x88, 0xf4, 0xae, 0x39, 0x81, 0xb6, 0x9b, 0x0f, - 0xd6, 0x1a, 0xf6, 0xa1, 0xb9, 0x05, 0xd7, 0x48, 0x2f, 0x76, 0x43, 0xa7, 0xa3, 0x9c, 0x2b, 0xc4, - 0xc9, 0xd5, 0x48, 0x9c, 0x5c, 0xed, 0x20, 0x78, 0xe5, 0xf9, 0x5d, 0x6e, 0xbb, 0xe8, 0x5b, 0xa2, - 0xfd, 0x83, 0xc1, 0xa4, 0x79, 0x1e, 0xc4, 0x4e, 0x9d, 0x5f, 0x90, 0x1f, 0xfa, 0x65, 0x28, 0x7a, - 0x03, 0xfa, 0x66, 0x87, 0x27, 0xf3, 0x6e, 0x2c, 0xb0, 0x77, 0x40, 0x0b, 0x9c, 0xf1, 0x36, 0xeb, - 0x55, 0x12, 0x4e, 0x9c, 0x1e, 0x2d, 0x42, 0xf5, 0xd0, 0x0e, 0x0e, 0x71, 0x77, 0x47, 0x30, 0x8f, - 0xa5, 0x3a, 0x1f, 0x5a, 0x89, 0x6e, 0x29, 0xfb, 0x03, 0x29, 0xfa, 0x13, 0x1c, 0x9e, 0x21, 0xba, - 0x9a, 0x1e, 0xbf, 0x2e, 0x86, 0xf0, 0xaa, 0xde, 0x45, 0x46, 0xfd, 0xd0, 0x80, 0xdb, 0x62, 0xd8, - 0xea, 0xa1, 0xed, 0x1e, 0x60, 0x21, 0xcc, 0xcf, 0xab, 0xaf, 0xf4, 0xa4, 0xb3, 0x17, 0x9c, 0xf4, - 0x53, 0xa8, 0x47, 0x93, 0xa6, 0x89, 0x15, 0xaf, 0xa7, 0x4e, 0x62, 0x18, 0xf0, 0x0d, 0x5e, 0xb2, - 0xe8, 0x6f, 0xd2, 0xe6, 0x7b, 0xbd, 0xe8, 0x4e, 0x43, 0x7e, 0x4b, 0x66, 0x9b, 0x70, 0x53, 0x30, - 0xe3, 0x99, 0x8e, 0x38, 0xb7, 0xd4, 0x9c, 0xce, 0xe4, 0xc6, 0xed, 0x41, 0x78, 0x9c, 0xbd, 0x94, - 0xb4, 0x43, 0xe2, 0x26, 0xa4, 0x28, 0x86, 0x0e, 0x65, 0x86, 0xed, 0x00, 0x22, 0xb3, 0x72, 0xfc, - 0x4c, 0xf5, 0x13, 0x96, 0xda, 0x7e, 0xbe, 0x04, 0x48, 0x7f, 0x6a, 0x09, 0x8c, 0x46, 0xc5, 0x30, - 0x13, 0x09, 0x4a, 0xd4, 0xbe, 0x83, 0xfd, 0xbe, 0x13, 0x04, 0x4a, 0x9d, 0x48, 0xa7, 0xae, 0xb7, - 0x20, 0x37, 0xc0, 0x3c, 0x16, 0x97, 0x97, 0x90, 0xd8, 0x13, 0xca, 0x60, 0xda, 0x2f, 0x61, 0xfa, - 0x70, 0x47, 0xc0, 0x30, 0x83, 0x68, 0x71, 0x92, 0x62, 0x8a, 0x4c, 0x76, 0x66, 0x44, 0x26, 0x3b, - 0x1b, 0xcf, 0x64, 0xc7, 0xce, 0x87, 0xaa, 0xa3, 0xba, 0x9a, 0xf3, 0x61, 0x9b, 0x19, 0x20, 0xf2, - 0x6f, 0x57, 0xc3, 0xf5, 0x0f, 0xb8, 0xa3, 0xba, 0xaa, 0xa8, 0x86, 0xe9, 0x9c, 0x45, 0x15, 0x51, - 0x7c, 0x22, 0x13, 0x2a, 0xc4, 0x48, 0x96, 0x9a, 0xe2, 0xcf, 0x59, 0xb1, 0x36, 0xe9, 0x8c, 0x8f, - 0x60, 0x3a, 0xee, 0x8c, 0x2f, 0x25, 0xd4, 0x34, 0xe4, 0x43, 0xef, 0x08, 0x8b, 0x40, 0xcb, 0x3e, - 0x52, 0x6a, 0x8d, 0x1c, 0xf5, 0xd5, 0xa8, 0xf5, 0x9b, 0x92, 0x2b, 0xdd, 0x80, 0x97, 0x9d, 0x01, - 0x59, 0x8e, 0xe2, 0x2a, 0xcb, 0x3e, 0x24, 0xd6, 0x27, 0x70, 0x23, 0xe9, 0x7c, 0xaf, 0x66, 0x12, - 0x7b, 0x6c, 0x73, 0xea, 0xdc, 0xf3, 0xd5, 0x00, 0x7c, 0x26, 0xfd, 0xa4, 0xe2, 0x74, 0xaf, 0x86, - 0xf7, 0xaf, 0x43, 0x43, 0xe7, 0x83, 0xaf, 0x74, 0x2f, 0x46, 0x2e, 0xf9, 0x6a, 0xb8, 0x7e, 0xdf, - 0x90, 0x6c, 0xd5, 0x55, 0xf3, 0xe1, 0x17, 0x61, 0x2b, 0x62, 0xdd, 0xfd, 0x68, 0xf9, 0x2c, 0x46, - 0xde, 0x32, 0xab, 0xf7, 0x96, 0x72, 0x08, 0x25, 0x14, 0xfb, 0x4f, 0xba, 0xfa, 0x2f, 0x73, 0xf5, - 0x72, 0x30, 0x19, 0x77, 0x2e, 0x0b, 0x46, 0xc2, 0x73, 0x04, 0x46, 0x3f, 0x52, 0x5b, 0x45, 0x0d, - 0x52, 0x57, 0x63, 0xba, 0xdf, 0x90, 0x01, 0x26, 0x15, 0xc7, 0xae, 0x06, 0xc1, 0x86, 0xd9, 0xd1, - 0x21, 0xec, 0x4a, 0x20, 0xe6, 0x9b, 0x50, 0x8a, 0x2e, 0xb2, 0xca, 0x43, 0xda, 0x32, 0x14, 0xb7, - 0xb6, 0x77, 0x77, 0x9a, 0xab, 0xe4, 0x9e, 0x36, 0x0d, 0xc5, 0xd5, 0x6d, 0xcb, 0x7a, 0xbe, 0xd3, - 0x26, 0x17, 0xb5, 0xe4, 0xbb, 0x9a, 0xa5, 0x9f, 0x66, 0x21, 0xf3, 0xf4, 0x05, 0xfa, 0x14, 0xf2, - 0xec, 0x5d, 0xd7, 0x19, 0xcf, 0xfb, 0x1a, 0x67, 0x3d, 0x5d, 0x33, 0x5f, 0xfb, 0xde, 0x7f, 0xfe, - 0xf4, 0x0f, 0x33, 0x53, 0x66, 0x65, 0xf1, 0x78, 0x79, 0xf1, 0xe8, 0x78, 0x91, 0x06, 0xd9, 0x47, - 0xc6, 0x3c, 0xfa, 0x1a, 0x64, 0x77, 0x86, 0x21, 0x1a, 0xf9, 0xec, 0xaf, 0x31, 0xfa, 0x35, 0x9b, - 0x79, 0x9d, 0x32, 0x9d, 0x34, 0x81, 0x33, 0x1d, 0x0c, 0x43, 0xc2, 0xf2, 0x5b, 0x50, 0x56, 0xdf, - 0xa2, 0x9d, 0xfb, 0x16, 0xb0, 0x71, 0xfe, 0x3b, 0x37, 0xf3, 0x36, 0x85, 0x7a, 0xcd, 0x44, 0x1c, - 0x8a, 0xbd, 0x96, 0x53, 0x67, 0xd1, 0x3e, 0x71, 0xd1, 0xc8, 0x97, 0x82, 0x8d, 0xd1, 0x4f, 0xdf, - 0x52, 0xb3, 0x08, 0x4f, 0x5c, 0xc2, 0xf2, 0x9b, 0xfc, 0x8d, 0x5b, 0x27, 0x44, 0x77, 0x34, 0x8f, - 0x94, 0xd4, 0xc7, 0x37, 0x8d, 0xd9, 0xd1, 0x04, 0x1c, 0xe4, 0x16, 0x05, 0xb9, 0x61, 0x4e, 0x71, - 0x90, 0x4e, 0x44, 0xf2, 0xc8, 0x98, 0x5f, 0xea, 0x40, 0x9e, 0x96, 0x82, 0xd1, 0x67, 0xe2, 0x47, - 0x43, 0x53, 0x64, 0x1f, 0x61, 0xe8, 0x58, 0x11, 0xd9, 0x9c, 0xa6, 0x40, 0x55, 0xb3, 0x44, 0x80, - 0x68, 0x21, 0xf8, 0x91, 0x31, 0x3f, 0x67, 0xdc, 0x37, 0x96, 0xfe, 0x3a, 0x0f, 0x79, 0x5a, 0x72, - 0x40, 0x47, 0x00, 0xb2, 0xe4, 0x99, 0x9c, 0x5d, 0xaa, 0x9a, 0x9a, 0x9c, 0x5d, 0xba, 0x5a, 0x6a, - 0x36, 0x28, 0xe8, 0xb4, 0x39, 0x49, 0x40, 0x69, 0x25, 0x63, 0x91, 0x16, 0x6e, 0x88, 0x1e, 0x7f, - 0x68, 0xf0, 0xda, 0x0b, 0xdb, 0x66, 0x48, 0xc7, 0x2d, 0x56, 0xee, 0x4c, 0x2e, 0x07, 0x4d, 0x85, - 0xd3, 0x7c, 0x48, 0x01, 0x17, 0xcd, 0x9a, 0x04, 0xf4, 0x29, 0xc5, 0x23, 0x63, 0xfe, 0xb3, 0xba, - 0x79, 0x8d, 0x6b, 0x39, 0xd1, 0x83, 0xbe, 0x03, 0xd5, 0x78, 0x61, 0x0e, 0xdd, 0xd5, 0x60, 0x25, - 0x0b, 0x7d, 0x8d, 0x37, 0xcf, 0x26, 0xe2, 0x32, 0xcd, 0x50, 0x99, 0x38, 0x38, 0x43, 0x3e, 0xc2, - 0x78, 0x60, 0x13, 0x22, 0x6e, 0x03, 0xf4, 0xa7, 0x06, 0xaf, 0xad, 0xca, 0xba, 0x1a, 0xd2, 0x71, - 0x4f, 0x95, 0xef, 0x1a, 0xf7, 0xce, 0xa1, 0xe2, 0x42, 0x7c, 0x48, 0x85, 0xf8, 0xc0, 0x9c, 0x96, - 0x42, 0x84, 0x4e, 0x1f, 0x87, 0x1e, 0x97, 0xe2, 0xb3, 0x5b, 0xe6, 0x6b, 0x31, 0xe5, 0xc4, 0x7a, - 0xa5, 0xb1, 0x58, 0xfd, 0x4b, 0x6b, 0xac, 0x58, 0x89, 0x4d, 0x6b, 0xac, 0x78, 0xf1, 0x4c, 0x67, - 0x2c, 0x5e, 0xed, 0xd2, 0x18, 0x2b, 0xea, 0x59, 0xfa, 0xdf, 0x1c, 0x14, 0x57, 0xd9, 0xdf, 0xca, - 0x20, 0x0f, 0x4a, 0x51, 0x45, 0x08, 0xcd, 0xe8, 0x92, 0xce, 0xf2, 0x2a, 0xd7, 0xb8, 0x33, 0xb2, - 0x9f, 0x0b, 0xf4, 0x06, 0x15, 0xe8, 0x75, 0xf3, 0x06, 0x41, 0xe6, 0x7f, 0x8e, 0xb3, 0xc8, 0x52, - 0x93, 0x8b, 0x76, 0xb7, 0x4b, 0x14, 0xf1, 0x9b, 0x50, 0x51, 0xeb, 0x33, 0xe8, 0x0d, 0x6d, 0xa2, - 0x5b, 0x2d, 0xf6, 0x34, 0xcc, 0xb3, 0x48, 0x38, 0xf2, 0x9b, 0x14, 0x79, 0xc6, 0xbc, 0xa9, 0x41, - 0xf6, 0x29, 0x69, 0x0c, 0x9c, 0x15, 0x52, 0xf4, 0xe0, 0xb1, 0x8a, 0x8d, 0x1e, 0x3c, 0x5e, 0x87, - 0x39, 0x13, 0x7c, 0x48, 0x49, 0x09, 0x78, 0x00, 0x20, 0x2b, 0x1d, 0x48, 0xab, 0x4b, 0xe5, 0xc2, - 0x9a, 0x74, 0x0e, 0xe9, 0x22, 0x89, 0x69, 0x52, 0x58, 0xbe, 0xee, 0x12, 0xb0, 0x3d, 0x27, 0x08, - 0xd9, 0xc6, 0x9c, 0x88, 0xd5, 0x29, 0x90, 0x76, 0x3e, 0xf1, 0xb2, 0x47, 0xe3, 0xee, 0x99, 0x34, - 0x1c, 0xfd, 0x1e, 0x45, 0xbf, 0x63, 0x36, 0x34, 0xe8, 0x03, 0x46, 0x4b, 0x16, 0xdb, 0xff, 0x17, - 0xa0, 0xfc, 0xcc, 0x76, 0xdc, 0x10, 0xbb, 0xb6, 0xdb, 0xc1, 0x68, 0x1f, 0xf2, 0x34, 0x76, 0x27, - 0x1d, 0xb1, 0x9a, 0x96, 0x4f, 0x3a, 0xe2, 0x58, 0x5e, 0xda, 0x9c, 0xa5, 0xc0, 0x0d, 0xf3, 0x3a, - 0x01, 0xee, 0x4b, 0xd6, 0x8b, 0x2c, 0xa3, 0x6d, 0xcc, 0xa3, 0x97, 0x50, 0xe0, 0xf5, 0xe8, 0x04, - 0xa3, 0x58, 0x52, 0xad, 0x71, 0x4b, 0xdf, 0xa9, 0x5b, 0xcb, 0x2a, 0x4c, 0x40, 0xe9, 0x08, 0xce, - 0x31, 0x80, 0x2c, 0xaf, 0x24, 0x2d, 0x9a, 0x2a, 0xcb, 0x34, 0x66, 0x47, 0x13, 0xe8, 0x74, 0xaa, - 0x62, 0x76, 0x23, 0x5a, 0x82, 0xfb, 0x0d, 0xc8, 0xad, 0xdb, 0xc1, 0x21, 0x4a, 0xc4, 0x5e, 0xe5, - 0x41, 0x68, 0xa3, 0xa1, 0xeb, 0xe2, 0x28, 0x77, 0x28, 0xca, 0x4d, 0xe6, 0xca, 0x54, 0x14, 0xfa, - 0x40, 0xd2, 0x98, 0x47, 0x5d, 0x28, 0xb0, 0xd7, 0xa0, 0x49, 0xfd, 0xc5, 0x9e, 0x96, 0x26, 0xf5, - 0x17, 0x7f, 0x40, 0x7a, 0x3e, 0xca, 0x00, 0xc6, 0xc5, 0x1b, 0x4b, 0x94, 0x78, 0x99, 0x92, 0x78, - 0x98, 0xd9, 0x98, 0x19, 0xd5, 0xcd, 0xb1, 0xee, 0x52, 0xac, 0xdb, 0x66, 0x3d, 0x65, 0x2b, 0x4e, - 0xf9, 0xc8, 0x98, 0xbf, 0x6f, 0xa0, 0xef, 0x00, 0xc8, 0xfa, 0x53, 0x6a, 0x07, 0x26, 0x6b, 0x5a, - 0xa9, 0x1d, 0x98, 0x2a, 0x5d, 0x99, 0x0b, 0x14, 0x77, 0xce, 0xbc, 0x9b, 0xc4, 0x0d, 0x7d, 0xdb, - 0x0d, 0x5e, 0x62, 0xff, 0x7d, 0x96, 0xfc, 0x0e, 0x0e, 0x9d, 0x01, 0x99, 0xb2, 0x0f, 0xa5, 0xa8, - 0x3c, 0x90, 0xf4, 0xb6, 0xc9, 0x42, 0x46, 0xd2, 0xdb, 0xa6, 0xea, 0x0a, 0x71, 0xb7, 0x13, 0x5b, - 0x2d, 0x82, 0x94, 0x6c, 0xc0, 0xbf, 0xa8, 0x41, 0x8e, 0x1c, 0xc8, 0xc9, 0xe1, 0x44, 0x26, 0x7b, - 0x92, 0xb3, 0x4f, 0xe5, 0xab, 0x93, 0xb3, 0x4f, 0xe7, 0x89, 0xe2, 0x87, 0x13, 0x72, 0x59, 0x5b, - 0x64, 0x59, 0x14, 0x32, 0x53, 0x0f, 0xca, 0x4a, 0x12, 0x08, 0x69, 0x98, 0xc5, 0xf3, 0xdf, 0xc9, - 0x70, 0xa7, 0xc9, 0x20, 0x99, 0xaf, 0x53, 0xbc, 0xeb, 0x2c, 0xdc, 0x51, 0xbc, 0x2e, 0xa3, 0x20, - 0x80, 0x7c, 0x76, 0x7c, 0xdf, 0x6b, 0x66, 0x17, 0xdf, 0xfb, 0xb3, 0xa3, 0x09, 0x46, 0xce, 0x4e, - 0x6e, 0xfc, 0x57, 0x50, 0x51, 0x13, 0x3f, 0x48, 0x23, 0x7c, 0x22, 0x43, 0x9f, 0x8c, 0x23, 0xba, - 0xbc, 0x51, 0xdc, 0xb3, 0x51, 0x48, 0x5b, 0x21, 0x23, 0xc0, 0x3d, 0x28, 0xf2, 0x04, 0x90, 0x4e, - 0xa5, 0xf1, 0x24, 0xbe, 0x4e, 0xa5, 0x89, 0xec, 0x51, 0xfc, 0xf4, 0x4c, 0x11, 0xc9, 0x45, 0x54, - 0xc4, 0x6a, 0x8e, 0xf6, 0x04, 0x87, 0xa3, 0xd0, 0x64, 0xd2, 0x76, 0x14, 0x9a, 0x92, 0x1f, 0x18, - 0x85, 0x76, 0x80, 0x43, 0xee, 0x0f, 0xc4, 0xe5, 0x1a, 0x8d, 0x60, 0xa6, 0xc6, 0x47, 0xf3, 0x2c, - 0x12, 0xdd, 0xe5, 0x46, 0x02, 0x8a, 0xe0, 0x78, 0x02, 0x20, 0x93, 0x51, 0xc9, 0x13, 0xab, 0xb6, - 0x4e, 0x90, 0x3c, 0xb1, 0xea, 0xf3, 0x59, 0x71, 0xdf, 0x27, 0x71, 0xd9, 0xdd, 0x8a, 0x20, 0xff, - 0xd8, 0x00, 0x94, 0x4e, 0x57, 0xa1, 0x77, 0xf5, 0xdc, 0xb5, 0x35, 0x87, 0xc6, 0x7b, 0x17, 0x23, - 0xd6, 0x85, 0x33, 0x29, 0x52, 0x87, 0x52, 0x0f, 0x5e, 0x11, 0xa1, 0xbe, 0x6b, 0xc0, 0x44, 0x2c, - 0xc5, 0x85, 0xde, 0x1a, 0x61, 0xd3, 0x44, 0xe1, 0xa1, 0xf1, 0xf6, 0xb9, 0x74, 0xba, 0xa3, 0xbc, - 0xb2, 0x02, 0xc4, 0x9d, 0xe6, 0x77, 0x0c, 0xa8, 0xc6, 0x33, 0x61, 0x68, 0x04, 0xef, 0x54, 0xbd, - 0xa2, 0x31, 0x77, 0x3e, 0xe1, 0xd9, 0xe6, 0x91, 0xd7, 0x99, 0x1e, 0x14, 0x79, 0xca, 0x4c, 0xb7, - 0xf0, 0xe3, 0x05, 0x0e, 0xdd, 0xc2, 0x4f, 0xe4, 0xdb, 0x34, 0x0b, 0xdf, 0xf7, 0x7a, 0x58, 0xd9, - 0x66, 0x3c, 0x93, 0x36, 0x0a, 0xed, 0xec, 0x6d, 0x96, 0x48, 0xc3, 0x8d, 0x42, 0x93, 0xdb, 0x4c, - 0x24, 0xcc, 0xd0, 0x08, 0x66, 0xe7, 0x6c, 0xb3, 0x64, 0xbe, 0x4d, 0xb3, 0xcd, 0x28, 0xa0, 0xb2, - 0xcd, 0x64, 0x22, 0x4b, 0xb7, 0xcd, 0x52, 0xb5, 0x18, 0xdd, 0x36, 0x4b, 0xe7, 0xc2, 0x34, 0x76, - 0xa4, 0xb8, 0xb1, 0x6d, 0x76, 0x4d, 0x93, 0xea, 0x42, 0xef, 0x8d, 0x50, 0xa2, 0xb6, 0xb2, 0xd3, - 0x78, 0xff, 0x82, 0xd4, 0x23, 0xd7, 0x38, 0x53, 0xbf, 0x58, 0xe3, 0x7f, 0x64, 0xc0, 0xb4, 0x2e, - 0x3b, 0x86, 0x46, 0xe0, 0x8c, 0x28, 0x04, 0x35, 0x16, 0x2e, 0x4a, 0x7e, 0xb6, 0xb6, 0xa2, 0x55, - 0xff, 0xb8, 0xf6, 0xaf, 0x9f, 0xcf, 0x18, 0xff, 0xf1, 0xf9, 0x8c, 0xf1, 0x5f, 0x9f, 0xcf, 0x18, - 0x3f, 0xf9, 0x9f, 0x99, 0xb1, 0xfd, 0x02, 0xfd, 0x1f, 0x30, 0x2c, 0xff, 0x2c, 0x00, 0x00, 0xff, - 0xff, 0x0d, 0x96, 0x9e, 0x56, 0x27, 0x42, 0x00, 0x00, + // 4411 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x5c, 0x4f, 0x6f, 0x1c, 0xc9, + 0x75, 0x67, 0xcf, 0x5f, 0xce, 0x9b, 0xe1, 0x70, 0x58, 0xa2, 0xb4, 0xa3, 0x59, 0x89, 0xe2, 0xb6, + 0x56, 0xbb, 0x5a, 0xed, 0x2e, 0x29, 0x91, 0x92, 0x37, 0x51, 0xb0, 0x1b, 0x8f, 0xc8, 0x59, 0x89, + 0x11, 0x45, 0xd2, 0xcd, 0x91, 0xd6, 0xbb, 0x01, 0xcc, 0x34, 0x67, 0x4a, 0x64, 0x9b, 0x33, 0xdd, + 0xe3, 0xee, 0x1e, 0x8a, 0x74, 0x0e, 0x76, 0x9c, 0x38, 0x86, 0x13, 0xc0, 0x40, 0x1c, 0x20, 0x30, + 0x82, 0xe4, 0x12, 0x04, 0x48, 0x0e, 0x4e, 0x90, 0x1c, 0x72, 0x08, 0x72, 0xc8, 0x21, 0x39, 0x24, + 0x87, 0x00, 0x01, 0xf2, 0x05, 0x92, 0x8d, 0x4f, 0xf9, 0x10, 0x81, 0x51, 0xff, 0xba, 0xaa, 0xbb, + 0xab, 0x49, 0xae, 0xc9, 0x85, 0x2f, 0xab, 0xe9, 0xaa, 0x57, 0xef, 0xf7, 0xea, 0xbd, 0xaa, 0xf7, + 0xaa, 0xde, 0x2b, 0x2e, 0x54, 0xfc, 0x51, 0x6f, 0x61, 0xe4, 0x7b, 0xa1, 0x87, 0x6a, 0x38, 0xec, + 0xf5, 0x03, 0xec, 0x1f, 0x62, 0x7f, 0xb4, 0xdb, 0x9a, 0xdd, 0xf3, 0xf6, 0x3c, 0xda, 0xb1, 0x48, + 0x7e, 0x31, 0x9a, 0x56, 0x93, 0xd0, 0x2c, 0xda, 0x23, 0x67, 0x71, 0x78, 0xd8, 0xeb, 0x8d, 0x76, + 0x17, 0x0f, 0x0e, 0x79, 0x4f, 0x2b, 0xea, 0xb1, 0xc7, 0xe1, 0xfe, 0x68, 0x97, 0xfe, 0xc3, 0xfb, + 0xe6, 0xa3, 0xbe, 0x43, 0xec, 0x07, 0x8e, 0xe7, 0x8e, 0x76, 0xc5, 0x2f, 0x4e, 0x71, 0x6d, 0xcf, + 0xf3, 0xf6, 0x06, 0x98, 0x8d, 0x77, 0x5d, 0x2f, 0xb4, 0x43, 0xc7, 0x73, 0x03, 0xd6, 0x6b, 0xfe, + 0xc8, 0x80, 0xba, 0x85, 0x83, 0x91, 0xe7, 0x06, 0xf8, 0x09, 0xb6, 0xfb, 0xd8, 0x47, 0xd7, 0x01, + 0x7a, 0x83, 0x71, 0x10, 0x62, 0x7f, 0xc7, 0xe9, 0x37, 0x8d, 0x79, 0xe3, 0x76, 0xc1, 0xaa, 0xf0, + 0x96, 0xb5, 0x3e, 0x7a, 0x1d, 0x2a, 0x43, 0x3c, 0xdc, 0x65, 0xbd, 0x39, 0xda, 0x3b, 0xc9, 0x1a, + 0xd6, 0xfa, 0xa8, 0x05, 0x93, 0x3e, 0x3e, 0x74, 0x08, 0x7c, 0x33, 0x3f, 0x6f, 0xdc, 0xce, 0x5b, + 0xd1, 0x37, 0x19, 0xe8, 0xdb, 0x2f, 0xc3, 0x9d, 0x10, 0xfb, 0xc3, 0x66, 0x81, 0x0d, 0x24, 0x0d, + 0x5d, 0xec, 0x0f, 0x1f, 0x96, 0xbf, 0xf7, 0x0f, 0xcd, 0xfc, 0xf2, 0xc2, 0x5d, 0xf3, 0x5f, 0x8a, + 0x50, 0xb3, 0x6c, 0x77, 0x0f, 0x5b, 0xf8, 0x5b, 0x63, 0x1c, 0x84, 0xa8, 0x01, 0xf9, 0x03, 0x7c, + 0x4c, 0xe5, 0xa8, 0x59, 0xe4, 0x27, 0x63, 0xe4, 0xee, 0xe1, 0x1d, 0xec, 0x32, 0x09, 0x6a, 0x84, + 0x91, 0xbb, 0x87, 0x3b, 0x6e, 0x1f, 0xcd, 0x42, 0x71, 0xe0, 0x0c, 0x9d, 0x90, 0xc3, 0xb3, 0x8f, + 0x98, 0x5c, 0x85, 0x84, 0x5c, 0x2b, 0x00, 0x81, 0xe7, 0x87, 0x3b, 0x9e, 0xdf, 0xc7, 0x7e, 0xb3, + 0x38, 0x6f, 0xdc, 0xae, 0x2f, 0xbd, 0xb9, 0xa0, 0x5a, 0x6c, 0x41, 0x15, 0x68, 0x61, 0xdb, 0xf3, + 0xc3, 0x4d, 0x42, 0x6b, 0x55, 0x02, 0xf1, 0x13, 0x7d, 0x0c, 0x55, 0xca, 0x24, 0xb4, 0xfd, 0x3d, + 0x1c, 0x36, 0x4b, 0x94, 0xcb, 0xad, 0x53, 0xb8, 0x74, 0x29, 0xb1, 0x45, 0xe1, 0xd9, 0x6f, 0x64, + 0x42, 0x2d, 0xc0, 0xbe, 0x63, 0x0f, 0x9c, 0x6f, 0xdb, 0xbb, 0x03, 0xdc, 0x2c, 0xcf, 0x1b, 0xb7, + 0x27, 0xad, 0x58, 0x1b, 0x99, 0xff, 0x01, 0x3e, 0x0e, 0x76, 0x3c, 0x77, 0x70, 0xdc, 0x9c, 0xa4, + 0x04, 0x93, 0xa4, 0x61, 0xd3, 0x1d, 0x1c, 0x53, 0xeb, 0x79, 0x63, 0x37, 0x64, 0xbd, 0x15, 0xda, + 0x5b, 0xa1, 0x2d, 0xb4, 0xfb, 0x1e, 0x34, 0x86, 0x8e, 0xbb, 0x33, 0xf4, 0xfa, 0x3b, 0x91, 0x42, + 0x80, 0x28, 0xe4, 0x51, 0xf9, 0x0f, 0xa8, 0x05, 0xee, 0x59, 0xf5, 0xa1, 0xe3, 0x3e, 0xf3, 0xfa, + 0x96, 0xd0, 0x0f, 0x19, 0x62, 0x1f, 0xc5, 0x87, 0x54, 0x93, 0x43, 0xec, 0x23, 0x75, 0xc8, 0x07, + 0x70, 0x89, 0xa0, 0xf4, 0x7c, 0x6c, 0x87, 0x58, 0x8e, 0xaa, 0xc5, 0x47, 0xcd, 0x0c, 0x1d, 0x77, + 0x85, 0x92, 0xc4, 0x06, 0xda, 0x47, 0xa9, 0x81, 0x53, 0xc9, 0x81, 0xf6, 0x51, 0x7c, 0xa0, 0xf9, + 0x01, 0x54, 0x22, 0xbb, 0xa0, 0x49, 0x28, 0x6c, 0x6c, 0x6e, 0x74, 0x1a, 0x13, 0x08, 0xa0, 0xd4, + 0xde, 0x5e, 0xe9, 0x6c, 0xac, 0x36, 0x0c, 0x54, 0x85, 0xf2, 0x6a, 0x87, 0x7d, 0xe4, 0x5a, 0xe5, + 0x1f, 0xf3, 0xf5, 0xf6, 0x14, 0x40, 0x9a, 0x02, 0x95, 0x21, 0xff, 0xb4, 0xf3, 0x69, 0x63, 0x82, + 0x10, 0xbf, 0xe8, 0x58, 0xdb, 0x6b, 0x9b, 0x1b, 0x0d, 0x83, 0x70, 0x59, 0xb1, 0x3a, 0xed, 0x6e, + 0xa7, 0x91, 0x23, 0x14, 0xcf, 0x36, 0x57, 0x1b, 0x79, 0x54, 0x81, 0xe2, 0x8b, 0xf6, 0xfa, 0xf3, + 0x4e, 0xa3, 0x10, 0x31, 0x93, 0xab, 0xf8, 0xcf, 0x0c, 0x98, 0xe2, 0xe6, 0x66, 0x7b, 0x0b, 0xdd, + 0x87, 0xd2, 0x3e, 0xdd, 0x5f, 0x74, 0x25, 0x57, 0x97, 0xae, 0x25, 0xd6, 0x46, 0x6c, 0x0f, 0x5a, + 0x9c, 0x16, 0x99, 0x90, 0x3f, 0x38, 0x0c, 0x9a, 0xb9, 0xf9, 0xfc, 0xed, 0xea, 0x52, 0x63, 0x81, + 0x79, 0x86, 0x85, 0xa7, 0xf8, 0xf8, 0x85, 0x3d, 0x18, 0x63, 0x8b, 0x74, 0x22, 0x04, 0x85, 0xa1, + 0xe7, 0x63, 0xba, 0xe0, 0x27, 0x2d, 0xfa, 0x9b, 0xec, 0x02, 0x6a, 0x73, 0xbe, 0xd8, 0xd9, 0x87, + 0x14, 0xef, 0x3f, 0x0c, 0x80, 0xad, 0x71, 0x98, 0xbd, 0xc5, 0x66, 0xa1, 0x78, 0x48, 0x10, 0xf8, + 0xf6, 0x62, 0x1f, 0x74, 0x6f, 0x61, 0x3b, 0xc0, 0xd1, 0xde, 0x22, 0x1f, 0x68, 0x1e, 0xca, 0x23, + 0x1f, 0x1f, 0xee, 0x1c, 0x1c, 0x52, 0xb4, 0x49, 0x69, 0xa7, 0x12, 0x69, 0x7f, 0x7a, 0x88, 0xee, + 0x40, 0xcd, 0xd9, 0x73, 0x3d, 0x1f, 0xef, 0x30, 0xa6, 0x45, 0x95, 0x6c, 0xc9, 0xaa, 0xb2, 0x4e, + 0x3a, 0x25, 0x85, 0x96, 0x41, 0x95, 0xb4, 0xb4, 0xeb, 0xa4, 0x4f, 0xce, 0xe7, 0xbb, 0x06, 0x54, + 0xe9, 0x7c, 0xce, 0xa5, 0xec, 0x25, 0x39, 0x91, 0x1c, 0x1d, 0x96, 0x52, 0x78, 0x6a, 0x6a, 0x52, + 0x04, 0x17, 0xd0, 0x2a, 0x1e, 0xe0, 0x10, 0x9f, 0xc7, 0x79, 0x29, 0xaa, 0xcc, 0x6b, 0x55, 0x29, + 0xf1, 0xfe, 0xd2, 0x80, 0x4b, 0x31, 0xc0, 0x73, 0x4d, 0xbd, 0x09, 0xe5, 0x3e, 0x65, 0xc6, 0x64, + 0xca, 0x5b, 0xe2, 0x13, 0xdd, 0x87, 0x49, 0x2e, 0x52, 0xd0, 0xcc, 0xeb, 0x97, 0xa1, 0x94, 0xb2, + 0xcc, 0xa4, 0x0c, 0xa4, 0x98, 0xff, 0x94, 0x83, 0x0a, 0x57, 0xc6, 0xe6, 0x08, 0xb5, 0x61, 0xca, + 0x67, 0x1f, 0x3b, 0x74, 0xce, 0x5c, 0xc6, 0x56, 0xb6, 0x9f, 0x7c, 0x32, 0x61, 0xd5, 0xf8, 0x10, + 0xda, 0x8c, 0x7e, 0x0d, 0xaa, 0x82, 0xc5, 0x68, 0x1c, 0x72, 0x43, 0x35, 0xe3, 0x0c, 0xe4, 0xd2, + 0x7e, 0x32, 0x61, 0x01, 0x27, 0xdf, 0x1a, 0x87, 0xa8, 0x0b, 0xb3, 0x62, 0x30, 0x9b, 0x1f, 0x17, + 0x23, 0x4f, 0xb9, 0xcc, 0xc7, 0xb9, 0xa4, 0xcd, 0xf9, 0x64, 0xc2, 0x42, 0x7c, 0xbc, 0xd2, 0x89, + 0x56, 0xa5, 0x48, 0xe1, 0x11, 0x8b, 0x2f, 0x29, 0x91, 0xba, 0x47, 0x2e, 0x67, 0x22, 0xb4, 0xb5, + 0xac, 0xc8, 0xd6, 0x3d, 0x72, 0x23, 0x95, 0x3d, 0xaa, 0x40, 0x99, 0x37, 0x9b, 0xff, 0x9e, 0x03, + 0x10, 0x16, 0xdb, 0x1c, 0xa1, 0x55, 0xa8, 0xfb, 0xfc, 0x2b, 0xa6, 0xbf, 0xd7, 0xb5, 0xfa, 0xe3, + 0x86, 0x9e, 0xb0, 0xa6, 0xc4, 0x20, 0x26, 0xee, 0x47, 0x50, 0x8b, 0xb8, 0x48, 0x15, 0x5e, 0xd5, + 0xa8, 0x30, 0xe2, 0x50, 0x15, 0x03, 0x88, 0x12, 0x3f, 0x81, 0xcb, 0xd1, 0x78, 0x8d, 0x16, 0xdf, + 0x38, 0x41, 0x8b, 0x11, 0xc3, 0x4b, 0x82, 0x83, 0xaa, 0xc7, 0xc7, 0x8a, 0x60, 0x52, 0x91, 0x57, + 0x35, 0x8a, 0x64, 0x44, 0xaa, 0x26, 0x23, 0x09, 0x63, 0xaa, 0x04, 0x12, 0xf6, 0x59, 0xbb, 0xf9, + 0xd7, 0x05, 0x28, 0xaf, 0x78, 0xc3, 0x91, 0xed, 0x93, 0x45, 0x54, 0xf2, 0x71, 0x30, 0x1e, 0x84, + 0x54, 0x81, 0xf5, 0xa5, 0x9b, 0x71, 0x0c, 0x4e, 0x26, 0xfe, 0xb5, 0x28, 0xa9, 0xc5, 0x87, 0x90, + 0xc1, 0x3c, 0xca, 0xe7, 0xce, 0x30, 0x98, 0xc7, 0x78, 0x3e, 0x44, 0x38, 0x84, 0xbc, 0x74, 0x08, + 0x2d, 0x28, 0xf3, 0x03, 0x1b, 0x73, 0xd6, 0x4f, 0x26, 0x2c, 0xd1, 0x80, 0xde, 0x81, 0xe9, 0x64, + 0x28, 0x2c, 0x72, 0x9a, 0x7a, 0x2f, 0x1e, 0x39, 0x6f, 0x42, 0x2d, 0x16, 0xa1, 0x4b, 0x9c, 0xae, + 0x3a, 0x54, 0xe2, 0xf2, 0x15, 0xe1, 0xd6, 0xc9, 0xb1, 0xa2, 0xf6, 0x64, 0x42, 0x38, 0xf6, 0x1b, + 0xc2, 0xb1, 0x4f, 0xaa, 0x81, 0x96, 0xe8, 0x95, 0xfb, 0xf8, 0x37, 0x55, 0xaf, 0xf5, 0x55, 0x32, + 0x38, 0x22, 0x92, 0xee, 0xcb, 0xb4, 0x60, 0x2a, 0xa6, 0x32, 0x12, 0x23, 0x3b, 0x5f, 0x7b, 0xde, + 0x5e, 0x67, 0x01, 0xf5, 0x31, 0x8d, 0xa1, 0x56, 0xc3, 0x20, 0x01, 0x7a, 0xbd, 0xb3, 0xbd, 0xdd, + 0xc8, 0xa1, 0x2b, 0x50, 0xd9, 0xd8, 0xec, 0xee, 0x30, 0xaa, 0x7c, 0xab, 0xfc, 0xa7, 0xcc, 0x93, + 0xc8, 0xf8, 0xfc, 0x69, 0xc4, 0x93, 0x87, 0x68, 0x25, 0x32, 0x4f, 0x28, 0x91, 0xd9, 0x10, 0x91, + 0x39, 0x27, 0x23, 0x73, 0x1e, 0x21, 0x28, 0xae, 0x77, 0xda, 0xdb, 0x34, 0x48, 0x33, 0xd6, 0xcb, + 0xe9, 0x68, 0xfd, 0xa8, 0x0e, 0x35, 0x66, 0x9e, 0x9d, 0xb1, 0x4b, 0x0e, 0x13, 0x3f, 0x35, 0x00, + 0xe4, 0x86, 0x45, 0x8b, 0x50, 0xee, 0x31, 0x11, 0x9a, 0x06, 0xf5, 0x80, 0x97, 0xb5, 0x16, 0xb7, + 0x04, 0x15, 0xba, 0x07, 0xe5, 0x60, 0xdc, 0xeb, 0xe1, 0x40, 0x44, 0xee, 0xd7, 0x92, 0x4e, 0x98, + 0x3b, 0x44, 0x4b, 0xd0, 0x91, 0x21, 0x2f, 0x6d, 0x67, 0x30, 0xa6, 0x71, 0xfc, 0xe4, 0x21, 0x9c, + 0x4e, 0xfa, 0xd8, 0xbf, 0x30, 0xa0, 0xaa, 0x6c, 0x8b, 0x5f, 0x30, 0x04, 0x5c, 0x83, 0x0a, 0x15, + 0x06, 0xf7, 0x79, 0x10, 0x98, 0xb4, 0x64, 0x03, 0xfa, 0x0a, 0x54, 0xc4, 0x4e, 0x12, 0x71, 0xa0, + 0xa9, 0x67, 0xbb, 0x39, 0xb2, 0x24, 0xa9, 0x14, 0xb2, 0x0b, 0x33, 0x54, 0x4f, 0x3d, 0x72, 0xfb, + 0x10, 0x9a, 0x55, 0x8f, 0xe5, 0x46, 0xe2, 0x58, 0xde, 0x82, 0xc9, 0xd1, 0xfe, 0x71, 0xe0, 0xf4, + 0xec, 0x01, 0x17, 0x27, 0xfa, 0x96, 0x5c, 0xb7, 0x01, 0xa9, 0x5c, 0xcf, 0xa3, 0x00, 0xc9, 0xf4, + 0x0a, 0x54, 0x9f, 0xd8, 0xc1, 0x3e, 0x17, 0x52, 0xb6, 0xdf, 0x87, 0x29, 0xd2, 0xfe, 0xf4, 0xc5, + 0x19, 0xc4, 0x17, 0xa3, 0x96, 0xe9, 0x0d, 0x4b, 0x0c, 0x3b, 0x97, 0x81, 0x10, 0x14, 0xf6, 0xed, + 0x60, 0x9f, 0x2a, 0x63, 0xca, 0xa2, 0xbf, 0xd1, 0x3b, 0xd0, 0xe8, 0xb1, 0xf9, 0xef, 0x24, 0xee, + 0x5d, 0xd3, 0xbc, 0xdd, 0x4a, 0x09, 0x64, 0x43, 0x8d, 0x4d, 0xef, 0xa2, 0xa5, 0x91, 0x9a, 0x6a, + 0xc1, 0xf4, 0xb6, 0x6b, 0x8f, 0x82, 0x7d, 0x2f, 0x4c, 0x68, 0x71, 0xd9, 0xfc, 0x7b, 0x03, 0x1a, + 0xb2, 0xf3, 0x5c, 0x32, 0xbc, 0x0d, 0xd3, 0x3e, 0x1e, 0xda, 0x8e, 0xeb, 0xb8, 0x7b, 0x3b, 0xbb, + 0xc7, 0x21, 0x0e, 0xf8, 0x85, 0xb4, 0x1e, 0x35, 0x3f, 0x22, 0xad, 0x44, 0xd8, 0xdd, 0x81, 0xb7, + 0xcb, 0xdd, 0x2e, 0xfd, 0x8d, 0xde, 0x88, 0xfb, 0xdd, 0x8a, 0x70, 0x68, 0x5f, 0x89, 0xdc, 0xaf, + 0x94, 0xf9, 0x27, 0x39, 0xa8, 0x7d, 0x62, 0x87, 0x3d, 0xb1, 0x26, 0xd0, 0x1a, 0xd4, 0x23, 0xc7, + 0x4c, 0x5b, 0xb8, 0xdc, 0x89, 0x23, 0x04, 0x1d, 0x23, 0x6e, 0x2a, 0xe2, 0x08, 0x31, 0xd5, 0x53, + 0x1b, 0x28, 0x2b, 0xdb, 0xed, 0xe1, 0x41, 0xc4, 0x2a, 0x97, 0xcd, 0x8a, 0x12, 0xaa, 0xac, 0xd4, + 0x06, 0xf4, 0x75, 0x68, 0x8c, 0x7c, 0x6f, 0xcf, 0xc7, 0x41, 0x10, 0x31, 0x63, 0x41, 0xd9, 0xd4, + 0x30, 0xdb, 0xe2, 0xa4, 0x89, 0x73, 0xc9, 0xfd, 0x27, 0x13, 0xd6, 0xf4, 0x28, 0xde, 0x27, 0x5d, + 0xe5, 0xb4, 0x3c, 0xc1, 0x31, 0x5f, 0xf9, 0x83, 0x3c, 0xa0, 0xf4, 0x34, 0xbf, 0xe8, 0xc1, 0xf7, + 0x16, 0xd4, 0x83, 0xd0, 0xf6, 0x53, 0xab, 0x78, 0x8a, 0xb6, 0x46, 0xf1, 0xeb, 0x6d, 0x88, 0x24, + 0xdb, 0x71, 0xbd, 0xd0, 0x79, 0x79, 0xcc, 0xae, 0x1c, 0x56, 0x5d, 0x34, 0x6f, 0xd0, 0x56, 0xb4, + 0x01, 0xe5, 0x97, 0xce, 0x20, 0xc4, 0x7e, 0xd0, 0x2c, 0xce, 0xe7, 0x6f, 0xd7, 0x97, 0xde, 0x3d, + 0xcd, 0x30, 0x0b, 0x1f, 0x53, 0xfa, 0xee, 0xf1, 0x48, 0x3d, 0xcf, 0x72, 0x26, 0xea, 0xc1, 0xbc, + 0xa4, 0xbf, 0xe3, 0x98, 0x30, 0xf9, 0x8a, 0x30, 0xdd, 0x71, 0xfa, 0x34, 0xba, 0x46, 0x51, 0xf4, + 0xbe, 0x55, 0xa6, 0x1d, 0x6b, 0x7d, 0x74, 0x13, 0x26, 0x5f, 0xfa, 0xf6, 0xde, 0x10, 0xbb, 0x21, + 0xbb, 0xb7, 0x4b, 0x9a, 0xa8, 0xc3, 0x5c, 0x00, 0x90, 0xa2, 0x90, 0x58, 0xb6, 0xb1, 0xb9, 0xf5, + 0xbc, 0xdb, 0x98, 0x40, 0x35, 0x98, 0xdc, 0xd8, 0x5c, 0xed, 0xac, 0x77, 0x48, 0xb4, 0x13, 0x51, + 0xec, 0x9e, 0xdc, 0x74, 0x6d, 0x61, 0x88, 0xd8, 0x9a, 0x50, 0xe5, 0x32, 0xe2, 0xd7, 0x68, 0x21, + 0x97, 0x60, 0x71, 0xcf, 0xbc, 0x01, 0xb3, 0xba, 0xa5, 0x21, 0x08, 0xee, 0x9b, 0xff, 0x9a, 0x83, + 0x29, 0xbe, 0x11, 0xce, 0xb5, 0x73, 0xaf, 0x2a, 0x52, 0xf1, 0x0b, 0x87, 0x50, 0x52, 0x13, 0xca, + 0x6c, 0x83, 0xf4, 0xf9, 0x8d, 0x56, 0x7c, 0x12, 0x77, 0xcb, 0xd6, 0x3b, 0xee, 0x73, 0xb3, 0x47, + 0xdf, 0x5a, 0x47, 0x58, 0xd4, 0x3a, 0x42, 0xf4, 0x1e, 0x4c, 0x45, 0x1b, 0xce, 0x0e, 0xf8, 0x51, + 0xa9, 0x22, 0x4d, 0x51, 0x13, 0x9b, 0x8a, 0x74, 0xc6, 0x6c, 0x56, 0xce, 0xb0, 0x19, 0xba, 0x05, + 0x25, 0x7c, 0x88, 0xdd, 0x30, 0x68, 0x56, 0x69, 0x68, 0x9c, 0x12, 0x57, 0xa4, 0x0e, 0x69, 0xb5, + 0x78, 0xa7, 0x34, 0xd5, 0x47, 0x30, 0x43, 0x6f, 0xb0, 0x8f, 0x7d, 0xdb, 0x55, 0x6f, 0xe1, 0xdd, + 0xee, 0x3a, 0x0f, 0x24, 0xe4, 0x27, 0xaa, 0x43, 0x6e, 0x6d, 0x95, 0xeb, 0x27, 0xb7, 0xb6, 0x2a, + 0xc7, 0xff, 0xa1, 0x01, 0x48, 0x65, 0x70, 0x2e, 0x5b, 0x24, 0x50, 0x84, 0x1c, 0x79, 0x29, 0xc7, + 0x2c, 0x14, 0xb1, 0xef, 0x7b, 0x3e, 0x73, 0x94, 0x16, 0xfb, 0x90, 0xd2, 0xbc, 0xcf, 0x85, 0xb1, + 0xf0, 0xa1, 0x77, 0x10, 0x79, 0x00, 0xc6, 0xd6, 0x48, 0x0b, 0xdf, 0x85, 0x4b, 0x31, 0xf2, 0x8b, + 0x09, 0xda, 0x9b, 0x30, 0x4d, 0xb9, 0xae, 0xec, 0xe3, 0xde, 0xc1, 0xc8, 0x73, 0xdc, 0x94, 0x04, + 0xe8, 0x26, 0xf1, 0x5d, 0x22, 0x5c, 0x90, 0x29, 0xb2, 0x39, 0xd7, 0xa2, 0xc6, 0x6e, 0x77, 0x5d, + 0x2e, 0xf5, 0x5d, 0xb8, 0x92, 0x60, 0x28, 0x66, 0xf6, 0xeb, 0x50, 0xed, 0x45, 0x8d, 0x01, 0x3f, + 0x13, 0x5e, 0x8f, 0x8b, 0x9b, 0x1c, 0xaa, 0x8e, 0x90, 0x18, 0x5f, 0x87, 0xd7, 0x52, 0x18, 0x17, + 0xa1, 0x8e, 0xfb, 0xe6, 0x5d, 0xb8, 0x4c, 0x39, 0x3f, 0xc5, 0x78, 0xd4, 0x1e, 0x38, 0x87, 0xa7, + 0x9b, 0xe5, 0x98, 0xcf, 0x57, 0x19, 0xf1, 0xe5, 0x2e, 0x2b, 0x09, 0xdd, 0xe1, 0xd0, 0x5d, 0x67, + 0x88, 0xbb, 0xde, 0x7a, 0xb6, 0xb4, 0x24, 0x90, 0x1f, 0xe0, 0xe3, 0x80, 0x1f, 0x08, 0xe9, 0x6f, + 0xe9, 0xbd, 0xfe, 0xd6, 0xe0, 0xea, 0x54, 0xf9, 0x7c, 0xc9, 0x5b, 0x63, 0x0e, 0x60, 0x8f, 0xec, + 0x41, 0xdc, 0x27, 0x1d, 0x2c, 0xdb, 0xa6, 0xb4, 0x44, 0x02, 0x93, 0x28, 0x54, 0x4b, 0x0a, 0x7c, + 0x9d, 0x6f, 0x1c, 0xfa, 0x9f, 0x20, 0x75, 0x52, 0x7a, 0x0b, 0xaa, 0xb4, 0x67, 0x3b, 0xb4, 0xc3, + 0x71, 0x90, 0x65, 0xb9, 0x65, 0xf3, 0x07, 0x06, 0xdf, 0x51, 0x82, 0xcf, 0xb9, 0xe6, 0x7c, 0x0f, + 0x4a, 0xf4, 0xce, 0x27, 0xee, 0x2e, 0x57, 0x35, 0x0b, 0x9b, 0x49, 0x64, 0x71, 0x42, 0xe5, 0x9c, + 0x64, 0x40, 0xe9, 0x19, 0xad, 0x05, 0x28, 0xd2, 0x16, 0x84, 0xe5, 0x5c, 0x7b, 0xc8, 0x12, 0x8a, + 0x15, 0x8b, 0xfe, 0xa6, 0x47, 0x7c, 0x8c, 0xfd, 0xe7, 0xd6, 0x3a, 0xbb, 0x53, 0x54, 0xac, 0xe8, + 0x9b, 0x28, 0xb6, 0x37, 0x70, 0xb0, 0x1b, 0xd2, 0xde, 0x02, 0xed, 0x55, 0x5a, 0xd0, 0x2d, 0xa8, + 0x38, 0xc1, 0x3a, 0xb6, 0x7d, 0x97, 0x27, 0xed, 0x15, 0xc7, 0x2c, 0x7b, 0xe4, 0x1a, 0xfb, 0x06, + 0x34, 0x98, 0x64, 0xed, 0x7e, 0x5f, 0x39, 0xbf, 0x47, 0xf8, 0x46, 0x02, 0x3f, 0xc6, 0x3f, 0x77, + 0x3a, 0xff, 0xbf, 0x33, 0x60, 0x46, 0x01, 0x38, 0x97, 0x09, 0xde, 0x83, 0x12, 0xab, 0xa8, 0xf0, + 0xa3, 0xe0, 0x6c, 0x7c, 0x14, 0x83, 0xb1, 0x38, 0x0d, 0x5a, 0x80, 0x32, 0xfb, 0x25, 0x2e, 0x66, + 0x7a, 0x72, 0x41, 0x24, 0x45, 0x5e, 0x80, 0x4b, 0xbc, 0x0f, 0x0f, 0x3d, 0xdd, 0x9e, 0x2b, 0xc4, + 0x3d, 0xc4, 0xf7, 0x0d, 0x98, 0x8d, 0x0f, 0x38, 0xd7, 0x2c, 0x15, 0xb9, 0x73, 0x5f, 0x48, 0xee, + 0xdf, 0x10, 0x72, 0x3f, 0x1f, 0xf5, 0x95, 0x23, 0x67, 0x72, 0xc5, 0xa9, 0xd6, 0xcd, 0xc5, 0xad, + 0x2b, 0x79, 0xfd, 0x28, 0x9a, 0x93, 0x60, 0x76, 0xae, 0x39, 0x7d, 0x70, 0xa6, 0x39, 0x29, 0x47, + 0xb0, 0xd4, 0xe4, 0xd6, 0xc4, 0x32, 0x5a, 0x77, 0x82, 0x28, 0xe2, 0xbc, 0x0b, 0xb5, 0x81, 0xe3, + 0x62, 0xdb, 0xe7, 0x55, 0x21, 0x43, 0x5d, 0x8f, 0x0f, 0xac, 0x58, 0xa7, 0x64, 0xf5, 0xbb, 0x06, + 0x20, 0x95, 0xd7, 0x2f, 0xc7, 0x5a, 0x8b, 0x42, 0xc1, 0x5b, 0xbe, 0x37, 0xf4, 0xc2, 0xd3, 0x96, + 0xd9, 0x7d, 0xf3, 0xf7, 0x0d, 0xb8, 0x9c, 0x18, 0xf1, 0xcb, 0x90, 0xfc, 0xbe, 0x79, 0x0d, 0x66, + 0x56, 0xb1, 0x38, 0xe3, 0xa5, 0xb2, 0x01, 0xdb, 0x80, 0xd4, 0xde, 0x8b, 0x39, 0xc5, 0xfc, 0x0a, + 0xcc, 0x3c, 0xf3, 0x0e, 0x89, 0x23, 0x27, 0xdd, 0xd2, 0x4d, 0xb1, 0xf4, 0x54, 0xa4, 0xaf, 0xe8, + 0x5b, 0xba, 0xde, 0x6d, 0x40, 0xea, 0xc8, 0x8b, 0x10, 0x67, 0xd9, 0xfc, 0x1f, 0x03, 0x6a, 0xed, + 0x81, 0xed, 0x0f, 0x85, 0x28, 0x1f, 0x41, 0x89, 0xe5, 0x5a, 0x78, 0xe2, 0xf4, 0xad, 0x38, 0x3f, + 0x95, 0x96, 0x7d, 0xb4, 0x59, 0x66, 0x86, 0x8f, 0x22, 0x53, 0xe1, 0xb5, 0xe2, 0xd5, 0x44, 0xed, + 0x78, 0x15, 0xbd, 0x0f, 0x45, 0x9b, 0x0c, 0xa1, 0xe1, 0xb5, 0x9e, 0x4c, 0x80, 0x51, 0x6e, 0xe4, + 0x4a, 0x64, 0x31, 0x2a, 0xf3, 0x43, 0xa8, 0x2a, 0x08, 0xa8, 0x0c, 0xf9, 0xc7, 0x1d, 0x7e, 0x4d, + 0x6a, 0xaf, 0x74, 0xd7, 0x5e, 0xb0, 0xa4, 0x60, 0x1d, 0x60, 0xb5, 0x13, 0x7d, 0xe7, 0x34, 0xa5, + 0x3a, 0x9b, 0xf3, 0xe1, 0x71, 0x4b, 0x95, 0xd0, 0xc8, 0x92, 0x30, 0x77, 0x16, 0x09, 0x25, 0xc4, + 0xef, 0x18, 0x30, 0xc5, 0x55, 0x73, 0xde, 0xd0, 0x4c, 0x39, 0x67, 0x84, 0x66, 0x65, 0x1a, 0x16, + 0x27, 0x94, 0x32, 0xfc, 0xb3, 0x01, 0x8d, 0x55, 0xef, 0x95, 0xbb, 0xe7, 0xdb, 0xfd, 0x68, 0x0f, + 0x7e, 0x9c, 0x30, 0xe7, 0x42, 0x22, 0x77, 0x9f, 0xa0, 0x97, 0x0d, 0x09, 0xb3, 0x36, 0x65, 0x2e, + 0x85, 0xc5, 0x77, 0xf1, 0x69, 0x7e, 0x15, 0xa6, 0x13, 0x83, 0x88, 0x81, 0x5e, 0xb4, 0xd7, 0xd7, + 0x56, 0x89, 0x41, 0x68, 0x06, 0xb7, 0xb3, 0xd1, 0x7e, 0xb4, 0xde, 0xe1, 0x75, 0xd6, 0xf6, 0xc6, + 0x4a, 0x67, 0x5d, 0x1a, 0xea, 0x81, 0x98, 0xc1, 0x03, 0x73, 0x00, 0x33, 0x8a, 0x40, 0xe7, 0x2d, + 0x77, 0xe9, 0xe5, 0x95, 0x68, 0x4d, 0x98, 0xe2, 0xa7, 0x9c, 0xe4, 0xc6, 0xff, 0x69, 0x1e, 0xea, + 0xa2, 0xeb, 0xcb, 0x91, 0x02, 0x5d, 0x81, 0x52, 0x7f, 0x77, 0xdb, 0xf9, 0xb6, 0xa8, 0xb4, 0xf2, + 0x2f, 0xd2, 0x3e, 0x60, 0x38, 0xec, 0xfd, 0x04, 0xff, 0x42, 0xd7, 0xd8, 0xd3, 0x8a, 0x35, 0xb7, + 0x8f, 0x8f, 0xe8, 0x61, 0xa8, 0x60, 0xc9, 0x06, 0x9a, 0xa6, 0xe4, 0xef, 0x2c, 0xe8, 0x5d, 0x57, + 0x79, 0x77, 0x81, 0x96, 0xa1, 0x41, 0x7e, 0xb7, 0x47, 0xa3, 0x81, 0x83, 0xfb, 0x8c, 0x01, 0xb9, + 0xe6, 0x16, 0xe4, 0x69, 0x27, 0x45, 0x80, 0x6e, 0x40, 0x89, 0x5e, 0x01, 0x83, 0xe6, 0x24, 0x89, + 0xab, 0x92, 0x94, 0x37, 0xa3, 0x77, 0xa0, 0xca, 0x24, 0x5e, 0x73, 0x9f, 0x07, 0x98, 0xbe, 0x42, + 0x50, 0xf2, 0x21, 0x6a, 0x5f, 0xfc, 0x9c, 0x05, 0x59, 0xe7, 0x2c, 0xb4, 0x08, 0xf5, 0x20, 0xf4, + 0x7c, 0x7b, 0x0f, 0xbf, 0xe0, 0x2a, 0xab, 0xc6, 0x93, 0x76, 0x89, 0x6e, 0x69, 0xae, 0x6b, 0x30, + 0xd3, 0x1e, 0x87, 0xfb, 0x1d, 0x97, 0x04, 0xc7, 0x94, 0x31, 0xaf, 0x03, 0x22, 0xbd, 0xab, 0x4e, + 0xa0, 0xed, 0xe6, 0x83, 0xb5, 0x2b, 0xe1, 0x81, 0xb9, 0x01, 0x97, 0x48, 0x2f, 0x76, 0x43, 0xa7, + 0xa7, 0x1c, 0x44, 0xc4, 0x51, 0xd7, 0x48, 0x1c, 0x75, 0xed, 0x20, 0x78, 0xe5, 0xf9, 0x7d, 0x6e, + 0xec, 0xe8, 0x5b, 0xa2, 0xfd, 0xa3, 0xc1, 0xa4, 0x79, 0x1e, 0xc4, 0x8e, 0xa9, 0x5f, 0x90, 0x1f, + 0xfa, 0x55, 0x28, 0x7b, 0x23, 0xfa, 0xc8, 0x87, 0x67, 0xff, 0xae, 0x2c, 0xb0, 0x87, 0x43, 0x0b, + 0x9c, 0xf1, 0x26, 0xeb, 0x55, 0x32, 0x54, 0x9c, 0x9e, 0xa8, 0x79, 0xdf, 0x0e, 0xf6, 0x71, 0x7f, + 0x4b, 0x30, 0x8f, 0xe5, 0x46, 0x1f, 0x58, 0x89, 0x6e, 0x29, 0xfb, 0x3d, 0x29, 0xfa, 0x63, 0x1c, + 0x9e, 0x20, 0xba, 0x9a, 0x4f, 0xbf, 0x2c, 0x86, 0xf0, 0x32, 0xe0, 0x59, 0x46, 0xfd, 0xd0, 0x80, + 0xeb, 0x62, 0xd8, 0xca, 0xbe, 0xed, 0xee, 0x61, 0x21, 0xcc, 0x2f, 0xaa, 0xaf, 0xf4, 0xa4, 0xf3, + 0x67, 0x9c, 0xf4, 0x53, 0x68, 0x46, 0x93, 0xa6, 0x99, 0x18, 0x6f, 0xa0, 0x4e, 0x62, 0x1c, 0x70, + 0x8f, 0x50, 0xb1, 0xe8, 0x6f, 0xd2, 0xe6, 0x7b, 0x83, 0xe8, 0x12, 0x44, 0x7e, 0x4b, 0x66, 0xeb, + 0x70, 0x55, 0x30, 0xe3, 0xa9, 0x91, 0x38, 0xb7, 0xd4, 0x9c, 0x4e, 0xe4, 0xc6, 0xed, 0x41, 0x78, + 0x9c, 0xbc, 0x94, 0xb4, 0x43, 0xe2, 0x26, 0xa4, 0x28, 0x86, 0x0e, 0x65, 0x8e, 0xed, 0x00, 0x22, + 0xb3, 0x72, 0x5e, 0x4d, 0xf5, 0x13, 0x96, 0xda, 0x7e, 0xbe, 0x04, 0x48, 0x7f, 0x6a, 0x09, 0x64, + 0xa3, 0x62, 0x98, 0x8b, 0x04, 0x25, 0x6a, 0xdf, 0xc2, 0xfe, 0xd0, 0x09, 0x02, 0xa5, 0xb0, 0xa4, + 0x53, 0xd7, 0x5b, 0x50, 0x18, 0x61, 0x1e, 0xbc, 0xab, 0x4b, 0x48, 0xec, 0x09, 0x65, 0x30, 0xed, + 0x97, 0x30, 0x43, 0xb8, 0x21, 0x60, 0x98, 0x41, 0xb4, 0x38, 0x49, 0x31, 0x45, 0xea, 0x3b, 0x97, + 0x91, 0xfa, 0xce, 0xc7, 0x53, 0xdf, 0xb1, 0x03, 0xa5, 0xea, 0xa8, 0x2e, 0xe6, 0x40, 0xd9, 0x65, + 0x06, 0x88, 0xfc, 0xdb, 0xc5, 0x70, 0xfd, 0x23, 0xee, 0xa8, 0x2e, 0x2a, 0x0c, 0x62, 0x3a, 0x67, + 0x51, 0x76, 0x14, 0x9f, 0xc8, 0x84, 0x1a, 0x31, 0x92, 0xa5, 0xd6, 0x04, 0x0a, 0x56, 0xac, 0x4d, + 0x3a, 0xe3, 0x03, 0x98, 0x8d, 0x3b, 0xe3, 0x73, 0x09, 0x35, 0x0b, 0xc5, 0xd0, 0x3b, 0xc0, 0x22, + 0x32, 0xb3, 0x8f, 0x94, 0x5a, 0x23, 0x47, 0x7d, 0x31, 0x6a, 0xfd, 0xa6, 0xe4, 0x4a, 0x37, 0xe0, + 0x79, 0x67, 0x40, 0x96, 0xa3, 0xb8, 0xfb, 0xb2, 0x0f, 0x89, 0xf5, 0x09, 0x5c, 0x49, 0x3a, 0xdf, + 0x8b, 0x99, 0xc4, 0x0e, 0xdb, 0x9c, 0x3a, 0xf7, 0x7c, 0x31, 0x00, 0x9f, 0x49, 0x3f, 0xa9, 0x38, + 0xdd, 0x8b, 0xe1, 0xfd, 0x9b, 0xd0, 0xd2, 0xf9, 0xe0, 0x0b, 0xdd, 0x8b, 0x91, 0x4b, 0xbe, 0x18, + 0xae, 0xdf, 0x37, 0x24, 0x5b, 0x75, 0xd5, 0x7c, 0xf8, 0x45, 0xd8, 0x8a, 0x58, 0x77, 0x37, 0x5a, + 0x3e, 0x8b, 0x91, 0xb7, 0xcc, 0xeb, 0xbd, 0xa5, 0x1c, 0x42, 0x09, 0xc5, 0xfe, 0x93, 0xae, 0xfe, + 0xcb, 0x5c, 0xbd, 0x1c, 0x4c, 0xc6, 0x9d, 0xf3, 0x82, 0x91, 0xf0, 0x1c, 0x81, 0xd1, 0x8f, 0xd4, + 0x56, 0x51, 0x83, 0xd4, 0xc5, 0x98, 0xee, 0xb7, 0x64, 0x80, 0x49, 0xc5, 0xb1, 0x8b, 0x41, 0xb0, + 0x61, 0x3e, 0x3b, 0x84, 0x5d, 0x08, 0xc4, 0x9d, 0x36, 0x54, 0xa2, 0x9b, 0xaf, 0xf2, 0xf2, 0xb6, + 0x0a, 0xe5, 0x8d, 0xcd, 0xed, 0xad, 0xf6, 0x0a, 0xb9, 0xd8, 0xcd, 0x42, 0x79, 0x65, 0xd3, 0xb2, + 0x9e, 0x6f, 0x75, 0xc9, 0xcd, 0x2e, 0xf9, 0x10, 0x67, 0xe9, 0x67, 0x79, 0xc8, 0x3d, 0x7d, 0x81, + 0x3e, 0x85, 0x22, 0x7b, 0x08, 0x76, 0xc2, 0x7b, 0xc0, 0xd6, 0x49, 0x6f, 0xdd, 0xcc, 0xd7, 0xbe, + 0xf7, 0x5f, 0x3f, 0xfb, 0xe3, 0xdc, 0x8c, 0x59, 0x5b, 0x3c, 0x5c, 0x5e, 0x3c, 0x38, 0x5c, 0xa4, + 0x41, 0xf6, 0xa1, 0x71, 0x07, 0x7d, 0x0d, 0xf2, 0x5b, 0xe3, 0x10, 0x65, 0xbe, 0x13, 0x6c, 0x65, + 0x3f, 0x7f, 0x33, 0x2f, 0x53, 0xa6, 0xd3, 0x26, 0x70, 0xa6, 0xa3, 0x71, 0x48, 0x58, 0x7e, 0x0b, + 0xaa, 0xea, 0xe3, 0xb5, 0x53, 0x1f, 0x0f, 0xb6, 0x4e, 0x7f, 0x18, 0x67, 0x5e, 0xa7, 0x50, 0xaf, + 0x99, 0x88, 0x43, 0xb1, 0xe7, 0x75, 0xea, 0x2c, 0xba, 0x47, 0x2e, 0xca, 0x7c, 0x5a, 0xd8, 0xca, + 0x7e, 0x2b, 0x97, 0x9a, 0x45, 0x78, 0xe4, 0x12, 0x96, 0xdf, 0xe4, 0x8f, 0xe2, 0x7a, 0x21, 0xba, + 0xa1, 0x79, 0xd5, 0xa4, 0xbe, 0xd6, 0x69, 0xcd, 0x67, 0x13, 0x70, 0x90, 0x6b, 0x14, 0xe4, 0x8a, + 0x39, 0xc3, 0x41, 0x7a, 0x11, 0xc9, 0x43, 0xe3, 0xce, 0x52, 0x0f, 0x8a, 0xb4, 0x76, 0x8c, 0x3e, + 0x13, 0x3f, 0x5a, 0x9a, 0xaa, 0x7c, 0x86, 0xa1, 0x63, 0x55, 0x67, 0x73, 0x96, 0x02, 0xd5, 0xcd, + 0x0a, 0x01, 0xa2, 0x95, 0xe3, 0x87, 0xc6, 0x9d, 0xdb, 0xc6, 0x5d, 0x63, 0xe9, 0x6f, 0x8a, 0x50, + 0xa4, 0x35, 0x0a, 0x74, 0x00, 0x20, 0x6b, 0xa4, 0xc9, 0xd9, 0xa5, 0xca, 0xaf, 0xc9, 0xd9, 0xa5, + 0xcb, 0xab, 0x66, 0x8b, 0x82, 0xce, 0x9a, 0xd3, 0x04, 0x94, 0x96, 0x3e, 0x16, 0x69, 0xa5, 0x87, + 0xe8, 0xf1, 0x87, 0x06, 0x2f, 0xd6, 0xb0, 0x6d, 0x86, 0x74, 0xdc, 0x62, 0xf5, 0xd1, 0xe4, 0x72, + 0xd0, 0x94, 0x44, 0xcd, 0x07, 0x14, 0x70, 0xd1, 0x6c, 0x48, 0x40, 0x9f, 0x52, 0x3c, 0x34, 0xee, + 0x7c, 0xd6, 0x34, 0x2f, 0x71, 0x2d, 0x27, 0x7a, 0xd0, 0x77, 0xa0, 0x1e, 0xaf, 0xe4, 0xa1, 0x9b, + 0x1a, 0xac, 0x64, 0x65, 0xb0, 0xf5, 0xe6, 0xc9, 0x44, 0x5c, 0xa6, 0x39, 0x2a, 0x13, 0x07, 0x67, + 0xc8, 0x07, 0x18, 0x8f, 0x6c, 0x42, 0xc4, 0x6d, 0x80, 0xfe, 0xdc, 0xe0, 0xc5, 0x58, 0x59, 0x88, + 0x43, 0x3a, 0xee, 0xa9, 0x7a, 0x5f, 0xeb, 0xd6, 0x29, 0x54, 0x5c, 0x88, 0x0f, 0xa9, 0x10, 0x1f, + 0x98, 0xb3, 0x52, 0x88, 0xd0, 0x19, 0xe2, 0xd0, 0xe3, 0x52, 0x7c, 0x76, 0xcd, 0x7c, 0x2d, 0xa6, + 0x9c, 0x58, 0xaf, 0x34, 0x16, 0x2b, 0x98, 0x69, 0x8d, 0x15, 0xab, 0xc9, 0x69, 0x8d, 0x15, 0xaf, + 0xb6, 0xe9, 0x8c, 0xc5, 0xcb, 0x63, 0x1a, 0x63, 0x45, 0x3d, 0x4b, 0xff, 0x57, 0x80, 0xf2, 0x0a, + 0xfb, 0xe3, 0x1a, 0xe4, 0x41, 0x25, 0x2a, 0x21, 0xa1, 0x39, 0x5d, 0x96, 0x5a, 0x5e, 0xe5, 0x5a, + 0x37, 0x32, 0xfb, 0xb9, 0x40, 0x6f, 0x50, 0x81, 0x5e, 0x37, 0xaf, 0x10, 0x64, 0xfe, 0xf7, 0x3b, + 0x8b, 0x2c, 0x97, 0xb9, 0x68, 0xf7, 0xfb, 0x44, 0x11, 0xbf, 0x0d, 0x35, 0xb5, 0xa0, 0x83, 0xde, + 0xd0, 0x66, 0xc6, 0xd5, 0xea, 0x50, 0xcb, 0x3c, 0x89, 0x84, 0x23, 0xbf, 0x49, 0x91, 0xe7, 0xcc, + 0xab, 0x1a, 0x64, 0x9f, 0x92, 0xc6, 0xc0, 0x59, 0xe5, 0x45, 0x0f, 0x1e, 0x2b, 0xf1, 0xe8, 0xc1, + 0xe3, 0x85, 0x9b, 0x13, 0xc1, 0xc7, 0x94, 0x94, 0x80, 0x07, 0x00, 0xb2, 0x34, 0x82, 0xb4, 0xba, + 0x54, 0x2e, 0xac, 0x49, 0xe7, 0x90, 0xae, 0xaa, 0x98, 0x26, 0x85, 0xe5, 0xeb, 0x2e, 0x01, 0x3b, + 0x70, 0x82, 0x90, 0x6d, 0xcc, 0xa9, 0x58, 0x61, 0x03, 0x69, 0xe7, 0x13, 0xaf, 0x93, 0xb4, 0x6e, + 0x9e, 0x48, 0xc3, 0xd1, 0x6f, 0x51, 0xf4, 0x1b, 0x66, 0x4b, 0x83, 0x3e, 0x62, 0xb4, 0x64, 0xb1, + 0xfd, 0x7f, 0x09, 0xaa, 0xcf, 0x6c, 0xc7, 0x0d, 0xb1, 0x6b, 0xbb, 0x3d, 0x8c, 0x76, 0xa1, 0x48, + 0x63, 0x77, 0xd2, 0x11, 0xab, 0x79, 0xfc, 0xa4, 0x23, 0x8e, 0x25, 0xb2, 0xcd, 0x79, 0x0a, 0xdc, + 0x32, 0x2f, 0x13, 0xe0, 0xa1, 0x64, 0xbd, 0xc8, 0x52, 0xe0, 0xc6, 0x1d, 0xf4, 0x12, 0x4a, 0xbc, + 0x80, 0x9d, 0x60, 0x14, 0x4b, 0xaa, 0xb5, 0xae, 0xe9, 0x3b, 0x75, 0x6b, 0x59, 0x85, 0x09, 0x28, + 0x1d, 0xc1, 0x39, 0x04, 0x90, 0xf5, 0x98, 0xa4, 0x45, 0x53, 0x75, 0x9c, 0xd6, 0x7c, 0x36, 0x81, + 0x4e, 0xa7, 0x2a, 0x66, 0x3f, 0xa2, 0x25, 0xb8, 0xdf, 0x80, 0xc2, 0x13, 0x3b, 0xd8, 0x47, 0x89, + 0xd8, 0xab, 0xbc, 0x20, 0x6d, 0xb5, 0x74, 0x5d, 0x1c, 0xe5, 0x06, 0x45, 0xb9, 0xca, 0x5c, 0x99, + 0x8a, 0x42, 0x5f, 0x54, 0x1a, 0x77, 0x50, 0x1f, 0x4a, 0xec, 0xf9, 0x68, 0x52, 0x7f, 0xb1, 0xb7, + 0xa8, 0x49, 0xfd, 0xc5, 0x5f, 0x9c, 0x9e, 0x8e, 0x32, 0x82, 0x49, 0xf1, 0x28, 0x13, 0x25, 0x9e, + 0xb2, 0x24, 0x5e, 0x72, 0xb6, 0xe6, 0xb2, 0xba, 0x39, 0xd6, 0x4d, 0x8a, 0x75, 0xdd, 0x6c, 0xa6, + 0x6c, 0xc5, 0x29, 0x1f, 0x1a, 0x77, 0xee, 0x1a, 0xe8, 0x3b, 0x00, 0xb2, 0x60, 0x95, 0xda, 0x81, + 0xc9, 0x22, 0x58, 0x6a, 0x07, 0xa6, 0x6a, 0x5d, 0xe6, 0x02, 0xc5, 0xbd, 0x6d, 0xde, 0x4c, 0xe2, + 0x86, 0xbe, 0xed, 0x06, 0x2f, 0xb1, 0xff, 0x3e, 0xcb, 0x96, 0x07, 0xfb, 0xce, 0x88, 0x4c, 0xd9, + 0x87, 0x4a, 0x54, 0x4f, 0x48, 0x7a, 0xdb, 0x64, 0xe5, 0x23, 0xe9, 0x6d, 0x53, 0x85, 0x88, 0xb8, + 0xdb, 0x89, 0xad, 0x16, 0x41, 0x4a, 0x36, 0xe0, 0x5f, 0x35, 0xa0, 0x40, 0x0e, 0xe4, 0xe4, 0x70, + 0x22, 0x93, 0x3d, 0xc9, 0xd9, 0xa7, 0xf2, 0xd5, 0xc9, 0xd9, 0xa7, 0xf3, 0x44, 0xf1, 0xc3, 0x09, + 0xb9, 0xac, 0x2d, 0xb2, 0x2c, 0x0a, 0x99, 0xa9, 0x07, 0x55, 0x25, 0x09, 0x84, 0x34, 0xcc, 0xe2, + 0xf9, 0xef, 0x64, 0xb8, 0xd3, 0x64, 0x90, 0xcc, 0xd7, 0x29, 0xde, 0x65, 0x16, 0xee, 0x28, 0x5e, + 0x9f, 0x51, 0x10, 0x40, 0x3e, 0x3b, 0xbe, 0xef, 0x35, 0xb3, 0x8b, 0xef, 0xfd, 0xf9, 0x6c, 0x82, + 0xcc, 0xd9, 0xc9, 0x8d, 0xff, 0x0a, 0x6a, 0x6a, 0xe2, 0x07, 0x69, 0x84, 0x4f, 0x64, 0xe8, 0x93, + 0x71, 0x44, 0x97, 0x37, 0x8a, 0x7b, 0x36, 0x0a, 0x69, 0x2b, 0x64, 0x04, 0x78, 0x00, 0x65, 0x9e, + 0x00, 0xd2, 0xa9, 0x34, 0x9e, 0xc4, 0xd7, 0xa9, 0x34, 0x91, 0x3d, 0x8a, 0x9f, 0x9e, 0x29, 0x22, + 0xb9, 0x88, 0x8a, 0x58, 0xcd, 0xd1, 0x1e, 0xe3, 0x30, 0x0b, 0x4d, 0x26, 0x6d, 0xb3, 0xd0, 0x94, + 0xfc, 0x40, 0x16, 0xda, 0x1e, 0x0e, 0xb9, 0x3f, 0x10, 0x97, 0x6b, 0x94, 0xc1, 0x4c, 0x8d, 0x8f, + 0xe6, 0x49, 0x24, 0xba, 0xcb, 0x8d, 0x04, 0x14, 0xc1, 0xf1, 0x08, 0x40, 0x26, 0xa3, 0x92, 0x27, + 0x56, 0x6d, 0x9d, 0x20, 0x79, 0x62, 0xd5, 0xe7, 0xb3, 0xe2, 0xbe, 0x4f, 0xe2, 0xb2, 0xbb, 0x15, + 0x41, 0xfe, 0xb1, 0x01, 0x28, 0x9d, 0xae, 0x42, 0xef, 0xea, 0xb9, 0x6b, 0x6b, 0x0e, 0xad, 0xf7, + 0xce, 0x46, 0xac, 0x0b, 0x67, 0x52, 0xa4, 0x1e, 0xa5, 0x1e, 0xbd, 0x22, 0x42, 0x7d, 0xd7, 0x80, + 0xa9, 0x58, 0x8a, 0x0b, 0xbd, 0x95, 0x61, 0xd3, 0x44, 0xe1, 0xa1, 0xf5, 0xf6, 0xa9, 0x74, 0xba, + 0xa3, 0xbc, 0xb2, 0x02, 0xc4, 0x9d, 0xe6, 0xf7, 0x0c, 0xa8, 0xc7, 0x33, 0x61, 0x28, 0x83, 0x77, + 0xaa, 0x5e, 0xd1, 0xba, 0x7d, 0x3a, 0xe1, 0xc9, 0xe6, 0x91, 0xd7, 0x99, 0x01, 0x94, 0x79, 0xca, + 0x4c, 0xb7, 0xf0, 0xe3, 0x05, 0x0e, 0xdd, 0xc2, 0x4f, 0xe4, 0xdb, 0x34, 0x0b, 0xdf, 0xf7, 0x06, + 0x58, 0xd9, 0x66, 0x3c, 0x93, 0x96, 0x85, 0x76, 0xf2, 0x36, 0x4b, 0xa4, 0xe1, 0xb2, 0xd0, 0xe4, + 0x36, 0x13, 0x09, 0x33, 0x94, 0xc1, 0xec, 0x94, 0x6d, 0x96, 0xcc, 0xb7, 0x69, 0xb6, 0x19, 0x05, + 0x54, 0xb6, 0x99, 0x4c, 0x64, 0xe9, 0xb6, 0x59, 0xaa, 0x16, 0xa3, 0xdb, 0x66, 0xe9, 0x5c, 0x98, + 0xc6, 0x8e, 0x14, 0x37, 0xb6, 0xcd, 0x2e, 0x69, 0x52, 0x5d, 0xe8, 0xbd, 0x0c, 0x25, 0x6a, 0x2b, + 0x3b, 0xad, 0xf7, 0xcf, 0x48, 0x9d, 0xb9, 0xc6, 0x99, 0xfa, 0xc5, 0x1a, 0xff, 0x13, 0x03, 0x66, + 0x75, 0xd9, 0x31, 0x94, 0x81, 0x93, 0x51, 0x08, 0x6a, 0x2d, 0x9c, 0x95, 0xfc, 0x64, 0x6d, 0x45, + 0xab, 0xfe, 0x51, 0xe3, 0xdf, 0x3e, 0x9f, 0x33, 0xfe, 0xf3, 0xf3, 0x39, 0xe3, 0xbf, 0x3f, 0x9f, + 0x33, 0x7e, 0xf2, 0xbf, 0x73, 0x13, 0xbb, 0x25, 0xfa, 0x7f, 0x6c, 0x58, 0xfe, 0x79, 0x00, 0x00, + 0x00, 0xff, 0xff, 0x68, 0xa6, 0xc7, 0xa8, 0x58, 0x42, 0x00, 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -11518,6 +11528,13 @@ func (m *StatusResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i -= len(m.XXX_unrecognized) copy(dAtA[i:], m.XXX_unrecognized) } + if len(m.StorageVersion) > 0 { + i -= len(m.StorageVersion) + copy(dAtA[i:], m.StorageVersion) + i = encodeVarintRpc(dAtA, i, uint64(len(m.StorageVersion))) + i-- + dAtA[i] = 0x5a + } if m.IsLearner { i-- if m.IsLearner { @@ -14430,6 +14447,10 @@ func (m *StatusResponse) Size() (n int) { if m.IsLearner { n += 2 } + l = len(m.StorageVersion) + if l > 0 { + n += 1 + l + sovRpc(uint64(l)) + } if m.XXX_unrecognized != nil { n += len(m.XXX_unrecognized) } @@ -22532,6 +22553,38 @@ func (m *StatusResponse) Unmarshal(dAtA []byte) error { } } m.IsLearner = bool(v != 0) + case 11: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field StorageVersion", wireType) + } + var stringLen uint64 + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowRpc + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + stringLen |= uint64(b&0x7F) << shift + if b < 0x80 { + break + } + } + intStringLen := int(stringLen) + if intStringLen < 0 { + return ErrInvalidLengthRpc + } + postIndex := iNdEx + intStringLen + if postIndex < 0 { + return ErrInvalidLengthRpc + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.StorageVersion = string(dAtA[iNdEx:postIndex]) + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipRpc(dAtA[iNdEx:]) diff --git a/api/etcdserverpb/rpc.proto b/api/etcdserverpb/rpc.proto index 29bca1fdf701..d9eeb5c86759 100644 --- a/api/etcdserverpb/rpc.proto +++ b/api/etcdserverpb/rpc.proto @@ -1158,6 +1158,8 @@ message StatusResponse { int64 dbSizeInUse = 9 [(versionpb.etcd_version_field)="3.4"]; // isLearner indicates if the member is raft learner. bool isLearner = 10 [(versionpb.etcd_version_field)="3.4"]; + // storageVersion is the version of the db file. + string storageVersion = 11 [(versionpb.etcd_version_field)="3.6"]; } message AuthEnableRequest { diff --git a/etcdctl/ctlv3/command/printer.go b/etcdctl/ctlv3/command/printer.go index 287f88984cb6..d5fc07377fbd 100644 --- a/etcdctl/ctlv3/command/printer.go +++ b/etcdctl/ctlv3/command/printer.go @@ -211,13 +211,14 @@ func makeEndpointHealthTable(healthList []epHealth) (hdr []string, rows [][]stri } func makeEndpointStatusTable(statusList []epStatus) (hdr []string, rows [][]string) { - hdr = []string{"endpoint", "ID", "version", "db size", "db size in use", "is leader", "is learner", "raft term", + hdr = []string{"endpoint", "ID", "version", "storage version", "db size", "db size in use", "is leader", "is learner", "raft term", "raft index", "raft applied index", "errors"} for _, status := range statusList { rows = append(rows, []string{ status.Ep, fmt.Sprintf("%x", status.Resp.Header.MemberId), status.Resp.Version, + status.Resp.StorageVersion, humanize.Bytes(uint64(status.Resp.DbSize)), humanize.Bytes(uint64(status.Resp.DbSizeInUse)), fmt.Sprint(status.Resp.Leader == status.Resp.Header.MemberId), diff --git a/etcdctl/ctlv3/command/printer_fields.go b/etcdctl/ctlv3/command/printer_fields.go index 2cb5def1b3c2..634ea5a8bcf2 100644 --- a/etcdctl/ctlv3/command/printer_fields.go +++ b/etcdctl/ctlv3/command/printer_fields.go @@ -155,6 +155,7 @@ func (p *fieldsPrinter) EndpointStatus(eps []epStatus) { for _, ep := range eps { p.hdr(ep.Resp.Header) fmt.Printf("\"Version\" : %q\n", ep.Resp.Version) + fmt.Printf("\"StorageVersion\" : %q\n", ep.Resp.StorageVersion) fmt.Println(`"DBSize" :`, ep.Resp.DbSize) fmt.Println(`"DBSizeInUse" :`, ep.Resp.DbSizeInUse) fmt.Println(`"Leader" :`, ep.Resp.Leader) diff --git a/scripts/etcd_version_annotations.txt b/scripts/etcd_version_annotations.txt index 155e9c47d4a8..17b587cf6271 100644 --- a/scripts/etcd_version_annotations.txt +++ b/scripts/etcd_version_annotations.txt @@ -384,6 +384,7 @@ etcdserverpb.StatusResponse.leader: "" etcdserverpb.StatusResponse.raftAppliedIndex: "3.4" etcdserverpb.StatusResponse.raftIndex: "" etcdserverpb.StatusResponse.raftTerm: "" +etcdserverpb.StatusResponse.storageVersion: "3.6" etcdserverpb.StatusResponse.version: "" etcdserverpb.TxnRequest: "3.0" etcdserverpb.TxnRequest.compare: "" diff --git a/server/etcdserver/adapters.go b/server/etcdserver/adapters.go index 110abd7e9519..c3a17c6ee205 100644 --- a/server/etcdserver/adapters.go +++ b/server/etcdserver/adapters.go @@ -26,12 +26,14 @@ import ( "go.etcd.io/etcd/server/v3/storage/schema" ) -// serverVersionAdapter implements Server interface needed by serverversion.Monitor +// serverVersionAdapter implements the interface Server defined in package +// go.etcd.io/etcd/server/v3/etcdserver/version, and it's needed by Monitor +// in the same package. type serverVersionAdapter struct { *EtcdServer } -func newServerVersionAdapter(s *EtcdServer) *serverVersionAdapter { +func NewServerVersionAdapter(s *EtcdServer) *serverVersionAdapter { return &serverVersionAdapter{ EtcdServer: s, } diff --git a/server/etcdserver/api/v3rpc/maintenance.go b/server/etcdserver/api/v3rpc/maintenance.go index 59732f619006..931f712db880 100644 --- a/server/etcdserver/api/v3rpc/maintenance.go +++ b/server/etcdserver/api/v3rpc/maintenance.go @@ -27,6 +27,7 @@ import ( "go.etcd.io/etcd/raft/v3" "go.etcd.io/etcd/server/v3/auth" "go.etcd.io/etcd/server/v3/etcdserver" + serverversion "go.etcd.io/etcd/server/v3/etcdserver/version" "go.etcd.io/etcd/server/v3/storage/backend" "go.etcd.io/etcd/server/v3/storage/mvcc" "go.etcd.io/etcd/server/v3/storage/schema" @@ -76,10 +77,11 @@ type maintenanceServer struct { hdr header cs ClusterStatusGetter d Downgrader + vs serverversion.Server } func NewMaintenanceServer(s *etcdserver.EtcdServer) pb.MaintenanceServer { - srv := &maintenanceServer{lg: s.Cfg.Logger, rg: s, kg: s, bg: s, a: s, lt: s, hdr: newHeader(s), cs: s, d: s} + srv := &maintenanceServer{lg: s.Cfg.Logger, rg: s, kg: s, bg: s, a: s, lt: s, hdr: newHeader(s), cs: s, d: s, vs: etcdserver.NewServerVersionAdapter(s)} if srv.lg == nil { srv.lg = zap.NewNop() } @@ -235,6 +237,9 @@ func (ms *maintenanceServer) Status(ctx context.Context, ar *pb.StatusRequest) ( DbSizeInUse: ms.bg.Backend().SizeInUse(), IsLearner: ms.cs.IsLearner(), } + if storageVersion := ms.vs.GetStorageVersion(); storageVersion != nil { + resp.StorageVersion = storageVersion.String() + } if resp.Leader == raft.None { resp.Errors = append(resp.Errors, etcdserver.ErrNoLeader.Error()) } diff --git a/server/etcdserver/server.go b/server/etcdserver/server.go index d10559a4e21d..e925f47d4501 100644 --- a/server/etcdserver/server.go +++ b/server/etcdserver/server.go @@ -2067,7 +2067,7 @@ func (s *EtcdServer) ClusterVersion() *semver.Version { // monitorClusterVersions every monitorVersionInterval checks if it's the leader and updates cluster version if needed. func (s *EtcdServer) monitorClusterVersions() { - monitor := serverversion.NewMonitor(s.Logger(), newServerVersionAdapter(s)) + monitor := serverversion.NewMonitor(s.Logger(), NewServerVersionAdapter(s)) for { select { case <-s.firstCommitInTerm.Receive(): @@ -2085,7 +2085,7 @@ func (s *EtcdServer) monitorClusterVersions() { // monitorStorageVersion every monitorVersionInterval updates storage version if needed. func (s *EtcdServer) monitorStorageVersion() { - monitor := serverversion.NewMonitor(s.Logger(), newServerVersionAdapter(s)) + monitor := serverversion.NewMonitor(s.Logger(), NewServerVersionAdapter(s)) for { select { case <-time.After(monitorVersionInterval): @@ -2175,7 +2175,7 @@ func (s *EtcdServer) updateClusterVersionV3(ver string) { // monitorDowngrade every DowngradeCheckTime checks if it's the leader and cancels downgrade if needed. func (s *EtcdServer) monitorDowngrade() { - monitor := serverversion.NewMonitor(s.Logger(), newServerVersionAdapter(s)) + monitor := serverversion.NewMonitor(s.Logger(), NewServerVersionAdapter(s)) t := s.Cfg.DowngradeCheckTime if t == 0 { return @@ -2294,5 +2294,5 @@ func (s *EtcdServer) raftStatus() raft.Status { } func (s *EtcdServer) Version() *serverversion.Manager { - return serverversion.NewManager(s.Logger(), newServerVersionAdapter(s)) + return serverversion.NewManager(s.Logger(), NewServerVersionAdapter(s)) } From d0688e0158045b773f1326a5c69a71bf89406531 Mon Sep 17 00:00:00 2001 From: ahrtr Date: Fri, 11 Mar 2022 11:09:43 +0800 Subject: [PATCH 08/30] update the example output of 'etcdctl endpoint status -w table' --- .../dev-guide/apispec/swagger/rpc.swagger.json | 2 +- api/etcdserverpb/rpc.pb.go | 2 +- api/etcdserverpb/rpc.proto | 2 +- etcdctl/README.md | 14 +++++++------- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/Documentation/dev-guide/apispec/swagger/rpc.swagger.json b/Documentation/dev-guide/apispec/swagger/rpc.swagger.json index 35cc83fbddcb..dd12cf315cc7 100644 --- a/Documentation/dev-guide/apispec/swagger/rpc.swagger.json +++ b/Documentation/dev-guide/apispec/swagger/rpc.swagger.json @@ -2772,7 +2772,7 @@ "format": "uint64" }, "storageVersion": { - "description": "storageVersion is the version of the db file.", + "description": "storageVersion is the version of the db file. It might be get updated with delay in relationship to the target cluster version.", "type": "string" }, "version": { diff --git a/api/etcdserverpb/rpc.pb.go b/api/etcdserverpb/rpc.pb.go index 52c53f8afcb1..3ca2b50d4ba6 100644 --- a/api/etcdserverpb/rpc.pb.go +++ b/api/etcdserverpb/rpc.pb.go @@ -4236,7 +4236,7 @@ type StatusResponse struct { DbSizeInUse int64 `protobuf:"varint,9,opt,name=dbSizeInUse,proto3" json:"dbSizeInUse,omitempty"` // isLearner indicates if the member is raft learner. IsLearner bool `protobuf:"varint,10,opt,name=isLearner,proto3" json:"isLearner,omitempty"` - // storageVersion is the version of the db file. + // storageVersion is the version of the db file. It might be get updated with delay in relationship to the target cluster version. StorageVersion string `protobuf:"bytes,11,opt,name=storageVersion,proto3" json:"storageVersion,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` diff --git a/api/etcdserverpb/rpc.proto b/api/etcdserverpb/rpc.proto index d9eeb5c86759..9d60c0b9e5db 100644 --- a/api/etcdserverpb/rpc.proto +++ b/api/etcdserverpb/rpc.proto @@ -1158,7 +1158,7 @@ message StatusResponse { int64 dbSizeInUse = 9 [(versionpb.etcd_version_field)="3.4"]; // isLearner indicates if the member is raft learner. bool isLearner = 10 [(versionpb.etcd_version_field)="3.4"]; - // storageVersion is the version of the db file. + // storageVersion is the version of the db file. It might be get updated with delay in relationship to the target cluster version. string storageVersion = 11 [(versionpb.etcd_version_field)="3.6"]; } diff --git a/etcdctl/README.md b/etcdctl/README.md index 990a3fdcd18f..e0a11e635d0e 100644 --- a/etcdctl/README.md +++ b/etcdctl/README.md @@ -809,13 +809,13 @@ Get the status for all endpoints in the cluster associated with the default endp ```bash ./etcdctl -w table endpoint --cluster status -+------------------------+------------------+----------------+---------+-----------+-----------+------------+ -| ENDPOINT | ID | VERSION | DB SIZE | IS LEADER | RAFT TERM | RAFT INDEX | -+------------------------+------------------+----------------+---------+-----------+-----------+------------+ -| http://127.0.0.1:2379 | 8211f1d0f64f3269 | 3.2.0-rc.1+git | 25 kB | false | 2 | 8 | -| http://127.0.0.1:22379 | 91bc3c398fb3c146 | 3.2.0-rc.1+git | 25 kB | false | 2 | 8 | -| http://127.0.0.1:32379 | fd422379fda50e48 | 3.2.0-rc.1+git | 25 kB | true | 2 | 8 | -+------------------------+------------------+----------------+---------+-----------+-----------+------------+ ++------------------------+------------------+---------------+-----------------+---------+----------------+-----------+------------+-----------+------------+--------------------+--------+ +| ENDPOINT | ID | VERSION | STORAGE VERSION | DB SIZE | DB SIZE IN USE | IS LEADER | IS LEARNER | RAFT TERM | RAFT INDEX | RAFT APPLIED INDEX | ERRORS | ++------------------------+------------------+---------------+-----------------+---------+----------------+-----------+------------+-----------+------------+--------------------+--------+ +| http://127.0.0.1:2379 | 8211f1d0f64f3269 | 3.6.0-alpha.0 | 3.6.0 | 25 kB | 25 kB | false | false | 2 | 8 | 8 | | +| http://127.0.0.1:22379 | 91bc3c398fb3c146 | 3.6.0-alpha.0 | 3.6.0 | 25 kB | 25 kB | true | false | 2 | 8 | 8 | | +| http://127.0.0.1:32379 | fd422379fda50e48 | 3.6.0-alpha.0 | 3.6.0 | 25 kB | 25 kB | false | false | 2 | 8 | 8 | | ++------------------------+------------------+---------------+-----------------+---------+----------------+-----------+------------+-----------+------------+--------------------+--------+ ``` ### ENDPOINT HASHKV From 66eb3dbbdcad156c8a122940486f6ad194aa6dfe Mon Sep 17 00:00:00 2001 From: kkkkun Date: Fri, 18 Mar 2022 19:17:48 +0800 Subject: [PATCH 09/30] tests: Migrate defrag tests to common framework --- tests/common/defrag_test.go | 46 ++++++++++++++++++ tests/e2e/ctl_v3_alarm_test.go | 2 +- tests/e2e/ctl_v3_compact_test.go | 77 ------------------------------- tests/e2e/ctl_v3_defrag_test.go | 24 ++++------ tests/e2e/ctl_v3_snapshot_test.go | 3 -- tests/framework/config/client.go | 7 ++- tests/framework/e2e/etcdctl.go | 14 ++++++ tests/framework/integration.go | 16 +++++++ tests/framework/interface.go | 1 + 9 files changed, 94 insertions(+), 96 deletions(-) create mode 100644 tests/common/defrag_test.go delete mode 100644 tests/e2e/ctl_v3_compact_test.go diff --git a/tests/common/defrag_test.go b/tests/common/defrag_test.go new file mode 100644 index 000000000000..6cfce4c68070 --- /dev/null +++ b/tests/common/defrag_test.go @@ -0,0 +1,46 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "testing" + "time" + + "go.etcd.io/etcd/tests/v3/framework/config" + "go.etcd.io/etcd/tests/v3/framework/testutils" +) + +func TestDefragOnline(t *testing.T) { + testRunner.BeforeTest(t) + options := config.DefragOption{Timeout: 10 * time.Second} + clus := testRunner.NewCluster(t, config.ClusterConfig{ClusterSize: 3}) + testutils.ExecuteWithTimeout(t, 10*time.Second, func() { + defer clus.Close() + var kvs = []testutils.KV{{Key: "key", Val: "val1"}, {Key: "key", Val: "val2"}, {Key: "key", Val: "val3"}} + for i := range kvs { + if err := clus.Client().Put(kvs[i].Key, kvs[i].Val); err != nil { + t.Fatalf("compactTest #%d: put kv error (%v)", i, err) + } + } + _, err := clus.Client().Compact(4, config.CompactOption{Physical: true, Timeout: 10 * time.Second}) + if err != nil { + t.Fatalf("defrag_test: compact with revision error (%v)", err) + } + + if err = clus.Client().Defragment(options); err != nil { + t.Fatalf("defrag_test: defrag error (%v)", err) + } + }) +} diff --git a/tests/e2e/ctl_v3_alarm_test.go b/tests/e2e/ctl_v3_alarm_test.go index f33654f000a4..b99d3a6a388e 100644 --- a/tests/e2e/ctl_v3_alarm_test.go +++ b/tests/e2e/ctl_v3_alarm_test.go @@ -21,7 +21,7 @@ import ( "testing" "time" - "go.etcd.io/etcd/client/v3" + clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/e2e" ) diff --git a/tests/e2e/ctl_v3_compact_test.go b/tests/e2e/ctl_v3_compact_test.go deleted file mode 100644 index f29e580d9f65..000000000000 --- a/tests/e2e/ctl_v3_compact_test.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2016 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package e2e - -import ( - "strconv" - "strings" - "testing" - - "go.etcd.io/etcd/tests/v3/framework/e2e" -) - -func TestCtlV3Compact(t *testing.T) { testCtl(t, compactTest) } -func TestCtlV3CompactPhysical(t *testing.T) { testCtl(t, compactTest, withCompactPhysical()) } - -func compactTest(cx ctlCtx) { - compactPhysical := cx.compactPhysical - if err := ctlV3Compact(cx, 2, compactPhysical); err != nil { - if !strings.Contains(err.Error(), "required revision is a future revision") { - cx.t.Fatal(err) - } - } else { - cx.t.Fatalf("expected '...future revision' error, got ") - } - - var kvs = []kv{{"key", "val1"}, {"key", "val2"}, {"key", "val3"}} - for i := range kvs { - if err := ctlV3Put(cx, kvs[i].key, kvs[i].val, ""); err != nil { - cx.t.Fatalf("compactTest #%d: ctlV3Put error (%v)", i, err) - } - } - - if err := ctlV3Get(cx, []string{"key", "--rev", "3"}, kvs[1:2]...); err != nil { - cx.t.Errorf("compactTest: ctlV3Get error (%v)", err) - } - - if err := ctlV3Compact(cx, 4, compactPhysical); err != nil { - cx.t.Fatal(err) - } - - if err := ctlV3Get(cx, []string{"key", "--rev", "3"}, kvs[1:2]...); err != nil { - if !strings.Contains(err.Error(), "required revision has been compacted") { - cx.t.Errorf("compactTest: ctlV3Get error (%v)", err) - } - } else { - cx.t.Fatalf("expected '...has been compacted' error, got ") - } - - if err := ctlV3Compact(cx, 2, compactPhysical); err != nil { - if !strings.Contains(err.Error(), "required revision has been compacted") { - cx.t.Fatal(err) - } - } else { - cx.t.Fatalf("expected '...has been compacted' error, got ") - } -} - -func ctlV3Compact(cx ctlCtx, rev int64, physical bool) error { - rs := strconv.FormatInt(rev, 10) - cmdArgs := append(cx.PrefixArgs(), "compact", rs) - if physical { - cmdArgs = append(cmdArgs, "--physical") - } - return e2e.SpawnWithExpectWithEnv(cmdArgs, cx.envMap, "compacted revision "+rs) -} diff --git a/tests/e2e/ctl_v3_defrag_test.go b/tests/e2e/ctl_v3_defrag_test.go index c47ed3f4cdfc..889bbbd078bf 100644 --- a/tests/e2e/ctl_v3_defrag_test.go +++ b/tests/e2e/ctl_v3_defrag_test.go @@ -15,13 +15,12 @@ package e2e import ( + "strconv" "testing" "go.etcd.io/etcd/tests/v3/framework/e2e" ) -func TestCtlV3DefragOnline(t *testing.T) { testCtl(t, defragOnlineTest) } - func TestCtlV3DefragOfflineEtcdutl(t *testing.T) { testCtlWithOffline(t, maintenanceInitKeys, defragOfflineTest, withEtcdutl()) } @@ -35,18 +34,6 @@ func maintenanceInitKeys(cx ctlCtx) { } } -func defragOnlineTest(cx ctlCtx) { - maintenanceInitKeys(cx) - - if err := ctlV3Compact(cx, 4, cx.compactPhysical); err != nil { - cx.t.Fatal(err) - } - - if err := ctlV3OnlineDefrag(cx); err != nil { - cx.t.Fatalf("defragTest ctlV3Defrag error (%v)", err) - } -} - func ctlV3OnlineDefrag(cx ctlCtx) error { cmdArgs := append(cx.PrefixArgs(), "defrag") lines := make([]string, cx.epc.Cfg.ClusterSize) @@ -67,3 +54,12 @@ func defragOfflineTest(cx ctlCtx) { cx.t.Fatalf("defragTest ctlV3Defrag error (%v)", err) } } + +func ctlV3Compact(cx ctlCtx, rev int64, physical bool) error { + rs := strconv.FormatInt(rev, 10) + cmdArgs := append(cx.PrefixArgs(), "compact", rs) + if physical { + cmdArgs = append(cmdArgs, "--physical") + } + return e2e.SpawnWithExpectWithEnv(cmdArgs, cx.envMap, "compacted revision "+rs) +} diff --git a/tests/e2e/ctl_v3_snapshot_test.go b/tests/e2e/ctl_v3_snapshot_test.go index 59e1dc770f27..c952b3aaed1a 100644 --- a/tests/e2e/ctl_v3_snapshot_test.go +++ b/tests/e2e/ctl_v3_snapshot_test.go @@ -294,9 +294,6 @@ func testIssue6361(t *testing.T, etcdutl bool) { // For storageVersion to be stored, all fields expected 3.6 fields need to be set. This happens after first WAL snapshot. // In this test we lower SnapshotCount to 1 to ensure WAL snapshot is triggered. -func TestCtlV3SnapshotVersion(t *testing.T) { - testCtl(t, snapshotVersionTest, withCfg(e2e.EtcdProcessClusterConfig{SnapshotCount: 1})) -} func TestCtlV3SnapshotVersionEtcdutl(t *testing.T) { testCtl(t, snapshotVersionTest, withEtcdutl(), withCfg(e2e.EtcdProcessClusterConfig{SnapshotCount: 1})) } diff --git a/tests/framework/config/client.go b/tests/framework/config/client.go index c41565b47186..6160d841da7f 100644 --- a/tests/framework/config/client.go +++ b/tests/framework/config/client.go @@ -15,8 +15,9 @@ package config import ( - clientv3 "go.etcd.io/etcd/client/v3" "time" + + clientv3 "go.etcd.io/etcd/client/v3" ) type GetOptions struct { @@ -41,3 +42,7 @@ type CompactOption struct { Physical bool Timeout time.Duration } + +type DefragOption struct { + Timeout time.Duration +} diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index 18c436748264..f11bdceadfc5 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -243,3 +243,17 @@ func (ctl *EtcdctlV3) Health() error { return SpawnWithExpects(args, map[string]string{}, lines...) } + +func (ctl *EtcdctlV3) Defragment(o config.DefragOption) error { + + args := append(ctl.cmdArgs(), "defrag") + if o.Timeout != 0 { + args = append(args, fmt.Sprintf("--command-timeout=%s", o.Timeout)) + } + lines := make([]string, len(ctl.endpoints)) + for i := range lines { + lines[i] = "Finished defragmenting etcd member" + } + _, err := SpawnWithExpectLines(args, map[string]string{}, lines...) + return err +} diff --git a/tests/framework/integration.go b/tests/framework/integration.go index 73e15f130ff0..94c1925b7c4c 100644 --- a/tests/framework/integration.go +++ b/tests/framework/integration.go @@ -196,3 +196,19 @@ func (c integrationClient) Health() error { } return nil } + +func (c integrationClient) Defragment(o config.DefragOption) error { + ctx := context.Background() + if o.Timeout != 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, o.Timeout) + defer cancel() + } + for _, ep := range c.Endpoints() { + _, err := c.Client.Defragment(ctx, ep) + if err != nil { + return err + } + } + return nil +} diff --git a/tests/framework/interface.go b/tests/framework/interface.go index f6982f321a89..db4b3b980cc3 100644 --- a/tests/framework/interface.go +++ b/tests/framework/interface.go @@ -41,4 +41,5 @@ type Client interface { Status() ([]*clientv3.StatusResponse, error) HashKV(rev int64) ([]*clientv3.HashKVResponse, error) Health() error + Defragment(opts config.DefragOption) error } From a5335847386ed5b4b729bf24c5a3b3e58f995d09 Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Wed, 16 Mar 2022 12:50:50 +0000 Subject: [PATCH 10/30] tests/framework: add basic lease operations --- tests/framework/config/client.go | 4 ++++ tests/framework/e2e/etcdctl.go | 36 +++++++++++++++++++++++++++++++- tests/framework/integration.go | 16 ++++++++++++++ tests/framework/interface.go | 3 ++- 4 files changed, 57 insertions(+), 2 deletions(-) diff --git a/tests/framework/config/client.go b/tests/framework/config/client.go index 6160d841da7f..c63dd5995002 100644 --- a/tests/framework/config/client.go +++ b/tests/framework/config/client.go @@ -46,3 +46,7 @@ type CompactOption struct { type DefragOption struct { Timeout time.Duration } + +type LeaseOption struct { + WithAttachedKeys bool +} diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index f11bdceadfc5..aae7a406e370 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -17,6 +17,7 @@ package e2e import ( "encoding/json" "fmt" + "strconv" "strings" clientv3 "go.etcd.io/etcd/client/v3" @@ -185,7 +186,6 @@ func (ctl *EtcdctlV3) Status() ([]*clientv3.StatusResponse, error) { if err != nil { return nil, err } - var epStatus []*struct { Endpoint string Status *clientv3.StatusResponse @@ -241,7 +241,41 @@ func (ctl *EtcdctlV3) Health() error { lines[i] = "is healthy" } return SpawnWithExpects(args, map[string]string{}, lines...) +} +func (ctl *EtcdctlV3) Grant(ttl int64) (*clientv3.LeaseGrantResponse, error) { + args := ctl.cmdArgs() + args = append(args, "lease", "grant", strconv.FormatInt(ttl, 10), "-w", "json") + cmd, err := SpawnCmd(args, nil) + if err != nil { + return nil, err + } + var resp clientv3.LeaseGrantResponse + line, err := cmd.Expect("ID") + if err != nil { + return nil, err + } + err = json.Unmarshal([]byte(line), &resp) + return &resp, err +} + +func (ctl *EtcdctlV3) TimeToLive(id clientv3.LeaseID, o config.LeaseOption) (*clientv3.LeaseTimeToLiveResponse, error) { + args := ctl.cmdArgs() + args = append(args, "lease", "timetolive", strconv.FormatInt(int64(id), 16), "-w", "json") + if o.WithAttachedKeys { + args = append(args, "--keys") + } + cmd, err := SpawnCmd(args, nil) + if err != nil { + return nil, err + } + var resp clientv3.LeaseTimeToLiveResponse + line, err := cmd.Expect("id") + if err != nil { + return nil, err + } + err = json.Unmarshal([]byte(line), &resp) + return &resp, err } func (ctl *EtcdctlV3) Defragment(o config.DefragOption) error { diff --git a/tests/framework/integration.go b/tests/framework/integration.go index 94c1925b7c4c..b0a7cd17a161 100644 --- a/tests/framework/integration.go +++ b/tests/framework/integration.go @@ -212,3 +212,19 @@ func (c integrationClient) Defragment(o config.DefragOption) error { } return nil } + +func (c integrationClient) Grant(ttl int64) (*clientv3.LeaseGrantResponse, error) { + ctx := context.Background() + return c.Client.Grant(ctx, ttl) +} + +func (c integrationClient) TimeToLive(id clientv3.LeaseID, o config.LeaseOption) (*clientv3.LeaseTimeToLiveResponse, error) { + ctx := context.Background() + + leaseOpts := []clientv3.LeaseOption{} + if o.WithAttachedKeys { + leaseOpts = append(leaseOpts, clientv3.WithAttachedKeys()) + } + + return c.Client.TimeToLive(ctx, id, leaseOpts...) +} diff --git a/tests/framework/interface.go b/tests/framework/interface.go index db4b3b980cc3..7ee691d99c74 100644 --- a/tests/framework/interface.go +++ b/tests/framework/interface.go @@ -37,9 +37,10 @@ type Client interface { Get(key string, opts config.GetOptions) (*clientv3.GetResponse, error) Delete(key string, opts config.DeleteOptions) (*clientv3.DeleteResponse, error) Compact(rev int64, opts config.CompactOption) (*clientv3.CompactResponse, error) - Status() ([]*clientv3.StatusResponse, error) HashKV(rev int64) ([]*clientv3.HashKVResponse, error) Health() error Defragment(opts config.DefragOption) error + Grant(ttl int64) (*clientv3.LeaseGrantResponse, error) + TimeToLive(id clientv3.LeaseID, opts config.LeaseOption) (*clientv3.LeaseTimeToLiveResponse, error) } From 6b7be72a4330e9ca3176c2c5401fbceac4ccf655 Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Wed, 16 Mar 2022 12:51:29 +0000 Subject: [PATCH 11/30] tests: migrate TestCtlV3LeaseGrantTimeToLive.* to common --- tests/common/lease_test.go | 71 ++++++++++++++++++++++++++++++++++ tests/e2e/ctl_v3_lease_test.go | 40 ------------------- 2 files changed, 71 insertions(+), 40 deletions(-) create mode 100644 tests/common/lease_test.go diff --git a/tests/common/lease_test.go b/tests/common/lease_test.go new file mode 100644 index 000000000000..df2c60a17340 --- /dev/null +++ b/tests/common/lease_test.go @@ -0,0 +1,71 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" + "go.etcd.io/etcd/tests/v3/framework/config" + "go.etcd.io/etcd/tests/v3/framework/testutils" +) + +func TestLeaseGrantTimeToLive(t *testing.T) { + testRunner.BeforeTest(t) + + tcs := []struct { + name string + config config.ClusterConfig + }{ + { + name: "NoTLS", + config: config.ClusterConfig{ClusterSize: 1}, + }, + { + name: "PeerTLS", + config: config.ClusterConfig{ClusterSize: 3, PeerTLS: config.ManualTLS}, + }, + { + name: "PeerAutoTLS", + config: config.ClusterConfig{ClusterSize: 3, PeerTLS: config.AutoTLS}, + }, + { + name: "ClientTLS", + config: config.ClusterConfig{ClusterSize: 1, ClientTLS: config.ManualTLS}, + }, + { + name: "ClientAutoTLS", + config: config.ClusterConfig{ClusterSize: 1, ClientTLS: config.AutoTLS}, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + clus := testRunner.NewCluster(t, tc.config) + defer clus.Close() + cc := clus.Client() + + testutils.ExecuteWithTimeout(t, 10*time.Second, func() { + ttl := int64(10) + leaseResp, err := cc.Grant(ttl) + require.NoError(t, err) + + ttlResp, err := cc.TimeToLive(leaseResp.ID, config.LeaseOption{}) + require.NoError(t, err) + require.Equal(t, ttl, ttlResp.GrantedTTL) + }) + }) + } +} diff --git a/tests/e2e/ctl_v3_lease_test.go b/tests/e2e/ctl_v3_lease_test.go index d13309bc5bdd..f1388c5a5ef2 100644 --- a/tests/e2e/ctl_v3_lease_test.go +++ b/tests/e2e/ctl_v3_lease_test.go @@ -24,20 +24,6 @@ import ( "go.etcd.io/etcd/tests/v3/framework/e2e" ) -func TestCtlV3LeaseGrantTimeToLive(t *testing.T) { testCtl(t, leaseTestGrantTimeToLive) } -func TestCtlV3LeaseGrantTimeToLiveNoTLS(t *testing.T) { - testCtl(t, leaseTestGrantTimeToLive, withCfg(*e2e.NewConfigNoTLS())) -} -func TestCtlV3LeaseGrantTimeToLiveClientTLS(t *testing.T) { - testCtl(t, leaseTestGrantTimeToLive, withCfg(*e2e.NewConfigClientTLS())) -} -func TestCtlV3LeaseGrantTimeToLiveClientAutoTLS(t *testing.T) { - testCtl(t, leaseTestGrantTimeToLive, withCfg(*e2e.NewConfigClientAutoTLS())) -} -func TestCtlV3LeaseGrantTimeToLivePeerTLS(t *testing.T) { - testCtl(t, leaseTestGrantTimeToLive, withCfg(*e2e.NewConfigPeerTLS())) -} - func TestCtlV3LeaseGrantLeases(t *testing.T) { testCtl(t, leaseTestGrantLeaseListed) } func TestCtlV3LeaseGrantLeasesNoTLS(t *testing.T) { testCtl(t, leaseTestGrantLeaseListed, withCfg(*e2e.NewConfigNoTLS())) @@ -108,32 +94,6 @@ func TestCtlV3LeaseRevokePeerTLS(t *testing.T) { testCtl(t, leaseTestRevoked, withCfg(*e2e.NewConfigPeerTLS())) } -func leaseTestGrantTimeToLive(cx ctlCtx) { - id, err := ctlV3LeaseGrant(cx, 10) - if err != nil { - cx.t.Fatalf("leaseTestGrantTimeToLive: ctlV3LeaseGrant error (%v)", err) - } - - cmdArgs := append(cx.PrefixArgs(), "lease", "timetolive", id, "--keys") - proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap) - if err != nil { - cx.t.Fatalf("leaseTestGrantTimeToLive: error (%v)", err) - } - line, err := proc.Expect(" granted with TTL(") - if err != nil { - cx.t.Fatalf("leaseTestGrantTimeToLive: error (%v)", err) - } - if err = proc.Close(); err != nil { - cx.t.Fatalf("leaseTestGrantTimeToLive: error (%v)", err) - } - if !strings.Contains(line, ", attached keys") { - cx.t.Fatalf("leaseTestGrantTimeToLive: expected 'attached keys', got %q", line) - } - if !strings.Contains(line, id) { - cx.t.Fatalf("leaseTestGrantTimeToLive: expected leaseID %q, got %q", id, line) - } -} - func leaseTestGrantLeaseListed(cx ctlCtx) { err := leaseTestGrantLeasesList(cx) if err != nil { From 68e6493977ccd7ab6d7fe180e6af3748dc9fc79c Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Wed, 16 Mar 2022 15:00:29 +0000 Subject: [PATCH 12/30] tests/framework: add Client.LeaseList --- tests/framework/e2e/etcdctl.go | 17 ++++++++++++++++- tests/framework/integration.go | 6 ++++++ tests/framework/interface.go | 1 + 3 files changed, 23 insertions(+), 1 deletion(-) diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index aae7a406e370..f0714aeb7ff2 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -279,7 +279,6 @@ func (ctl *EtcdctlV3) TimeToLive(id clientv3.LeaseID, o config.LeaseOption) (*cl } func (ctl *EtcdctlV3) Defragment(o config.DefragOption) error { - args := append(ctl.cmdArgs(), "defrag") if o.Timeout != 0 { args = append(args, fmt.Sprintf("--command-timeout=%s", o.Timeout)) @@ -291,3 +290,19 @@ func (ctl *EtcdctlV3) Defragment(o config.DefragOption) error { _, err := SpawnWithExpectLines(args, map[string]string{}, lines...) return err } + +func (ctl *EtcdctlV3) LeaseList() (*clientv3.LeaseLeasesResponse, error) { + args := ctl.cmdArgs() + args = append(args, "lease", "list", "-w", "json") + cmd, err := SpawnCmd(args, nil) + if err != nil { + return nil, err + } + var resp clientv3.LeaseLeasesResponse + line, err := cmd.Expect("id") + if err != nil { + return nil, err + } + err = json.Unmarshal([]byte(line), &resp) + return &resp, err +} diff --git a/tests/framework/integration.go b/tests/framework/integration.go index b0a7cd17a161..6a5eac295bbb 100644 --- a/tests/framework/integration.go +++ b/tests/framework/integration.go @@ -228,3 +228,9 @@ func (c integrationClient) TimeToLive(id clientv3.LeaseID, o config.LeaseOption) return c.Client.TimeToLive(ctx, id, leaseOpts...) } + +func (c integrationClient) LeaseList() (*clientv3.LeaseLeasesResponse, error) { + ctx := context.Background() + + return c.Client.Leases(ctx) +} diff --git a/tests/framework/interface.go b/tests/framework/interface.go index 7ee691d99c74..c4fa9bda1c6c 100644 --- a/tests/framework/interface.go +++ b/tests/framework/interface.go @@ -43,4 +43,5 @@ type Client interface { Defragment(opts config.DefragOption) error Grant(ttl int64) (*clientv3.LeaseGrantResponse, error) TimeToLive(id clientv3.LeaseID, opts config.LeaseOption) (*clientv3.LeaseTimeToLiveResponse, error) + LeaseList() (*clientv3.LeaseLeasesResponse, error) } From b7beaf9c62e5472b7168a0a565af289f73a143f2 Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Wed, 16 Mar 2022 15:03:36 +0000 Subject: [PATCH 13/30] tests: migrate TestCtlV3LeaseGrantLeases.* to common --- tests/common/lease_test.go | 78 ++++++++++++++++++++++++++++++++++ tests/e2e/ctl_v3_auth_test.go | 18 ++++++++ tests/e2e/ctl_v3_lease_test.go | 39 ----------------- 3 files changed, 96 insertions(+), 39 deletions(-) diff --git a/tests/common/lease_test.go b/tests/common/lease_test.go index df2c60a17340..5676fbf24789 100644 --- a/tests/common/lease_test.go +++ b/tests/common/lease_test.go @@ -19,6 +19,7 @@ import ( "time" "github.com/stretchr/testify/require" + clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) @@ -69,3 +70,80 @@ func TestLeaseGrantTimeToLive(t *testing.T) { }) } } + +func TestLeaseGrantAndList(t *testing.T) { + testRunner.BeforeTest(t) + + tcs := []struct { + name string + config config.ClusterConfig + }{ + { + name: "NoTLS", + config: config.ClusterConfig{ClusterSize: 1}, + }, + { + name: "PeerTLS", + config: config.ClusterConfig{ClusterSize: 3, PeerTLS: config.ManualTLS}, + }, + { + name: "PeerAutoTLS", + config: config.ClusterConfig{ClusterSize: 3, PeerTLS: config.AutoTLS}, + }, + { + name: "ClientTLS", + config: config.ClusterConfig{ClusterSize: 1, ClientTLS: config.ManualTLS}, + }, + { + name: "ClientAutoTLS", + config: config.ClusterConfig{ClusterSize: 1, ClientTLS: config.AutoTLS}, + }, + } + for _, tc := range tcs { + nestedCases := []struct { + name string + leaseCount int + }{ + { + name: "no_leases", + leaseCount: 0, + }, + { + name: "one_lease", + leaseCount: 1, + }, + { + name: "many_leases", + leaseCount: 3, + }, + } + + for _, nc := range nestedCases { + t.Run(tc.name+"/"+nc.name, func(t *testing.T) { + clus := testRunner.NewCluster(t, tc.config) + defer clus.Close() + cc := clus.Client() + + testutils.ExecuteWithTimeout(t, 10*time.Second, func() { + createdLeases := []clientv3.LeaseID{} + for i := 0; i < nc.leaseCount; i++ { + leaseResp, err := cc.Grant(10) + require.NoError(t, err) + createdLeases = append(createdLeases, leaseResp.ID) + } + + resp, err := cc.LeaseList() + require.NoError(t, err) + require.Len(t, resp.Leases, nc.leaseCount) + + returnedLeases := make([]clientv3.LeaseID, 0, nc.leaseCount) + for _, status := range resp.Leases { + returnedLeases = append(returnedLeases, status.ID) + } + + require.ElementsMatch(t, createdLeases, returnedLeases) + }) + }) + } + } +} diff --git a/tests/e2e/ctl_v3_auth_test.go b/tests/e2e/ctl_v3_auth_test.go index fd15a9e617ec..8fd64097200f 100644 --- a/tests/e2e/ctl_v3_auth_test.go +++ b/tests/e2e/ctl_v3_auth_test.go @@ -862,6 +862,24 @@ func authLeaseTestLeaseGrantLeases(cx ctlCtx) { } } +func leaseTestGrantLeasesList(cx ctlCtx) error { + id, err := ctlV3LeaseGrant(cx, 10) + if err != nil { + return fmt.Errorf("ctlV3LeaseGrant error (%v)", err) + } + + cmdArgs := append(cx.PrefixArgs(), "lease", "list") + proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap) + if err != nil { + return fmt.Errorf("lease list failed (%v)", err) + } + _, err = proc.Expect(id) + if err != nil { + return fmt.Errorf("lease id not in returned list (%v)", err) + } + return proc.Close() +} + func authLeaseTestLeaseRevoke(cx ctlCtx) { cx.user, cx.pass = "root", "root" authSetupTestUser(cx) diff --git a/tests/e2e/ctl_v3_lease_test.go b/tests/e2e/ctl_v3_lease_test.go index f1388c5a5ef2..4772ef2159c6 100644 --- a/tests/e2e/ctl_v3_lease_test.go +++ b/tests/e2e/ctl_v3_lease_test.go @@ -24,20 +24,6 @@ import ( "go.etcd.io/etcd/tests/v3/framework/e2e" ) -func TestCtlV3LeaseGrantLeases(t *testing.T) { testCtl(t, leaseTestGrantLeaseListed) } -func TestCtlV3LeaseGrantLeasesNoTLS(t *testing.T) { - testCtl(t, leaseTestGrantLeaseListed, withCfg(*e2e.NewConfigNoTLS())) -} -func TestCtlV3LeaseGrantLeasesClientTLS(t *testing.T) { - testCtl(t, leaseTestGrantLeaseListed, withCfg(*e2e.NewConfigClientTLS())) -} -func TestCtlV3LeaseGrantLeasesClientAutoTLS(t *testing.T) { - testCtl(t, leaseTestGrantLeaseListed, withCfg(*e2e.NewConfigClientAutoTLS())) -} -func TestCtlV3LeaseGrantLeasesPeerTLS(t *testing.T) { - testCtl(t, leaseTestGrantLeaseListed, withCfg(*e2e.NewConfigPeerTLS())) -} - func TestCtlV3LeaseTestTimeToLiveExpired(t *testing.T) { testCtl(t, leaseTestTimeToLiveExpired) } func TestCtlV3LeaseTestTimeToLiveExpiredNoTLS(t *testing.T) { testCtl(t, leaseTestTimeToLiveExpired, withCfg(*e2e.NewConfigNoTLS())) @@ -94,31 +80,6 @@ func TestCtlV3LeaseRevokePeerTLS(t *testing.T) { testCtl(t, leaseTestRevoked, withCfg(*e2e.NewConfigPeerTLS())) } -func leaseTestGrantLeaseListed(cx ctlCtx) { - err := leaseTestGrantLeasesList(cx) - if err != nil { - cx.t.Fatalf("leaseTestGrantLeasesList: (%v)", err) - } -} - -func leaseTestGrantLeasesList(cx ctlCtx) error { - id, err := ctlV3LeaseGrant(cx, 10) - if err != nil { - return fmt.Errorf("ctlV3LeaseGrant error (%v)", err) - } - - cmdArgs := append(cx.PrefixArgs(), "lease", "list") - proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap) - if err != nil { - return fmt.Errorf("lease list failed (%v)", err) - } - _, err = proc.Expect(id) - if err != nil { - return fmt.Errorf("lease id not in returned list (%v)", err) - } - return proc.Close() -} - func leaseTestTimeToLiveExpired(cx ctlCtx) { err := leaseTestTimeToLiveExpire(cx, 3) if err != nil { From dca5874d4402ca5660d8a8bc2b63dbdda473727a Mon Sep 17 00:00:00 2001 From: nic-chen Date: Sat, 19 Mar 2022 00:46:42 +0800 Subject: [PATCH 14/30] tests: Migrate key value Get to common framework --- tests/common/alarm_test.go | 93 +++++++++++++++++++++++++++++++ tests/framework/config/cluster.go | 7 ++- tests/framework/e2e/etcdctl.go | 6 ++ tests/framework/integration.go | 9 +++ tests/framework/interface.go | 2 +- 5 files changed, 113 insertions(+), 4 deletions(-) create mode 100644 tests/common/alarm_test.go diff --git a/tests/common/alarm_test.go b/tests/common/alarm_test.go new file mode 100644 index 000000000000..1d5708663438 --- /dev/null +++ b/tests/common/alarm_test.go @@ -0,0 +1,93 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "os" + "strings" + "testing" + "time" + + "go.etcd.io/etcd/tests/v3/framework/config" +) + +func TestAlarm(t *testing.T) { + testRunner.BeforeTest(t) + + clus := testRunner.NewCluster(t, config.ClusterConfig{ClusterSize: 3, QuotaBackendBytes: int64(13 * os.Getpagesize())}) + defer clus.Close() + + // test small put still works + smallbuf := strings.Repeat("a", 64) + if err := clus.Client().Put("1st_test", smallbuf); err != nil { + t.Fatalf("alarmTest: put kv error (%v)", err) + } + + // write some chunks to fill up the database + buf := strings.Repeat("b", os.Getpagesize()) + for { + if err := clus.Client().Put("2nd_test", buf); err != nil { + if !strings.Contains(err.Error(), "etcdserver: mvcc: database space exceeded") { + t.Fatal(err) + } + break + } + } + + // quota alarm should now be on + _, err := clus.Client().Alarm("list") + if err != nil { + t.Fatalf("alarmTest: Alarm error (%v)", err) + } + + // endpoint should not healthy + if err := clus.Client().Health(); err == nil { + t.Fatalf("endpoint should not healthy") + } + + // check that Put is rejected when alarm is on + if err := clus.Client().Put("3rd_test", smallbuf); err != nil { + if !strings.Contains(err.Error(), "etcdserver: mvcc: database space exceeded") { + t.Fatal(err) + } + } + + // get latest revision to compact + sresp, err := clus.Client().Status() + if err != nil { + t.Fatalf("get endpoint status error: %v", err) + } + + // make some space + _, err = clus.Client().Compact(sresp[0].Header.Revision, config.CompactOption{Physical: true, Timeout: 10 * time.Second}) + if err != nil { + t.Fatalf("alarmTest: Compact error (%v)", err) + } + + if err = clus.Client().Defragment(config.DefragOption{Timeout: 10 * time.Second}); err != nil { + t.Fatalf("alarmTest: defrag error (%v)", err) + } + + // turn off alarm + _, err = clus.Client().Alarm("disarm") + if err != nil { + t.Fatalf("alarmTest: Alarm error (%v)", err) + } + + // put one more key below quota + if err := clus.Client().Put("4th_test", smallbuf); err != nil { + t.Fatal(err) + } +} diff --git a/tests/framework/config/cluster.go b/tests/framework/config/cluster.go index c9deab294ebe..f18c6e5c4061 100644 --- a/tests/framework/config/cluster.go +++ b/tests/framework/config/cluster.go @@ -23,7 +23,8 @@ const ( ) type ClusterConfig struct { - ClusterSize int - PeerTLS TLSConfig - ClientTLS TLSConfig + ClusterSize int + PeerTLS TLSConfig + ClientTLS TLSConfig + QuotaBackendBytes int64 } diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index f11bdceadfc5..3f0fb87a7b35 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -241,7 +241,13 @@ func (ctl *EtcdctlV3) Health() error { lines[i] = "is healthy" } return SpawnWithExpects(args, map[string]string{}, lines...) +} + +func (ctl *EtcdctlV3) Alarm(cmd string) (*clientv3.AlarmResponse, error) { + args := ctl.cmdArgs() + args = append(args, "alarm", cmd) + return nil, SpawnWithExpect(args, "alarm:NOSPACE") } func (ctl *EtcdctlV3) Defragment(o config.DefragOption) error { diff --git a/tests/framework/integration.go b/tests/framework/integration.go index 94c1925b7c4c..e674de78511d 100644 --- a/tests/framework/integration.go +++ b/tests/framework/integration.go @@ -45,6 +45,7 @@ func (e integrationRunner) NewCluster(t testing.TB, cfg config.ClusterConfig) Cl var integrationCfg integration.ClusterConfig integrationCfg.Size = cfg.ClusterSize integrationCfg.ClientTLS, err = tlsInfo(t, cfg.ClientTLS) + integrationCfg.QuotaBackendBytes = cfg.QuotaBackendBytes if err != nil { t.Fatalf("ClientTLS: %s", err) } @@ -159,6 +160,14 @@ func (c integrationClient) Compact(rev int64, o config.CompactOption) (*clientv3 return c.Client.Compact(ctx, rev, clientOpts...) } +func (c integrationClient) Alarm(cmd string) (*clientv3.AlarmResponse, error) { + ctx := context.Background() + if cmd == "list" { + return c.Client.AlarmList(ctx) + } + return c.Client.AlarmDisarm(ctx, nil) +} + func (c integrationClient) Status() ([]*clientv3.StatusResponse, error) { endpoints := c.Client.Endpoints() var resp []*clientv3.StatusResponse diff --git a/tests/framework/interface.go b/tests/framework/interface.go index db4b3b980cc3..9b1b424f695e 100644 --- a/tests/framework/interface.go +++ b/tests/framework/interface.go @@ -37,9 +37,9 @@ type Client interface { Get(key string, opts config.GetOptions) (*clientv3.GetResponse, error) Delete(key string, opts config.DeleteOptions) (*clientv3.DeleteResponse, error) Compact(rev int64, opts config.CompactOption) (*clientv3.CompactResponse, error) - Status() ([]*clientv3.StatusResponse, error) HashKV(rev int64) ([]*clientv3.HashKVResponse, error) Health() error Defragment(opts config.DefragOption) error + Alarm(cmd string) (*clientv3.AlarmResponse, error) } From b50f10299b6f9aa9cef2ba5cf2dedd7fcb390841 Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Wed, 16 Mar 2022 15:26:08 +0000 Subject: [PATCH 15/30] tests/framework: Add PutOptions Put can take a leaseid to associate a value with a lease. This adds the ability for tests to make use of this. --- tests/common/compact_test.go | 2 +- tests/common/defrag_test.go | 2 +- tests/common/kv_test.go | 6 +++--- tests/e2e/ctl_v3_grpc_test.go | 3 ++- tests/framework/config/client.go | 4 ++++ tests/framework/e2e/etcdctl.go | 9 +++++++-- tests/framework/integration.go | 8 ++++++-- tests/framework/interface.go | 2 +- 8 files changed, 25 insertions(+), 11 deletions(-) diff --git a/tests/common/compact_test.go b/tests/common/compact_test.go index 8effa94e1277..d21d206aac3a 100644 --- a/tests/common/compact_test.go +++ b/tests/common/compact_test.go @@ -47,7 +47,7 @@ func TestCompact(t *testing.T) { testutils.ExecuteWithTimeout(t, 10*time.Second, func() { var kvs = []testutils.KV{{Key: "key", Val: "val1"}, {Key: "key", Val: "val2"}, {Key: "key", Val: "val3"}} for i := range kvs { - if err := clus.Client().Put(kvs[i].Key, kvs[i].Val); err != nil { + if err := clus.Client().Put(kvs[i].Key, kvs[i].Val, config.PutOptions{}); err != nil { t.Fatalf("compactTest #%d: put kv error (%v)", i, err) } } diff --git a/tests/common/defrag_test.go b/tests/common/defrag_test.go index 6cfce4c68070..2c7db2b7cd1a 100644 --- a/tests/common/defrag_test.go +++ b/tests/common/defrag_test.go @@ -30,7 +30,7 @@ func TestDefragOnline(t *testing.T) { defer clus.Close() var kvs = []testutils.KV{{Key: "key", Val: "val1"}, {Key: "key", Val: "val2"}, {Key: "key", Val: "val3"}} for i := range kvs { - if err := clus.Client().Put(kvs[i].Key, kvs[i].Val); err != nil { + if err := clus.Client().Put(kvs[i].Key, kvs[i].Val, config.PutOptions{}); err != nil { t.Fatalf("compactTest #%d: put kv error (%v)", i, err) } } diff --git a/tests/common/kv_test.go b/tests/common/kv_test.go index 2a4eccacc913..6f5849ee9599 100644 --- a/tests/common/kv_test.go +++ b/tests/common/kv_test.go @@ -60,7 +60,7 @@ func TestKVPut(t *testing.T) { testutils.ExecuteWithTimeout(t, 10*time.Second, func() { key, value := "foo", "bar" - if err := cc.Put(key, value); err != nil { + if err := cc.Put(key, value, config.PutOptions{}); err != nil { t.Fatalf("count not put key %q, err: %s", key, err) } resp, err := cc.Get(key, config.GetOptions{Serializable: true}) @@ -123,7 +123,7 @@ func TestKVGet(t *testing.T) { ) for i := range kvs { - if err := cc.Put(kvs[i], "bar"); err != nil { + if err := cc.Put(kvs[i], "bar", config.PutOptions{}); err != nil { t.Fatalf("count not put key %q, err: %s", kvs[i], err) } } @@ -246,7 +246,7 @@ func TestKVDelete(t *testing.T) { } for _, tt := range tests { for i := range kvs { - if err := cc.Put(kvs[i], "bar"); err != nil { + if err := cc.Put(kvs[i], "bar", config.PutOptions{}); err != nil { t.Fatalf("count not put key %q, err: %s", kvs[i], err) } } diff --git a/tests/e2e/ctl_v3_grpc_test.go b/tests/e2e/ctl_v3_grpc_test.go index 8c8be2c698ef..878dd7948285 100644 --- a/tests/e2e/ctl_v3_grpc_test.go +++ b/tests/e2e/ctl_v3_grpc_test.go @@ -24,6 +24,7 @@ import ( "time" "github.com/stretchr/testify/assert" + "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/e2e" "go.etcd.io/etcd/tests/v3/framework/testutils" ) @@ -100,7 +101,7 @@ func TestAuthority(t *testing.T) { endpoints := templateEndpoints(t, tc.clientURLPattern, epc) client := e2e.NewEtcdctl(cfg, endpoints) - err = client.Put("foo", "bar") + err = client.Put("foo", "bar", config.PutOptions{}) if err != nil { t.Fatal(err) } diff --git a/tests/framework/config/client.go b/tests/framework/config/client.go index c63dd5995002..ef2ef9c273c0 100644 --- a/tests/framework/config/client.go +++ b/tests/framework/config/client.go @@ -32,6 +32,10 @@ type GetOptions struct { SortBy clientv3.SortTarget } +type PutOptions struct { + LeaseID clientv3.LeaseID +} + type DeleteOptions struct { Prefix bool FromKey bool diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index f0714aeb7ff2..2d30a83e0562 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -107,8 +107,13 @@ func (ctl *EtcdctlV3) Get(key string, o config.GetOptions) (*clientv3.GetRespons return &resp, err } -func (ctl *EtcdctlV3) Put(key, value string) error { - return SpawnWithExpect(ctl.cmdArgs("put", key, value), "OK") +func (ctl *EtcdctlV3) Put(key, value string, opts config.PutOptions) error { + args := ctl.cmdArgs() + args = append(args, "put", key, value) + if opts.LeaseID != 0 { + args = append(args, "--lease", strconv.FormatInt(int64(opts.LeaseID), 16)) + } + return SpawnWithExpect(args, "OK") } func (ctl *EtcdctlV3) Delete(key string, o config.DeleteOptions) (*clientv3.DeleteResponse, error) { diff --git a/tests/framework/integration.go b/tests/framework/integration.go index 6a5eac295bbb..3c4cf2567669 100644 --- a/tests/framework/integration.go +++ b/tests/framework/integration.go @@ -126,8 +126,12 @@ func (c integrationClient) Get(key string, o config.GetOptions) (*clientv3.GetRe return c.Client.Get(context.Background(), key, clientOpts...) } -func (c integrationClient) Put(key, value string) error { - _, err := c.Client.Put(context.Background(), key, value) +func (c integrationClient) Put(key, value string, opts config.PutOptions) error { + clientOpts := []clientv3.OpOption{} + if opts.LeaseID != 0 { + clientOpts = append(clientOpts, clientv3.WithLease(opts.LeaseID)) + } + _, err := c.Client.Put(context.Background(), key, value, clientOpts...) return err } diff --git a/tests/framework/interface.go b/tests/framework/interface.go index c4fa9bda1c6c..08ead6a80def 100644 --- a/tests/framework/interface.go +++ b/tests/framework/interface.go @@ -33,7 +33,7 @@ type Cluster interface { } type Client interface { - Put(key, value string) error + Put(key, value string, opts config.PutOptions) error Get(key string, opts config.GetOptions) (*clientv3.GetResponse, error) Delete(key string, opts config.DeleteOptions) (*clientv3.DeleteResponse, error) Compact(rev int64, opts config.CompactOption) (*clientv3.CompactResponse, error) From c12e03c8e64837eecf048b27dd7fcc9c4a8f7c89 Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Wed, 16 Mar 2022 15:27:51 +0000 Subject: [PATCH 16/30] tests: migrate TestCtlV3LeaseTestTimeToLiveExpired.* to common --- tests/common/lease_test.go | 60 ++++++++++++++++++++++++++++++++++ tests/e2e/ctl_v3_auth_test.go | 22 +++++++++++++ tests/e2e/ctl_v3_lease_test.go | 44 ------------------------- 3 files changed, 82 insertions(+), 44 deletions(-) diff --git a/tests/common/lease_test.go b/tests/common/lease_test.go index 5676fbf24789..caeabf7da0bf 100644 --- a/tests/common/lease_test.go +++ b/tests/common/lease_test.go @@ -147,3 +147,63 @@ func TestLeaseGrantAndList(t *testing.T) { } } } + +func TestLeaseGrantTimeToLiveExpired(t *testing.T) { + testRunner.BeforeTest(t) + + tcs := []struct { + name string + config config.ClusterConfig + }{ + { + name: "NoTLS", + config: config.ClusterConfig{ClusterSize: 1}, + }, + { + name: "PeerTLS", + config: config.ClusterConfig{ClusterSize: 3, PeerTLS: config.ManualTLS}, + }, + { + name: "PeerAutoTLS", + config: config.ClusterConfig{ClusterSize: 3, PeerTLS: config.AutoTLS}, + }, + { + name: "ClientTLS", + config: config.ClusterConfig{ClusterSize: 1, ClientTLS: config.ManualTLS}, + }, + { + name: "ClientAutoTLS", + config: config.ClusterConfig{ClusterSize: 1, ClientTLS: config.AutoTLS}, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + clus := testRunner.NewCluster(t, tc.config) + defer clus.Close() + cc := clus.Client() + + testutils.ExecuteWithTimeout(t, 10*time.Second, func() { + leaseResp, err := cc.Grant(2) + require.NoError(t, err) + + err = cc.Put("foo", "bar", config.PutOptions{LeaseID: leaseResp.ID}) + require.NoError(t, err) + + getResp, err := cc.Get("foo", config.GetOptions{}) + require.NoError(t, err) + require.Equal(t, int64(1), getResp.Count) + + time.Sleep(3 * time.Second) + + ttlResp, err := cc.TimeToLive(leaseResp.ID, config.LeaseOption{}) + require.NoError(t, err) + require.Equal(t, int64(-1), ttlResp.TTL) + + getResp, err = cc.Get("foo", config.GetOptions{}) + require.NoError(t, err) + // Value should expire with the lease + require.Equal(t, int64(0), getResp.Count) + }) + }) + } +} diff --git a/tests/e2e/ctl_v3_auth_test.go b/tests/e2e/ctl_v3_auth_test.go index 8fd64097200f..6c75f5b06347 100644 --- a/tests/e2e/ctl_v3_auth_test.go +++ b/tests/e2e/ctl_v3_auth_test.go @@ -853,6 +853,28 @@ func authLeaseTestTimeToLiveExpired(cx ctlCtx) { } } +func leaseTestTimeToLiveExpire(cx ctlCtx, ttl int) error { + leaseID, err := ctlV3LeaseGrant(cx, ttl) + if err != nil { + return fmt.Errorf("ctlV3LeaseGrant error (%v)", err) + } + + if err = ctlV3Put(cx, "key", "val", leaseID); err != nil { + return fmt.Errorf("ctlV3Put error (%v)", err) + } + // eliminate false positive + time.Sleep(time.Duration(ttl+1) * time.Second) + cmdArgs := append(cx.PrefixArgs(), "lease", "timetolive", leaseID) + exp := fmt.Sprintf("lease %s already expired", leaseID) + if err = e2e.SpawnWithExpectWithEnv(cmdArgs, cx.envMap, exp); err != nil { + return fmt.Errorf("lease not properly expired: (%v)", err) + } + if err := ctlV3Get(cx, []string{"key"}); err != nil { + return fmt.Errorf("ctlV3Get error (%v)", err) + } + return nil +} + func authLeaseTestLeaseGrantLeases(cx ctlCtx) { cx.user, cx.pass = "root", "root" authSetupTestUser(cx) diff --git a/tests/e2e/ctl_v3_lease_test.go b/tests/e2e/ctl_v3_lease_test.go index 4772ef2159c6..0c1173cd2763 100644 --- a/tests/e2e/ctl_v3_lease_test.go +++ b/tests/e2e/ctl_v3_lease_test.go @@ -19,25 +19,10 @@ import ( "strconv" "strings" "testing" - "time" "go.etcd.io/etcd/tests/v3/framework/e2e" ) -func TestCtlV3LeaseTestTimeToLiveExpired(t *testing.T) { testCtl(t, leaseTestTimeToLiveExpired) } -func TestCtlV3LeaseTestTimeToLiveExpiredNoTLS(t *testing.T) { - testCtl(t, leaseTestTimeToLiveExpired, withCfg(*e2e.NewConfigNoTLS())) -} -func TestCtlV3LeaseTestTimeToLiveExpiredClientTLS(t *testing.T) { - testCtl(t, leaseTestTimeToLiveExpired, withCfg(*e2e.NewConfigClientTLS())) -} -func TestCtlV3LeaseTestTimeToLiveExpiredClientAutoTLS(t *testing.T) { - testCtl(t, leaseTestTimeToLiveExpired, withCfg(*e2e.NewConfigClientAutoTLS())) -} -func TestCtlV3LeaseTestTimeToLiveExpiredPeerTLS(t *testing.T) { - testCtl(t, leaseTestTimeToLiveExpired, withCfg(*e2e.NewConfigPeerTLS())) -} - func TestCtlV3LeaseKeepAlive(t *testing.T) { testCtl(t, leaseTestKeepAlive) } func TestCtlV3LeaseKeepAliveNoTLS(t *testing.T) { testCtl(t, leaseTestKeepAlive, withCfg(*e2e.NewConfigNoTLS())) @@ -80,35 +65,6 @@ func TestCtlV3LeaseRevokePeerTLS(t *testing.T) { testCtl(t, leaseTestRevoked, withCfg(*e2e.NewConfigPeerTLS())) } -func leaseTestTimeToLiveExpired(cx ctlCtx) { - err := leaseTestTimeToLiveExpire(cx, 3) - if err != nil { - cx.t.Fatalf("leaseTestTimeToLiveExpire: (%v)", err) - } -} - -func leaseTestTimeToLiveExpire(cx ctlCtx, ttl int) error { - leaseID, err := ctlV3LeaseGrant(cx, ttl) - if err != nil { - return fmt.Errorf("ctlV3LeaseGrant error (%v)", err) - } - - if err = ctlV3Put(cx, "key", "val", leaseID); err != nil { - return fmt.Errorf("ctlV3Put error (%v)", err) - } - // eliminate false positive - time.Sleep(time.Duration(ttl+1) * time.Second) - cmdArgs := append(cx.PrefixArgs(), "lease", "timetolive", leaseID) - exp := fmt.Sprintf("lease %s already expired", leaseID) - if err = e2e.SpawnWithExpectWithEnv(cmdArgs, cx.envMap, exp); err != nil { - return fmt.Errorf("lease not properly expired: (%v)", err) - } - if err := ctlV3Get(cx, []string{"key"}); err != nil { - return fmt.Errorf("ctlV3Get error (%v)", err) - } - return nil -} - func leaseTestKeepAlive(cx ctlCtx) { // put with TTL 10 seconds and keep-alive leaseID, err := ctlV3LeaseGrant(cx, 10) From 353b011f5943cdeb1e427f77972dd55e7b187e75 Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Wed, 16 Mar 2022 16:00:05 +0000 Subject: [PATCH 17/30] tests/framework: Add Client.LeaseKeepAliveOnce --- tests/framework/e2e/etcdctl.go | 16 ++++++++++++++++ tests/framework/integration.go | 6 ++++++ tests/framework/interface.go | 1 + 3 files changed, 23 insertions(+) diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index 2d30a83e0562..aa315ddf0fa0 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -311,3 +311,19 @@ func (ctl *EtcdctlV3) LeaseList() (*clientv3.LeaseLeasesResponse, error) { err = json.Unmarshal([]byte(line), &resp) return &resp, err } + +func (ctl *EtcdctlV3) LeaseKeepAliveOnce(id clientv3.LeaseID) (*clientv3.LeaseKeepAliveResponse, error) { + args := ctl.cmdArgs() + args = append(args, "lease", "keep-alive", strconv.FormatInt(int64(id), 16), "--once", "-w", "json") + cmd, err := SpawnCmd(args, nil) + if err != nil { + return nil, err + } + var resp clientv3.LeaseKeepAliveResponse + line, err := cmd.Expect("ID") + if err != nil { + return nil, err + } + err = json.Unmarshal([]byte(line), &resp) + return &resp, err +} diff --git a/tests/framework/integration.go b/tests/framework/integration.go index 3c4cf2567669..e5c8c01a445b 100644 --- a/tests/framework/integration.go +++ b/tests/framework/integration.go @@ -238,3 +238,9 @@ func (c integrationClient) LeaseList() (*clientv3.LeaseLeasesResponse, error) { return c.Client.Leases(ctx) } + +func (c integrationClient) LeaseKeepAliveOnce(id clientv3.LeaseID) (*clientv3.LeaseKeepAliveResponse, error) { + ctx := context.Background() + + return c.Client.KeepAliveOnce(ctx, id) +} diff --git a/tests/framework/interface.go b/tests/framework/interface.go index 08ead6a80def..ca7fb18decd8 100644 --- a/tests/framework/interface.go +++ b/tests/framework/interface.go @@ -44,4 +44,5 @@ type Client interface { Grant(ttl int64) (*clientv3.LeaseGrantResponse, error) TimeToLive(id clientv3.LeaseID, opts config.LeaseOption) (*clientv3.LeaseTimeToLiveResponse, error) LeaseList() (*clientv3.LeaseLeasesResponse, error) + LeaseKeepAliveOnce(id clientv3.LeaseID) (*clientv3.LeaseKeepAliveResponse, error) } From 36279e0797dbd2df4d658c20caceb4c2746e293e Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Wed, 16 Mar 2022 16:00:44 +0000 Subject: [PATCH 18/30] tests: migrate TestCtlV3LeaseKeepAliveOnce.* to common --- tests/common/lease_test.go | 52 ++++++++++++++++++++++++++++++++++ tests/e2e/ctl_v3_lease_test.go | 45 ----------------------------- 2 files changed, 52 insertions(+), 45 deletions(-) diff --git a/tests/common/lease_test.go b/tests/common/lease_test.go index caeabf7da0bf..98e382cebedd 100644 --- a/tests/common/lease_test.go +++ b/tests/common/lease_test.go @@ -207,3 +207,55 @@ func TestLeaseGrantTimeToLiveExpired(t *testing.T) { }) } } + +func TestLeaseGrantKeepAliveOnce(t *testing.T) { + testRunner.BeforeTest(t) + + tcs := []struct { + name string + config config.ClusterConfig + }{ + { + name: "NoTLS", + config: config.ClusterConfig{ClusterSize: 1}, + }, + { + name: "PeerTLS", + config: config.ClusterConfig{ClusterSize: 3, PeerTLS: config.ManualTLS}, + }, + { + name: "PeerAutoTLS", + config: config.ClusterConfig{ClusterSize: 3, PeerTLS: config.AutoTLS}, + }, + { + name: "ClientTLS", + config: config.ClusterConfig{ClusterSize: 1, ClientTLS: config.ManualTLS}, + }, + { + name: "ClientAutoTLS", + config: config.ClusterConfig{ClusterSize: 1, ClientTLS: config.AutoTLS}, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + clus := testRunner.NewCluster(t, tc.config) + defer clus.Close() + cc := clus.Client() + + testutils.ExecuteWithTimeout(t, 10*time.Second, func() { + leaseResp, err := cc.Grant(2) + require.NoError(t, err) + + _, err = cc.LeaseKeepAliveOnce(leaseResp.ID) + require.NoError(t, err) + + time.Sleep(2 * time.Second) // Wait for the original lease to expire + + ttlResp, err := cc.TimeToLive(leaseResp.ID, config.LeaseOption{}) + require.NoError(t, err) + // We still have a lease! + require.Greater(t, int64(2), ttlResp.TTL) + }) + }) + } +} diff --git a/tests/e2e/ctl_v3_lease_test.go b/tests/e2e/ctl_v3_lease_test.go index 0c1173cd2763..905351c934ef 100644 --- a/tests/e2e/ctl_v3_lease_test.go +++ b/tests/e2e/ctl_v3_lease_test.go @@ -37,20 +37,6 @@ func TestCtlV3LeaseKeepAlivePeerTLS(t *testing.T) { testCtl(t, leaseTestKeepAlive, withCfg(*e2e.NewConfigPeerTLS())) } -func TestCtlV3LeaseKeepAliveOnce(t *testing.T) { testCtl(t, leaseTestKeepAliveOnce) } -func TestCtlV3LeaseKeepAliveOnceNoTLS(t *testing.T) { - testCtl(t, leaseTestKeepAliveOnce, withCfg(*e2e.NewConfigNoTLS())) -} -func TestCtlV3LeaseKeepAliveOnceClientTLS(t *testing.T) { - testCtl(t, leaseTestKeepAliveOnce, withCfg(*e2e.NewConfigClientTLS())) -} -func TestCtlV3LeaseKeepAliveOnceClientAutoTLS(t *testing.T) { - testCtl(t, leaseTestKeepAliveOnce, withCfg(*e2e.NewConfigClientAutoTLS())) -} -func TestCtlV3LeaseKeepAliveOncePeerTLS(t *testing.T) { - testCtl(t, leaseTestKeepAliveOnce, withCfg(*e2e.NewConfigPeerTLS())) -} - func TestCtlV3LeaseRevoke(t *testing.T) { testCtl(t, leaseTestRevoked) } func TestCtlV3LeaseRevokeNoTLS(t *testing.T) { testCtl(t, leaseTestRevoked, withCfg(*e2e.NewConfigNoTLS())) @@ -82,23 +68,6 @@ func leaseTestKeepAlive(cx ctlCtx) { } } -func leaseTestKeepAliveOnce(cx ctlCtx) { - // put with TTL 10 seconds and keep-alive once - leaseID, err := ctlV3LeaseGrant(cx, 10) - if err != nil { - cx.t.Fatalf("leaseTestKeepAlive: ctlV3LeaseGrant error (%v)", err) - } - if err := ctlV3Put(cx, "key", "val", leaseID); err != nil { - cx.t.Fatalf("leaseTestKeepAlive: ctlV3Put error (%v)", err) - } - if err := ctlV3LeaseKeepAliveOnce(cx, leaseID); err != nil { - cx.t.Fatalf("leaseTestKeepAlive: ctlV3LeaseKeepAliveOnce error (%v)", err) - } - if err := ctlV3Get(cx, []string{"key"}, kv{"key", "val"}); err != nil { - cx.t.Fatalf("leaseTestKeepAlive: ctlV3Get error (%v)", err) - } -} - func leaseTestRevoked(cx ctlCtx) { err := leaseTestRevoke(cx) if err != nil { @@ -161,20 +130,6 @@ func ctlV3LeaseKeepAlive(cx ctlCtx, leaseID string) error { return proc.Stop() } -func ctlV3LeaseKeepAliveOnce(cx ctlCtx, leaseID string) error { - cmdArgs := append(cx.PrefixArgs(), "lease", "keep-alive", "--once", leaseID) - - proc, err := e2e.SpawnCmd(cmdArgs, nil) - if err != nil { - return err - } - - if _, err = proc.Expect(fmt.Sprintf("lease %s keepalived with TTL(", leaseID)); err != nil { - return err - } - return proc.Stop() -} - func ctlV3LeaseRevoke(cx ctlCtx, leaseID string) error { cmdArgs := append(cx.PrefixArgs(), "lease", "revoke", leaseID) return e2e.SpawnWithExpectWithEnv(cmdArgs, cx.envMap, fmt.Sprintf("lease %s revoked", leaseID)) From ab3353582d0f4b3f89da53fe2974d01f623cc226 Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Wed, 16 Mar 2022 16:21:12 +0000 Subject: [PATCH 19/30] tests/framework: Add Client.LeaseRevoke --- tests/framework/e2e/etcdctl.go | 16 ++++++++++++++++ tests/framework/integration.go | 6 ++++++ tests/framework/interface.go | 1 + 3 files changed, 23 insertions(+) diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index aa315ddf0fa0..546c9c305107 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -327,3 +327,19 @@ func (ctl *EtcdctlV3) LeaseKeepAliveOnce(id clientv3.LeaseID) (*clientv3.LeaseKe err = json.Unmarshal([]byte(line), &resp) return &resp, err } + +func (ctl *EtcdctlV3) LeaseRevoke(id clientv3.LeaseID) (*clientv3.LeaseRevokeResponse, error) { + args := ctl.cmdArgs() + args = append(args, "lease", "revoke", strconv.FormatInt(int64(id), 16), "-w", "json") + cmd, err := SpawnCmd(args, nil) + if err != nil { + return nil, err + } + var resp clientv3.LeaseRevokeResponse + line, err := cmd.Expect("header") + if err != nil { + return nil, err + } + err = json.Unmarshal([]byte(line), &resp) + return &resp, err +} diff --git a/tests/framework/integration.go b/tests/framework/integration.go index e5c8c01a445b..28bb414220e2 100644 --- a/tests/framework/integration.go +++ b/tests/framework/integration.go @@ -244,3 +244,9 @@ func (c integrationClient) LeaseKeepAliveOnce(id clientv3.LeaseID) (*clientv3.Le return c.Client.KeepAliveOnce(ctx, id) } + +func (c integrationClient) LeaseRevoke(id clientv3.LeaseID) (*clientv3.LeaseRevokeResponse, error) { + ctx := context.Background() + + return c.Client.Revoke(ctx, id) +} diff --git a/tests/framework/interface.go b/tests/framework/interface.go index ca7fb18decd8..cde4c8a3a72e 100644 --- a/tests/framework/interface.go +++ b/tests/framework/interface.go @@ -45,4 +45,5 @@ type Client interface { TimeToLive(id clientv3.LeaseID, opts config.LeaseOption) (*clientv3.LeaseTimeToLiveResponse, error) LeaseList() (*clientv3.LeaseLeasesResponse, error) LeaseKeepAliveOnce(id clientv3.LeaseID) (*clientv3.LeaseKeepAliveResponse, error) + LeaseRevoke(id clientv3.LeaseID) (*clientv3.LeaseRevokeResponse, error) } From 87740f6c7be7ca6fba8764a9bbf7d835387e1770 Mon Sep 17 00:00:00 2001 From: Danielle Lancashire Date: Wed, 16 Mar 2022 16:21:43 +0000 Subject: [PATCH 20/30] tests: Migrate TestCtlV3LeaseRevoke.* to common --- tests/common/lease_test.go | 61 ++++++++++++++++++++++++++++++++++ tests/e2e/ctl_v3_lease_test.go | 39 ---------------------- 2 files changed, 61 insertions(+), 39 deletions(-) diff --git a/tests/common/lease_test.go b/tests/common/lease_test.go index 98e382cebedd..e64f3b86e348 100644 --- a/tests/common/lease_test.go +++ b/tests/common/lease_test.go @@ -259,3 +259,64 @@ func TestLeaseGrantKeepAliveOnce(t *testing.T) { }) } } + +func TestLeaseGrantRevoke(t *testing.T) { + testRunner.BeforeTest(t) + + tcs := []struct { + name string + config config.ClusterConfig + }{ + { + name: "NoTLS", + config: config.ClusterConfig{ClusterSize: 1}, + }, + { + name: "PeerTLS", + config: config.ClusterConfig{ClusterSize: 3, PeerTLS: config.ManualTLS}, + }, + { + name: "PeerAutoTLS", + config: config.ClusterConfig{ClusterSize: 3, PeerTLS: config.AutoTLS}, + }, + { + name: "ClientTLS", + config: config.ClusterConfig{ClusterSize: 1, ClientTLS: config.ManualTLS}, + }, + { + name: "ClientAutoTLS", + config: config.ClusterConfig{ClusterSize: 1, ClientTLS: config.AutoTLS}, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + clus := testRunner.NewCluster(t, tc.config) + defer clus.Close() + cc := clus.Client() + + testutils.ExecuteWithTimeout(t, 10*time.Second, func() { + leaseResp, err := cc.Grant(20) + require.NoError(t, err) + + err = cc.Put("foo", "bar", config.PutOptions{LeaseID: leaseResp.ID}) + require.NoError(t, err) + + getResp, err := cc.Get("foo", config.GetOptions{}) + require.NoError(t, err) + require.Equal(t, int64(1), getResp.Count) + + _, err = cc.LeaseRevoke(leaseResp.ID) + require.NoError(t, err) + + ttlResp, err := cc.TimeToLive(leaseResp.ID, config.LeaseOption{}) + require.NoError(t, err) + require.Equal(t, int64(-1), ttlResp.TTL) + + getResp, err = cc.Get("foo", config.GetOptions{}) + require.NoError(t, err) + // Value should expire with the lease + require.Equal(t, int64(0), getResp.Count) + }) + }) + } +} diff --git a/tests/e2e/ctl_v3_lease_test.go b/tests/e2e/ctl_v3_lease_test.go index 905351c934ef..6ac44c44f240 100644 --- a/tests/e2e/ctl_v3_lease_test.go +++ b/tests/e2e/ctl_v3_lease_test.go @@ -37,20 +37,6 @@ func TestCtlV3LeaseKeepAlivePeerTLS(t *testing.T) { testCtl(t, leaseTestKeepAlive, withCfg(*e2e.NewConfigPeerTLS())) } -func TestCtlV3LeaseRevoke(t *testing.T) { testCtl(t, leaseTestRevoked) } -func TestCtlV3LeaseRevokeNoTLS(t *testing.T) { - testCtl(t, leaseTestRevoked, withCfg(*e2e.NewConfigNoTLS())) -} -func TestCtlV3LeaseRevokeClientTLS(t *testing.T) { - testCtl(t, leaseTestRevoked, withCfg(*e2e.NewConfigClientTLS())) -} -func TestCtlV3LeaseRevokeClientAutoTLS(t *testing.T) { - testCtl(t, leaseTestRevoked, withCfg(*e2e.NewConfigClientAutoTLS())) -} -func TestCtlV3LeaseRevokePeerTLS(t *testing.T) { - testCtl(t, leaseTestRevoked, withCfg(*e2e.NewConfigPeerTLS())) -} - func leaseTestKeepAlive(cx ctlCtx) { // put with TTL 10 seconds and keep-alive leaseID, err := ctlV3LeaseGrant(cx, 10) @@ -68,31 +54,6 @@ func leaseTestKeepAlive(cx ctlCtx) { } } -func leaseTestRevoked(cx ctlCtx) { - err := leaseTestRevoke(cx) - if err != nil { - cx.t.Fatalf("leaseTestRevoke: (%v)", err) - } -} - -func leaseTestRevoke(cx ctlCtx) error { - // put with TTL 10 seconds and revoke - leaseID, err := ctlV3LeaseGrant(cx, 10) - if err != nil { - return fmt.Errorf("ctlV3LeaseGrant error (%v)", err) - } - if err := ctlV3Put(cx, "key", "val", leaseID); err != nil { - return fmt.Errorf("ctlV3Put error (%v)", err) - } - if err := ctlV3LeaseRevoke(cx, leaseID); err != nil { - return fmt.Errorf("ctlV3LeaseRevoke error (%v)", err) - } - if err := ctlV3Get(cx, []string{"key"}); err != nil { // expect no output - return fmt.Errorf("ctlV3Get error (%v)", err) - } - return nil -} - func ctlV3LeaseGrant(cx ctlCtx, ttl int) (string, error) { cmdArgs := append(cx.PrefixArgs(), "lease", "grant", strconv.Itoa(ttl)) proc, err := e2e.SpawnCmd(cmdArgs, cx.envMap) From 3e657bdc5035c35e5c0c46ef1ca8598a828993a4 Mon Sep 17 00:00:00 2001 From: nic-chen Date: Sat, 19 Mar 2022 00:51:29 +0800 Subject: [PATCH 21/30] tests: Migrate key value Get to common framework --- tests/common/alarm_test.go | 103 +++++++++++++++++++------------------ 1 file changed, 52 insertions(+), 51 deletions(-) diff --git a/tests/common/alarm_test.go b/tests/common/alarm_test.go index 1d5708663438..9312703bb20b 100644 --- a/tests/common/alarm_test.go +++ b/tests/common/alarm_test.go @@ -21,73 +21,74 @@ import ( "time" "go.etcd.io/etcd/tests/v3/framework/config" + "go.etcd.io/etcd/tests/v3/framework/testutils" ) func TestAlarm(t *testing.T) { testRunner.BeforeTest(t) - clus := testRunner.NewCluster(t, config.ClusterConfig{ClusterSize: 3, QuotaBackendBytes: int64(13 * os.Getpagesize())}) defer clus.Close() + testutils.ExecuteWithTimeout(t, 10*time.Second, func() { + // test small put still works + smallbuf := strings.Repeat("a", 64) + if err := clus.Client().Put("1st_test", smallbuf); err != nil { + t.Fatalf("alarmTest: put kv error (%v)", err) + } - // test small put still works - smallbuf := strings.Repeat("a", 64) - if err := clus.Client().Put("1st_test", smallbuf); err != nil { - t.Fatalf("alarmTest: put kv error (%v)", err) - } - - // write some chunks to fill up the database - buf := strings.Repeat("b", os.Getpagesize()) - for { - if err := clus.Client().Put("2nd_test", buf); err != nil { - if !strings.Contains(err.Error(), "etcdserver: mvcc: database space exceeded") { - t.Fatal(err) + // write some chunks to fill up the database + buf := strings.Repeat("b", os.Getpagesize()) + for { + if err := clus.Client().Put("2nd_test", buf); err != nil { + if !strings.Contains(err.Error(), "etcdserver: mvcc: database space exceeded") { + t.Fatal(err) + } + break } - break } - } - // quota alarm should now be on - _, err := clus.Client().Alarm("list") - if err != nil { - t.Fatalf("alarmTest: Alarm error (%v)", err) - } + // quota alarm should now be on + _, err := clus.Client().Alarm("list") + if err != nil { + t.Fatalf("alarmTest: Alarm error (%v)", err) + } - // endpoint should not healthy - if err := clus.Client().Health(); err == nil { - t.Fatalf("endpoint should not healthy") - } + // endpoint should not healthy + if err := clus.Client().Health(); err == nil { + t.Fatalf("endpoint should not healthy") + } - // check that Put is rejected when alarm is on - if err := clus.Client().Put("3rd_test", smallbuf); err != nil { - if !strings.Contains(err.Error(), "etcdserver: mvcc: database space exceeded") { - t.Fatal(err) + // check that Put is rejected when alarm is on + if err := clus.Client().Put("3rd_test", smallbuf); err != nil { + if !strings.Contains(err.Error(), "etcdserver: mvcc: database space exceeded") { + t.Fatal(err) + } } - } - // get latest revision to compact - sresp, err := clus.Client().Status() - if err != nil { - t.Fatalf("get endpoint status error: %v", err) - } + // get latest revision to compact + sresp, err := clus.Client().Status() + if err != nil { + t.Fatalf("get endpoint status error: %v", err) + } - // make some space - _, err = clus.Client().Compact(sresp[0].Header.Revision, config.CompactOption{Physical: true, Timeout: 10 * time.Second}) - if err != nil { - t.Fatalf("alarmTest: Compact error (%v)", err) - } + // make some space + _, err = clus.Client().Compact(sresp[0].Header.Revision, config.CompactOption{Physical: true, Timeout: 10 * time.Second}) + if err != nil { + t.Fatalf("alarmTest: Compact error (%v)", err) + } - if err = clus.Client().Defragment(config.DefragOption{Timeout: 10 * time.Second}); err != nil { - t.Fatalf("alarmTest: defrag error (%v)", err) - } + if err = clus.Client().Defragment(config.DefragOption{Timeout: 10 * time.Second}); err != nil { + t.Fatalf("alarmTest: defrag error (%v)", err) + } - // turn off alarm - _, err = clus.Client().Alarm("disarm") - if err != nil { - t.Fatalf("alarmTest: Alarm error (%v)", err) - } + // turn off alarm + _, err = clus.Client().Alarm("disarm") + if err != nil { + t.Fatalf("alarmTest: Alarm error (%v)", err) + } - // put one more key below quota - if err := clus.Client().Put("4th_test", smallbuf); err != nil { - t.Fatal(err) - } + // put one more key below quota + if err := clus.Client().Put("4th_test", smallbuf); err != nil { + t.Fatal(err) + } + }) } From e9fa171665662860be9a951959dad1a533f02d13 Mon Sep 17 00:00:00 2001 From: nic-chen Date: Sat, 19 Mar 2022 16:31:19 +0800 Subject: [PATCH 22/30] fix test fail --- tests/common/alarm_test.go | 9 ++++++++- tests/e2e/ctl_v3_test.go | 4 ---- tests/framework/e2e.go | 5 +++-- 3 files changed, 11 insertions(+), 7 deletions(-) diff --git a/tests/common/alarm_test.go b/tests/common/alarm_test.go index 9312703bb20b..e96e99fc6e5f 100644 --- a/tests/common/alarm_test.go +++ b/tests/common/alarm_test.go @@ -69,9 +69,16 @@ func TestAlarm(t *testing.T) { if err != nil { t.Fatalf("get endpoint status error: %v", err) } + var rvs int64 + for _, resp := range sresp { + if resp != nil && resp.Header != nil { + rvs = resp.Header.Revision + break + } + } // make some space - _, err = clus.Client().Compact(sresp[0].Header.Revision, config.CompactOption{Physical: true, Timeout: 10 * time.Second}) + _, err = clus.Client().Compact(rvs, config.CompactOption{Physical: true, Timeout: 10 * time.Second}) if err != nil { t.Fatalf("alarmTest: Compact error (%v)", err) } diff --git a/tests/e2e/ctl_v3_test.go b/tests/e2e/ctl_v3_test.go index fc544603d5d3..ee309cdeb37a 100644 --- a/tests/e2e/ctl_v3_test.go +++ b/tests/e2e/ctl_v3_test.go @@ -181,10 +181,6 @@ func withQuota(b int64) ctlOption { return func(cx *ctlCtx) { cx.quotaBackendBytes = b } } -func withCompactPhysical() ctlOption { - return func(cx *ctlCtx) { cx.compactPhysical = true } -} - func withInitialCorruptCheck() ctlOption { return func(cx *ctlCtx) { cx.initialCorruptCheck = true } } diff --git a/tests/framework/e2e.go b/tests/framework/e2e.go index d0e799163e6f..8e068e3a846a 100644 --- a/tests/framework/e2e.go +++ b/tests/framework/e2e.go @@ -40,8 +40,9 @@ func (e e2eRunner) BeforeTest(t testing.TB) { func (e e2eRunner) NewCluster(t testing.TB, cfg config.ClusterConfig) Cluster { e2eConfig := e2e.EtcdProcessClusterConfig{ - InitialToken: "new", - ClusterSize: cfg.ClusterSize, + InitialToken: "new", + ClusterSize: cfg.ClusterSize, + QuotaBackendBytes: cfg.QuotaBackendBytes, } switch cfg.ClientTLS { case config.NoTLS: From 527edd39a6d7805efcc21da92664a42b9df7267a Mon Sep 17 00:00:00 2001 From: nic-chen Date: Sun, 20 Mar 2022 01:40:53 +0800 Subject: [PATCH 23/30] fix alarm test --- tests/common/alarm_test.go | 20 +++++++++++--------- tests/framework/e2e/etcdctl.go | 17 +++++++++++++---- tests/framework/integration.go | 4 ++-- tests/framework/interface.go | 2 +- 4 files changed, 27 insertions(+), 16 deletions(-) diff --git a/tests/common/alarm_test.go b/tests/common/alarm_test.go index 1188bab36a86..4bf12f994c74 100644 --- a/tests/common/alarm_test.go +++ b/tests/common/alarm_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + clientv3 "go.etcd.io/etcd/client/v3" "go.etcd.io/etcd/tests/v3/framework/config" "go.etcd.io/etcd/tests/v3/framework/testutils" ) @@ -47,16 +48,11 @@ func TestAlarm(t *testing.T) { } // quota alarm should now be on - _, err := clus.Client().Alarm("list") + alarmResp, err := clus.Client().Alarm("list", nil) if err != nil { t.Fatalf("alarmTest: Alarm error (%v)", err) } - // endpoint should not healthy - if err := clus.Client().Health(); err == nil { - t.Fatalf("endpoint should not healthy") - } - // check that Put is rejected when alarm is on if err := clus.Client().Put("3rd_test", smallbuf, config.PutOptions{}); err != nil { if !strings.Contains(err.Error(), "etcdserver: mvcc: database space exceeded") { @@ -88,9 +84,15 @@ func TestAlarm(t *testing.T) { } // turn off alarm - _, err = clus.Client().Alarm("disarm") - if err != nil { - t.Fatalf("alarmTest: Alarm error (%v)", err) + for _, alarm := range alarmResp.Alarms { + m := &clientv3.AlarmMember{ + MemberID: alarm.MemberID, + Alarm: alarm.Alarm, + } + _, err = clus.Client().Alarm("disarm", m) + if err != nil { + t.Fatalf("alarmTest: Alarm error (%v)", err) + } } // put one more key below quota diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index ee2ceacac1f8..25ea0eaf18b8 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -344,9 +344,18 @@ func (ctl *EtcdctlV3) LeaseRevoke(id clientv3.LeaseID) (*clientv3.LeaseRevokeRes return &resp, err } -func (ctl *EtcdctlV3) Alarm(cmd string) (*clientv3.AlarmResponse, error) { +func (ctl *EtcdctlV3) Alarm(cmd string, _ *clientv3.AlarmMember) (*clientv3.AlarmResponse, error) { args := ctl.cmdArgs() - args = append(args, "alarm", cmd) - - return nil, SpawnWithExpect(args, "alarm:NOSPACE") + args = append(args, "alarm", cmd, "-w", "json") + ep, err := SpawnCmd(args, nil) + if err != nil { + return nil, err + } + var resp clientv3.AlarmResponse + line, err := ep.Expect("alarm") + if err != nil { + return nil, err + } + err = json.Unmarshal([]byte(line), &resp) + return &resp, err } diff --git a/tests/framework/integration.go b/tests/framework/integration.go index b7403d81d4de..4ce431ff9dca 100644 --- a/tests/framework/integration.go +++ b/tests/framework/integration.go @@ -164,12 +164,12 @@ func (c integrationClient) Compact(rev int64, o config.CompactOption) (*clientv3 return c.Client.Compact(ctx, rev, clientOpts...) } -func (c integrationClient) Alarm(cmd string) (*clientv3.AlarmResponse, error) { +func (c integrationClient) Alarm(cmd string, member *clientv3.AlarmMember) (*clientv3.AlarmResponse, error) { ctx := context.Background() if cmd == "list" { return c.Client.AlarmList(ctx) } - return c.Client.AlarmDisarm(ctx, nil) + return c.Client.AlarmDisarm(ctx, member) } func (c integrationClient) Status() ([]*clientv3.StatusResponse, error) { diff --git a/tests/framework/interface.go b/tests/framework/interface.go index f925682df3c2..2864a22c6b4e 100644 --- a/tests/framework/interface.go +++ b/tests/framework/interface.go @@ -41,7 +41,7 @@ type Client interface { HashKV(rev int64) ([]*clientv3.HashKVResponse, error) Health() error Defragment(opts config.DefragOption) error - Alarm(cmd string) (*clientv3.AlarmResponse, error) + Alarm(cmd string, member *clientv3.AlarmMember) (*clientv3.AlarmResponse, error) Grant(ttl int64) (*clientv3.LeaseGrantResponse, error) TimeToLive(id clientv3.LeaseID, opts config.LeaseOption) (*clientv3.LeaseTimeToLiveResponse, error) LeaseList() (*clientv3.LeaseLeasesResponse, error) From bb7856090a8f3d08bfbb2c85c859de880fe0dd0f Mon Sep 17 00:00:00 2001 From: nic-chen Date: Sun, 20 Mar 2022 02:01:30 +0800 Subject: [PATCH 24/30] fix: set cluster size to 1 to avoid unstable --- tests/common/alarm_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/common/alarm_test.go b/tests/common/alarm_test.go index 4bf12f994c74..bb66fd727c1b 100644 --- a/tests/common/alarm_test.go +++ b/tests/common/alarm_test.go @@ -27,7 +27,7 @@ import ( func TestAlarm(t *testing.T) { testRunner.BeforeTest(t) - clus := testRunner.NewCluster(t, config.ClusterConfig{ClusterSize: 3, QuotaBackendBytes: int64(13 * os.Getpagesize())}) + clus := testRunner.NewCluster(t, config.ClusterConfig{ClusterSize: 1, QuotaBackendBytes: int64(13 * os.Getpagesize())}) defer clus.Close() testutils.ExecuteWithTimeout(t, 10*time.Second, func() { // test small put still works From 89146aa157c91b774bade97a7c354a73820bc5e4 Mon Sep 17 00:00:00 2001 From: nic-chen Date: Sun, 20 Mar 2022 02:06:42 +0800 Subject: [PATCH 25/30] chore: delete useless file --- tests/e2e/ctl_v3_alarm_test.go | 106 --------------------------------- 1 file changed, 106 deletions(-) delete mode 100644 tests/e2e/ctl_v3_alarm_test.go diff --git a/tests/e2e/ctl_v3_alarm_test.go b/tests/e2e/ctl_v3_alarm_test.go deleted file mode 100644 index b99d3a6a388e..000000000000 --- a/tests/e2e/ctl_v3_alarm_test.go +++ /dev/null @@ -1,106 +0,0 @@ -// Copyright 2016 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package e2e - -import ( - "context" - "os" - "strings" - "testing" - "time" - - clientv3 "go.etcd.io/etcd/client/v3" - "go.etcd.io/etcd/tests/v3/framework/e2e" -) - -func TestCtlV3Alarm(t *testing.T) { - // The boltdb minimum working set is six pages. - testCtl(t, alarmTest, withQuota(int64(13*os.Getpagesize()))) -} - -func alarmTest(cx ctlCtx) { - // test small put still works - smallbuf := strings.Repeat("a", 64) - if err := ctlV3Put(cx, "1st_test", smallbuf, ""); err != nil { - cx.t.Fatal(err) - } - - // write some chunks to fill up the database - buf := strings.Repeat("b", os.Getpagesize()) - for { - if err := ctlV3Put(cx, "2nd_test", buf, ""); err != nil { - if !strings.Contains(err.Error(), "etcdserver: mvcc: database space exceeded") { - cx.t.Fatal(err) - } - break - } - } - - // quota alarm should now be on - if err := ctlV3Alarm(cx, "list", "alarm:NOSPACE"); err != nil { - cx.t.Fatal(err) - } - - // '/health' handler should return 'false' - if err := e2e.CURLGet(cx.epc, e2e.CURLReq{Endpoint: "/health", Expected: `{"health":"false","reason":"ALARM NOSPACE"}`}); err != nil { - cx.t.Fatalf("failed get with curl (%v)", err) - } - - // check that Put is rejected when alarm is on - if err := ctlV3Put(cx, "3rd_test", smallbuf, ""); err != nil { - if !strings.Contains(err.Error(), "etcdserver: mvcc: database space exceeded") { - cx.t.Fatal(err) - } - } - - eps := cx.epc.EndpointsV3() - - // get latest revision to compact - cli, err := clientv3.New(clientv3.Config{ - Endpoints: eps, - DialTimeout: 3 * time.Second, - }) - if err != nil { - cx.t.Fatal(err) - } - defer cli.Close() - sresp, err := cli.Status(context.TODO(), eps[0]) - if err != nil { - cx.t.Fatal(err) - } - - // make some space - if err := ctlV3Compact(cx, sresp.Header.Revision, true); err != nil { - cx.t.Fatal(err) - } - if err := ctlV3OnlineDefrag(cx); err != nil { - cx.t.Fatal(err) - } - - // turn off alarm - if err := ctlV3Alarm(cx, "disarm", "alarm:NOSPACE"); err != nil { - cx.t.Fatal(err) - } - - // put one more key below quota - if err := ctlV3Put(cx, "4th_test", smallbuf, ""); err != nil { - cx.t.Fatal(err) - } -} - -func ctlV3Alarm(cx ctlCtx, cmd string, as ...string) error { - cmdArgs := append(cx.PrefixArgs(), "alarm", cmd) - return e2e.SpawnWithExpects(cmdArgs, cx.envMap, as...) -} From c6488a2c56acf3d59168a603ca568a369edb1eea Mon Sep 17 00:00:00 2001 From: nic-chen Date: Sun, 20 Mar 2022 10:09:27 +0800 Subject: [PATCH 26/30] fix review --- tests/common/alarm_test.go | 6 +++--- tests/framework/e2e/etcdctl.go | 20 ++++++++++++++++++-- tests/framework/integration.go | 12 ++++++------ tests/framework/interface.go | 3 ++- 4 files changed, 29 insertions(+), 12 deletions(-) diff --git a/tests/common/alarm_test.go b/tests/common/alarm_test.go index bb66fd727c1b..9873743c2ff5 100644 --- a/tests/common/alarm_test.go +++ b/tests/common/alarm_test.go @@ -48,7 +48,7 @@ func TestAlarm(t *testing.T) { } // quota alarm should now be on - alarmResp, err := clus.Client().Alarm("list", nil) + alarmResp, err := clus.Client().AlarmList() if err != nil { t.Fatalf("alarmTest: Alarm error (%v)", err) } @@ -85,11 +85,11 @@ func TestAlarm(t *testing.T) { // turn off alarm for _, alarm := range alarmResp.Alarms { - m := &clientv3.AlarmMember{ + alarmMember := &clientv3.AlarmMember{ MemberID: alarm.MemberID, Alarm: alarm.Alarm, } - _, err = clus.Client().Alarm("disarm", m) + _, err = clus.Client().AlarmDisarm(alarmMember) if err != nil { t.Fatalf("alarmTest: Alarm error (%v)", err) } diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index 25ea0eaf18b8..dd5f57e35027 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -344,9 +344,25 @@ func (ctl *EtcdctlV3) LeaseRevoke(id clientv3.LeaseID) (*clientv3.LeaseRevokeRes return &resp, err } -func (ctl *EtcdctlV3) Alarm(cmd string, _ *clientv3.AlarmMember) (*clientv3.AlarmResponse, error) { +func (ctl *EtcdctlV3) AlarmList() (*clientv3.AlarmResponse, error) { args := ctl.cmdArgs() - args = append(args, "alarm", cmd, "-w", "json") + args = append(args, "alarm", "list", "-w", "json") + ep, err := SpawnCmd(args, nil) + if err != nil { + return nil, err + } + var resp clientv3.AlarmResponse + line, err := ep.Expect("alarm") + if err != nil { + return nil, err + } + err = json.Unmarshal([]byte(line), &resp) + return &resp, err +} + +func (ctl *EtcdctlV3) AlarmDisarm(_ *clientv3.AlarmMember) (*clientv3.AlarmResponse, error) { + args := ctl.cmdArgs() + args = append(args, "alarm", "disarm", "-w", "json") ep, err := SpawnCmd(args, nil) if err != nil { return nil, err diff --git a/tests/framework/integration.go b/tests/framework/integration.go index 4ce431ff9dca..f5867da4633d 100644 --- a/tests/framework/integration.go +++ b/tests/framework/integration.go @@ -164,12 +164,12 @@ func (c integrationClient) Compact(rev int64, o config.CompactOption) (*clientv3 return c.Client.Compact(ctx, rev, clientOpts...) } -func (c integrationClient) Alarm(cmd string, member *clientv3.AlarmMember) (*clientv3.AlarmResponse, error) { - ctx := context.Background() - if cmd == "list" { - return c.Client.AlarmList(ctx) - } - return c.Client.AlarmDisarm(ctx, member) +func (c integrationClient) AlarmList() (*clientv3.AlarmResponse, error) { + return c.Client.AlarmList(context.Background()) +} + +func (c integrationClient) AlarmDisarm(alarmMember *clientv3.AlarmMember) (*clientv3.AlarmResponse, error) { + return c.Client.AlarmDisarm(context.Background(), alarmMember) } func (c integrationClient) Status() ([]*clientv3.StatusResponse, error) { diff --git a/tests/framework/interface.go b/tests/framework/interface.go index 2864a22c6b4e..8460483dd572 100644 --- a/tests/framework/interface.go +++ b/tests/framework/interface.go @@ -41,7 +41,8 @@ type Client interface { HashKV(rev int64) ([]*clientv3.HashKVResponse, error) Health() error Defragment(opts config.DefragOption) error - Alarm(cmd string, member *clientv3.AlarmMember) (*clientv3.AlarmResponse, error) + AlarmList() (*clientv3.AlarmResponse, error) + AlarmDisarm(alarmMember *clientv3.AlarmMember) (*clientv3.AlarmResponse, error) Grant(ttl int64) (*clientv3.LeaseGrantResponse, error) TimeToLive(id clientv3.LeaseID, opts config.LeaseOption) (*clientv3.LeaseTimeToLiveResponse, error) LeaseList() (*clientv3.LeaseLeasesResponse, error) From 7a0c254b73b47c589f53643d3845c4d368eaa69a Mon Sep 17 00:00:00 2001 From: Hitoshi Mitake Date: Mon, 21 Mar 2022 22:25:58 +0900 Subject: [PATCH 27/30] tests/integration: re-enable TestV3AuthOldRevConcurrent --- tests/integration/v3_auth_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/v3_auth_test.go b/tests/integration/v3_auth_test.go index 4f4e53e97f04..741ddccb824e 100644 --- a/tests/integration/v3_auth_test.go +++ b/tests/integration/v3_auth_test.go @@ -358,7 +358,6 @@ func TestV3AuthNonAuthorizedRPCs(t *testing.T) { } func TestV3AuthOldRevConcurrent(t *testing.T) { - t.Skip() // TODO(jingyih): re-enable the test when #10408 is fixed. integration.BeforeTest(t) clus := integration.NewCluster(t, &integration.ClusterConfig{Size: 1}) defer clus.Terminate(t) From 44aad460db1ce67e543d0b425d7a01227d3b6d47 Mon Sep 17 00:00:00 2001 From: Marek Siarkowicz Date: Wed, 2 Mar 2022 12:09:44 +0100 Subject: [PATCH 28/30] tests: Migrate noquorum kv tests to common framework --- tests/common/kv_test.go | 39 +++++++++++++ tests/e2e/ctl_v3_kv_no_quorum_test.go | 83 --------------------------- tests/framework/config/client.go | 1 + tests/framework/e2e.go | 24 ++++++++ tests/framework/e2e/etcdctl.go | 3 + tests/framework/integration.go | 34 ++++++++++- tests/framework/interface.go | 7 +++ 7 files changed, 106 insertions(+), 85 deletions(-) delete mode 100644 tests/e2e/ctl_v3_kv_no_quorum_test.go diff --git a/tests/common/kv_test.go b/tests/common/kv_test.go index 6f5849ee9599..93bc8d774925 100644 --- a/tests/common/kv_test.go +++ b/tests/common/kv_test.go @@ -266,3 +266,42 @@ func TestKVDelete(t *testing.T) { }) } } + +func TestKVGetNoQuorum(t *testing.T) { + testRunner.BeforeTest(t) + tcs := []struct { + name string + options config.GetOptions + + wantError bool + }{ + { + name: "Serializable", + options: config.GetOptions{Serializable: true}, + }, + { + name: "Linearizable", + options: config.GetOptions{Serializable: false, Timeout: time.Second}, + wantError: true, + }, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + clus := testRunner.NewCluster(t, config.ClusterConfig{ClusterSize: 3}) + defer clus.Close() + + clus.Members()[0].Stop() + clus.Members()[1].Stop() + + cc := clus.Members()[2].Client() + testutils.ExecuteWithTimeout(t, 10*time.Second, func() { + key := "foo" + _, err := cc.Get(key, tc.options) + gotError := err != nil + if gotError != tc.wantError { + t.Fatalf("Unexpeted result, wantError: %v, gotErr: %v, err: %s", tc.wantError, gotError, err) + } + }) + }) + } +} diff --git a/tests/e2e/ctl_v3_kv_no_quorum_test.go b/tests/e2e/ctl_v3_kv_no_quorum_test.go deleted file mode 100644 index dbc599b49765..000000000000 --- a/tests/e2e/ctl_v3_kv_no_quorum_test.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2021 The etcd Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// When the quorum isn't satisfied, then each etcd member isn't able to -// publish/register server information(i.e., clientURL) into the cluster. -// Accordingly, the v2 proxy can't get any member's clientURL, so this -// case will fail for sure in this case. -// -// todo(ahrtr): When v2 proxy is removed, then we can remove the go build -// lines below. -//go:build !cluster_proxy -// +build !cluster_proxy - -package e2e - -import ( - "testing" - - "go.etcd.io/etcd/tests/v3/framework/e2e" -) - -func TestSerializableReadWithoutQuorum(t *testing.T) { - tcs := []struct { - name string - testFunc func(cx ctlCtx) - }{ - { - name: "serializableReadTest", - testFunc: serializableReadTest, - }, - { - name: "linearizableReadTest", - testFunc: linearizableReadTest, - }, - } - for _, tc := range tcs { - t.Run(tc.name, func(t *testing.T) { - // Initialize a cluster with 3 members - epc, err := e2e.InitEtcdProcessCluster(t, e2e.NewConfigAutoTLS()) - if err != nil { - t.Fatalf("Failed to initilize the etcd cluster: %v", err) - } - - // Remove two members, so that only one etcd will get started - epc.Procs = epc.Procs[:1] - - // Start the etcd cluster with only one member - if err := epc.Start(); err != nil { - t.Fatalf("Failed to start the etcd cluster: %v", err) - } - - // construct the ctl context - cx := getDefaultCtlCtx(t) - cx.epc = epc - runCtlTest(t, tc.testFunc, nil, cx) - }) - } -} - -func serializableReadTest(cx ctlCtx) { - cx.quorum = false - if err := ctlV3Get(cx, []string{"key1"}, []kv{}...); err != nil { - cx.t.Errorf("serializableReadTest failed: %v", err) - } -} - -func linearizableReadTest(cx ctlCtx) { - cx.quorum = true - if err := ctlV3GetWithErr(cx, []string{"key"}, []string{"retrying of unary invoker failed"}); err != nil { // expect errors - cx.t.Fatalf("ctlV3GetWithErr error (%v)", err) - } -} diff --git a/tests/framework/config/client.go b/tests/framework/config/client.go index ef2ef9c273c0..63ee8bf54e2a 100644 --- a/tests/framework/config/client.go +++ b/tests/framework/config/client.go @@ -30,6 +30,7 @@ type GetOptions struct { Limit int Order clientv3.SortOrder SortBy clientv3.SortTarget + Timeout time.Duration } type PutOptions struct { diff --git a/tests/framework/e2e.go b/tests/framework/e2e.go index 8e068e3a846a..01528aed0e58 100644 --- a/tests/framework/e2e.go +++ b/tests/framework/e2e.go @@ -84,6 +84,30 @@ func (c *e2eCluster) Client() Client { return e2eClient{e2e.NewEtcdctl(c.Cfg, c.EndpointsV3())} } +func (c *e2eCluster) Members() (ms []Member) { + for _, proc := range c.EtcdProcessCluster.Procs { + ms = append(ms, e2eMember{EtcdProcess: proc, Cfg: c.Cfg}) + } + return ms +} + type e2eClient struct { *e2e.EtcdctlV3 } + +type e2eMember struct { + e2e.EtcdProcess + Cfg *e2e.EtcdProcessClusterConfig +} + +func (m e2eMember) Client() Client { + return e2eClient{e2e.NewEtcdctl(m.Cfg, m.EndpointsV3())} +} + +func (m e2eMember) Start() error { + return m.Restart() +} + +func (m e2eMember) Stop() { + m.EtcdProcess.Stop() +} diff --git a/tests/framework/e2e/etcdctl.go b/tests/framework/e2e/etcdctl.go index dd5f57e35027..6d697582fa20 100644 --- a/tests/framework/e2e/etcdctl.go +++ b/tests/framework/e2e/etcdctl.go @@ -42,6 +42,9 @@ func (ctl *EtcdctlV3) DowngradeEnable(version string) error { func (ctl *EtcdctlV3) Get(key string, o config.GetOptions) (*clientv3.GetResponse, error) { args := ctl.cmdArgs() + if o.Timeout != 0 { + args = append(args, fmt.Sprintf("--command-timeout=%s", o.Timeout)) + } if o.Serializable { args = append(args, "--consistency", "s") } diff --git a/tests/framework/integration.go b/tests/framework/integration.go index f5867da4633d..c580a19f0623 100644 --- a/tests/framework/integration.go +++ b/tests/framework/integration.go @@ -81,6 +81,30 @@ type integrationCluster struct { t testing.TB } +func (c *integrationCluster) Members() (ms []Member) { + for _, m := range c.Cluster.Members { + ms = append(ms, integrationMember{m, c.t}) + } + return ms +} + +type integrationMember struct { + *integration.Member + t testing.TB +} + +func (m integrationMember) Client() Client { + return integrationClient{m.Member.Client} +} + +func (m integrationMember) Start() error { + return m.Member.Restart(m.t) +} + +func (m integrationMember) Stop() { + m.Member.Stop(m.t) +} + func (c *integrationCluster) Close() error { c.Terminate(c.t) return nil @@ -91,7 +115,7 @@ func (c *integrationCluster) Client() Client { if err != nil { c.t.Fatal(err) } - return &integrationClient{cc} + return integrationClient{cc} } type integrationClient struct { @@ -99,6 +123,12 @@ type integrationClient struct { } func (c integrationClient) Get(key string, o config.GetOptions) (*clientv3.GetResponse, error) { + ctx := context.Background() + if o.Timeout != 0 { + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, o.Timeout) + defer cancel() + } clientOpts := []clientv3.OpOption{} if o.Revision != 0 { clientOpts = append(clientOpts, clientv3.WithRev(int64(o.Revision))) @@ -124,7 +154,7 @@ func (c integrationClient) Get(key string, o config.GetOptions) (*clientv3.GetRe if o.SortBy != clientv3.SortByKey || o.Order != clientv3.SortNone { clientOpts = append(clientOpts, clientv3.WithSort(o.SortBy, o.Order)) } - return c.Client.Get(context.Background(), key, clientOpts...) + return c.Client.Get(ctx, key, clientOpts...) } func (c integrationClient) Put(key, value string, opts config.PutOptions) error { diff --git a/tests/framework/interface.go b/tests/framework/interface.go index 8460483dd572..c391056c4cd5 100644 --- a/tests/framework/interface.go +++ b/tests/framework/interface.go @@ -28,8 +28,15 @@ type testRunner interface { } type Cluster interface { + Members() []Member + Client() Client Close() error +} + +type Member interface { Client() Client + Start() error + Stop() } type Client interface { From 37804358312c886a260331ad246f41bb1e1a6e6d Mon Sep 17 00:00:00 2001 From: ahrtr Date: Tue, 22 Mar 2022 06:24:32 +0800 Subject: [PATCH 29/30] added unit test for newClientCfg --- etcdctl/ctlv3/command/global_test.go | 131 +++++++++++++++++++++++++++ etcdctl/go.mod | 4 + etcdctl/go.sum | 3 + 3 files changed, 138 insertions(+) create mode 100644 etcdctl/ctlv3/command/global_test.go diff --git a/etcdctl/ctlv3/command/global_test.go b/etcdctl/ctlv3/command/global_test.go new file mode 100644 index 000000000000..0dc56abdd25f --- /dev/null +++ b/etcdctl/ctlv3/command/global_test.go @@ -0,0 +1,131 @@ +// Copyright 2022 The etcd Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package command + +import ( + "crypto/tls" + + "go.uber.org/zap" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "go.etcd.io/etcd/client/pkg/v3/transport" + clientv3 "go.etcd.io/etcd/client/v3" +) + +func TestNewClientConfig(t *testing.T) { + cases := []struct { + name string + spec clientv3.ConfigSpec + expectedConf clientv3.Config + }{ + { + name: "default secure transport", + spec: clientv3.ConfigSpec{ + Endpoints: []string{"http://192.168.0.10:2379"}, + DialTimeout: 2 * time.Second, + KeepAliveTime: 3 * time.Second, + KeepAliveTimeout: 5 * time.Second, + Secure: &clientv3.SecureConfig{ + InsecureTransport: false, + }, + }, + expectedConf: clientv3.Config{ + Endpoints: []string{"http://192.168.0.10:2379"}, + DialTimeout: 2 * time.Second, + DialKeepAliveTime: 3 * time.Second, + DialKeepAliveTimeout: 5 * time.Second, + TLS: &tls.Config{}, + }, + }, + { + name: "default secure transport and auth enabled", + spec: clientv3.ConfigSpec{ + Endpoints: []string{"http://192.168.0.12:2379"}, + DialTimeout: 1 * time.Second, + KeepAliveTime: 4 * time.Second, + KeepAliveTimeout: 6 * time.Second, + Secure: &clientv3.SecureConfig{ + InsecureTransport: false, + }, + Auth: &clientv3.AuthConfig{ + Username: "test", + Password: "changeme", + }, + }, + expectedConf: clientv3.Config{ + Endpoints: []string{"http://192.168.0.12:2379"}, + DialTimeout: 1 * time.Second, + DialKeepAliveTime: 4 * time.Second, + DialKeepAliveTimeout: 6 * time.Second, + TLS: &tls.Config{}, + Username: "test", + Password: "changeme", + }, + }, + { + name: "default secure transport and skip TLS verification", + spec: clientv3.ConfigSpec{ + Endpoints: []string{"http://192.168.0.13:2379"}, + DialTimeout: 1 * time.Second, + KeepAliveTime: 3 * time.Second, + KeepAliveTimeout: 5 * time.Second, + Secure: &clientv3.SecureConfig{ + InsecureTransport: false, + InsecureSkipVerify: true, + }, + }, + expectedConf: clientv3.Config{ + Endpoints: []string{"http://192.168.0.13:2379"}, + DialTimeout: 1 * time.Second, + DialKeepAliveTime: 3 * time.Second, + DialKeepAliveTimeout: 5 * time.Second, + TLS: &tls.Config{ + InsecureSkipVerify: true, + }, + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + cfg, err := newClientCfg(tc.spec.Endpoints, tc.spec.DialTimeout, tc.spec.KeepAliveTime, tc.spec.KeepAliveTimeout, tc.spec.Secure, tc.spec.Auth) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + assert.Equal(t, tc.expectedConf, *cfg) + }) + } +} + +func TestNewClientConfigWithSecureCfg(t *testing.T) { + tls, err := transport.SelfCert(zap.NewNop(), t.TempDir(), []string{"localhost"}, 1) + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + scfg := &clientv3.SecureConfig{ + Cert: tls.CertFile, + Key: tls.KeyFile, + Cacert: tls.TrustedCAFile, + } + + cfg, err := newClientCfg([]string{"http://192.168.0.13:2379"}, 2*time.Second, 3*time.Second, 5*time.Second, scfg, nil) + if cfg == nil || err != nil { + t.Fatalf("Unexpected result client config: %v", err) + } +} diff --git a/etcdctl/go.mod b/etcdctl/go.mod index d6b770a4d614..49ded5f42e76 100644 --- a/etcdctl/go.mod +++ b/etcdctl/go.mod @@ -8,6 +8,7 @@ require ( github.com/olekukonko/tablewriter v0.0.5 github.com/spf13/cobra v1.2.1 github.com/spf13/pflag v1.0.5 + github.com/stretchr/testify v1.7.0 github.com/urfave/cli v1.22.4 go.etcd.io/etcd/api/v3 v3.6.0-alpha.0 go.etcd.io/etcd/client/pkg/v3 v3.6.0-alpha.0 @@ -27,6 +28,7 @@ require ( github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd/v22 v22.3.2 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang-jwt/jwt v3.2.2+incompatible // indirect github.com/golang/protobuf v1.5.2 // indirect @@ -35,6 +37,7 @@ require ( github.com/jonboulle/clockwork v0.2.2 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_golang v1.11.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.26.0 // indirect @@ -56,6 +59,7 @@ require ( golang.org/x/text v0.3.7 // indirect google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1 // indirect google.golang.org/protobuf v1.27.1 // indirect + gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect sigs.k8s.io/json v0.0.0-20211020170558-c049b76a60c6 // indirect ) diff --git a/etcdctl/go.sum b/etcdctl/go.sum index af19c33748f5..5e8427fda7ab 100644 --- a/etcdctl/go.sum +++ b/etcdctl/go.sum @@ -242,9 +242,11 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/magiconair/properties v1.8.5/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= @@ -729,6 +731,7 @@ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.28 h1:n1tBJnnK2r7g9OW2btFH91V92STTUevLXYFb8gy9EMk= gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= From 780e3ae935c4964ad16d18337a5d9a43ad40fd3b Mon Sep 17 00:00:00 2001 From: ahrtr Date: Wed, 23 Mar 2022 14:22:37 +0800 Subject: [PATCH 30/30] update 3.5 changelog to cover the fix for the watchablestore runlock bug --- CHANGELOG/CHANGELOG-3.5.md | 1 + CHANGELOG/CHANGELOG-3.6.md | 1 + 2 files changed, 2 insertions(+) diff --git a/CHANGELOG/CHANGELOG-3.5.md b/CHANGELOG/CHANGELOG-3.5.md index 526e86147c0a..6bf16109d38e 100644 --- a/CHANGELOG/CHANGELOG-3.5.md +++ b/CHANGELOG/CHANGELOG-3.5.md @@ -31,6 +31,7 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.1...v3.5.2) and - Add [`etcd --experimental-enable-lease-checkpoint-persist`](https://github.com/etcd-io/etcd/pull/13508) flag to enable checkpoint persisting. - Fix [Lease checkpoints don't prevent to reset ttl on leader change](https://github.com/etcd-io/etcd/pull/13508), requires enabling checkpoint persisting. - Fix [assertion failed due to tx closed when recovering v3 backend from a snapshot db](https://github.com/etcd-io/etcd/pull/13501) +- Fix [segmentation violation(SIGSEGV) error due to premature unlocking of watchableStore](https://github.com/etcd-io/etcd/pull/13541)
diff --git a/CHANGELOG/CHANGELOG-3.6.md b/CHANGELOG/CHANGELOG-3.6.md index f33b697bf5a3..6d6e1f1787a8 100644 --- a/CHANGELOG/CHANGELOG-3.6.md +++ b/CHANGELOG/CHANGELOG-3.6.md @@ -57,6 +57,7 @@ See [code changes](https://github.com/etcd-io/etcd/compare/v3.5.0...v3.6.0). - Fix [etcd gateway doesn't format the endpoint of IPv6 address correctly](https://github.com/etcd-io/etcd/pull/13551) - Fix [A client can cause a nil dereference in etcd by passing an invalid SortTarget](https://github.com/etcd-io/etcd/pull/13555) - Fix [Grant lease with negative ID can possibly cause db out of sync](https://github.com/etcd-io/etcd/pull/13676) +- Fix [segmentation violation(SIGSEGV) error due to premature unlocking of watchableStore](https://github.com/etcd-io/etcd/pull/13505) ### tools/benchmark