Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
43565: kvnemeses: begin scaffolding for a jepsen-style kv test r=nvanbenschoten/tbg a=danhhz Package kvnemeses exercises the KV api with random traffic and then validates that the observed behaviors are consistent with our guarantees. A set of Operations are generated which represent usage of the public KV api. These include both "workload" operations like Gets and Puts as well as "admin" operations like rebalances. These Operations can be handed to an Applier, which runs them against the KV api and records the results. Operations do allow for concurrency (this testing is much less interesting otherwise), which means that the state of the KV map is not recoverable from _only_ the input. TODO(dan): We can use RangeFeed to recover the exact KV history. This plus some Kyle magic can be used to check our transactional guarantees. TODO (in later commits) - Validate the log - CPut/InitPut/Increment/Delete - DeleteRange/ClearRange/RevertRange/Scan/ReverseScan - ChangeReplicas/TransferLease - ExportRequest - AddSSTable - Root and leaf transactions - GCRequest - Protected timestamps Release note: None 44144: colexec: fix multiple starts of the wrapped processors r=yuzefovich a=yuzefovich **colexec: fix multiple starts of the wrapped processors** Previously, wrapped processors could be started multiple times if they were in the input chain for the bufferOp (each of the CASE arms will initialize its input - the bufferOp). Now this is fixed by tracking in both Columnarizer and bufferOp whether Init has already been called. Previous behavior could lead to a crash when rowexec.valuesProcessor was wrapped because it sends a "bogus" metadata header on each call to Start, and only single header is expected whereas with multiple Inits they would be multiple headers. Fixes: #44133. Release note (bug fix): Previously, CockroachDB could crash in special circumstances when vectorized execution engine is used (it was more likely to happen if `vectorize=experimental_on` setting was used). Now this has been fixed. **execerror: catch panics coming from sql/execinfra package** sql/execinfra is definitely a part of the vectorized engine as a whole, so we should be catching panics coming from it when running vectorized flows. Release note: None 44169: sql/opt/optbuilder: resolve remaining comments from #44015 r=nvanbenschoten a=nvanbenschoten This commit resolves a few typos that were missed before #44015 was merged. Release note: None 44172: Include "/cockroach" into PATH in Docker image r=vladdy a=vladdy This adds "/cockroach" into environment's PATH in Docker image to require less typing when invoking "cockroach" commands via running container. Fixes: #44189 Co-authored-by: Daniel Harrison <daniel.harrison@gmail.com> Co-authored-by: Yahor Yuzefovich <yahor@cockroachlabs.com> Co-authored-by: Nathan VanBenschoten <nvanbenschoten@gmail.com> Co-authored-by: Vlad Artamonov <742047+vladdy@users.noreply.github.com>
- Loading branch information
Showing
33 changed files
with
4,893 additions
and
134 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,166 @@ | ||
| // Copyright 2020 The Cockroach Authors. | ||
| // | ||
| // Use of this software is governed by the Business Source License | ||
| // included in the file licenses/BSL.txt. | ||
| // | ||
| // As of the Change Date specified in that file, in accordance with | ||
| // the Business Source License, use of this software will be governed | ||
| // by the Apache License, Version 2.0, included in the file | ||
| // licenses/APL.txt. | ||
|
|
||
| package kvnemeses | ||
|
|
||
| import ( | ||
| "context" | ||
|
|
||
| "github.com/cockroachdb/cockroach/pkg/internal/client" | ||
| "github.com/cockroachdb/cockroach/pkg/util/hlc" | ||
| "github.com/cockroachdb/cockroach/pkg/util/syncutil" | ||
| "github.com/cockroachdb/errors" | ||
| ) | ||
|
|
||
| // Applier executes Steps. | ||
| type Applier struct { | ||
| db *client.DB | ||
| mu struct { | ||
| syncutil.Mutex | ||
| txns map[string]*client.Txn | ||
| } | ||
| } | ||
|
|
||
| // MakeApplier constructs an Applier that executes against the given DB. | ||
| func MakeApplier(db *client.DB) *Applier { | ||
| a := &Applier{ | ||
| db: db, | ||
| } | ||
| a.mu.txns = make(map[string]*client.Txn) | ||
| return a | ||
| } | ||
|
|
||
| // Apply executes the given Step and mutates it with the result of execution. An | ||
| // error is only returned from Apply if there is an internal coding error within | ||
| // Applier, errors from a Step execution are saved in the Step itself. | ||
| func (a *Applier) Apply(ctx context.Context, step *Step) (retErr error) { | ||
| step.Before = a.db.Clock().Now() | ||
| defer func() { | ||
| step.After = a.db.Clock().Now() | ||
| if p := recover(); p != nil { | ||
| retErr = errors.Errorf(`panic applying step %s: %v`, step, p) | ||
| } | ||
| }() | ||
| a.applyOp(ctx, &step.Op) | ||
| return nil | ||
| } | ||
|
|
||
| func (a *Applier) applyOp(ctx context.Context, op *Operation) { | ||
| switch o := op.GetValue().(type) { | ||
| case *GetOperation, *PutOperation, *BatchOperation: | ||
| applyBatchOp(ctx, a.db, op) | ||
| case *SplitOperation: | ||
| err := a.db.AdminSplit(ctx, o.Key, o.Key, hlc.MaxTimestamp) | ||
| o.Result = resultError(ctx, err) | ||
| case *MergeOperation: | ||
| err := a.db.AdminMerge(ctx, o.Key) | ||
| o.Result = resultError(ctx, err) | ||
| case *ClosureTxnOperation: | ||
| txnErr := a.db.Txn(ctx, func(ctx context.Context, txn *client.Txn) error { | ||
| for i := range o.Ops { | ||
| op := &o.Ops[i] | ||
| applyBatchOp(ctx, txn, op) | ||
| // The KV api disallows use of a txn after an operation on it errors. | ||
| if r := op.Result(); r.Type == ResultType_Error { | ||
| return errors.DecodeError(ctx, *r.Err) | ||
| } | ||
| } | ||
| switch o.Type { | ||
| case ClosureTxnType_Commit: | ||
| return nil | ||
| case ClosureTxnType_Rollback: | ||
| return errors.New("rollback") | ||
| default: | ||
| panic(errors.AssertionFailedf(`unknown closure txn type: %s`, o.Type)) | ||
| } | ||
| }) | ||
| o.Result = resultError(ctx, txnErr) | ||
| default: | ||
| panic(errors.AssertionFailedf(`unknown operation type: %T %v`, o, o)) | ||
| } | ||
| } | ||
|
|
||
| type clientI interface { | ||
| Get(context.Context, interface{}) (client.KeyValue, error) | ||
| Put(context.Context, interface{}, interface{}) error | ||
| Run(context.Context, *client.Batch) error | ||
| } | ||
|
|
||
| func applyBatchOp(ctx context.Context, db clientI, op *Operation) { | ||
| switch o := op.GetValue().(type) { | ||
| case *GetOperation: | ||
| result, err := db.Get(ctx, o.Key) | ||
| if err != nil { | ||
| o.Result = resultError(ctx, err) | ||
| } else { | ||
| o.Result.Type = ResultType_Value | ||
| if result.Value != nil { | ||
| if value, err := result.Value.GetBytes(); err != nil { | ||
| panic(errors.Wrapf(err, "decoding %x", result.Value.RawBytes)) | ||
| } else { | ||
| o.Result.Value = value | ||
| } | ||
| } | ||
| } | ||
| case *PutOperation: | ||
| err := db.Put(ctx, o.Key, o.Value) | ||
| o.Result = resultError(ctx, err) | ||
| case *BatchOperation: | ||
| b := &client.Batch{} | ||
| for i := range o.Ops { | ||
| switch subO := o.Ops[i].GetValue().(type) { | ||
| case *GetOperation: | ||
| b.Get(subO.Key) | ||
| case *PutOperation: | ||
| b.Put(subO.Key, subO.Value) | ||
| default: | ||
| panic(errors.AssertionFailedf(`unknown batch operation type: %T %v`, subO, subO)) | ||
| } | ||
| } | ||
| runErr := db.Run(ctx, b) | ||
| o.Result = resultError(ctx, runErr) | ||
| for i := range o.Ops { | ||
| switch subO := o.Ops[i].GetValue().(type) { | ||
| case *GetOperation: | ||
| if b.Results[i].Err != nil { | ||
| subO.Result = resultError(ctx, b.Results[i].Err) | ||
| } else { | ||
| subO.Result.Type = ResultType_Value | ||
| result := b.Results[i].Rows[0] | ||
| if result.Value != nil { | ||
| if value, err := result.Value.GetBytes(); err != nil { | ||
| panic(errors.Wrapf(err, "decoding %x", result.Value.RawBytes)) | ||
| } else { | ||
| subO.Result.Value = value | ||
| } | ||
| } | ||
| } | ||
| case *PutOperation: | ||
| err := b.Results[i].Err | ||
| subO.Result = resultError(ctx, err) | ||
| default: | ||
| panic(errors.AssertionFailedf(`unknown batch operation type: %T %v`, subO, subO)) | ||
| } | ||
| } | ||
| default: | ||
| panic(errors.AssertionFailedf(`unknown batch operation type: %T %v`, o, o)) | ||
| } | ||
| } | ||
|
|
||
| func resultError(ctx context.Context, err error) Result { | ||
| if err == nil { | ||
| return Result{Type: ResultType_NoError} | ||
| } | ||
| ee := errors.EncodeError(ctx, err) | ||
| return Result{ | ||
| Type: ResultType_Error, | ||
| Err: &ee, | ||
| } | ||
| } |
Oops, something went wrong.