cherry-pick 1.1: sql: don't enable "kv tracing" for SHOW TRACE FOR <stmt>

.github/bors.toml

@@ -0,0 +1,13 @@
status = [
  "GitHub CI (Cockroach)"
]
pr_status = [
  "license/cla"
]
block_labels = [
  "do-not-merge"
]

[committer]
name = "craig[bot]"
email = "bors@cockroachlabs.com"

.go-version

@@ -1 +1 @@
GOVERS = go1\.[89].*
GOVERS = go1\.([89]|10).*

Gopkg.toml

@@ -51,4 +51,9 @@ ignored = [

[[constraint]]
  name = "github.com/coreos/etcd"
  branch = "master"
  source = "https://github.com/cockroachdb/etcd"
  branch = "crdb-release-1.1"

[[constraint]]
  name = "github.com/rlmcpherson/s3gof3r"
  revision = "864ae0bf7cf2e20c0002b7ea17f4d84fec1abc14"

LICENSE

@@ -16,10 +16,10 @@ CockroachDB Community License Agreement
     Agreement, the terms below have the following meanings.

    (a) "CockroachDB" shall mean the SQL database software provided by Cockroach
        Labs, including both CockroachDB Community and CockroachDB Enterprise
        Labs, including both CockroachDB Core and CockroachDB Enterprise
        editions, as defined below.

    (b) "CockroachDB Community Edition" shall mean the open source version of
    (b) "CockroachDB Core" shall mean the open source version of
        CockroachDB, available free of charge at

            https://github.com/cockroachdb/cockroach
@@ -93,25 +93,25 @@ CockroachDB Community License Agreement

  2. Licenses.

    (a) License to CockroachDB Community Edition.  The License for CockroachDB
        Community Edition is the Apache License, Version 2.0 ("Apache License").
    (a) License to CockroachDB Core.  The License for CockroachDB
        Core is the Apache License, Version 2.0 ("Apache License").
        The Apache License includes a grant of patent license, as well as
        redistribution rights that are contingent on several requirements.
        Please see

            http://www.apache.org/licenses/LICENSE-2.0

        for full terms.  CockroachDB Community Edition is a no-cost, entry-level
        for full terms.  CockroachDB Core is a no-cost, entry-level
        license and as such, contains the following disclaimers: NOTWITHSTANDING
        ANYTHING TO THE CONTRARY HEREIN, COCKROACHDB COMMUNITY EDITION IS
        ANYTHING TO THE CONTRARY HEREIN, COCKROACHDB CORE IS
        PROVIDED "AS IS" AND "AS AVAILABLE", AND ALL EXPRESS OR IMPLIED
        WARRANTIES ARE EXCLUDED AND DISCLAIMED, INCLUDING WITHOUT LIMITATION THE
        IMPLIED WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE,
        NON-INFRINGEMENT, AND ANY WARRANTIES ARISING BY STATUTE OR OTHERWISE IN
        LAW OR FROM COURSE OF DEALING, COURSE OF PERFORMANCE, OR USE IN TRADE.
        For clarity, the terms of this Agreement, other than the relevant
        definitions in Section 1 and this Section 2(a) do not apply to
        CockroachDB Community Edition.
        CockroachDB Core.

    (b) License to CockroachDB Enterprise Edition.

Makefile

@@ -28,7 +28,7 @@ TESTS        :=.## Tests to run for use with `make test`.
BENCHES      :=## Benchmarks to run for use with `make bench`.
FILES        :=## Space delimited list of logic test files to run, for make testlogic.
TESTTIMEOUT  := 4m## Test timeout to use for regular tests.
RACETIMEOUT  := 15m## Test timeout to use for race tests.
RACETIMEOUT  := 25m## Test timeout to use for race tests.
ACCEPTANCETIMEOUT := 30m## Test timeout to use for acceptance tests.
BENCHTIMEOUT := 5m## Test timeout to use for benchmarks.
TESTFLAGS    :=## Extra flags to pass to the go test runner, e.g. "-v --vmodule=raft=1"
@@ -361,8 +361,7 @@ $(ARCHIVE).tmp: .buildinfo/tag .buildinfo/rev .buildinfo/basebranch
# For details, see the "Possible timestamp problems with diff-files?" thread on
# the Git mailing list (http://marc.info/?l=git&m=131687596307197).
.buildinfo/tag: | .buildinfo
	@{ git describe --tags --exact-match 2> /dev/null || git rev-parse --short HEAD; } | tr -d \\n > $@
	@git diff --quiet HEAD || echo -dirty >> $@
	@{ git describe --tags --dirty 2> /dev/null || git rev-parse --short HEAD; } | tr -d \\n > $@

.buildinfo/basebranch: | .buildinfo
	@git describe --tags --abbrev=0 | tr -d \\n > $@

build/common.mk

@@ -430,7 +430,7 @@ libprotobuf: $(PROTOBUF_DIR)/Makefile

.PHONY: libsnappy
libsnappy: $(SNAPPY_DIR)/Makefile
	@$(MAKE) --no-print-directory -C $(SNAPPY_DIR)
	@$(MAKE) --no-print-directory -C $(SNAPPY_DIR) snappy

.PHONY: librocksdb
librocksdb: $(ROCKSDB_DIR)/Makefile

build/deploy/Dockerfile

@@ -1,7 +1,14 @@
FROM debian:8.7
FROM debian:8.9

MAINTAINER Tobias Schottdorf <tobias.schottdorf@gmail.com>

# Install root CAs so we can make SSL connections to phone home and
# do backups to GCE/AWS/Azure.
RUN apt-get update && \
	apt-get -y upgrade && \
	apt-get install -y ca-certificates  && \
	rm -rf /var/lib/apt/lists/*

RUN mkdir -p /cockroach
COPY cockroach.sh cockroach /cockroach/
# Set working directory so that relative paths

build/style_test.go

@@ -23,6 +23,7 @@ import (
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"testing"
@@ -186,7 +187,7 @@ func TestStyle(t *testing.T) {

	t.Run("TestTodoStyle", func(t *testing.T) {
		t.Parallel()
		cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `\sTODO\([^)]*\)[^:]`, "--", "*.go")
		cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `\sTODO\([^)]+\)[^:]`, "--", "*.go")
		if err != nil {
			t.Fatal(err)
		}
@@ -210,7 +211,7 @@ func TestStyle(t *testing.T) {

	t.Run("TestTimeutil", func(t *testing.T) {
		t.Parallel()
		cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `time\.(Now|Since)`, "--", "*.go")
		cmd, stderr, filter, err := dirCmd(pkg.Dir, "git", "grep", "-nE", `time\.(Now|Since|Unix\())`, "--", "*.go", ":!pkg/security/securitytest/embedded.go")
		if err != nil {
			t.Fatal(err)
		}
@@ -221,7 +222,7 @@ func TestStyle(t *testing.T) {

		if err := stream.ForEach(stream.Sequence(
			filter,
			stream.GrepNot(`^util/(log|syncutil|timeutil|tracing)/\w+\.go\b`),
			stream.GrepNot(`^util/(syncutil|timeutil|tracing)/\w+\.go\b`),
		), func(s string) {
			t.Errorf(`%s <- forbidden; use "timeutil" instead`, s)
		}); err != nil {
@@ -523,6 +524,26 @@ func TestStyle(t *testing.T) {

	t.Run("TestForbiddenImports", func(t *testing.T) {
		t.Parallel()

		// forbiddenImportPkg -> permittedReplacementPkg
		forbiddenImports := map[string]string{
			"context": "golang.org/x/net/context",
			"log":     "util/log",
			"path":    "path/filepath",
			"github.com/golang/protobuf/proto": "github.com/gogo/protobuf/proto",
			"github.com/satori/go.uuid":        "util/uuid",
			"golang.org/x/sync/singleflight":   "github.com/cockroachdb/cockroach/pkg/util/syncutil/singleflight",
		}

		// grepBuf creates a grep string that matches any forbidden import pkgs.
		var grepBuf bytes.Buffer
		grepBuf.WriteByte('(')
		for forbiddenPkg := range forbiddenImports {
			grepBuf.WriteByte('|')
			grepBuf.WriteString(regexp.QuoteMeta(forbiddenPkg))
		}
		grepBuf.WriteString(")$")

		filter := stream.FilterFunc(func(arg stream.Arg) error {
			for _, useAllFiles := range []bool{false, true} {
				buildContext := build.Default
@@ -559,31 +580,30 @@ func TestStyle(t *testing.T) {
			filter,
			stream.Sort(),
			stream.Uniq(),
			stream.Grep(`^`+settingsPkgPrefix+`: | `+grepBuf.String()),
			stream.GrepNot(`cockroach/pkg/cmd/`),
			stream.Grep(`^`+settingsPkgPrefix+`: | (github\.com/golang/protobuf/proto|github\.com/satori/go\.uuid|log|path|context|syscall)$`),
			stream.GrepNot(`cockroach/pkg/(cli|security): syscall$`),
			stream.GrepNot(`cockroach/pkg/(base|security|util/(log|randutil|stop)): log$`),
			stream.GrepNot(`cockroach/pkg/(server/serverpb|ts/tspb): github\.com/golang/protobuf/proto$`),
			stream.GrepNot(`cockroach/pkg/util/caller: path$`),
			stream.GrepNot(`cockroach/pkg/ccl/storageccl: path$`),
			stream.GrepNot(`cockroach/pkg/util/uuid: github\.com/satori/go\.uuid$`),
		), func(s string) {
			switch {
			case strings.HasSuffix(s, " path"):
				t.Errorf(`%s <- please use "path/filepath" instead of "path"`, s)
			case strings.HasSuffix(s, " log"):
				t.Errorf(`%s <- please use "util/log" instead of "log"`, s)
			case strings.HasSuffix(s, " github.com/golang/protobuf/proto"):
				t.Errorf(`%s <- please use "github.com/gogo/protobuf/proto" instead of "github.com/golang/protobuf/proto"`, s)
			case strings.HasSuffix(s, " github.com/satori/go.uuid"):
				t.Errorf(`%s <- please use "util/uuid" instead of "github.com/satori/go.uuid"`, s)
			case strings.HasSuffix(s, " context"):
				t.Errorf(`%s <- please use "golang.org/x/net/context" instead of "context"`, s)
			case strings.HasSuffix(s, " syscall"):
				t.Errorf(`%s <- please use "golang.org/x/sys" instead of "syscall"`, s)
			case strings.HasPrefix(s, settingsPkgPrefix+": github.com/cockroachdb/cockroach"):
				if !strings.HasSuffix(s, "testutils") && !strings.HasSuffix(s, "humanizeutil") &&
					!strings.HasSuffix(s, settingsPkgPrefix) {
			pkgStr := strings.Split(s, ": ")
			importingPkg, importedPkg := pkgStr[0], pkgStr[1]

			// Test that a disallowed package is not imported.
			if replPkg, ok := forbiddenImports[importedPkg]; ok {
				t.Errorf(`%s <- please use %q instead of %q`, s, replPkg, importedPkg)
			}

			// Test that the settings package does not import CRDB dependencies.
			if importingPkg == settingsPkgPrefix && strings.HasPrefix(importedPkg, cockroachDB) {
				switch {
				case strings.HasSuffix(s, "testutils"):
				case strings.HasSuffix(s, "humanizeutil"):
				case strings.HasSuffix(s, settingsPkgPrefix):
				default:
					t.Errorf("%s <- please don't add CRDB dependencies to settings pkg", s)
				}
			}

build/teamcity-check.sh

@@ -16,7 +16,14 @@ build/builder.sh env \
build/builder.sh make lint 2>&1 | tee artifacts/lint.log | go-test-teamcity

build/builder.sh make generate
build/builder.sh /bin/bash -c '! git status --porcelain | read || (git status; git diff -a 1>&2; exit 1)'

# The workspace is clean iff `git status --porcelain` produces no output. Any
# output is either an error message or a listing of an untracked/dirty file.
if [[ "$(git status --porcelain 2>&1)" != "" ]]; then
  git status >&2 || true
  git diff -a >&2 || true
  exit 1
fi

# Run the UI tests. This logically belongs in teamcity-test.sh, but we do it
# here to minimize total build time since the rest of this script completes

build/teamcity-post-failures.py

@@ -0,0 +1,96 @@
#!/usr/bin/env python3
"""Post failures from the current teamcity job as github issues.

Requires the following environment variables:
- TC_API_PASSWORD
- TC_BUILD_BRANCH
- TC_BUILD_ID
- GITHUB_API_TOKEN
"""

import json
import os
import urllib.error
import urllib.request
import xml.etree.ElementTree as ET

from urllib.parse import urljoin, urlencode

BASEURL = "https://teamcity.cockroachdb.com/httpAuth/app/rest/"

auth_handler = urllib.request.HTTPBasicAuthHandler()
auth_handler.add_password(realm='TeamCity',
                          uri='https://teamcity.cockroachdb.com',
                          user='robot',
                          passwd=os.environ['TC_API_PASSWORD'])
opener = urllib.request.build_opener(auth_handler)


def tc_url(path, **params):
    return urljoin(BASEURL, path) + '?' + urlencode(params)


def collect_build_results(build_id):
    """Yield a sequence of (name, log) pairs for all failed tests.

    Looks at the given build ID and all its dependencies.
    """
    dep_data = ET.parse(opener.open(tc_url('builds/{0}'.format(build_id),
                                           fields='snapshot-dependencies(build(id,status))')))
    for b in dep_data.findall("./snapshot-dependencies/build"):
        if b.attrib['status'] != 'SUCCESS':
            yield from collect_build_results(b.attrib['id'])

    test_data = ET.parse(opener.open(tc_url('testOccurrences',
                                            locator='count:100,status:FAILURE,build:(id:{0})'.format(build_id),
                                            fields='testOccurrence(details,name,duration,build(buildType(name)))')))
    for o in test_data.findall('./testOccurrence'):
        test_name = '{0}/{1}'.format(o.find('build/buildType').attrib['name'], o.attrib['name'])

        test_log = '--- FAIL: {0}/{1} ({2:.3f}s)\n{3}\n'.format(
            o.find("build/buildType").attrib["name"],
            o.attrib["name"],
            int(o.attrib["duration"])/1000.,
            o.find("details").text)
        yield (test_name, test_log)


def create_issue(build_id, failed_tests):
    """Format a list of failed tests as an issue.

    Returns a dict which should be encoded as json for posting to the
    github API.
    """
    return {
        'title': 'teamcity: failed tests on {0}: {1}'.format(os.environ['TC_BUILD_BRANCH'],
                                                             ', '.join(t[0] for t in failed_tests)),
        'body': '''\
The following tests appear to have failed:

[#{0}](https://teamcity.cockroachdb.com/viewLog.html?buildId={0}):


```
{1:.60000}
```

Please assign, take a look and update the issue accordingly.
'''.format(build_id, ''.join(t[1] for t in failed_tests)),
        'labels': ['C-test-failure', 'O-robot'],
    }


def post_issue(issue):
    req = urllib.request.Request(
        'https://api.github.com/repos/cockroachdb/cockroach/issues',
        data=json.dumps(issue).encode('utf-8'),
        headers={'Authorization': 'token {0}'.format(os.environ['GITHUB_API_TOKEN'])})
    opener.open(req).read()


if __name__ == '__main__':
    build_id = os.environ['TC_BUILD_ID']
    failed_tests = list(collect_build_results(build_id))
    if failed_tests:
        issue = create_issue(build_id, failed_tests)
        post_issue(issue)