…gators

Cherrypicking #15688 for #15660

The groupNode code was creating unnecessary `IdentAggregators`, even for
constant values. This is not only inefficient, but it doesn't work
correctly in the case where there are no rows to be processed.

Fixes #15686.
Updates #12525.

.github/bors.toml

@@ -0,0 +1,13 @@
status = [
  "GitHub CI (Cockroach)"
]
pr_status = [
  "license/cla"
]
block_labels = [
  "do-not-merge"
]

[committer]
name = "craig[bot]"
email = "bors@cockroachlabs.com"

build/Dockerfile

@@ -104,8 +104,8 @@ RUN git clone --depth 1 https://github.com/tpoechtrager/osxcross.git \
# BEGIN https://github.com/docker-library/golang/blob/master/1.8/alpine/Dockerfile

COPY parallelbuilds-go1.8.patch /
RUN curl -fsSL https://storage.googleapis.com/golang/go1.8.1.src.tar.gz -o golang.tar.gz \
 && echo '33daf4c03f86120fdfdc66bddf6bfff4661c7ca11c5da473e537f4d69b470e57  golang.tar.gz' | sha256sum -c - \
RUN curl -fsSL https://storage.googleapis.com/golang/go1.8.3.src.tar.gz -o golang.tar.gz \
 && echo '5f5dea2447e7dcfdc50fa6b94c512e58bfba5673c039259fd843f68829d99fa6  golang.tar.gz' | sha256sum -c - \
 && tar -C /usr/local -xzf golang.tar.gz \
 && rm golang.tar.gz \
 && cd /usr/local/go/src \

build/common.mk

@@ -181,7 +181,7 @@ endif
#     TARGET-NAME: [export] VARIABLE [:+?]=
#
# The additional complexity below handles whitespace and comments.
$(REPO_ROOT)/build/variables.mk: $(REPO_ROOT)/Makefile $(REPO_ROOT)/.go-version $(REPO_ROOT)/build/common.mk
$(REPO_ROOT)/build/variables.mk: $(REPO_ROOT)/Makefile $(REPO_ROOT)/.go-version $(REPO_ROOT)/build/common.mk $(REPO_ROOT)/build/archive/contents/Makefile
	@echo '# This file is auto-generated by Make.' > $@
	@echo '# DO NOT EDIT!' >> $@
	@echo 'define VALID_VARS' >> $@
@@ -206,6 +206,8 @@ HOST_TRIPLE := $(shell $$($(GO) env CC) -dumpmachine)
CONFIGURE_FLAGS :=
CMAKE_FLAGS := $(if $(MINGW),-G 'MSYS Makefiles')

ENABLE_ROCKSDB_ASSERTIONS := $(findstring race,$(TAGS))

ifdef XHOST_TRIPLE

# Darwin wants clang, so special treatment is in order.
@@ -254,7 +256,7 @@ endif

JEMALLOC_DIR := $(BUILD_DIR)/jemalloc
PROTOBUF_DIR := $(BUILD_DIR)/protobuf
ROCKSDB_DIR  := $(BUILD_DIR)/rocksdb
ROCKSDB_DIR  := $(BUILD_DIR)/rocksdb$(if $(ENABLE_ROCKSDB_ASSERTIONS),_assert)
SNAPPY_DIR   := $(BUILD_DIR)/snappy
# Can't share with protobuf because protoc is always built for the host.
PROTOC_DIR := $(GOPATH)/native/$(HOST_TRIPLE)/protobuf
@@ -292,7 +294,7 @@ CGO_FLAGS_FILES := $(CGO_UNSUFFIXED_FLAGS_FILES) $(CGO_SUFFIXED_FLAGS_FILES)

$(CGO_UNSUFFIXED_FLAGS_FILES): .ALWAYS_REBUILD

$(CGO_FLAGS_FILES):
$(CGO_FLAGS_FILES): $(REPO_ROOT)/build/common.mk
	@echo 'GEN $@'
	@echo '// GENERATED FILE DO NOT EDIT' > $@
	@echo >> $@
@@ -346,7 +348,10 @@ $(JEMALLOC_DIR)/Makefile: $(C_DEPS_DIR)/jemalloc-rebuild $(JEMALLOC_SRC_DIR)/.ex
	mkdir -p $(JEMALLOC_DIR)
	@# NOTE: If you change the configure flags below, bump the version in
	@# $(C_DEPS_DIR)/jemalloc-rebuild. See above for rationale.
	cd $(JEMALLOC_DIR) && $(JEMALLOC_SRC_DIR)/configure $(CONFIGURE_FLAGS) --enable-prof
	@#
	@# jemalloc profiling deadlocks when built against musl. See
	@# https://github.com/jemalloc/jemalloc/issues/585.
	cd $(JEMALLOC_DIR) && $(JEMALLOC_SRC_DIR)/configure $(CONFIGURE_FLAGS) $(if $(findstring musl,$(TARGET_TRIPLE)),,--enable-prof)

$(PROTOBUF_DIR)/Makefile: $(C_DEPS_DIR)/protobuf-rebuild $(PROTOBUF_SRC_DIR)/.extracted
	rm -rf $(PROTOBUF_DIR)
@@ -372,9 +377,13 @@ $(ROCKSDB_DIR)/Makefile: $(C_DEPS_DIR)/rocksdb-rebuild $(ROCKSDB_SRC_DIR)/.extra
	cd $(ROCKSDB_DIR) && cmake $(CMAKE_FLAGS) $(ROCKSDB_SRC_DIR) \
	  $(if $(findstring release,$(TYPE)),,-DWITH_$(if $(findstring mingw,$(TARGET_TRIPLE)),AVX2,SSE42)=OFF) \
	  -DSNAPPY_LIBRARIES=$(SNAPPY_DIR)/.libs/libsnappy.a -DSNAPPY_INCLUDE_DIR=$(SNAPPY_SRC_DIR) -DWITH_SNAPPY=ON \
	  -DJEMALLOC_LIBRARIES=$(JEMALLOC_DIR)/lib/libjemalloc.a -DJEMALLOC_INCLUDE_DIR=$(JEMALLOC_DIR)/include -DWITH_JEMALLOC=ON
	  -DJEMALLOC_LIBRARIES=$(JEMALLOC_DIR)/lib/libjemalloc.a -DJEMALLOC_INCLUDE_DIR=$(JEMALLOC_DIR)/include -DWITH_JEMALLOC=ON \
	  $(if $(ENABLE_ROCKSDB_ASSERTIONS),,-DCMAKE_CXX_FLAGS=-DNDEBUG)
	@# TODO(benesch): Tweak how we pass -DNDEBUG above when we upgrade to a
	@# RocksDB release that includes https://github.com/facebook/rocksdb/pull/2300.

$(SNAPPY_DIR)/Makefile: $(C_DEPS_DIR)/snappy-rebuild $(SNAPPY_SRC_DIR)/.extracted
	rm -rf $(SNAPPY_DIR)
	mkdir -p $(SNAPPY_DIR)
	@# NOTE: If you change the configure flags below, bump the version in
	@# $(C_DEPS_DIR)/snappy-rebuild. See above for rationale.

build/protobuf.mk

@@ -113,7 +113,10 @@ $(UI_JS): $(GO_PROTOS) $(COREOS_RAFT_PROTOS) $(YARN_INSTALLED_TARGET)
	echo '// GENERATED FILE DO NOT EDIT' > $@
	pbjs -t static-module -w es6 --strict-long --keep-case --path $(ORG_ROOT) --path $(GOGO_PROTOBUF_PATH) --path $(COREOS_PATH) --path $(GRPC_GATEWAY_GOOGLEAPIS_PATH) $(GW_PROTOS) >> $@

$(UI_TS): $(UI_JS)
$(UI_TS): $(UI_JS) $(YARN_INSTALLED_TARGET)
	# Install a known-good version of jsdoc; see
	# https://github.com/dcodeIO/protobuf.js/issues/716.
	(cd $(UI_ROOT)/node_modules/protobufjs/cli && npm install --silent jsdoc@3.4.3)
	# Add comment recognized by reviewable.
	echo '// GENERATED FILE DO NOT EDIT' > $@
	pbts $(UI_JS) >> $@

build/style_test.go

@@ -458,6 +458,8 @@ func TestStyle(t *testing.T) {
				"Eventf:1",
				"ErrEvent:1",
				"ErrEventf:1",
				"NewError:1",
				"NewErrorf:1",
				"VEvent:2",
				"VEventf:2",
				"UnimplementedWithIssueErrorf:1",

build/teamcity-post-failures.py

@@ -0,0 +1,96 @@
#!/usr/bin/env python3
"""Post failures from the current teamcity job as github issues.

Requires the following environment variables:
- TC_API_PASSWORD
- TC_BUILD_BRANCH
- TC_BUILD_ID
- GITHUB_API_TOKEN
"""

import json
import os
import urllib.error
import urllib.request
import xml.etree.ElementTree as ET

from urllib.parse import urljoin, urlencode

BASEURL = "https://teamcity.cockroachdb.com/httpAuth/app/rest/"

auth_handler = urllib.request.HTTPBasicAuthHandler()
auth_handler.add_password(realm='TeamCity',
                          uri='https://teamcity.cockroachdb.com',
                          user='robot',
                          passwd=os.environ['TC_API_PASSWORD'])
opener = urllib.request.build_opener(auth_handler)


def tc_url(path, **params):
    return urljoin(BASEURL, path) + '?' + urlencode(params)


def collect_build_results(build_id):
    """Yield a sequence of (name, log) pairs for all failed tests.

    Looks at the given build ID and all its dependencies.
    """
    dep_data = ET.parse(opener.open(tc_url('builds/{0}'.format(build_id),
                                           fields='snapshot-dependencies(build(id,status))')))
    for b in dep_data.findall("./snapshot-dependencies/build"):
        if b.attrib['status'] != 'SUCCESS':
            yield from collect_build_results(b.attrib['id'])

    test_data = ET.parse(opener.open(tc_url('testOccurrences',
                                            locator='count:100,status:FAILURE,build:(id:{0})'.format(build_id),
                                            fields='testOccurrence(details,name,duration,build(buildType(name)))')))
    for o in test_data.findall('./testOccurrence'):
        test_name = '{0}/{1}'.format(o.find('build/buildType').attrib['name'], o.attrib['name'])

        test_log = '--- FAIL: {0}/{1} ({2:.3f}s)\n{3}\n'.format(
            o.find("build/buildType").attrib["name"],
            o.attrib["name"],
            int(o.attrib["duration"])/1000.,
            o.find("details").text)
        yield (test_name, test_log)


def create_issue(build_id, failed_tests):
    """Format a list of failed tests as an issue.

    Returns a dict which should be encoded as json for posting to the
    github API.
    """
    return {
        'title': 'teamcity: failed tests on {0}: {1}'.format(os.environ['TC_BUILD_BRANCH'],
                                                             ', '.join(t[0] for t in failed_tests)),
        'body': '''\
The following tests appear to have failed:

[#{0}](https://teamcity.cockroachdb.com/viewLog.html?buildId={0}):


```
{1:.60000}
```

Please assign, take a look and update the issue accordingly.
'''.format(build_id, ''.join(t[1] for t in failed_tests)),
        'labels': ['C-test-failure', 'O-robot'],
    }


def post_issue(issue):
    req = urllib.request.Request(
        'https://api.github.com/repos/cockroachdb/cockroach/issues',
        data=json.dumps(issue).encode('utf-8'),
        headers={'Authorization': 'token {0}'.format(os.environ['GITHUB_API_TOKEN'])})
    opener.open(req).read()


if __name__ == '__main__':
    build_id = os.environ['TC_BUILD_ID']
    failed_tests = list(collect_build_results(build_id))
    if failed_tests:
        issue = create_issue(build_id, failed_tests)
        post_issue(issue)

build/teamcity-release-upload.sh → build/teamcity-publish-artifacts.sh

@@ -1,32 +1,15 @@
#!/usr/bin/env bash
set -euxo pipefail

export BUILDER_HIDE_GOPATH_SRC=1

echo 'starting release build'
cat .buildinfo/tag || true
cat .buildinfo/rev || true
build/builder.sh git status
# Any arguments to this script are passed through unmodified to
# ./build/teamcity-publish-s3-binaries.

build/builder.sh go install ./pkg/cmd/release-upload

echo 'installed release builder'
cat .buildinfo/tag || true
cat .buildinfo/rev || true
build/builder.sh git status
set -euxo pipefail

build/builder.sh env \
	AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
	AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
	TC_BUILD_BRANCH="$TC_BUILD_BRANCH" \
	release-upload
export BUILDER_HIDE_GOPATH_SRC=1

echo 'built and uploaded artifacts'
cat .buildinfo/tag || true
cat .buildinfo/rev || true
build/builder.sh git status
build/teamcity-publish-s3-binaries.sh "$@"

if [ "$TC_BUILD_BRANCH" != master ]; then
if [[ "$TC_BUILD_BRANCH" != *alpha* ]] && [ "$TEAMCITY_BUILDCONF_NAME" == 'Publish Releases' ]; then
	image=docker.io/cockroachdb/cockroach

	cp cockroach-linux-2.6.32-gnu-amd64 build/deploy/cockroach

build/teamcity-publish-s3-binaries.sh

@@ -0,0 +1,31 @@
#!/usr/bin/env bash

# Any arguments to this script are passed through unmodified to
# ./pkg/cmd/publish-artifacts.

set -euxo pipefail

export BUILDER_HIDE_GOPATH_SRC=1

echo 'starting release build'
cat .buildinfo/tag || true
cat .buildinfo/rev || true
build/builder.sh git status

build/builder.sh go install ./pkg/cmd/publish-artifacts

echo 'installed release builder'
cat .buildinfo/tag || true
cat .buildinfo/rev || true
build/builder.sh git status

build/builder.sh env \
	AWS_ACCESS_KEY_ID="$AWS_ACCESS_KEY_ID" \
	AWS_SECRET_ACCESS_KEY="$AWS_SECRET_ACCESS_KEY" \
	TC_BUILD_BRANCH="$TC_BUILD_BRANCH" \
	publish-artifacts "$@"

echo 'built and uploaded artifacts'
cat .buildinfo/tag || true
cat .buildinfo/rev || true
build/builder.sh git status

build/teamcity-testrace.sh

@@ -12,7 +12,11 @@ build/builder.sh env \
	TARGET=stressrace \
	github-pull-request-make

# Due to a limit on the number of running goroutines in the race
# detector, testrace fails when run on 16-CPU machines. Set GOMAXPROCS
# to work around this.
build/builder.sh env \
        GOMAXPROCS=8 \
	make testrace \
	TESTFLAGS='-v' \
	2>&1 \

build/variables.mk

@@ -25,6 +25,7 @@ define VALID_VARS
  C_LIBS
  C_LIBS_SRCS
  DUPLFLAGS
  ENABLE_ROCKSDB_ASSERTIONS
  FILES
  FIND_RELEVANT
  GITHOOKS
@@ -37,6 +38,7 @@ define VALID_VARS
  GOVERS
  GO_INSTALL
  HOST_TRIPLE
  INSTALL
  ISDARWIN
  JEMALLOC_DIR
  JEMALLOC_SRC_DIR
@@ -80,5 +82,7 @@ define VALID_VARS
  XHOST_BIN_DIR
  XHOST_TRIPLE
  YARN_INSTALLED_TARGET
  bindir
  prefix
  space
endef

c-deps/jemalloc-rebuild

@@ -1,4 +1,4 @@
Bump the version below when changing jemalloc configure flags. Search for "BUILD
ARTIFACT CACHING" in build/common.mk for rationale.

1
2