Merge branch 'master' into kubeflow-blog-3

.dockerignore

@@ -1,2 +1,4 @@
-*
-!etc/compile/GO_VERSION
+/.git
+/doc
+/etc
+/examples

.gitignore

@@ -36,7 +36,6 @@ _cgo_export.*
 _testmain.go
 
 *.exe
-*.test
 
 *.swp
 *.swo

.goxc.json.template

@@ -6,7 +6,7 @@
 	],
 	"BuildConstraints": "linux,amd64 darwin,amd64",
     "BuildSettings" : {
-        "LdFlags": "-X github.com/pachyderm/pachyderm/src/server/vendor/github.com/pachyderm/pachyderm/src/client/version.AdditionalVersion=%%VERSION_ADDITIONAL%%"
+        "LdFlags": "-X github.com/pachyderm/pachyderm/src/client/version.AdditionalVersion=%%VERSION_ADDITIONAL%%"
     },
 	"TaskSettings": {
 		"publish-github": {

.travis.yml

@@ -1,14 +1,13 @@
 dist: xenial
 sudo: required
-services:
-  - docker
 cache:
   directories:
     # If any directories are added here, they must also be added to
     # etc/testing/travis_cache.sh
     - ${HOME}/.cache
     - ${HOME}/cached-deps
     - ${HOME}/gopath/pkg/mod
+    - ${HOME}/docker
     - "$(python3 -c 'import site; print(site.USER_BASE)')"
 language: go
 go:
@@ -37,6 +36,7 @@ env:
 before_install:
   # Make sure cache dirs exist and are writable
   - etc/testing/travis_cache.sh
+  - etc/testing/travis_before_install.sh
 install:
   - etc/testing/travis_install.sh
 before_script:
@@ -47,6 +47,7 @@ before_cache:
   # Make sure cache dirs have the necessary permissions for Travis to traverse
   # & copy them all elsewhere
   - etc/testing/travis_cache.sh
+  - etc/testing/travis_before_cache.sh
 after_failure:
   - kubectl version
   - kubectl get all
@@ -60,7 +61,10 @@ branches:
   only:
   # Trigger only for PR's to master
   - master
-  # Trigger CI tess for all branches that end with "-ci-trigger"
+  # Trigger CI tests for all release branches
+  # Eg. 1.9.x, 1.10.x
+  - /^\d+\.\d+\.x/
+  # Trigger CI tests for all branches that end with "-ci-trigger"
   # ruby reg-ex except for things before/after "/"
   - /.*(?i:ci-trigger)$/
   # For all branches uncomment the line below or remove the "branches" section

.vscode/tasks.json

@@ -4,13 +4,13 @@
     {
       "label": "docker-build-proto",
       "type": "shell",
-      "command": "docker build -t pachyderm_proto etc/proto",
+      "command": "eval $(docker-machine env --shell bash); docker build -t pachyderm_proto etc/proto",
       "problemMatcher": []
     },
     {
       "label": "build-proto",
       "type": "shell",
-      "command": "find src -regex \".*\\.proto\" | xargs tar cf - | docker run -i pachyderm_proto | tar xf -",
+      "command": "eval $(docker-machine env --shell bash); find src -regex \".*\\.proto\" | xargs tar cf - | docker run -i pachyderm_proto | tar xf -",
       "problemMatcher": []
     },
     {
@@ -34,7 +34,7 @@
     {
       "label": "docker-build-pull",
       "type": "shell",
-      "command": "docker pull pachyderm/compile:go1.12.1"
+      "command": "eval $(docker-machine env --shell bash); docker pull pachyderm/compile:go1.12.1"
     },
     {
       "label": "docker-build-all",
@@ -51,7 +51,8 @@
     {
       "label": "docker-build-pachd",
       "type": "shell",
-      "command": "./etc/compile/vscode_compile_task.sh pachd"
+      "command": "./etc/compile/vscode_compile_task.sh pachd",
+      "problemMatcher": []
     },
     {
       "label": "launch-dev",

CHANGELOG.md

@@ -1,5 +1,62 @@
 # Changelog
 
+## 1.10.0
+
+- Change Pachyderm license from Apache 2.0 to Pachyderm Community License
+- Changes to how resources are applied to pipeline containers (#4675)
+- Changes to GitHook and Prometheus ports (#4537)
+- Changes to handle S3 credentials passed to S3 gateway when Auth is disabled (#4585)
+- Changes to add support for ‘zsh’ shell (#4494)
+- Changes to allow only critical servers to startup with `--required-critical-servers-only` (#4536)
+- Changes to improve job logging (#4538)
+- Changes to support copying files from output repo to input repos (#4475)
+- Changes to ‘flush job’ CLI to support streaming output with --raw option (#4569)
+- Changes to remove cluster ID check (#4532)
+- Adds annotations and labels to top-level pipeline spec (#4608) (NOTE: If your pipeline spec specifies “service.annotations”, it is recommended that you follow the upgrade path and manually update the pipelines specs to include annotations under the new metadata tag)
+- Adds support for S3 inputs & outputs in pipeline specs (#4605, #4660)
+- New interactive Pachyderm Shell. The shell provides an easier way to interact with pachctl, including advanced auto-completion support (#4485, #4557)
+- Adds support for creating secrets through Pachyderm. (#4483)
+- Adds support for disabling commit progress indicator to reduce load on etcd (#4696)
+- Fixes a bug that ignored the EDITOR environment variable (#4672)
+- Fixes a bug that would cause restore failures from v1.8.x version to v1.9.x+ version (#4662)
+- Fixes a bug that would result in missing output data, under specific conditions, when a job resumes processing (#4656)
+- Fixes a bug that caused errors when specifying a branch name as the provenance of a new commit (#4657)
+- Fixes a bug that would leave a stats commit open under some failure conditions during run pipeline (#4637)
+- Fixes a bug that resulted in a stuck merge process when some commits are left in an unfinished state (#4595)
+- Fixes a bug that ignored the cron pipeline overwrite value when ‘run cron’ is called from the command line (#4517)
+- Fixes a bug that caused `edit pipeline` command to open an empty file (#4526)
+- Fixes a bug where some unfinished commit finish times displayed the Unix Epoch time. (#4539)
+- Fixes a family of bugs and edge conditions with spout marker (#4487)
+- Fixes a bug that would cause crash in ‘diff file’ command (#4601)
+- Fixes a bug that caused a crash when `run pipeline` is executed with stats enabled (#4615)
+- Fixes a bug that incorrectly skips duplicate datums in a union, under specific conditions (#4691)
+- Fixes a bug that ignored the logging level set in the environment variable (#4706) 
+
+
+## 1.9.12
+
+- New configuration for deployments (exposed through pachctl deploy flags):
+  - Only require critical servers to startup and run without error (--require-critical-servers-only). (#4512)
+- Improved job logging. (#4523)
+- Fixes a bug where some unfinished commit finish times displayed the Unix Epoch time. (#4524) 
+- Fixes a bug with edit pipeline. (#4530) 
+- Removed cluster id check. (#4534)
+- Fixes a bug with spout markers. (#4487)
+
+## 1.9.11
+
+- New configuration for deployments (exposed through pachctl deploy flags):
+  - Object storage upload concurrency limit (--upload-concurrency-limit). (#4393)
+- Various configuration improvements. (#4442)
+- Fixes a bug that would cause workers to segfault. (#4459)
+- Upgrades pachyderm to go 1.13.5. (#4472)
+- New configuration for amazon and custom deployments (exposed through pachctl deploy amazon/custom flags):
+  - Disabling ssl (--disable-ssl) (#4473)
+  - Skipping certificate verification (--no-verify-ssl) (#4473)
+- Further improves the logging and error reporting during pachd startup. (#4486)
+- Removes pprof http server from pachd (debugging should happen through the debug api). (#4496)
+- Removes k8s api access from worker code. (#4498)
+
 ## 1.9.10
 
 - Fixes a bug that causes `pachctl` to connect to the wrong cluster (#4416)
@@ -123,6 +180,21 @@
 - `atom` inputs have been removed and use `pfs` inputs instead. (#3639)
 - The `ADDRESS` env var for connecting to pachd has been removed, use `PACHD_ADDRESS` instead. (#3638)
 
+## 1.8.8
+
+- Fixes a bug that caused pipelines to recompute everything when they were restored. (#4079)
+
+## 1.8.7
+
+- Make the 'put file' directory traversal change backwards compatible for legacy branches (#3707)
+- Several fixes to provenance (#3734):
+    - Force provenance to be transitively closed
+    - Propagate all affected branches on deleteCommit
+    - Fix weird two branches with one commit bugs
+- Added a new fsck utility for PFS (#3734)
+- Make stats somewhat toggleable (#3758)
+- Example of spouts using kafka (#3752)
+- Refactor/fix some of the PFS upload steps (#3750)
 
 ## 1.8.6
 

Dockerfile

@@ -1,33 +1,12 @@
-FROM ubuntu:18.04
-LABEL maintainer="jdoliner@pachyderm.io"
-
-RUN \
-  apt-get update -yq && \
-  apt-get install -yq --no-install-recommends \
-    build-essential \
-    ca-certificates \
-    curl \
-    git \
-    libssl-dev \
-    pkg-config && \
-  apt-get clean && \
-  rm -rf /var/lib/apt
-COPY etc/compile/GO_VERSION GO_VERSION
-RUN \
-  curl -fsSL https://get.docker.com/builds/Linux/x86_64/docker-1.12.1.tgz | tar -C /bin -xz docker/docker --strip-components=1 && \
-  chmod +x /bin/docker
-RUN \
-  curl -sSL https://storage.googleapis.com/golang/$(cat GO_VERSION).linux-amd64.tar.gz | tar -C /tmp -xz && \
-  mkdir -p /usr/local/go && \
-  mv /tmp/go/bin /usr/local/go && \
-  mv /tmp/go/src /usr/local/go && \
-  mkdir -p /usr/local/go/pkg/tool/linux_amd64 && \
-  mv /tmp/go/pkg/include /usr/local/go/pkg && \
-  mv /tmp/go/pkg/tool/linux_amd64/compile /usr/local/go/pkg/tool/linux_amd64 && \
-  mv /tmp/go/pkg/tool/linux_amd64/asm /usr/local/go/pkg/tool/linux_amd64 && \
-  mv /tmp/go/pkg/tool/linux_amd64/link /usr/local/go/pkg/tool/linux_amd64 && \
-  rm -rf /tmp/go && \
-  mkdir -p /go/bin
-ENV PATH /go/bin:/usr/local/go/bin:$PATH
-ENV GOPATH /go
-RUN go get github.com/kisielk/errcheck golang.org/x/lint/golint
+# syntax=docker/dockerfile:1.0-experimental
+ARG GO_VERSION
+FROM golang:${GO_VERSION}
+RUN go get github.com/go-bindata/go-bindata/...
+WORKDIR /app
+COPY . .
+ARG LD_FLAGS
+RUN --mount=type=cache,target=/go/pkg/mod \
+    --mount=type=cache,target=/root/.cache/go-build \
+    go-bindata -o src/server/cmd/worker/assets/assets.go -pkg assets /etc/ssl/certs/... && \
+    CGO_ENABLED=0 go build -ldflags "${LD_FLAGS}" -o pachd "src/server/cmd/pachd/main.go" && \
+    CGO_ENABLED=0 go build -ldflags "${LD_FLAGS}" -o worker "src/server/cmd/worker/main.go"