Merge branch 'develop' into feature/testsSholdNotFailFast

.circleci/config.yml

@@ -4,10 +4,10 @@ workflows:
   build-and-deploy:
     jobs:
       - build
-      - push:     
+      - push:
           filters:
             tags:
-              only: /^v.*/   
+              only: /^v.*/
             branches:
               ignore: /.*/
 jobs:
@@ -30,7 +30,7 @@ jobs:
     resource_class: large
     steps:
       - checkout
-      - run: 
+      - run:
           name: semver check
           command: |
             MAJOR=$(echo ${CIRCLE_TAG#v} | cut -d"." -f1)
@@ -46,8 +46,8 @@ jobs:
 
             This job uses the semver from the git TAG as the public version to publish.
 
-            - This should only run on workflows triggered by a tag. 
-            - The tag name should be a semver like 'v1.2.3' 
+            - This should only run on workflows triggered by a tag.
+            - The tag name should be a semver like 'v1.2.3'
             - The version should follow conventions documented at https://github.com/fsaintjacques/semver-tool
             EOF
             exit 1

.github/workflows/cli.yml

@@ -12,22 +12,26 @@ on:
 
 jobs:
   build:
-    name: Build and test ${{ matrix.os }}
+    name: Build and test ${{ matrix.os }} ${{ matrix.arch }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
         os: [linux, darwin, windows]
+        arch: [amd64, arm64]
+        exclude:
+          - os: windows
+            arch: arm64
     env:
       VAULT_VERSION: "1.1.3"
       VAULT_TOKEN: "root"
       VAULT_ADDR: "http://127.0.0.1:8200"
     steps:
       - name: Install dependencies
         run: sudo apt-get update && sudo apt-get install git -y
-      - name: Set up Go 1.13
+      - name: Set up Go 1.17
         uses: actions/setup-go@v2
         with:
-          go-version: 1.13
+          go-version: 1.17
         id: go
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
@@ -37,13 +41,24 @@ jobs:
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
-      - name: Build
+      - name: Build Linux and Darwin
+        if: matrix.os != 'windows'
+        run: GOOS=${{ matrix.os }} GOARCH=${{ matrix.arch }} go build -o sops-${{ matrix.os }}-${{ matrix.arch }}-${{ github.sha }} -v ./cmd/sops
+      - name: Build Windows
+        if: matrix.os == 'windows'
         run: GOOS=${{ matrix.os }} go build -o sops-${{ matrix.os }}-${{ github.sha }} -v ./cmd/sops
       - name: Import test GPG keys
         run: for i in 1 2 3 4 5; do gpg --import pgp/sops_functional_tests_key.asc && break || sleep 15; done
       - name: Test
         run: make test
-      - name: Upload artifact
+      - name: Upload artifact for Linux and Darwin
+        if: matrix.os != 'windows'
+        uses: actions/upload-artifact@v2
+        with:
+          name: sops-${{ matrix.os }}-${{ matrix.arch }}-${{ github.sha }}
+          path: sops-${{ matrix.os }}-${{ matrix.arch }}-${{ github.sha }}
+      - name: Upload artifact for Windows
+        if: matrix.os == 'windows'
         uses: actions/upload-artifact@v2
         with:
           name: sops-${{ matrix.os }}-${{ github.sha }}
@@ -63,9 +78,9 @@ jobs:
         uses: actions/checkout@v2
       - uses: actions/download-artifact@v2
         with:
-          name: sops-linux-${{ github.sha }}
+          name: sops-linux-amd64-${{ github.sha }}
       - name: Move SOPS binary
-        run: mv sops-linux-${{ github.sha }} ./functional-tests/sops
+        run: mv sops-linux-amd64-${{ github.sha }} ./functional-tests/sops
       - name: Make SOPS binary executable
         run: chmod +x ./functional-tests/sops
       - name: Download Vault

.github/workflows/release.yml

@@ -15,13 +15,15 @@ jobs:
         run: sudo apt-get update && sudo apt-get install git ruby rpm -y
       - name: Install fpm
         run: gem install fpm || sudo gem install fpm
-      - name: Set up Go 1.15
+      - name: Set up Go 1.17
         uses: actions/setup-go@v2
         with:
-          go-version: 1.15
+          go-version: 1.17
         id: go
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
+      - name: Go vendor
+        run: go mod vendor
       - name: Make release directory
         run: mkdir dist
       - name: Build deb and rpm
@@ -32,20 +34,32 @@ jobs:
         run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       - name: Set RELEASE_NUMBER
         run: echo "RELEASE_NUMBER=$(echo $RELEASE_VERSION | cut -c2-)" >> $GITHUB_ENV
-      - name: Build darwin binary
-        run: GOOS=darwin CGO_ENABLED=0 go build -mod vendor -o dist/sops-${{ env.RELEASE_VERSION }}.darwin go.mozilla.org/sops/v3/cmd/sops
+      - name: Build linux arm64 binary
+        run: GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -mod vendor -o dist/sops-${{ env.RELEASE_VERSION }}.linux.arm64 go.mozilla.org/sops/v3/cmd/sops
+      - name: Build darwin amd64 binary
+        run: GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build -mod vendor -o dist/sops-${{ env.RELEASE_VERSION }}.darwin.amd64 go.mozilla.org/sops/v3/cmd/sops
+      - name: Copy darwin amd64 to have a no-architecture labeled version
+        run: cp dist/sops-${{ env.RELEASE_VERSION }}.darwin.amd64 dist/sops-${{ env.RELEASE_VERSION }}.darwin
+      - name: Build darwin arm64 binary
+        run: GOOS=darwin GOARCH=arm64 CGO_ENABLED=0 go build -mod vendor -o dist/sops-${{ env.RELEASE_VERSION }}.darwin.arm64 go.mozilla.org/sops/v3/cmd/sops
       - name: Build windows binary
         run: GOOS=windows CGO_ENABLED=0 go build -mod vendor -o dist/sops-${{ env.RELEASE_VERSION }}.exe go.mozilla.org/sops/v3/cmd/sops
       - name: Copy already built linux binary
-        run: cp tmppkg/usr/local/bin/sops dist/sops-${{ env.RELEASE_VERSION }}.linux
+        run: cp tmppkg/usr/local/bin/sops dist/sops-${{ env.RELEASE_VERSION }}.linux && cp tmppkg/usr/local/bin/sops dist/sops-${{ env.RELEASE_VERSION }}.linux.amd64
       - name: Create release
         uses: "mozilla/action-automatic-releases@latest"
         with:
           repo_token: "${{ secrets.GITHUB_TOKEN }}"
           prerelease: true
           files: |
             dist/sops-${{ env.RELEASE_VERSION }}.exe
+            dist/sops-${{ env.RELEASE_VERSION }}.darwin.amd64
+            dist/sops-${{ env.RELEASE_VERSION }}.darwin.arm64
             dist/sops-${{ env.RELEASE_VERSION }}.darwin
+            dist/sops-${{ env.RELEASE_VERSION }}.linux.amd64
+            dist/sops-${{ env.RELEASE_VERSION }}.linux.arm64
             dist/sops-${{ env.RELEASE_VERSION }}.linux
             dist/sops_${{ env.RELEASE_NUMBER }}_amd64.deb
+            dist/sops_${{ env.RELEASE_NUMBER }}_arm64.deb
             dist/sops-${{ env.RELEASE_NUMBER }}-1.x86_64.rpm
+            dist/sops-${{ env.RELEASE_NUMBER }}-1.aarch64.rpm

Dockerfile

@@ -1,10 +1,10 @@
-FROM golang:1.15
+FROM golang:1.17
 
 COPY . /go/src/go.mozilla.org/sops
 WORKDIR /go/src/go.mozilla.org/sops
 
 RUN CGO_ENABLED=1 make install
 RUN apt-get update
-RUN apt-get install -y vim python-pip emacs
+RUN apt-get install -y vim python3-pip emacs
 RUN pip install awscli
 ENV EDITOR vim

Dockerfile.alpine

@@ -1,4 +1,5 @@
 FROM golang:1.12-alpine3.10 AS builder
+FROM golang:1.17-alpine3.15 AS builder
 
 RUN apk --no-cache add make
 
@@ -8,7 +9,7 @@ WORKDIR /go/src/go.mozilla.org/sops
 RUN CGO_ENABLED=1 make install
 
 
-FROM alpine:3.10
+FROM alpine:3.15
 
 RUN apk --no-cache add \
   vim ca-certificates

Makefile

@@ -3,7 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 PROJECT		:= go.mozilla.org/sops/v3
-GO 		:= GO15VENDOREXPERIMENT=1 GO111MODULE=on GOPROXY=https://proxy.golang.org go
+GO 		:= GOPROXY=https://proxy.golang.org go
 GOLINT 		:= golint
 
 all: test vet generate install functional-tests
@@ -48,31 +48,47 @@ functional-tests-all:
 	$(GO) build -o functional-tests/sops go.mozilla.org/sops/v3/cmd/sops
 	cd functional-tests && cargo test && cargo test -- --ignored
 
-deb-pkg: vendor
+# Creates variables during target re-definition. Basically this block allows the particular variables to be used in the final target
+build-deb-%: OS = $(word 1,$(subst -, ,$*))
+build-deb-%: ARCH = $(word 2,$(subst -, ,$*))
+build-deb-%: FPM_ARCH = $(word 3,$(subst -, ,$*))
+# Poor-mans function with parameters being split out from the variable part of it's name
+build-deb-%:
 	rm -rf tmppkg
 	mkdir -p tmppkg/usr/local/bin
-	GOOS=linux CGO_ENABLED=0 go build -mod vendor -o tmppkg/usr/local/bin/sops go.mozilla.org/sops/v3/cmd/sops
+	GOOS=$(OS) GOARCH="$(ARCH)" CGO_ENABLED=0 go build -mod vendor -o tmppkg/usr/local/bin/sops go.mozilla.org/sops/v3/cmd/sops
 	fpm -C tmppkg -n sops --license MPL2.0 --vendor mozilla \
 		--description "Sops is an editor of encrypted files that supports YAML, JSON and BINARY formats and encrypts with AWS KMS and PGP." \
 		-m "AJ Bahnken <ajvb+sops@mozilla.com>" \
 		--url https://go.mozilla.org/sops \
-		--architecture x86_64 \
+		--architecture $(FPM_ARCH) \
 		-v "$$(grep '^const Version' version/version.go |cut -d \" -f 2)" \
 		-s dir -t deb .
 
-rpm-pkg: vendor
+# Create .deb packages for multiple architectures
+deb-pkg: vendor build-deb-linux-amd64-x86_64 build-deb-linux-arm64-arm64
+
+# Creates variables during target re-definition. Basically this block allows the particular variables to be used in the final target
+build-rpm-%: OS = $(word 1,$(subst -, ,$*))
+build-rpm-%: ARCH = $(word 2,$(subst -, ,$*))
+build-rpm-%: FPM_ARCH = $(word 3,$(subst -, ,$*))
+# Poor-mans function with parameters being split out from the variable part of it's name
+build-rpm-%:
 	rm -rf tmppkg
 	mkdir -p tmppkg/usr/local/bin
-	GOOS=linux CGO_ENABLED=0 go build -mod vendor -o tmppkg/usr/local/bin/sops go.mozilla.org/sops/v3/cmd/sops
+	GOOS=$(OS) GOARCH="$(ARCH)" CGO_ENABLED=0 go build -mod vendor -o tmppkg/usr/local/bin/sops go.mozilla.org/sops/v3/cmd/sops
 	fpm -C tmppkg -n sops --license MPL2.0 --vendor mozilla \
 		--description "Sops is an editor of encrypted files that supports YAML, JSON and BINARY formats and encrypts with AWS KMS and PGP." \
 		-m "AJ Bahnken <ajvb+sops@mozilla.com>" \
 		--url https://go.mozilla.org/sops \
-		--architecture x86_64 \
-		--rpm-os linux \
+		--architecture $(FPM_ARCH) \
+		--rpm-os $(OS) \
 		-v "$$(grep '^const Version' version/version.go |cut -d \" -f 2)" \
 		-s dir -t rpm .
 
+# Create .rpm packages for multiple architectures
+rpm-pkg: vendor build-rpm-linux-amd64-x86_64 build-rpm-linux-arm64-arm64
+
 dmg-pkg: install
 ifneq ($(OS),darwin)
 		echo 'you must be on MacOS and set OS=darwin on the make command line to build an OSX package'
@@ -82,7 +98,7 @@ else
 	cp $$GOPATH/bin/sops tmppkg/usr/local/bin/
 	fpm -C tmppkg -n sops --license MPL2.0 --vendor mozilla \
 		--description "Sops is an editor of encrypted files that supports YAML, JSON and BINARY formats and encrypts with AWS KMS and PGP." \
-		-m "Julien Vehent <jvehent+sops@mozilla.com>" \
+		-m "Mozilla Security <security@mozilla.org>" \
 		--url https://go.mozilla.org/sops \
 		--architecture x86_64 \
 		-v "$$(grep '^const Version' version/version.go |cut -d \" -f 2)" \

README.rst

@@ -9,11 +9,8 @@ formats and encrypts with AWS KMS, GCP KMS, Azure Key Vault, age, and PGP.
 
 ------------
 
-.. image:: https://godoc.org/go.mozilla.org/sops?status.svg
-	:target: https://godoc.org/go.mozilla.org/sops
-
-.. image:: https://travis-ci.org/mozilla/sops.svg?branch=master
-	:target: https://travis-ci.org/mozilla/sops
+.. image:: https://pkg.go.dev/badge/go.mozilla.org/sops/v3.svg
+	:target: https://pkg.go.dev/go.mozilla.org/sops/v3
 
 Download
 --------
@@ -28,12 +25,13 @@ For the adventurous, unstable features are available in the `develop` branch, wh
 
 .. code:: bash
 
-	$ go get -u go.mozilla.org/sops/v3/cmd/sops
+	$ mkdir -p $GOPATH/src/go.mozilla.org/sops/
+        $ git clone https://github.com/mozilla/sops.git $GOPATH/src/go.mozilla.org/sops/
         $ cd $GOPATH/src/go.mozilla.org/sops/
         $ git checkout develop
         $ make install
 
-(requires Go >= 1.13)
+(requires Go >= 1.17)
 
 If you don't have Go installed, set it up with:
 
@@ -46,12 +44,7 @@ If you don't have Go installed, set it up with:
 
 Or whatever variation of the above fits your system and shell.
 
-To use **sops** as a library, take a look at the `decrypt package <https://godoc.org/go.mozilla.org/sops/decrypt>`_.
-
-**What happened to Python Sops?** We rewrote Sops in Go to solve a number of
-deployment issues, but the Python branch still exists under ``python-sops``. We
-will keep maintaining it for a while, and you can still ``pip install sops``,
-but we strongly recommend you use the Go version instead.
+To use **sops** as a library, take a look at the `decrypt package <https://pkg.go.dev/go.mozilla.org/sops/v3/decrypt>`_.
 
 .. sectnum::
 .. contents:: Table of Contents
@@ -197,6 +190,8 @@ configuration directory. On Linux, this would be ``$XDG_CONFIG_HOME/sops/age/key
 On macOS, this would be ``$HOME/Library/Application Support/sops/age/keys.txt``. On
 Windows, this would be ``%AppData%\sops\age\keys.txt``. You can specify the location
 of this file manually by setting the environment variable **SOPS_AGE_KEY_FILE**.
+Alternatively you can provide the the key(s) directly by setting the **SOPS_AGE_KEY**
+environment variable.
 
 The contents of this key file should be a list of age X25519 identities, one
 per line. Lines beginning with ``#`` are considered comments and ignored. Each
@@ -704,12 +699,6 @@ By default, ``sops`` uses the key server ``keys.openpgp.org`` to retrieve the GP
 keys that are not present in the local keyring.
 This is no longer configurable. You can learn more about why from this write-up: `SKS Keyserver Network Under Attack <https://gist.github.com/rjhansen/67ab921ffb4084c865b3618d6955275f>`_.
 
-Example: place the following in your ``~/.bashrc``
-
-.. code:: bash
-
-	SOPS_GPG_KEYSERVER = 'gpg.example.com'
-
 
 Key groups
 ~~~~~~~~~~
@@ -1207,7 +1196,7 @@ This file will not work in sops:
 	  - array
 	  - elements
 
-But this one will because because the ``sops`` key can be added at the same level as the
+But this one will work because the ``sops`` key can be added at the same level as the
 ``data`` key.
 
 .. code:: yaml
@@ -1374,26 +1363,6 @@ The value must be formatted as json.
 
 	$ sops --set '["an_array"][1] {"uid1":null,"uid2":1000,"uid3":["bob"]}' ~/git/svc/sops/example.yaml
 
-Using sops as a library in a python script
-~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
-
-You can import sops as a module and use it in your python program.
-
-.. code:: python
-
-	import sops
-
-	pathtype = sops.detect_filetype(path)
-	tree = sops.load_file_into_tree(path, pathtype)
-	sops_key, tree = sops.get_key(tree)
-	tree = sops.walk_and_decrypt(tree, sops_key)
-	sops.write_file(tree, path=path, filetype=pathtype)
-
-Note: this uses the previous implementation of `sops` written in python,
-
-and so doesn't support newer features such as GCP-KMS.
-To use the current version, call out to ``sops`` using ``subprocess.run``
-
 Showing diffs in cleartext in git
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 
@@ -1709,8 +1678,8 @@ file format introduced in **1.0**.
 Security
 --------
 
-Please report security issues to jvehent at mozilla dot com, or by using one
-of the contact method available on keybase: `https://keybase.io/jvehent <https://keybase.io/jvehent>`_
+Please report security issues to security at mozilla dot org, or by using one
+of the contact method available here: `https://www.mozilla.org/en-US/security/#For_Developers <https://www.mozilla.org/en-US/security/#For_Developers>`_
 
 License
 -------
@@ -1721,9 +1690,12 @@ Authors
 
 The core team is composed of:
 
+* AJ Banhken @ajvb
+
+The original authors were:
+
 * Adrian Utrilla @autrilla
 * Julien Vehent @jvehent
-* AJ Banhken @ajvb
 
 And a whole bunch of `contributors <https://github.com/mozilla/sops/graphs/contributors>`_