Support for Yandex Cloud KMS

README.md

@@ -164,6 +164,14 @@ To configure AWS KMS key region for client-side encryption and decryption (i.e.,
 To configure the compression method used for backups. Possible options are: `lz4`, 'lzma', 'brotli'. The default method is `lz4`. LZ4 is the fastest method, but the compression ratio is bad.
 LZMA is way much slower. However, it compresses backups about 6 times better than LZ4. Brotli is a good trade-off between speed and compression ratio, which is about 3 times better than LZ4.
 
+* `YC_CSE_KMS_KEY_ID`
+
+To configure Yandex Cloud KMS key for client-side encryption and decryption. By default, no encryption is used.
+
+* `YC_SERVICE_ACCOUNT_KEY_FILE`
+
+To configure the name of a file containing private key of Yandex Cloud Service Account. If not set a token from the metadata service (http://169.254.169.254) will be used to make API calls to Yandex Cloud KMS. 
+
 **More options are available for the chosen database. See it in [Databases](#databases)**
 
 Usage

go.mod

@@ -55,6 +55,8 @@ require (
 	github.com/wal-g/storages v0.0.0-20201214053810-2f4f68a084d2
 	github.com/wal-g/tracelog v0.0.0-20190824100002-0ab2b054ff30
 	github.com/xdg/stringprep v1.0.1-0.20180714160509-73f8eece6fdc // indirect
+	github.com/yandex-cloud/go-genproto v0.0.0-20201102102956-0c505728b6f0
+	github.com/yandex-cloud/go-sdk v0.0.0-20201109103511-a86298d3fea5
 	go.mongodb.org/mongo-driver v1.3.4
 	golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899
 	golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5 // indirect

go.sum

@@ -67,6 +67,7 @@ github.com/aws/aws-sdk-go v1.34.3 h1:pkbLkV9Q/KY86rbV/WG+yzjNektJbjNRdsTNGtNDZcY
 github.com/aws/aws-sdk-go v1.34.3/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0=
 github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
+github.com/c2h5oh/datasize v0.0.0-20200112174442-28bbd4740fee/go.mod h1:S/7n9copUssQ56c7aAgHqftWO4LTf4xY6CGWt8Bc+3M=
 github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4=
 github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
@@ -111,6 +112,7 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
 github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo=
+github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk=
 github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE=
 github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a h1:FQqoVvjbiUioBBFUL5up+h+GdCa/AnJsL/1bIs/veSI=
@@ -229,6 +231,10 @@ github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoA
 github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
 github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
+github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
@@ -335,6 +341,7 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
 github.com/minio/sio v0.2.0 h1:NCRCFLx0r5pRbXf65LVNjxbCGZgNQvNFQkgX3XF4BoA=
 github.com/minio/sio v0.2.0/go.mod h1:nKM5GIWSrqbOZp0uhyj6M1iA0X6xQzSGtYSaTKSCut0=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mongodb/mongo-tools-common v2.0.1+incompatible h1:7s09mrgAmrbJrkNfj1r8Gu6d4WP+u96oVtkBwOyPNWE=
@@ -455,6 +462,10 @@ github.com/xdg/stringprep v1.0.1-0.20180714160509-73f8eece6fdc h1:vIp1tjhVogU0yB
 github.com/xdg/stringprep v1.0.1-0.20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
 github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
+github.com/yandex-cloud/go-genproto v0.0.0-20201102102956-0c505728b6f0 h1:QwTWqr9AopGtz6MEUBglexQxwirTVdE6S+0NspUqC84=
+github.com/yandex-cloud/go-genproto v0.0.0-20201102102956-0c505728b6f0/go.mod h1:HEUYX/p8966tMUHHT+TsS0hF/Ca/NYwqprC5WXSDMfE=
+github.com/yandex-cloud/go-sdk v0.0.0-20201109103511-a86298d3fea5 h1:3pzMBmRkBDrmCUwUXFVHY0y5OzrDZiL2luZH9gCMZBk=
+github.com/yandex-cloud/go-sdk v0.0.0-20201109103511-a86298d3fea5/go.mod h1:EpFnwwuvMXhv3bb9oL4xyq4HowFHCk2QH08brsoFI74=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -549,6 +560,7 @@ golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200320220750-118fecf932d8/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
@@ -729,6 +741,7 @@ google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfG
 google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
+google.golang.org/genproto v0.0.0-20200323114720-3f67cca34472/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
@@ -778,6 +791,8 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=
+gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

internal/config.go

@@ -96,6 +96,9 @@ const (
 
 	AwsAccessKeyId     = "AWS_ACCESS_KEY_ID"
 	AwsSecretAccessKey = "AWS_SECRET_ACCESS_KEY"
+
+	YcKmsKeyIdSetting  = "YC_CSE_KMS_KEY_ID"
+	YcSaKeyFileSetting = "YC_SERVICE_ACCOUNT_KEY_FILE"
 )
 
 var (
@@ -217,6 +220,10 @@ var (
 		"WALG_GS_PREFIX":                 true,
 		"GOOGLE_APPLICATION_CREDENTIALS": true,
 
+		// Yandex Cloud
+		YcSaKeyFileSetting: true,
+		YcKmsKeyIdSetting:  true,
+
 		// SH
 		"WALG_SSH_PREFIX":      true,
 		"SSH_PORT":             true,

internal/configure.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"github.com/wal-g/wal-g/internal/crypto/yckms"
 	"os"
 	"os/exec"
 	"path/filepath"
@@ -309,6 +310,10 @@ func ConfigureCrypter() crypto.Crypter {
 		return awskms.CrypterFromKeyID(viper.GetString(CseKmsIDSetting), viper.GetString(CseKmsRegionSetting))
 	}
 
+	if viper.IsSet(YcKmsKeyIdSetting) {
+		return yckms.YcCrypterFromKeyIdAndCredential(viper.GetString(YcKmsKeyIdSetting), viper.GetString(YcSaKeyFileSetting))
+	}
+
 	if crypter := configureLibsodiumCrypter(); crypter != nil {
 		return crypter
 	}

internal/crypto/yckms/crypter.go

@@ -0,0 +1,60 @@
+package yckms
+
+import (
+	"bufio"
+	"context"
+	"github.com/minio/sio"
+	"github.com/wal-g/tracelog"
+	"github.com/wal-g/wal-g/internal/crypto"
+	"github.com/wal-g/wal-g/internal/ioextensions"
+	ycsdk "github.com/yandex-cloud/go-sdk"
+	"io"
+)
+
+type YcCrypter struct {
+	SymmetricKey ycSymmetricKeyInterface
+}
+
+func (crypter *YcCrypter) Encrypt(writer io.Writer) (io.WriteCloser, error) {
+	if crypter.SymmetricKey.GetKey() == nil {
+		err := crypter.SymmetricKey.CreateKey()
+		tracelog.ErrorLogger.FatalfOnError("Can't generate symmetric key: %v", err)
+	}
+
+	bufferedWriter := bufio.NewWriter(writer)
+	_, err := bufferedWriter.Write(crypter.SymmetricKey.GetEncryptedKey())
+
+	if err != nil {
+		tracelog.ErrorLogger.Printf("Can't write encryption key to buffer: %v", err)
+		return nil, err
+	}
+
+	encryptedWriter, err := sio.EncryptWriter(bufferedWriter, sio.Config{Key: crypter.SymmetricKey.GetKey(), CipherSuites: []byte{sio.AES_256_GCM}})
+
+	if err != nil {
+		tracelog.ErrorLogger.Printf("YC KMS can't create encrypted writer: %v", err)
+		return nil, err
+	}
+
+	return ioextensions.NewOnCloseFlusher(encryptedWriter, bufferedWriter), nil
+}
+
+func (crypter *YcCrypter) Decrypt(reader io.Reader) (io.Reader, error) {
+	err := crypter.SymmetricKey.ReadEncryptedKey(reader)
+	tracelog.ErrorLogger.FatalfOnError("Can't read encryption key from archive file header: %v", err)
+
+	err = crypter.SymmetricKey.Decrypt()
+	tracelog.ErrorLogger.FatalfOnError("Can't decrypt data encryption key from archive file header: %v", err)
+
+	return sio.DecryptReader(reader, sio.Config{Key: crypter.SymmetricKey.GetKey(), CipherSuites: []byte{sio.AES_256_GCM}})
+}
+
+func YcCrypterFromKeyIdAndCredential(keyId string, saFilePath string) crypto.Crypter {
+	credentials := resolveCredentials(saFilePath)
+	sdk, err := ycsdk.Build(context.Background(), ycsdk.Config{
+		Credentials: credentials,
+	})
+	tracelog.ErrorLogger.FatalfOnError("Can't initialize yc sdk: %v", err)
+
+	return &YcCrypter{SymmetricKey: YcSymmetricKeyFromKeyIdAndSdk(keyId, sdk)}
+}

internal/crypto/yckms/crypter_test.go

@@ -0,0 +1,83 @@
+package yckms
+
+import (
+	"bytes"
+	"github.com/stretchr/testify/assert"
+	"github.com/wal-g/wal-g/internal/crypto"
+	"io"
+	"io/ioutil"
+	"testing"
+)
+
+const (
+	testSecretString = "this is a very secret string used in our tests"
+)
+
+type mockedSymmetricKey struct {
+	key          []byte
+	encryptedKey []byte
+}
+
+func (m *mockedSymmetricKey) GetKey() []byte {
+	return m.key
+}
+
+func (m *mockedSymmetricKey) Decrypt() error {
+	m.key = make([]byte, 32)
+	for i := range m.key {
+		m.key[i] = 0xbb
+	}
+	return nil
+}
+
+func (m *mockedSymmetricKey) GetEncryptedKey() []byte {
+	return m.encryptedKey
+}
+
+func (m *mockedSymmetricKey) ReadEncryptedKey(r io.Reader) error {
+	m.encryptedKey = make([]byte, 64)
+	_, err := r.Read(m.encryptedKey)
+	return err
+}
+
+func (m *mockedSymmetricKey) CreateKey() error {
+	m.encryptedKey = make([]byte, 64)
+	for i := range m.encryptedKey {
+		m.encryptedKey[i] = 0xaa
+	}
+	m.key = make([]byte, 32)
+	for i := range m.key {
+		m.key[i] = 0xbb
+	}
+	return nil
+}
+
+func MockedYcCrypter() crypto.Crypter {
+	return &YcCrypter{
+		SymmetricKey: &mockedSymmetricKey{
+			key:          nil,
+			encryptedKey: nil,
+		},
+	}
+}
+
+func TestYcCrypterEncryptionCycle(t *testing.T) {
+	crypter := MockedYcCrypter()
+	buffer := new(bytes.Buffer)
+
+	encrypt, err := crypter.Encrypt(buffer)
+	assert.NoErrorf(t, err, "YcCrypter encryption error: %v", err)
+
+	_, err = encrypt.Write([]byte(testSecretString))
+	assert.NoErrorf(t, err, "YcCrypter writing error: %v", err)
+	err = encrypt.Close()
+	assert.NoErrorf(t, err, "YcCrypter closing error: %v", err)
+
+	decrypt, err := crypter.Decrypt(buffer)
+	assert.NoErrorf(t, err, "YcCrypter decryption error: %v", err)
+
+	decryptedData, err := ioutil.ReadAll(decrypt)
+	assert.NoErrorf(t, err, "YcCryptor reading decrypted data error: %v", err)
+
+	assert.Equal(t, testSecretString, string(decryptedData), "Decrypted text not equal to plain text")
+}