Merge pull request #89 from cyberark/update-secrets-batch-request

Update secrets batch request

CHANGELOG.md

@@ -8,6 +8,11 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 ### Changed
 - Updated Go versions to 1.15
 
+### Added
+- RetrieveBatchSecretsSafe method which allows the user to specify use of the `Accept: base64`
+  header in batch retrieval requests. This allows for the retrieval of binary secrets from Conjur.
+  [cyberark/conjur-api-go#88](https://github.com/cyberark/conjur-api-go/issues/88)
+
 ## [0.6.1] - 2020-12-02
 ### Changed
 - Errors from YAML parsing are now breaking and visible in logs.

conjurapi/client.go

@@ -36,7 +36,7 @@ type Router interface {
 	LoadPolicyRequest(mode PolicyMode, policyID string, policy io.Reader) (*http.Request, error)
 	ResourceRequest(resourceID string) (*http.Request, error)
 	ResourcesRequest(filter *ResourceFilter) (*http.Request, error)
-	RetrieveBatchSecretsRequest(variableIDs []string) (*http.Request, error)
+	RetrieveBatchSecretsRequest(variableIDs []string, base64Flag bool) (*http.Request, error)
 	RetrieveSecretRequest(variableID string) (*http.Request, error)
 	RotateAPIKeyRequest(roleID string) (*http.Request, error)
 }

conjurapi/router_v4.go

@@ -86,7 +86,11 @@ func (r RouterV4) AddSecretRequest(variableID, secretValue string) (*http.Reques
 	return nil, fmt.Errorf("AddSecret is not supported for Conjur V4")
 }
 
-func (r RouterV4) RetrieveBatchSecretsRequest(variableIDs []string) (*http.Request, error) {
+func (r RouterV4) RetrieveBatchSecretsRequest(variableIDs []string, base64Flag bool) (*http.Request, error) {
+	if base64Flag {
+		return nil, fmt.Errorf("Batch retrieving Base64-encoded secrets is not supported in Conjur V4")
+	}
+
 	return http.NewRequest(
 		"GET",
 		r.batchVariableURL(variableIDs),

conjurapi/router_v5.go

@@ -131,18 +131,28 @@ func (r RouterV5) LoadPolicyRequest(mode PolicyMode, policyID string, policy io.
 	)
 }
 
-func (r RouterV5) RetrieveBatchSecretsRequest(variableIDs []string) (*http.Request, error) {
+func (r RouterV5) RetrieveBatchSecretsRequest(variableIDs []string, base64Flag bool) (*http.Request, error) {
 	fullVariableIDs := []string{}
 	for _, variableID := range variableIDs {
 		fullVariableID := makeFullId(r.Config.Account, "variable", variableID)
 		fullVariableIDs = append(fullVariableIDs, fullVariableID)
 	}
 
-	return http.NewRequest(
+	request, err := http.NewRequest(
 		"GET",
 		r.batchVariableURL(fullVariableIDs),
 		nil,
 	)
+
+	if err != nil {
+		return nil, err
+	}
+
+	if base64Flag {
+		request.Header.Add("Accept", "base64")
+	}
+
+	return request, nil
 }
 
 func (r RouterV5) RetrieveSecretRequest(variableID string) (*http.Request, error) {
@@ -168,11 +178,17 @@ func (r RouterV5) AddSecretRequest(variableID, secretValue string) (*http.Reques
 		return nil, err
 	}
 
-	return http.NewRequest(
+	request, err := http.NewRequest(
 		"POST",
 		variableURL,
 		strings.NewReader(secretValue),
 	)
+	if err != nil {
+		return nil, err
+	}
+
+	request.Header.Add("Content-Type", "application/x-www-form-urlencoded")
+	return request, nil
 }
 
 func (r RouterV5) variableURL(variableID string) (string, error) {

conjurapi/variable.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 
 	"encoding/json"
+	"encoding/base64"
 
 	"github.com/cyberark/conjur-api-go/conjurapi/response"
 )
@@ -14,25 +15,39 @@ import (
 //
 // The authenticated user must have execute privilege on all variables.
 func (c *Client) RetrieveBatchSecrets(variableIDs []string) (map[string][]byte, error) {
-	resp, err := c.retrieveBatchSecrets(variableIDs)
+	jsonResponse, err := c.retrieveBatchSecrets(variableIDs, false)
 	if err != nil {
 		return nil, err
 	}
 
-	data, err := response.DataResponse(resp)
-	if err != nil {
-		return nil, err
+	resolvedVariables := map[string][]byte{}
+	for id, value := range jsonResponse {
+		resolvedVariables[id] = []byte(value)
 	}
 
-	jsonResponse := map[string]string{}
-	err = json.Unmarshal(data, &jsonResponse)
+	return resolvedVariables, nil
+}
+
+// RetrieveBatchSecretsSafe fetches values for all variables in a slice using a
+// single API call. This version of the method will automatically base64-encode
+// the secrets on the server side allowing the retrieval of binary values in
+// batch requests. Secrets are NOT base64 encoded in the returned map.
+//
+// The authenticated user must have execute privilege on all variables.
+func (c *Client) RetrieveBatchSecretsSafe(variableIDs []string) (map[string][]byte, error) {
+	jsonResponse, err := c.retrieveBatchSecrets(variableIDs, true)
 	if err != nil {
 		return nil, err
 	}
 
 	resolvedVariables := map[string][]byte{}
+	var decodedValue []byte
 	for id, value := range jsonResponse {
-		resolvedVariables[id] = []byte(value)
+		decodedValue, err = base64.StdEncoding.DecodeString(value)
+		if err != nil {
+			return nil, err
+		}
+		resolvedVariables[id] = decodedValue
 	}
 
 	return resolvedVariables, nil
@@ -63,13 +78,29 @@ func (c *Client) RetrieveSecretReader(variableID string) (io.ReadCloser, error)
 	return response.SecretDataResponse(resp)
 }
 
-func (c *Client) retrieveBatchSecrets(variableIDs []string) (*http.Response, error) {
-	req, err := c.router.RetrieveBatchSecretsRequest(variableIDs)
+func (c *Client) retrieveBatchSecrets(variableIDs []string, base64Flag bool) (map[string]string, error) {
+	req, err := c.router.RetrieveBatchSecretsRequest(variableIDs, base64Flag)
 	if err != nil {
 		return nil, err
 	}
 
-	return c.SubmitRequest(req)
+	resp, err := c.SubmitRequest(req)
+	if err != nil {
+		return nil, err
+	}
+
+	data, err := response.DataResponse(resp)
+	if err != nil {
+		return nil, err
+	}
+
+	jsonResponse := map[string]string{}
+	err = json.Unmarshal(data, &jsonResponse)
+	if err != nil {
+		return nil, err
+	}
+
+	return jsonResponse, nil
 }
 
 func (c *Client) retrieveSecret(variableID string) (*http.Response, error) {

conjurapi/variable_test.go

@@ -94,12 +94,21 @@ func TestClient_RetrieveSecret(t *testing.T) {
 				"onemore":              "{\"json\": \"object\"}",
 				"a/ b /c":              "somevalue",
 			}
+			binaryVariables := map[string]string{
+				"binary1":              "test\xf0\xf1",
+				"binary2":              "tes\xf0t\xf1i\xf2ng",
+				"nonBinary":            "testing",
+			}
 
 			policy := ""
 			for id := range variables {
 				policy = fmt.Sprintf("%s- !variable %s\n", policy, id)
 			}
 
+			for id := range binaryVariables {
+				policy = fmt.Sprintf("%s- !variable %s\n", policy, id)
+			}
+
 			conjur, err := NewClientFromKey(*config, authn.LoginPair{Login: login, APIKey: apiKey})
 			So(err, ShouldBeNil)
 
@@ -114,6 +123,11 @@ func TestClient_RetrieveSecret(t *testing.T) {
 				So(err, ShouldBeNil)
 			}
 
+			for id, value := range binaryVariables {
+				err = conjur.AddSecret(id, value)
+				So(err, ShouldBeNil)
+			}
+
 			Convey("Fetch many secrets in a single batch retrieval", func() {
 				variableIds := []string{}
 				for id := range variables {
@@ -130,6 +144,36 @@ func TestClient_RetrieveSecret(t *testing.T) {
 					So(string(fetchedValue), ShouldEqual, value)
 				}
 			})
+
+			Convey("Fetch binary secrets in a batch request", func(){
+				variableIds := []string{}
+				for id := range binaryVariables {
+					variableIds = append(variableIds, id)
+				}
+
+				secrets, err := conjur.RetrieveBatchSecretsSafe(variableIds)
+				So(err, ShouldBeNil)
+
+				for id, value := range binaryVariables {
+					fullyQualifiedID := fmt.Sprintf("%s:variable:%s", config.Account, id)
+					fetchedValue, ok := secrets[fullyQualifiedID]
+					So(ok, ShouldBeTrue)
+					So(string(fetchedValue), ShouldEqual, value)
+				}
+			})
+
+			Convey("Fail to fetch binary secrets in batch request", func(){
+				variableIds := []string{}
+				for id := range binaryVariables {
+					variableIds = append(variableIds, id)
+				}
+
+				_, err := conjur.RetrieveBatchSecrets(variableIds)
+				So(err, ShouldNotBeNil)
+				So(err.Error(), ShouldContainSubstring, "Issue encoding secret into JSON format")
+				conjurError := err.(*response.ConjurError)
+				So(conjurError.Code, ShouldEqual, 500)
+			})
 		})
 
 		Convey("Token authenticator can be used to fetch a secret", func() {

docker-compose.yml

@@ -4,8 +4,7 @@ services:
     image: postgres:9.3
 
   conjur:
-    image: cyberark/conjur:1.3
-
+    image: cyberark/conjur:edge
     command: server -a cucumber
     environment:
       DATABASE_URL: postgres://postgres@postgres/postgres