Skip to content
This repository has been archived by the owner on Feb 20, 2020. It is now read-only.

Commit

Permalink
Bug 1375200 - fetch secrets from taskcluster secrets service rather t…
Browse files Browse the repository at this point in the history 
…han from provisioner
  • Loading branch information
@petemoore
petemoore committed Feb 5, 2019
1 parent ef584fd commit 2e068d2
Show file tree
Hide file tree
Showing 18 changed files with 556 additions and 326 deletions.
2 changes: 1 addition & 1 deletion .travis.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ script:
- "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || git status"
- "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || test $(git status --porcelain | wc -l) == 0"
- "go get -ldflags \"-X main.revision=$(git rev-parse HEAD)\" -v -t ./..."
- "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || GORACE=history_size=7 travis_wait 30 ./gotestcover.sh coverage.report"
- "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || GORACE=history_size=7 travis_wait 45 ./gotestcover.sh coverage.report"
- "test $GIMME_OS.$GIMME_ARCH != linux.amd64 || ${GOPATH}/bin/ineffassign ."

after_script:
Expand Down
119 changes: 84 additions & 35 deletions aws.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,13 +69,20 @@ type UserData struct {
ProvisionerBaseURL string `json:"provisionerBaseUrl"`
TaskclusterRootURL string `json:"taskclusterRootUrl"`
SecurityToken string `json:"securityToken"`
// GenericWorker could be defined as type PublicHostSetup, but then
// we wouldn't have a way to call dec.DisallowUnknownFields() without also
// affecting unpacking of UserData struct (which may have unknown fields).
GenericWorker json.RawMessage `json:"genericWorker"`
}

type Secrets struct {
GenericWorker struct {
Config json.RawMessage `json:"config"`
} `json:"generic-worker"`
Files []File `json:"files"`
type PublicHostSetup struct {
Config gwconfig.PublicConfig `json:"config"`
Files []File `json:"files"`
}

type PrivateHostSetup struct {
Config gwconfig.PrivateConfig `json:"config"`
Files []File `json:"files"`
}

type File struct {
Expand Down Expand Up @@ -190,6 +197,7 @@ func updateConfigWithAmazonSettings(c *gwconfig.Config) error {

userData, err := queryUserData()
if err != nil {
// if we can't read user data, this is a serious problem
return err
}
c.ProvisionerID = userData.ProvisionerID
Expand All @@ -203,21 +211,27 @@ func updateConfigWithAmazonSettings(c *gwconfig.Config) error {
awsprov.Authenticate = false
awsprov.Credentials = nil

secToken, getErr := awsprov.GetSecret(userData.SecurityToken)
// remove secrets even if we couldn't retrieve them!
removeErr := awsprov.RemoveSecret(userData.SecurityToken)
if getErr != nil {
return getErr
}
if removeErr != nil {
return removeErr
}
// Retrieving credentials from provisioner only happens on first run. After
// that, they are not available, so only look for them if we need them.
if c.AccessToken == "" || c.ClientID == "" {
secToken, getErr := awsprov.GetSecret(userData.SecurityToken)
// remove secrets even if we couldn't retrieve them!
removeErr := awsprov.RemoveSecret(userData.SecurityToken)
if getErr != nil {
// serious error
return getErr
}
if removeErr != nil {
// security risk if we can't delete secret, so return err
return removeErr
}

c.AccessToken = secToken.Credentials.AccessToken
c.Certificate = secToken.Credentials.Certificate
c.ClientID = secToken.Credentials.ClientID
c.WorkerGroup = userData.Region
c.WorkerType = userData.WorkerType
c.AccessToken = secToken.Credentials.AccessToken
c.Certificate = secToken.Credentials.Certificate
c.ClientID = secToken.Credentials.ClientID
c.WorkerGroup = userData.Region
c.WorkerType = userData.WorkerType
}

awsMetadata := map[string]interface{}{}
for _, url := range []string{
Expand All @@ -232,6 +246,7 @@ func updateConfigWithAmazonSettings(c *gwconfig.Config) error {
key := url[strings.LastIndex(url, "/")+1:]
value, err := queryMetaData(url)
if err != nil {
// not being able to read metadata is serious error
return err
}
awsMetadata[key] = value
Expand All @@ -244,20 +259,46 @@ func updateConfigWithAmazonSettings(c *gwconfig.Config) error {
c.InstanceType = awsMetadata["instance-type"].(string)
c.AvailabilityZone = awsMetadata["availability-zone"].(string)

secrets := new(Secrets)
err = json.Unmarshal(secToken.Data, secrets)
// Parse the config before applying it, to ensure that no disallowed fields
// are included.
publicHostSetup, err := userData.PublicHostSetup()
if err != nil {
return err
}

// Now overlay existing config with values in secrets
err = c.MergeInJSON([]byte(secrets.GenericWorker.Config))
// Host setup per worker type "userData" section.
//
// Note, we first update configuration from public host setup, before
// calling tc-secrets to get private host setup, in case secretsBaseURL is
// configured in userdata.
c.MergeInJSON(userData.GenericWorker, func(a map[string]interface{}) map[string]interface{} {
return a["config"].(map[string]interface{})
})

// Fetch additional (secret) host setup from taskcluster-secrets service.
// See: https://bugzil.la/1375200
tcsec := c.Secrets()
secretName := "worker-type:" + c.ProvisionerID + "/" + c.WorkerType
sec, err := tcsec.Get(secretName)
if err != nil {
return err
}
b := bytes.NewBuffer([]byte(sec.Secret))
d := json.NewDecoder(b)
d.DisallowUnknownFields()
var privateHostSetup PrivateHostSetup
err = d.Decode(&privateHostSetup)
if err != nil {
return err
}

// Now overlay existing config
c.MergeInJSON(sec.Secret, func(a map[string]interface{}) map[string]interface{} {
return a["config"].(map[string]interface{})
})

// Now put secret files in place...
for _, f := range secrets.Files {
// Now put files in place...
for _, f := range append(publicHostSetup.Files, privateHostSetup.Files...) {
err := f.Extract()
if err != nil {
return err
Expand All @@ -277,23 +318,22 @@ func deploymentIDUpdated() bool {
log.Printf("**** Can't reach provisioner to see if there is a new deploymentId: %v", err)
return false
}
secrets := new(Secrets)
err = json.Unmarshal(wtr.Secrets, secrets)
userData := new(UserData)
err = json.Unmarshal(wtr.UserData, &userData)
if err != nil {
log.Printf("**** Can't unmarshal worker type secrets - probably somebody has botched a worker type update - not shutting down as in such a case, that would kill entire pool!")
log.Printf("WARNING: Can't unmarshal custom worker type userdata - probably somebody has botched a worker type update - not shutting down as in such a case, that would kill entire pool!")
return false
}
c := new(gwconfig.Config)
err = json.Unmarshal(secrets.GenericWorker.Config, c)
publicHostSetup, err := userData.PublicHostSetup()
if err != nil {
log.Printf("**** Can't unmarshal config - probably somebody has botched a worker type update - not shutting down as in such a case, that would kill entire pool!")
return false
log.Printf("WARNING: Can't extract public host setup from latest userdata for worker type %v - not shutting down as latest user data is probably botched", config.WorkerType)
}
if c.DeploymentID == config.DeploymentID {
log.Printf("No change to deploymentId - %q == %q", config.DeploymentID, c.DeploymentID)
latestDeploymentID := publicHostSetup.Config.DeploymentID
if latestDeploymentID == config.DeploymentID {
log.Printf("No change to deploymentId - %q == %q", config.DeploymentID, latestDeploymentID)
return false
}
log.Printf("New deploymentId found! %q => %q - therefore shutting down!", config.DeploymentID, c.DeploymentID)
log.Printf("New deploymentId found! %q => %q - therefore shutting down!", config.DeploymentID, latestDeploymentID)
return true
}

Expand Down Expand Up @@ -321,3 +361,12 @@ func handleWorkerShutdown(abort func()) func() {
}()
return ticker.Stop
}

func (userData *UserData) PublicHostSetup() (publicHostSetup *PublicHostSetup, err error) {
publicHostSetup = &PublicHostSetup{}
b := bytes.NewBuffer([]byte(userData.GenericWorker))
d := json.NewDecoder(b)
d.DisallowUnknownFields()
err = d.Decode(publicHostSetup)
return
}
119 changes: 77 additions & 42 deletions aws_helper_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"fmt"
"log"
"net/http"
"os"
"path/filepath"
Expand All @@ -17,6 +18,14 @@ type MockAWSProvisionedEnvironment struct {
SecretFiles []map[string]string
Terminating bool
PretendMetadata string
// PrivateHostSetupFunc is an optional function to mock the content of the
// worker type secret from the taskcluster secrets service.
// If nil, m.PrivateHostSetup will be called instead.
PrivateHostSetupFunc func(t *testing.T) interface{}
// PublicHostSetupFunc is an optional function to mock the content of the
// property genericWorker in the AWS userdata.
// If nil, m.PublicHostSetup will be called instead.
PublicHostSetupFunc func(t *testing.T) interface{}
}

func WriteJSON(t *testing.T, w http.ResponseWriter, resp interface{}) {
Expand All @@ -27,8 +36,8 @@ func WriteJSON(t *testing.T, w http.ResponseWriter, resp interface{}) {
w.Write(bytes)
}

func (m *MockAWSProvisionedEnvironment) Setup(t *testing.T) func() {
teardown := setupEnvironment(t)
func (m *MockAWSProvisionedEnvironment) Setup(t *testing.T) (teardown func(), err error) {
td := setupEnvironment(t)
workerType := slugid.Nice()
configureForAWS = true
oldEC2MetadataBaseURL := EC2MetadataBaseURL
Expand All @@ -39,18 +48,20 @@ func (m *MockAWSProvisionedEnvironment) Setup(t *testing.T) func() {
// use http.DefaultServeMux.
ec2MetadataHandler := http.NewServeMux()
ec2MetadataHandler.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
switch req.URL.Path {
switch req.URL.EscapedPath() {

// simulate provisioner endpoints

case "/provisioner/worker-type/" + workerType:
// just return some json - no tests are currently using the
// contents, but require something to be returned
resp := map[string]interface{}{
"secrets": m.Secrets(t),
"foo": "bar",
}
WriteJSON(t, w, resp)

case "/provisioner/secret/12345":
resp := map[string]interface{}{
"data": m.Secrets(t),
"credentials": map[string]string{
"clientId": os.Getenv("TASKCLUSTER_CLIENT_ID"),
"certificate": os.Getenv("TASKCLUSTER_CERTIFICATE"),
Expand All @@ -60,6 +71,19 @@ func (m *MockAWSProvisionedEnvironment) Setup(t *testing.T) func() {
}
WriteJSON(t, w, resp)

// simulate taskcluster secrets endpoints

case "/secrets/secret/worker-type%3Atest-provisioner%2F" + workerType:
pri := m.PrivateHostSetup
if m.PrivateHostSetupFunc != nil {
pri = m.PrivateHostSetupFunc
}
resp := map[string]interface{}{
"secret": pri(t),
"expires": "2077-08-19T00:00:00.000Z",
}
WriteJSON(t, w, resp)

// simulate AWS endpoints

case "/latest/meta-data/ami-id":
Expand All @@ -83,6 +107,10 @@ func (m *MockAWSProvisionedEnvironment) Setup(t *testing.T) func() {
case "/latest/meta-data/public-ipv4":
fmt.Fprint(w, "12.34.56.78")
case "/latest/user-data":
pub := m.PublicHostSetup
if m.PublicHostSetupFunc != nil {
pub = m.PublicHostSetupFunc
}
resp := map[string]interface{}{
"data": map[string]interface{}{},
"capacity": 1,
Expand All @@ -98,11 +126,13 @@ func (m *MockAWSProvisionedEnvironment) Setup(t *testing.T) func() {
"lastModified": time.Now().Add(time.Minute * -30),
"provisionerBaseUrl": "http://localhost:13243/provisioner",
"securityToken": "12345",
"genericWorker": pub(t),
}
WriteJSON(t, w, resp)
default:
w.WriteHeader(400)
fmt.Fprintf(w, "Cannot serve URL %v", req.URL)
log.Printf("Cannot serve URL %v", req.URL)
t.Fatalf("Cannot serve URL %v", req.URL)
}
})
Expand All @@ -117,59 +147,64 @@ func (m *MockAWSProvisionedEnvironment) Setup(t *testing.T) func() {
s.ListenAndServe()
t.Log("HTTP server for mock Provisioner and EC2 metadata endpoints stopped")
}()
var err error
config, err = loadConfig(filepath.Join(testdataDir, t.Name(), "generic-worker.config"), true, false)
if err != nil {
t.Fatalf("Error: %v", err)
}
return func() {
teardown()
td()
err := s.Shutdown(context.Background())
if err != nil {
t.Fatalf("Error shutting down http server: %v", err)
}
EC2MetadataBaseURL = oldEC2MetadataBaseURL
configureForAWS = false
}
}, err
}

func (m *MockAWSProvisionedEnvironment) Secrets(t *testing.T) interface{} {
func (m *MockAWSProvisionedEnvironment) PublicHostSetup(t *testing.T) interface{} {

gwConfig := map[string]interface{}{
"config": map[string]interface{}{
// Need common caches directory across tests, since files
// directory-caches.json and file-caches.json are not per-test.
"cachesDir": filepath.Join(cwd, "caches"),
"cleanUpTaskDirs": false,
"deploymentId": "sdkfjh4zxmnf",
"disableReboots": true,
// Need common downloads directory across tests, since files
// directory-caches.json and file-caches.json are not per-test.
"downloadsDir": filepath.Join(cwd, "downloads"),
"idleTimeoutSecs": 60,
"livelogSecret": "I have to confess, when me and my friends sort of used to run through the fields of wheat, um, the farmers weren't too pleased about that.",
"numberOfTasksToRun": 1,
// should be enough for tests, and travis-ci.org CI environments
// don't have a lot of free disk
"requiredDiskSpaceMegabytes": 16,
"runTasksAsCurrentUser": os.Getenv("GW_TESTS_RUN_AS_TASK_USER") == "",
"sentryProject": "generic-worker-tests",
"shutdownMachineOnIdle": false,
"shutdownMachineOnInternalError": false,
"openpgpSigningKeyLocation": filepath.Join(testdataDir, "private-opengpg-key"),
"ed25519SigningKeyLocation": filepath.Join(testdataDir, "ed25519_private_key"),
"subdomain": "taskcluster-worker.net",
"tasksDir": filepath.Join(testdataDir, t.Name()),
"workerTypeMetadata": map[string]interface{}{
"machine-setup": map[string]string{
"pretend-metadata": m.PretendMetadata,
},
// Need common caches directory across tests, since files
// directory-caches.json and file-caches.json are not per-test.
"cachesDir": filepath.Join(cwd, "caches"),
"cleanUpTaskDirs": false,
"deploymentId": "sdkfjh4zxmnf",
"disableReboots": true,
// Need common downloads directory across tests, since files
// directory-caches.json and file-caches.json are not per-test.
"downloadsDir": filepath.Join(cwd, "downloads"),
"idleTimeoutSecs": 60,
"numberOfTasksToRun": 1,
// should be enough for tests, and travis-ci.org CI environments
// don't have a lot of free disk
"requiredDiskSpaceMegabytes": 16,
"runTasksAsCurrentUser": os.Getenv("GW_TESTS_RUN_AS_TASK_USER") == "",
"secretsBaseUrl": "http://localhost:13243/secrets",
"sentryProject": "generic-worker-tests",
"shutdownMachineOnIdle": false,
"shutdownMachineOnInternalError": false,
"openpgpSigningKeyLocation": filepath.Join(testdataDir, "private-opengpg-key"),
"ed25519SigningKeyLocation": filepath.Join(testdataDir, "ed25519_private_key"),
"subdomain": "taskcluster-worker.net",
"tasksDir": filepath.Join(testdataDir, t.Name()),
"workerTypeMetadata": map[string]interface{}{
"machine-setup": map[string]string{
"pretend-metadata": m.PretendMetadata,
},
},
}

return map[string]interface{}{
"files": m.SecretFiles,
"generic-worker": gwConfig,
"config": gwConfig,
}
}

func (m *MockAWSProvisionedEnvironment) PrivateHostSetup(t *testing.T) interface{} {

gwConfig := map[string]interface{}{
"livelogSecret": "I have to confess, when me and my friends sort of used to run through the fields of wheat, um, the farmers weren't too pleased about that.",
}

return map[string]interface{}{
"files": m.SecretFiles,
"config": gwConfig,
}
}

0 comments on commit 2e068d2

Please sign in to comment.