-
Notifications
You must be signed in to change notification settings - Fork 2.3k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
[gcp-deployer] add service account key handler and a check health handler #1329
Changes from 2 commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,100 @@ | ||
package app | ||
|
||
import ( | ||
"golang.org/x/net/context" | ||
"cloud.google.com/go/container/apiv1" | ||
iamadmin "cloud.google.com/go/iam/admin/apiv1" | ||
"google.golang.org/api/option" | ||
"golang.org/x/oauth2" | ||
"k8s.io/client-go/rest" | ||
containerpb "google.golang.org/genproto/googleapis/container/v1" | ||
"google.golang.org/genproto/googleapis/iam/admin/v1" | ||
"fmt" | ||
log "github.com/sirupsen/logrus" | ||
"k8s.io/api/core/v1" | ||
meta_v1 "k8s.io/apimachinery/pkg/apis/meta/v1" | ||
clientset "k8s.io/client-go/kubernetes" | ||
"encoding/base64" | ||
) | ||
|
||
|
||
type InsertSaKeyRequest struct { | ||
Cluster string | ||
Namespace string | ||
Project string | ||
SecretKey string | ||
SecretName string | ||
ServiceAccount string | ||
Token string | ||
Zone string | ||
} | ||
|
||
func buildClusterConfig(ctx context.Context, ts oauth2.TokenSource, request InsertSaKeyRequest) (*rest.Config, error) { | ||
c, err := container.NewClusterManagerClient(ctx, option.WithTokenSource(ts)) | ||
if err != nil { | ||
return nil, err | ||
} | ||
req := &containerpb.GetClusterRequest{ | ||
ProjectId: request.Project, | ||
Zone: request.Zone, | ||
ClusterId: request.Cluster, | ||
} | ||
resp, err := c.GetCluster(ctx, req) | ||
if err != nil { | ||
return nil, err | ||
} | ||
Token, err := ts.Token() | ||
caDec, _ := base64.StdEncoding.DecodeString(resp.MasterAuth.ClusterCaCertificate) | ||
return &rest.Config{ | ||
Host: "https://" + resp.Endpoint, | ||
BearerToken: Token.AccessToken, | ||
TLSClientConfig: rest.TLSClientConfig { | ||
CAData: []byte(string(caDec)), | ||
}, | ||
}, nil | ||
} | ||
|
||
func (s *ksServer) InsertSaKey(ctx context.Context, request InsertSaKeyRequest) error { | ||
ts := oauth2.StaticTokenSource(&oauth2.Token{ | ||
AccessToken: request.Token, | ||
}) | ||
k8sConfig, err := buildClusterConfig(ctx, ts, request) | ||
if err != nil { | ||
log.Errorf("Failed getting GKE cluster info: %v", err) | ||
return err | ||
} | ||
|
||
c, err := iamadmin.NewIamClient(ctx, option.WithTokenSource(ts)) | ||
if err != nil { | ||
log.Errorf("Cannot create iam admin client: %v", err) | ||
return err | ||
} | ||
createServiceAccountKeyRequest := admin.CreateServiceAccountKeyRequest{ | ||
Name: fmt.Sprintf("projects/%v/serviceAccounts/%v", request.Project, request.ServiceAccount), | ||
} | ||
|
||
s.iamMux.Lock() | ||
defer s.iamMux.Unlock() | ||
|
||
createdKey, err := c.CreateServiceAccountKey(ctx, &createServiceAccountKeyRequest) | ||
if err != nil { | ||
log.Errorf("Failed creating sa key: %v", err) | ||
return err | ||
} | ||
k8sClientset, err := clientset.NewForConfig(k8sConfig) | ||
secretData := make(map[string][]byte) | ||
secretData[request.SecretKey] = createdKey.PrivateKeyData | ||
_, err = k8sClientset.CoreV1().Secrets(request.Namespace).Create( | ||
&v1.Secret{ | ||
ObjectMeta: meta_v1.ObjectMeta{ | ||
Namespace: request.Namespace, | ||
Name: request.SecretName, | ||
}, | ||
Data: secretData, | ||
}) | ||
if err != nil { | ||
log.Errorf("Failed creating secret in GKE cluster: %v", err) | ||
return err | ||
} | ||
return nil | ||
} |
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -32,6 +32,7 @@ const JUPYTER_PROTOTYPE = "jupyterhub" | |
type KsService interface { | ||
// CreateApp creates a ksonnet application. | ||
CreateApp(context.Context, CreateRequest) error | ||
InsertSaKey(context.Context, InsertSaKeyRequest) error | ||
} | ||
|
||
// appInfo keeps track of information about apps. | ||
|
@@ -58,6 +59,7 @@ type ksServer struct { | |
|
||
apps map[string]*appInfo | ||
appsMux sync.Mutex | ||
iamMux sync.Mutex | ||
} | ||
|
||
// NewServer constructs a ksServer. | ||
|
@@ -134,11 +136,19 @@ type CreateRequest struct { | |
AutoConfigure bool | ||
} | ||
|
||
// createRequest is the response to a createRequest | ||
type createResponse struct { | ||
// basicServerResponse is general response contains nil if handler raise no error, otherwise an error message. | ||
type basicServerResponse struct { | ||
Err string `json:"err,omitempty"` // errors don't JSON-marshal, so we use a string | ||
} | ||
|
||
type LiveRequest struct { | ||
Msg string | ||
} | ||
|
||
type LiveResponse struct { | ||
Reply string | ||
} | ||
|
||
// Request to apply an app. | ||
type ApplyRequest struct { | ||
// Name of the app to apply | ||
|
@@ -515,8 +525,30 @@ func makeCreateAppEndpoint(svc KsService) endpoint.Endpoint { | |
req := request.(CreateRequest) | ||
err := svc.CreateApp(ctx, req) | ||
|
||
r := &createResponse{} | ||
r := &basicServerResponse{} | ||
|
||
if err != nil { | ||
r.Err = err.Error() | ||
} | ||
return r, nil | ||
} | ||
} | ||
|
||
func makeHealthzEndpoint(svc KsService) endpoint.Endpoint { | ||
return func(ctx context.Context, request interface{}) (interface{}, error) { | ||
req := request.(LiveRequest) | ||
r := &LiveResponse{} | ||
r.Reply = req.Msg + "accepted! Sill alive!" | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Are there no other checks we want to perform here to make sure the entire system is up and running? If the endpoint returns a success code, can the caller then assume that all other services are up.. etc? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Currently this one only indicate backend is reachable and have cycles to respond to requests. We can keep a server status variable to indicate backend status, which should kill the pod if needed so we have auto-start. |
||
log.Info("response info: " + r.Reply) | ||
return r, nil | ||
} | ||
} | ||
|
||
func makeSaKeyEndpoint(svc KsService) endpoint.Endpoint { | ||
return func(ctx context.Context, request interface{}) (interface{}, error) { | ||
req := request.(InsertSaKeyRequest) | ||
err := svc.InsertSaKey(ctx, req) | ||
r := &basicServerResponse{} | ||
if err != nil { | ||
r.Err = err.Error() | ||
} | ||
|
@@ -550,6 +582,35 @@ func (s *ksServer) StartHttp(port int) { | |
encodeResponse, | ||
) | ||
|
||
healthzHandler := httptransport.NewServer( | ||
makeHealthzEndpoint(s), | ||
func (_ context.Context, r *http.Request) (interface{}, error) { | ||
var request LiveRequest | ||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil { | ||
log.Info("Err decoding request: " + err.Error()) | ||
return nil, err | ||
} | ||
log.Info("Request received: " + request.Msg) | ||
return request, nil | ||
}, | ||
encodeResponse, | ||
) | ||
|
||
insertSaKeyHandler := httptransport.NewServer( | ||
makeSaKeyEndpoint(s), | ||
func (_ context.Context, r *http.Request) (interface{}, error) { | ||
var request InsertSaKeyRequest | ||
if err := json.NewDecoder(r.Body).Decode(&request); err != nil { | ||
return nil, err | ||
} | ||
return request, nil | ||
}, | ||
encodeResponse, | ||
) | ||
|
||
http.Handle("/apps/create", createAppHandler) | ||
http.Handle("/healthz", healthzHandler) | ||
http.Handle("/iam/insertSaKey", insertSaKeyHandler) | ||
|
||
log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil)) | ||
} |
Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is this call synchronous? What does the caller of the SA key endpoint expect to happen after making the API call? Do they then have to wait or poll somehow if they want to block on this operation?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
It's synchronous. When handler returns, either the sa key already inserted into GKE or an error happened and included in response.