Skip to content

Commit

Permalink
Build logs for functions (#406)
Browse files Browse the repository at this point in the history
  • Loading branch information
Dominic Wong committed Mar 17, 2022
1 parent d68feef commit 3f11ab2
Show file tree
Hide file tree
Showing 6 changed files with 374 additions and 75 deletions.
13 changes: 13 additions & 0 deletions function/examples.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -191,5 +191,18 @@
"url": "https://europe-west1-m3o-apis.cloudfunctions.net/helloworld"
}
}
],
"logs": [
{
"title": "Retrieve build logs for a function",
"run_check": false,
"request": {
"name": "helloworld",
"logs_type": "build"
},
"response": {
"logs": "starting build \"a85d3407-bcbf-472e-bb52-f6ddbc9cdff8\"\n\nStep #2 - \"post-buildpack\": Retagging eu.gcr.io/m3o-apis/gcf/europe-west1/540f08ba-4d25-449e-b629-c926d647d42f:domtestfn_version-1 as eu.gcr.io/m3o-apis/gcf/europe-west1/540f08ba-4d25-449e-b629-c926d647d42f:latest\nStep #2 - \"post-buildpack\": Image eu.gcr.io/m3o-apis/gcf/europe-west1/540f08ba-4d25-449e-b629-c926d647d42f:domtestfn_version-1 copied to eu.gcr.io/m3o-apis/gcf/europe-west1/540f08ba-4d25-449e-b629-c926d647d42f:latest\nStep #2 - \"post-buildpack\": Image eu.gcr.io/m3o-apis/gcf/europe-west1/540f08ba-4d25-449e-b629-c926d647d42f/cache:a85d3407-bcbf-472e-bb52-f6ddbc9cdff8 does not exist\nStep #2 - \"post-buildpack\": Already have image (with digest): eu.gcr.io/fn-img/utilities/buildpack-shim:base_20220304_18_04_RC00"
}
}
]
}
1 change: 1 addition & 0 deletions function/handler/function.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ var (
FunctionKey = "function/func/"
OwnerKey = "function/owner/"
ReservationKey = "function/reservation/"
BuildLogsKey = "function/buildlogs/"
)

type Function struct{}
130 changes: 121 additions & 9 deletions function/handler/google.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ import (
"os"
"os/exec"
"path/filepath"
"regexp"
"sort"
"strings"
"time"

Expand All @@ -26,6 +28,10 @@ import (
"github.com/teris-io/shortid"
)

var (
buildLogsRegex = regexp.MustCompile("For Cloud Build Logs, visit: https://console.cloud.google.com/cloud-build/builds;region=.*/(.*)\\?project=")
)

type GoogleFunction struct {
project string
// eg. https://us-central1-m3o-apis.cloudfunctions.net/
Expand Down Expand Up @@ -74,13 +80,13 @@ var (
"php74": "index.php",
}

GitIgnore = []string{
".git",
"dist",
"node_modules",
"vendor",
"*.jar",
}
GitIgnore = []string{
".git",
"dist",
"node_modules",
"vendor",
"*.jar",
}
)

var (
Expand Down Expand Up @@ -189,8 +195,8 @@ func NewFunction() *GoogleFunction {
}

func (e *GoogleFunction) WriteGcloudIgnore(dir string) error {
data := []byte(strings.Join(GitIgnore, "\n"))
return ioutil.WriteFile(filepath.Join(dir, ".gcloudignore"), data, 0644)
data := []byte(strings.Join(GitIgnore, "\n"))
return ioutil.WriteFile(filepath.Join(dir, ".gcloudignore"), data, 0644)
}

func (e *GoogleFunction) Deploy(ctx context.Context, req *function.DeployRequest, rsp *function.DeployResponse) error {
Expand Down Expand Up @@ -258,6 +264,7 @@ func (e *GoogleFunction) Deploy(ctx context.Context, req *function.DeployRequest
if err != nil {
return errors.InternalServerError("function.deploy", "Failed to create source dir")
}

if err := os.WriteFile(filepath.Join(dir, SourceFile[req.Runtime]), []byte(req.Source), 0644); err != nil {
return errors.InternalServerError("function.deploy", "Failed to save source")
}
Expand Down Expand Up @@ -395,6 +402,40 @@ func (e *GoogleFunction) Deploy(ctx context.Context, req *function.DeployRequest
e.WriteGcloudIgnore(cmd.Dir)

outp, err := cmd.CombinedOutput()

var buildID string
reRes := buildLogsRegex.FindStringSubmatch(string(outp))
if len(reRes) > 1 {
buildID = reRes[1]
}
// Logs are a nice to have so don't error out
logCmd := exec.Command("gcloud", "logging", "read", "--project", e.project, "--format", "json", fmt.Sprintf(`resource.type=build AND resource.labels.build_id=%s`, buildID))
logOutp, logErr := logCmd.CombinedOutput()
if logErr != nil {
log.Errorf("Failed to retrieve logs for %s %s", buildID, logErr, string(logOutp))
} else {
// logs are returned in reverse chronological order as json
var logs []map[string]interface{}
if err := json.Unmarshal(logOutp, &logs); err != nil {
log.Errorf("Error unmarshalling logs %s", err)
} else {
filteredLogs := []string{}
for _, l := range logs {
if tp, ok := l["textPayload"]; ok {
filteredLogs = append(filteredLogs, tp.(string))
}
}
sort.Sort(sort.Reverse(sort.StringSlice(filteredLogs)))
// store it
logsKey := BuildLogsKey + tenantId + "/" + req.Name

if err := store.Write(store.NewRecord(logsKey, strings.Join(filteredLogs, "\n"))); err != nil {
log.Errorf("Error writing logs to store %s", err)
}
}

}

if err != nil {
log.Error(fmt.Errorf(string(outp)))
fn.Status = "DeploymentError"
Expand Down Expand Up @@ -557,6 +598,41 @@ func (e *GoogleFunction) Update(ctx context.Context, req *function.UpdateRequest
e.WriteGcloudIgnore(cmd.Dir)

outp, err := cmd.CombinedOutput()

var buildID string
reRes := buildLogsRegex.FindStringSubmatch(string(outp))
if len(reRes) > 1 {
buildID = reRes[1]
}

// Logs are a nice to have so don't error out
logCmd := exec.Command("gcloud", "logging", "read", "--project", e.project, "--format", "json", fmt.Sprintf(`resource.type=build AND resource.labels.build_id=%s`, buildID))
logOutp, logErr := logCmd.CombinedOutput()
if logErr != nil {
log.Errorf("Failed to retrieve logs for %s %s", buildID, logErr, string(logOutp))
} else {
// logs are returned in reverse chronological order as json
var logs []map[string]interface{}
if err := json.Unmarshal(logOutp, &logs); err != nil {
log.Errorf("Error unmarshalling logs %s", err)
} else {
filteredLogs := []string{}
for _, l := range logs {
if tp, ok := l["textPayload"]; ok {
filteredLogs = append(filteredLogs, tp.(string))
}
}
sort.Sort(sort.Reverse(sort.StringSlice(filteredLogs)))
// store it
logsKey := BuildLogsKey + tenantId + "/" + req.Name

if err := store.Write(store.NewRecord(logsKey, strings.Join(filteredLogs, "\n"))); err != nil {
log.Errorf("Error writing logs to store %s", err)
}
}

}

if err != nil {
log.Error(fmt.Errorf(string(outp)))
fn.Status = "DeploymentError"
Expand Down Expand Up @@ -733,6 +809,9 @@ func (e *GoogleFunction) deleteFunction(fn *function.Func, key string) {

// delete the global key
store.Delete(FunctionKey + fn.Id)

// delete the build logs
store.Delete(strings.ReplaceAll(key, OwnerKey, BuildLogsKey))
}

func (e *GoogleFunction) List(ctx context.Context, req *function.ListRequest, rsp *function.ListResponse) error {
Expand Down Expand Up @@ -922,3 +1001,36 @@ func (e *GoogleFunction) DeleteData(ctx context.Context, request *adminpb.Delete
log.Infof("Deleted %d functions for %s", len(recs), request.TenantId)
return nil
}

var (
logsFuncMap = map[string]func(e *GoogleFunction, ctx context.Context, req *function.LogsRequest, rsp *function.LogsResponse) error{
"build": buildLogs, // TODO add runtime logs
}
)

func (e *GoogleFunction) Logs(ctx context.Context, req *function.LogsRequest, rsp *function.LogsResponse) error {
f, ok := logsFuncMap[req.LogsType]
if !ok {
return errors.BadRequest("app.Logs", "Invalid logs_type specified")
}
return f(e, ctx, req, rsp)
}

func buildLogs(e *GoogleFunction, ctx context.Context, req *function.LogsRequest, rsp *function.LogsResponse) error {
id, ok := tenant.FromContext(ctx)
if !ok {
return errors.Unauthorized("app.Logs", "Unauthorized")
}
logsKey := BuildLogsKey + id + "/" + req.Name
recs, err := store.Read(logsKey)
if err != nil {
return errors.NotFound("app.Logs", "Build logs not found")
}
var ret string
if err := json.Unmarshal(recs[0].Value, &ret); err != nil {
log.Errorf("Error unmarshalling logs %s", err)
return errors.NotFound("app.Logs", "Build logs not found")
}
rsp.Logs = ret
return nil
}

0 comments on commit 3f11ab2

Please sign in to comment.