Skip to content

Commit

Permalink
sandbox: add gvisor runsc-based sandbox
Browse files Browse the repository at this point in the history
This creates a VM (running Container-Optimized OS) with configuration
such that it boots up and downloads/configures the runsc Docker
runtime, reloading the existing Docker daemon on the VM, and then
creates a new privileged Docker container with the host's
/var/run/docker.sock available to the container. From within that
container it's then possible for the new sandbox HTTP server to create
its own Docker containers running under gvisor (using docker run
--runtime=runsc).

This then adds a regional us-central1 load balancer and instance group
manager & instane template to run these VMs automatically across
us-central1. Then the play.golang.org frontend can hit that URL
(http://sandbox.play-sandbox-fwd.il4.us-central1.lb.golang-org.internal)

Fixes golang/go#25224
Updates golang/go#30439 (remove nacl)
Updates golang/go#33629 (this CL makes the playground support 2 versions)

Change-Id: I56c8a86875abcde9d29fa7592b23c0ecd3861458
Reviewed-on: https://go-review.googlesource.com/c/playground/+/195983
Run-TryBot: Brad Fitzpatrick <bradfitz@golang.org>
Reviewed-by: Alexander Rakoczy <alex@golang.org>
Reviewed-by: Emmanuel Odeke <emm.odeke@gmail.com>
  • Loading branch information
bradfitz committed Jan 8, 2020
1 parent a46a9c2 commit 4d36241
Show file tree
Hide file tree
Showing 18 changed files with 1,066 additions and 58 deletions.
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
.terraform
25 changes: 22 additions & 3 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -2,12 +2,14 @@
# Use of this source code is governed by a BSD-style
# license that can be found in the LICENSE file.

############################################################################
FROM debian:stretch AS nacl

RUN apt-get update && apt-get install -y --no-install-recommends curl bzip2 ca-certificates

RUN curl -s https://storage.googleapis.com/nativeclient-mirror/nacl/nacl_sdk/trunk.544461/naclsdk_linux.tar.bz2 | tar -xj -C /tmp --strip-components=2 pepper_67/tools/sel_ldr_x86_64

############################################################################
FROM debian:stretch AS build
LABEL maintainer="golang-dev@googlegroups.com"

Expand Down Expand Up @@ -64,12 +66,32 @@ COPY . /go/src/playground/
WORKDIR /go/src/playground
RUN go install

############################################################################
# Temporary Docker stage to add a pre-Go1.14 $GOROOT into our
# container for early linux/amd64 testing.
FROM golang:1.13 AS temp_pre_go14

ENV BUILD_DEPS 'curl git gcc patch libc6-dev ca-certificates'
RUN apt-get update && apt-get install -y --no-install-recommends ${BUILD_DEPS}

# go1.14beta1:
ENV GO_REV a5bfd9da1d1b24f326399b6b75558ded14514f23

RUN cd /usr/local && git clone https://go.googlesource.com/go go1.14 && cd go1.14 && git reset --hard ${GO_REV}
WORKDIR /usr/local/go1.14/src
RUN ./make.bash
ENV GOROOT /usr/local/go1.14
RUN ../bin/go install --tags=faketime std

############################################################################
# Final stage.
FROM debian:stretch

RUN apt-get update && apt-get install -y git ca-certificates --no-install-recommends

COPY --from=build /usr/local/go /usr/local/go
COPY --from=nacl /tmp/sel_ldr_x86_64 /usr/local/bin
COPY --from=temp_pre_go14 /usr/local/go1.14 /usr/local/go1.14

ENV GOPATH /go
ENV PATH /usr/local/go/bin:$GOPATH/bin:$PATH
Expand Down Expand Up @@ -101,9 +123,6 @@ COPY edit.html /app
COPY static /app/static
WORKDIR /app

# Run tests
RUN /app/playground test

# Whether we allow third-party imports via proxy.golang.org:
ENV ALLOW_PLAY_MODULE_DOWNLOADS true

Expand Down
23 changes: 19 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -9,12 +9,27 @@ GCLOUD_ACCESS_TOKEN := $(shell gcloud auth print-access-token)
docker:
docker build -t golang/playground .

test:
runlocal:
docker network create sandnet || true
docker kill play_dev || true
docker run --name=play_dev --rm --network=sandnet -ti -p 127.0.0.1:8081:8080/tcp golang/playground

test_go:
# Run fast tests first: (and tests whether, say, things compile)
GO111MODULE=on go test -v
# Then run the slower tests, which happen as one of the
# Dockerfile RUN steps:
docker build -t golang/playground .

test_nacl: docker
docker kill sandbox_front_test || true
docker run --rm --name=sandbox_front_test --network=sandnet -t golang/playground testnacl

test_gvisor: docker
docker kill sandbox_front_test || true
docker run --rm --name=sandbox_front_test --network=sandnet -t golang/playground test

# Note: test_gvisor is not included in "test" yet, because it requires
# running a separate server first ("make runlocal" in the sandbox
# directory)
test: test_go test_nacl

update-cloudbuild-trigger:
# The gcloud CLI doesn't yet support updating a trigger.
Expand Down
8 changes: 0 additions & 8 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -59,14 +59,12 @@ golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190502183928-7f726cade0ab/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421 h1:Wo7BWFiOk0QRFMLYMqJGFMd9CgUAcGx7V+qEg/h5IBI=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a h1:tImsplftrFpALCYumobsd0K86vlAs/eXGFms2txfJfA=
golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
Expand All @@ -75,16 +73,13 @@ golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJ
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82 h1:vsphBvatvfbhlb4PO1BYSr9dzugGxJ/SQHoNufZJq1w=
golang.org/x/sys v0.0.0-20190502175342-a43fa875dd82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2 h1:z99zHgr7hKfrUcX/KsoJk5FJfjTceCKIp96+biqP4To=
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
Expand All @@ -103,17 +98,14 @@ golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8T
google.golang.org/api v0.4.0 h1:KKgc1aqhV8wDPbDzlDtpvyjZFY3vjz85FP7p4wcQUyI=
google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7 h1:ZUjXAXmrAyrmmCPHgCA/vChHcpsX27MZ3yBonD/z1KE=
google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873 h1:nfPFGzJkUDX6uBmpN/pSw7MbOAWegH5QDQuoXFHedLg=
google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE=
google.golang.org/grpc v1.19.0 h1:cfg4PD8YEdSFnm7qLV4++93WcmhH2nIUhMjhdCvl3j8=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU=
google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38=
Expand Down
4 changes: 4 additions & 0 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,10 @@ func main() {
s.test()
return
}
if len(os.Args) > 1 && os.Args[1] == "testnacl" {
s.testNacl()
return
}

port := os.Getenv("PORT")
if port == "" {
Expand Down
122 changes: 95 additions & 27 deletions sandbox.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"context"
"crypto/sha256"
"encoding/json"
"errors"
"fmt"
"go/ast"
"go/doc"
Expand All @@ -26,11 +27,12 @@ import (
"runtime"
"strconv"
"strings"
"syscall"
"text/template"
"time"

"cloud.google.com/go/compute/metadata"
"github.com/bradfitz/gomemcache/memcache"
"golang.org/x/playground/sandbox/sandboxtypes"
)

const (
Expand Down Expand Up @@ -79,7 +81,7 @@ type response struct {
// If there is no cached *response for the combination of cachePrefix and request.Body,
// handler calls cmdFunc and in case of a nil error, stores the value of *response in the cache.
// The handler returned supports Cross-Origin Resource Sharing (CORS) from any domain.
func (s *server) commandHandler(cachePrefix string, cmdFunc func(*request) (*response, error)) http.HandlerFunc {
func (s *server) commandHandler(cachePrefix string, cmdFunc func(context.Context, *request) (*response, error)) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
cachePrefix := cachePrefix // so we can modify it below
w.Header().Set("Access-Control-Allow-Origin", "*")
Expand Down Expand Up @@ -110,7 +112,7 @@ func (s *server) commandHandler(cachePrefix string, cmdFunc func(*request) (*res
if err != memcache.ErrCacheMiss {
s.log.Errorf("s.cache.Get(%q, &response): %v", key, err)
}
resp, err = cmdFunc(&req)
resp, err = cmdFunc(r.Context(), &req)
if err != nil {
s.log.Errorf("cmdFunc error: %v", err)
http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
Expand Down Expand Up @@ -315,7 +317,7 @@ var failedTestPattern = "--- FAIL"
// The output of successfully ran program is returned in *response.Events.
// If a program cannot be built or has timed out,
// *response.Errors contains an explanation for a user.
func compileAndRun(req *request) (*response, error) {
func compileAndRun(ctx context.Context, req *request) (*response, error) {
// TODO(andybons): Add semaphore to limit number of running programs at once.
tmpDir, err := ioutil.TempDir("", "sandbox")
if err != nil {
Expand Down Expand Up @@ -368,15 +370,34 @@ func compileAndRun(req *request) (*response, error) {
}
}

// TODO: remove all this once Go 1.14 is out. This is a transitional/debug step
// to support both nacl & gvisor temporarily.
useGvisor := os.Getenv("GO_VERSION") >= "go1.14" ||
os.Getenv("DEBUG_FORCE_GVISOR") == "1" ||
strings.Contains(req.Body, "//play:gvisor\n")

exe := filepath.Join(tmpDir, "a.out")
goCache := filepath.Join(tmpDir, "gocache")

ctx, cancel := context.WithTimeout(context.Background(), maxCompileTime)
buildCtx, cancel := context.WithTimeout(ctx, maxCompileTime)
defer cancel()
cmd := exec.CommandContext(ctx, "go", "build", "-o", exe, buildPkgArg)
goBin := "go"
if useGvisor {
goBin = "/usr/local/go1.14/bin/go"
}
cmd := exec.CommandContext(buildCtx, goBin,
"build",
"-o", exe,
"-tags=faketime", // required for Go 1.14+, no-op before
buildPkgArg)
cmd.Dir = tmpDir
var goPath string
cmd.Env = []string{"GOOS=nacl", "GOARCH=amd64p32", "GOCACHE=" + goCache}
if useGvisor {
cmd.Env = []string{"GOOS=linux", "GOARCH=amd64", "GOROOT=/usr/local/go1.14"}
} else {
cmd.Env = []string{"GOOS=nacl", "GOARCH=amd64p32"}
}
cmd.Env = append(cmd.Env, "GOCACHE="+goCache)
if useModules {
// Create a GOPATH just for modules to be downloaded
// into GOPATH/pkg/mod.
Expand Down Expand Up @@ -411,28 +432,62 @@ func compileAndRun(req *request) (*response, error) {
}
return nil, fmt.Errorf("error building go source: %v", err)
}
ctx, cancel = context.WithTimeout(context.Background(), maxRunTime)
runCtx, cancel := context.WithTimeout(ctx, maxRunTime)
defer cancel()
cmd = exec.CommandContext(ctx, "sel_ldr_x86_64", "-l", "/dev/null", "-S", "-e", exe, testParam)
rec := new(Recorder)
cmd.Stdout = rec.Stdout()
cmd.Stderr = rec.Stderr()
var status int
if err := cmd.Run(); err != nil {
if ctx.Err() == context.DeadlineExceeded {
// Send what was captured before the timeout.
events, err := rec.Events()
if err != nil {
return nil, fmt.Errorf("error decoding events: %v", err)
}
return &response{Errors: "process took too long", Events: events}, nil
var exitCode int
if useGvisor {
f, err := os.Open(exe)
if err != nil {
return nil, err
}
exitErr, ok := err.(*exec.ExitError)
if !ok {
return nil, fmt.Errorf("error running sandbox: %v", err)
defer f.Close()
req, err := http.NewRequestWithContext(ctx, "POST", sandboxBackendURL(), f)
if err != nil {
return nil, err
}
req.Header.Add("Idempotency-Key", "1") // lets Transport do retries with a POST
if testParam != "" {
req.Header.Add("X-Argument", testParam)
}
req.GetBody = func() (io.ReadCloser, error) { return os.Open(exe) }
res, err := http.DefaultClient.Do(req)
if err != nil {
return nil, err
}
if ws, ok := exitErr.Sys().(syscall.WaitStatus); ok {
status = ws.ExitStatus()
defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return nil, fmt.Errorf("unexpected response from backend: %v", res.Status)
}
var execRes sandboxtypes.Response
if err := json.NewDecoder(res.Body).Decode(&execRes); err != nil {
log.Printf("JSON decode error from backend: %v", err)
return nil, errors.New("error parsing JSON from backend")
}
if execRes.Error != "" {
return &response{Errors: execRes.Error}, nil
}
exitCode = execRes.ExitCode
rec.Stdout().Write(execRes.Stdout)
rec.Stderr().Write(execRes.Stderr)
} else {
cmd := exec.CommandContext(runCtx, "sel_ldr_x86_64", "-l", "/dev/null", "-S", "-e", exe, testParam)
cmd.Stdout = rec.Stdout()
cmd.Stderr = rec.Stderr()
if err := cmd.Run(); err != nil {
if ctx.Err() == context.DeadlineExceeded {
// Send what was captured before the timeout.
events, err := rec.Events()
if err != nil {
return nil, fmt.Errorf("error decoding events: %v", err)
}
return &response{Errors: "process took too long", Events: events}, nil
}
exitErr, ok := err.(*exec.ExitError)
if !ok {
return nil, fmt.Errorf("error running sandbox: %v", err)
}
exitCode = exitErr.ExitCode()
}
}
events, err := rec.Events()
Expand All @@ -455,7 +510,7 @@ func compileAndRun(req *request) (*response, error) {
}
return &response{
Events: events,
Status: status,
Status: exitCode,
IsTest: testParam != "",
TestsFailed: fails,
VetErrors: vetOut,
Expand Down Expand Up @@ -490,7 +545,8 @@ func playgroundGoproxy() string {
}

func (s *server) healthCheck() error {
resp, err := compileAndRun(&request{Body: healthProg})
ctx := context.Background() // TODO: cap it to some reasonable timeout
resp, err := compileAndRun(ctx, &request{Body: healthProg})
if err != nil {
return err
}
Expand All @@ -503,6 +559,18 @@ func (s *server) healthCheck() error {
return nil
}

func sandboxBackendURL() string {
if v := os.Getenv("SANDBOX_BACKEND_URL"); v != "" {
return v
}
id, _ := metadata.ProjectID()
switch id {
case "golang-org":
return "http://sandbox.play-sandbox-fwd.il4.us-central1.lb.golang-org.internal/run"
}
panic(fmt.Sprintf("no SANDBOX_BACKEND_URL environment and no default defined for project %q", id))
}

const healthProg = `
package main
Expand Down
1 change: 1 addition & 0 deletions sandbox/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
*.yaml.expanded
34 changes: 34 additions & 0 deletions sandbox/Dockerfile
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
# This is the sandbox backend server.
#
# When it's run, the host maps in /var/run/docker.sock to this
# environment so the play-sandbox server can connect to the host's
# docker daemon, which has the gvisor "runsc" runtime available.

FROM golang:1.13 AS build

COPY . /go/src/playground
WORKDIR /go/src/playground/sandbox
RUN go install

FROM debian:buster

RUN apt-get update

# Extra stuff for occasional debugging:
RUN apt-get install --yes strace lsof emacs25-nox net-tools tcpdump procps

# Install Docker CLI:
RUN apt-get install --yes \
apt-transport-https \
ca-certificates \
curl \
gnupg2 \
software-properties-common
RUN bash -c "curl -fsSL https://download.docker.com/linux/debian/gpg | apt-key add -"
RUN add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian buster stable"
RUN apt-get update
RUN apt-get install --yes docker-ce-cli

COPY --from=build /go/bin/sandbox /usr/local/bin/play-sandbox

ENTRYPOINT ["/usr/local/bin/play-sandbox"]
Loading

0 comments on commit 4d36241

Please sign in to comment.