Skip to content

Commit

Permalink
organize and document the execution engine package
Browse files Browse the repository at this point in the history
Change-Id: I43c482b90d9364f5e9f0cf9350a61d75ab554701
Reviewed-on: https://gerrit.instructure.com/58501
Reviewed-by: Simon Williams <simon@instructure.com>
Tested-by: Jenkins
  • Loading branch information
codekitchen committed Jul 16, 2015
1 parent 9df8140 commit c070d9b
Show file tree
Hide file tree
Showing 5 changed files with 105 additions and 80 deletions.
12 changes: 10 additions & 2 deletions engine/engine.go
Expand Up @@ -5,14 +5,19 @@ import (
"path/filepath"
)

// Engine is a code execution engine, supporting a set of languages for
// sandboxed code execution.
type Engine struct {
languages []*Language
}

// Languages returns the list of supported execution languages.
func (eng *Engine) Languages() []*Language {
return eng.languages
}

// Run executes the given source code with the given options, using the
// specified language.
func (eng *Engine) Run(languageName string, opts *RunOptions) (*RunResult, error) {
lang := eng.findLanguage(languageName)
if lang == nil {
Expand All @@ -22,6 +27,9 @@ func (eng *Engine) Run(languageName string, opts *RunOptions) (*RunResult, error
return lang.Run(opts)
}

// New creates a new execution engine using the yaml config files in the
// specified directory. Only disableApparmor for development, docker alone isn't
// sufficient to fully sandbox the code.
func New(confPath string, disableApparmor bool) (result *Engine, err error) {
result = &Engine{}
configs, err := filepath.Glob(confPath + "/lang-*.yml")
Expand Down Expand Up @@ -51,8 +59,8 @@ func New(confPath string, disableApparmor bool) (result *Engine, err error) {
return
}

func (engine *Engine) findLanguage(name string) *Language {
for _, lang := range engine.Languages() {
func (eng *Engine) findLanguage(name string) *Language {
for _, lang := range eng.Languages() {
if lang.Name == name {
return lang
}
Expand Down
12 changes: 9 additions & 3 deletions engine/execution.go
Expand Up @@ -14,11 +14,17 @@ var (
tempdir = "/tmp"
)

// ExecutionResult contains the results from one step of code execution --
// either compilation or running.
type ExecutionResult struct {
ExitCode int
// The exit status code of the process. 0 is success, anything else is failure.
ExitCode int
// The output streams.
Stdout, Stderr string
RunTime time.Duration
ErrorString string
// How long this step ran.
RunTime time.Duration
// If an error occured, this error string will be non-empty.
ErrorString string
}

type execution struct {
Expand Down
81 changes: 7 additions & 74 deletions engine/language.go
Expand Up @@ -4,20 +4,10 @@ import (
"fmt"
"io/ioutil"
"os"
"regexp"

"gopkg.in/yaml.v2"
)

type test struct {
Source, Stdin, Stdout, Stderr string
ExitStatus int
}

type tests struct {
Simple, Apparmor, Rlimit test
}

// Language represents a supported execution language, read from the config yml
// files. It provides a Run method for sandboxed code execution.
type Language struct {
Expand All @@ -27,16 +17,20 @@ type Language struct {
Filename string
DockerImage string `yaml:"docker_image"`
compileStep bool
ApparmorProfile string `yaml:"apparmor_profile"`
CompilerProfile string `yaml:"compiler_profile"`
Tests tests
ApparmorProfile string `yaml:"apparmor_profile"`
CompilerProfile string `yaml:"compiler_profile"`
Checks checks `yaml:"tests"`
FileExtensions []string `yaml:"file_extensions"`
}

// RunResult contains the full results for each executed step of a code run.
// Some languages don't have a compile step, and the run step will be nil if
// compilation failed.
type RunResult struct {
CompileStep, RunStep *ExecutionResult
}

// RunOptions is configuration for a Language Run.
type RunOptions struct {
Source, Stdin string
Timeout int64
Expand Down Expand Up @@ -124,67 +118,6 @@ func loadLanguage(configName string) (lang *Language, err error) {
return
}

// RunTests does sanity checks on all supported Languages, including checks that
// stdin/stdout work as expected, and basic verification that the AppArmor
// profile is in effect.
func (lang *Language) RunTests() (err error) {
err = lang.runTest("simple", &lang.Tests.Simple)
if lang.ApparmorProfile == "" {
// skip the apparmor tests when it's been disabled
return
}

if err == nil {
err = lang.runTest("apparmor", &lang.Tests.Apparmor)
}
if err == nil {
err = lang.runTest("rlimit", &lang.Tests.Rlimit)
}
return
}

func (lang *Language) runTest(testName string, test *test) error {
result, err := lang.Run(&RunOptions{
Source: test.Source,
Stdin: test.Stdin,
Timeout: 30,
})

if err != nil {
return fmt.Errorf("Failure testing '%s' (%s): %v", lang.Name, testName, err)
}

output, _ := yaml.Marshal(result)

errorString := fmt.Sprintf("for '%s' (%s).\n%s", lang.Name, testName, output)

if result.RunStep == nil {
return fmt.Errorf("Didn't run %s", errorString)
}

if result.RunStep.ExitCode != test.ExitStatus {
return fmt.Errorf("Incorrect exit code %s", errorString)
}

match, err := regexp.MatchString(test.Stderr, result.RunStep.Stderr)
if err != nil {
return err
}
if match == false {
return fmt.Errorf("Incorrect stderr %s", errorString)
}

match, err = regexp.MatchString(test.Stdout, result.RunStep.Stdout)
if err != nil {
return err
}
if match == false {
return fmt.Errorf("Incorrect stdout %s", errorString)
}

return nil
}

func (lang *Language) disableAppArmor() {
lang.ApparmorProfile = ""
lang.CompilerProfile = ""
Expand Down
78 changes: 78 additions & 0 deletions engine/language_checks.go
@@ -0,0 +1,78 @@
package engine

import (
"fmt"
"regexp"

"gopkg.in/yaml.v2"
)

// RunChecks does sanity checks on the Language, including checks that
// stdin/stdout work as expected, and basic verification that the AppArmor
// profile is in effect.
func (lang *Language) RunChecks() (err error) {
err = lang.runCheck("simple", &lang.Checks.Simple)
if lang.ApparmorProfile == "" {
// skip the apparmor tests when it's been disabled
return
}

if err == nil {
err = lang.runCheck("apparmor", &lang.Checks.Apparmor)
}
if err == nil {
err = lang.runCheck("rlimit", &lang.Checks.Rlimit)
}
return
}

type check struct {
Source, Stdin, Stdout, Stderr string
ExitStatus int
}

type checks struct {
Simple, Apparmor, Rlimit check
}

func (lang *Language) runCheck(testName string, check *check) error {
result, err := lang.Run(&RunOptions{
Source: check.Source,
Stdin: check.Stdin,
Timeout: 30,
})

if err != nil {
return fmt.Errorf("Failure testing '%s' (%s): %v", lang.Name, testName, err)
}

output, _ := yaml.Marshal(result)

errorString := fmt.Sprintf("for '%s' (%s).\n%s", lang.Name, testName, output)

if result.RunStep == nil {
return fmt.Errorf("Didn't run %s", errorString)
}

if result.RunStep.ExitCode != check.ExitStatus {
return fmt.Errorf("Incorrect exit code %s", errorString)
}

match, err := regexp.MatchString(check.Stderr, result.RunStep.Stderr)
if err != nil {
return err
}
if match == false {
return fmt.Errorf("Incorrect stderr %s", errorString)
}

match, err = regexp.MatchString(check.Stdout, result.RunStep.Stdout)
if err != nil {
return err
}
if match == false {
return fmt.Errorf("Incorrect stdout %s", errorString)
}

return nil
}
2 changes: 1 addition & 1 deletion main.go
Expand Up @@ -32,7 +32,7 @@ func runLanguageTests(engine *engine.Engine, langToRun string) {
for _, lang := range engine.Languages() {
if langToRun == "" || langToRun == lang.Name {
fmt.Printf("Testing %s\n", lang.VisibleName)
err := lang.RunTests()
err := lang.RunChecks()
if err != nil {
panic(err)
}
Expand Down

0 comments on commit c070d9b

Please sign in to comment.