From a288b5dd70ec14dc5b3c6422dcc1fc4543977860 Mon Sep 17 00:00:00 2001 From: Scott Haseley Date: Fri, 12 Feb 2016 08:03:02 -0500 Subject: [PATCH] Rewrote build logic for the submission process. Initial commit for submissions. --- build.go | 293 +++++++++++++++++++++++++++++++++++++------------ build_test.go | 39 ++++--- conf.go | 6 +- environment.go | 4 +- groups.go | 40 +------ output.go | 11 +- persistence.go | 55 ++++++++++ run.go | 43 +++++--- run_test.go | 48 ++++++++ stats.go | 2 +- submission.go | 86 +++++++++++++++ target.go | 14 ++- 12 files changed, 490 insertions(+), 151 deletions(-) create mode 100644 persistence.go create mode 100644 submission.go diff --git a/build.go b/build.go index f3b6eb1..d340003 100644 --- a/build.go +++ b/build.go @@ -1,8 +1,11 @@ package test161 import ( + "crypto/sha256" + "encoding/hex" "errors" "fmt" + "gopkg.in/mgo.v2/bson" "io/ioutil" "os" "os/exec" @@ -10,109 +13,263 @@ import ( "strings" ) +// BuildTest is a variant of a Test, and specifies how the build process should work. +// We obey the same schema so the front end tools can treat this like any other test. +type BuildTest struct { + + // Mongo ID + ID bson.ObjectId `yaml:"-" json:"id" bson:"_id,omitempty"` + + // Metadata + Name string `yaml:"name" json:"name"` + Description string `yaml:"description" json:"description"` + + Commands []*BuildCommand `json:"commands"` // Protected by L + Status []Status `json:"status"` // Protected by L + Result TestResult `json:"result"` // Protected by L + + // Dependency data + DependencyID string `json:"depid"` + IsDependency bool `json:"isdependency"` + + // Grading. These are set when the test is being run as part of a Target. + PointsAvailable uint `json:"points_avail" bson:"points_avail"` + PointsEarned uint `json:"points_earned" bson:"points_earned"` + ScoringMethod string `json:"scoring_method" bson:"scoring_method"` + + startTime TimeFixedPoint + dir string // The base (temp) directory for the build. + wasCached bool // Was the based directory cached + srcDir string // Directory for the os161 source code (dir/src) + rootDir string // Directory for compilation output (dir/root) + + conf *BuildConf + + updateChan chan *TestUpdateMsg // Nonblocking write, may be nil +} + +// A variant of a Test Command for builds +type BuildCommand struct { + Type string `json:"type"` + Input InputLine `json:"input"` + + // Set during target init + PointsAvailable uint `json:"points_avail" bson:"points_avail"` + PointsEarned uint `json:"points_earned" bson:"points_earned"` + + // Set during testing + Output []*OutputLine `json:"output"` + + // Set during evaluation + Status string `json:"status"` + + startDir string // The directory to run this command in + handler func(*BuildTest, *BuildCommand) error // Invoke after command exits to determine success +} + // BuildConf specifies the configuration for building os161. type BuildConf struct { - Repo string // The git repository to clone - CommitID string // The git commit id (HEAD, hash, etc.) to check out - Config string // The os161 kernel config file for the build - - dir string // The base (temp) directory for the build. - srcDir string // Directory for the os161 source code (dir/src) - rootDir string // Directory for compilation output (dir/root) + Repo string // The git repository to clone + CommitID string // The git commit id (HEAD, hash, etc.) to check out + KConfig string // The os161 kernel config file for the build + RequiredCommit string // A commit required to be in git log + CacheDir string } -func NewBuildConf(repo, commit, config string) (*BuildConf, error) { - tdir, err := ioutil.TempDir("", "os161") - if err != nil { +// Use the BuildConf to create a sequence of commands that will build an os161 kernel +func (b *BuildConf) ToBuildTest() (*BuildTest, error) { + + t := &BuildTest{ + ID: bson.NewObjectId(), + Name: "build", + Description: "Clone Git repository and build kernel", + Commands: make([]*BuildCommand, 0), + Result: TEST_RESULT_NONE, + DependencyID: "build", + IsDependency: true, + PointsAvailable: uint(0), + PointsEarned: uint(0), + ScoringMethod: TEST_SCORING_ENTIRE, + conf: b, + } + + if err := t.initDirs(); err != nil { return nil, err } - return &BuildConf{ - dir: tdir, - srcDir: path.Join(tdir, "src"), - rootDir: path.Join(tdir, "root"), - Repo: repo, - CommitID: commit, - Config: config, - }, nil + t.addGitCommands() + t.addOverlayCommands() + t.addBuildCommands() + + return t, nil } // Get the root directory of the build output -func (b *BuildConf) RootDir() string { - return b.rootDir +func (t *BuildTest) RootDir() string { + return t.rootDir } -// CleanUp removes any temp resources created during the build. Only call this -// when completely done with the compilation output. -func (b *BuildConf) CleanUp() { - os.RemoveAll(b.dir) -} +func (cmd *BuildCommand) Run() error { + tokens := strings.Split(cmd.Input.Line, " ") + if len(tokens) < 1 { + return errors.New("BuildCommand: Empty command") + } -func (b *BuildConf) getSources() (string, error) { + c := exec.Command(tokens[0], tokens[1:]...) + c.Dir = cmd.startDir - cmds := []*buildCommand{ - &buildCommand{fmt.Sprintf("git clone %v src", b.Repo), b.dir}, - &buildCommand{fmt.Sprintf("git reset --hard %v", b.CommitID), b.srcDir}, + output, err := c.CombinedOutput() + if err != nil { + return err + } + + lines := strings.Split(string(output), "\n") + cmd.Output = make([]*OutputLine, len(lines)) + for i, l := range lines { + cmd.Output[i] = &OutputLine{ + Line: l, + SimTime: TimeFixedPoint(i), + WallTime: TimeFixedPoint(i), + } } - return commandBatch(cmds) + return nil } -// Build OS161. This assumes the sources have been pulled. -func (b *BuildConf) buildOS161() (string, error) { +type BuildResults struct { + RootDir string + KeyMap map[string]string +} + +// Figure out the build directory location, create it if it doesn't exist, and +// lock it. +func (t *BuildTest) initDirs() (err error) { - confDir := path.Join(b.srcDir, "kern/conf") - compDir := path.Join(path.Join(b.srcDir, "kern/compile"), b.Config) + buildDir := "" + + // Try the cache directory first + if len(t.conf.CacheDir) > 0 { + if _, err = os.Stat(t.conf.CacheDir); err == nil { + hashbytes := sha256.Sum256([]byte(t.conf.Repo)) + hash := strings.ToLower(hex.EncodeToString(hashbytes[:])) + buildDir = path.Join(t.conf.CacheDir, hash) + if _, err = os.Stat(buildDir); err != nil { + if err = os.Mkdir(buildDir, 0770); err != nil { + return + } + } + } + } - cmds := []*buildCommand{ - &buildCommand{"./configure --ostree=" + b.rootDir, b.srcDir}, - &buildCommand{"bmake", b.srcDir}, - &buildCommand{"bmake install", b.srcDir}, - &buildCommand{"./config " + b.Config, confDir}, - &buildCommand{"bmake", compDir}, - &buildCommand{"bmake depend", compDir}, - &buildCommand{"bmake install", compDir}, + // Use a temp directory instead + if len(buildDir) == 0 { + if buildDir, err = ioutil.TempDir("", "os161"); err != nil { + return + } } - return commandBatch(cmds) + t.dir = buildDir + t.srcDir = path.Join(buildDir, "src") + t.rootDir = path.Join(buildDir, "root") + + // TODO: Lock the build directory + + return } -func (b *BuildConf) GitAndBuild() (string, error) { - o, e := b.getSources() - if e != nil { - return o, e +func (t *BuildTest) Run() (*BuildResults, error) { + var err error + + t.Result = TEST_RESULT_RUNNING + + for _, c := range t.Commands { + c.Status = COMMAND_STATUS_RUNNING + + if err = c.Run(); err != nil { + c.Status = COMMAND_STATUS_INCORRECT + t.Result = TEST_RESULT_INCORRECT + return nil, err + } else { + if c.handler != nil { + err = c.handler(t, c) + if err != nil { + c.Status = COMMAND_STATUS_INCORRECT + t.Result = TEST_RESULT_INCORRECT + return nil, err + } + } + c.Status = COMMAND_STATUS_CORRECT + // TODO: Broadcast + } } - return b.buildOS161() + t.Result = TEST_RESULT_CORRECT + + res := &BuildResults{ + RootDir: t.rootDir, + KeyMap: nil, + } + return res, nil } -type buildCommand struct { - command string - dir string +func commitCheckHandler(t *BuildTest, command *BuildCommand) error { + for _, l := range command.Output { + if t.conf.RequiredCommit == l.Line { + return nil + } + } + + return errors.New("Cannot find required commit id") } -func singleCommand(cmd *buildCommand) (string, error) { - tokens := strings.Split(cmd.command, " ") - if len(tokens) < 1 { - return "", errors.New("buildConf: Empty command") +func (t *BuildTest) addGitCommands() { + + // If we have the repo cached, try a simple checkout. + if _, err := os.Stat(t.srcDir); err == nil { + t.addCommand(fmt.Sprintf("git checkout -f %v", t.conf.CommitID), t.srcDir) + t.wasCached = true + } else { + t.addCommand(fmt.Sprintf("git clone %v src", t.conf.Repo), t.dir) + t.addCommand(fmt.Sprintf("git checkout %v", t.conf.CommitID), t.srcDir) } - c := exec.Command(tokens[0], tokens[1:]...) - c.Dir = cmd.dir - output, err := c.CombinedOutput() - return string(output), err + // Before building, we may need to check for a specific commit + if len(t.conf.RequiredCommit) > 0 { + cmd := t.addCommand(fmt.Sprintf("git log --pretty=format:%v", "%H"), t.srcDir) + cmd.handler = commitCheckHandler + } } -func commandBatch(cmds []*buildCommand) (string, error) { - allOutput := "" +func (t *BuildTest) addOverlayCommands() { - for _, cmd := range cmds { - output, err := singleCommand(cmd) - allOutput += output + "\n" - if err != nil { - return allOutput, err - } +} + +func (t *BuildTest) addBuildCommands() error { + confDir := path.Join(t.srcDir, "kern/conf") + compDir := path.Join(path.Join(t.srcDir, "kern/compile"), t.conf.KConfig) + + t.addCommand("./configure --ostree="+t.rootDir, t.srcDir) + t.addCommand("bmake", t.srcDir) + t.addCommand("bmake install", t.srcDir) + t.addCommand("./config "+t.conf.KConfig, confDir) + t.addCommand("bmake", compDir) + t.addCommand("bmake depend", compDir) + t.addCommand("bmake install", compDir) + + return nil +} + +func (t *BuildTest) addCommand(cmdLine string, dir string) *BuildCommand { + cmd := &BuildCommand{ + Type: "build", + Output: []*OutputLine{}, + Status: COMMAND_STATUS_NONE, } + cmd.Input.Line = cmdLine + cmd.startDir = dir + cmd.handler = nil + + t.Commands = append(t.Commands, cmd) - return allOutput, nil + return cmd } diff --git a/build_test.go b/build_test.go index c55e980..33f62d2 100644 --- a/build_test.go +++ b/build_test.go @@ -5,6 +5,7 @@ import ( "testing" ) +/* func TestBuildGitOnly(t *testing.T) { t.Parallel() assert := assert.New(t) @@ -27,38 +28,35 @@ func TestBuildGitOnly(t *testing.T) { t.Log(e) t.Log(o) } +*/ func TestBuildFull(t *testing.T) { t.Parallel() assert := assert.New(t) - conf, err := NewBuildConf("", "", "") - assert.Nil(err) - assert.NotNil(conf) - if conf == nil { - return - } - - defer conf.CleanUp() - + conf := &BuildConf{} conf.Repo = "git@gitlab.ops-class.org:staff/sol3.git" conf.CommitID = "HEAD" - conf.Config = "SOL3" + conf.KConfig = "SOL3" + conf.CacheDir = "/home/shaseley/cache" + conf.RequiredCommit = "29b635f4b8393fda987244b45ab0e32a61ea5dcb" - o, e := conf.getSources() - assert.Nil(e) - t.Log(e) - t.Log(o) - if e != nil { - return + test, err := conf.ToBuildTest() + assert.Nil(err) + assert.NotNil(test) + + if test == nil { + t.Log(err) + t.FailNow() } - o, e = conf.buildOS161() - assert.Nil(e) - t.Log(e) - t.Log(o) + _, err = test.Run() + assert.Nil(err) + + t.Log(test.OutputJSON()) } +/* type confDetail struct { repo string commit string @@ -93,3 +91,4 @@ func TestBuildFailures(t *testing.T) { conf.CleanUp() } } +*/ diff --git a/conf.go b/conf.go index fa39560..3250206 100644 --- a/conf.go +++ b/conf.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/ericaro/frontmatter" "github.com/imdario/mergo" + "gopkg.in/mgo.v2/bson" "io/ioutil" "math/rand" "reflect" @@ -28,7 +29,7 @@ type Sys161Conf struct { RAM string `yaml:"ram" json:"ram"` Disk1 DiskConf `yaml:"disk1" json:"disk1"` Disk2 DiskConf `yaml:"disk2" json:"disk2"` - Random uint32 `yaml:"-" json:"randomseed"` + Random uint32 `yaml:"-" json:"randomseed" bson:"randomseed"` } type DiskConf struct { @@ -61,7 +62,7 @@ type MiscConf struct { CommandRetries uint `yaml:"commandretries" json:"commandretries"` PromptTimeout float32 `yaml:"prompttimeout" json:"prompttimeout"` CharacterTimeout uint `yaml:"charactertimeout" json:"charactertimeout"` - TempDir string `yaml:"tempdir" json:"-"` + TempDir string `yaml:"tempdir" json:"-" bson:"-"` RetryCharacters string `yaml:"retrycharacters" json:"retrycharacters"` KillOnExit string `yaml:"killonexit" json:"killonexit"` } @@ -153,6 +154,7 @@ func TestFromString(data string) (*Test, error) { // Check for empty commands and expand syntatic sugar before getting // started. Doing this first makes the main loop and retry logic simpler. + t.ID = bson.NewObjectId() err = t.initCommands() if err != nil { return nil, err diff --git a/environment.go b/environment.go index bd958ed..c590f87 100644 --- a/environment.go +++ b/environment.go @@ -16,7 +16,9 @@ type TestEnvironment struct { manager *manager - // These do depend on the TestGroup/Target + CacheDir string + + // These depend on the TestGroup/Target KeyMap map[string]string RootDir string } diff --git a/groups.go b/groups.go index 8f45f87..372d5c0 100644 --- a/groups.go +++ b/groups.go @@ -1,7 +1,6 @@ package test161 import ( - "encoding/json" "errors" "fmt" "github.com/bmatcuk/doublestar" @@ -9,7 +8,6 @@ import ( "path" "path/filepath" "strings" - "sync" ) // GroupConfig specifies how a group of tests should be created and run. @@ -17,55 +15,19 @@ type GroupConfig struct { Name string `json:name` UseDeps bool `json:"usedeps"` Tests []string `json:"tests"` - Env *TestEnvironment `json:"-"` + Env *TestEnvironment `json:"-" "bson:"-"` } // A group of tests to be run, which is the result of expanding a GroupConfig. type TestGroup struct { - id uint64 Tests map[string]*Test Config *GroupConfig } -// Since we don't allow id to be accessed directly, we have a special -// type for JSON output that exports the ID. -type jsonTestGroup struct { - Id uint64 `json:"id"` - Config *GroupConfig `json:"config"` - Tests map[string]*Test `json:"tests"` -} - -// Group ID counter and protection -var idLock = &sync.Mutex{} -var curID uint64 = 1 - -// Id retrieves the protected group id -func (t *TestGroup) Id() uint64 { - return t.id -} - -// Custom JSON marshaling to deal with our read-only id -func (tg *TestGroup) MarshalJSON() ([]byte, error) { - return json.Marshal(jsonTestGroup{tg.Id(), tg.Config, tg.Tests}) -} - -// Increments the global counter and returns the previous value. -func incrementId() (res uint64) { - idLock.Lock() - res = curID - curID += 1 - if curID == 0 { - curID = 1 - } - idLock.Unlock() - return -} - // EmptyGroup creates an empty TestGroup that can be used to add groups from // strings. func EmptyGroup() *TestGroup { tg := &TestGroup{} - tg.id = incrementId() tg.Tests = make(map[string]*Test) return tg } diff --git a/output.go b/output.go index 95a0173..076379a 100644 --- a/output.go +++ b/output.go @@ -55,8 +55,7 @@ func (tg *TestGroup) OutputJSON() (string, error) { func (tg *TestGroup) OutputString() string { var output string - output = fmt.Sprintf("\ngroup: id = %v\n", tg.Id()) - output += fmt.Sprintf("group: name = %v\n", tg.Config.Name) + output += fmt.Sprintf("\ngroup: name = %v\n", tg.Config.Name) output += fmt.Sprintf("group: rootdir = %v\n", tg.Config.Env.RootDir) output += fmt.Sprintf("group: testdir = %v\n", tg.Config.Env.TestDir) output += fmt.Sprintf("group: usedeps = %v\n", tg.Config.UseDeps) @@ -69,3 +68,11 @@ func (tg *TestGroup) OutputString() string { return output } + +func (t *BuildTest) OutputJSON() (string, error) { + outputBytes, err := json.MarshalIndent(t, "", " ") + if err != nil { + return "", err + } + return string(outputBytes), nil +} diff --git a/persistence.go b/persistence.go new file mode 100644 index 0000000..16b2bd1 --- /dev/null +++ b/persistence.go @@ -0,0 +1,55 @@ +package test161 + +const ( + MSG_SUBMISSION_CREATE = iota + MSG_SUBMISSION_SCORE + MSG_SUBMISSION_STATUS +) + +const ( + MSG_TEST_STATUS = iota + MSG_TEST_SCORE +) + +const ( + MSG_COMMAND_STATUS = iota + MSG_COMMAND_SCORE + MSG_COMMAND_OUTPUT +) + +// Each Submission has at most one PersistenceManager, and it is pinged when a +// variety of events occur. These callbacks are invoked synchronously, so it's +// up to the PersistenceManager to not slow down the tests. We do this because +// the PersistenceManager can create goroutines if applicable, but we can't +// make an asynchronous call synchronous when it might be needed. So, be kind +// ye PersistenceManagers. +type PersistenceManager interface { + SubmissionChanged(s *Submission, msg int) error + TestChanged(t *Test, msg int) error + CommandChanged(t *Test, c *Command, l *OutputLine) error + BuildTestChanged(t *BuildTest, msg int) error + BuildCommandChanged(t *BuildTest, c *BuildCommand, l *OutputLine) error +} + +type MongoPersistence struct { +} + +func (m *MongoPersistence) SubmissionChanged(s *Submission, msg int) error { + return nil +} + +func (m *MongoPersistence) TestChanged(t *Test, msg int) error { + return nil +} + +func (m *MongoPersistence) CommandChanged(t *Test, c *Command, l *OutputLine) error { + return nil +} + +func (m *MongoPersistence) BuildTestChanged(t *BuildTest, msg int) error { + return nil +} + +func (m *MongoPersistence) BuildCommandChanged(t *BuildTest, c *BuildCommand, l *OutputLine) error { + return nil +} diff --git a/run.go b/run.go index cca84ef..b3f7436 100644 --- a/run.go +++ b/run.go @@ -15,6 +15,8 @@ import ( "github.com/kr/pty" "github.com/ops-class/test161/expect" "github.com/termie/go-shutil" + // "gopkg.in/mgo.v2" + "gopkg.in/mgo.v2/bson" "io" "io/ioutil" "os" @@ -29,6 +31,9 @@ import ( type Test struct { + // Mongo ID + ID bson.ObjectId `yaml:"-" json:"id" bson:"_id,omitempty"` + // Input // Metadata @@ -45,10 +50,10 @@ type Test struct { Misc MiscConf `yaml:"misc" json:"misc"` // Actual test commands to run - Content string `fm:"content" yaml:"-" json:"-"` + Content string `fm:"content" yaml:"-" json:"-" bson:"-"` // Big lock that protects most fields shared between Run and getStats - L *sync.Mutex `json:"-"` + L *sync.Mutex `json:"-" bson:"-"` // Output @@ -61,7 +66,7 @@ type Test struct { // Dependency data DependencyID string `json:"depid"` - ExpandedDeps map[string]*Test `json:"-"` + ExpandedDeps map[string]*Test `json:"-" bson:"-"` IsDependency bool `json:"isdependency"` // Grading. These are set when the test is being run as part of a Target. @@ -117,12 +122,12 @@ const ( type Command struct { // Set during init Type string `json:"type"` - PromptPattern *regexp.Regexp `json:"-"` + PromptPattern *regexp.Regexp `json:"-" bson:"-"` Input InputLine `json:"input"` // Set during target init - PointsAvailable uint `json:"points_avail"` - PointsEarned uint `json:"points_earned"` + PointsAvailable uint `json:"points_avail" bson:"points_avail"` + PointsEarned uint `json:"points_earned" bson:"points_earned"` // Set during run init Panic string `json:"panic"` @@ -146,12 +151,10 @@ type InputLine struct { type OutputLine struct { WallTime TimeFixedPoint `json:"walltime"` SimTime TimeFixedPoint `json:"simtime"` - Buffer bytes.Buffer `json:"-"` + Buffer bytes.Buffer `json:"-" bson"-"` Line string `json:"line"` - - // TODO These don't need to be serialized for production - Trusted bool `json:"trusted"` - KeyName string `json:"keyname"` + Trusted bool `json:"trusted"` + KeyName string `json:"keyname"` } type Status struct { @@ -167,6 +170,7 @@ type TestResult string const ( TEST_RESULT_NONE TestResult = "none" // Hasn't run (initial status) + TEST_RESULT_RUNNING TestResult = "running" // Running TEST_RESULT_CORRECT TestResult = "correct" // Met the output criteria TEST_RESULT_INCORRECT TestResult = "incorrect" // Possibly some partial points, but didn't complete everything successfully TEST_RESULT_ABORT TestResult = "abort" // Aborted - internal error @@ -191,12 +195,13 @@ func (t *Test) Run(env *TestEnvironment) (err error) { // Save the test environment for other pieces that need it t.env = env - // We've aborted unless we hear otherwise - t.Result = TEST_RESULT_ABORT + t.Result = TEST_RESULT_RUNNING // Set the instance-specific input and expected output for _, c := range t.Commands { if err = c.instantiate(env); err != nil { + t.addStatus("aborted", "") + t.Result = TEST_RESULT_ABORT return } } @@ -205,6 +210,7 @@ func (t *Test) Run(env *TestEnvironment) (err error) { err = t.MergeConf(CONF_DEFAULTS) if err != nil { t.addStatus("aborted", "") + t.Result = TEST_RESULT_ABORT return err } @@ -212,6 +218,7 @@ func (t *Test) Run(env *TestEnvironment) (err error) { tempRoot, err := ioutil.TempDir(t.Misc.TempDir, "test161") if err != nil { t.addStatus("aborted", "") + t.Result = TEST_RESULT_ABORT return err } defer os.RemoveAll(tempRoot) @@ -221,6 +228,7 @@ func (t *Test) Run(env *TestEnvironment) (err error) { err = shutil.CopyTree(env.RootDir, t.tempDir, nil) if err != nil { t.addStatus("aborted", "") + t.Result = TEST_RESULT_ABORT return err } @@ -229,6 +237,7 @@ func (t *Test) Run(env *TestEnvironment) (err error) { _, err = os.Stat(kernelTarget) if err != nil { t.addStatus("aborted", "") + t.Result = TEST_RESULT_ABORT return err } @@ -237,15 +246,18 @@ func (t *Test) Run(env *TestEnvironment) (err error) { t.ConfString, err = t.PrintConf() if err != nil { t.addStatus("aborted", "") + t.Result = TEST_RESULT_ABORT return err } err = ioutil.WriteFile(confTarget, []byte(t.ConfString), 0440) if err != nil { t.addStatus("aborted", "") + t.Result = TEST_RESULT_ABORT return err } if _, err := os.Stat(confTarget); os.IsNotExist(err) { t.addStatus("aborted", "") + t.Result = TEST_RESULT_ABORT return err } @@ -256,6 +268,7 @@ func (t *Test) Run(env *TestEnvironment) (err error) { err = create.Run() if err != nil { t.addStatus("aborted", "") + t.Result = TEST_RESULT_ABORT return err } } @@ -265,6 +278,7 @@ func (t *Test) Run(env *TestEnvironment) (err error) { err = create.Run() if err != nil { t.addStatus("aborted", "") + t.Result = TEST_RESULT_ABORT return err } } @@ -288,6 +302,7 @@ func (t *Test) Run(env *TestEnvironment) (err error) { err = t.start161() if err != nil { t.addStatus("aborted", "") + t.Result = TEST_RESULT_ABORT return err } defer t.stop161() @@ -386,6 +401,8 @@ func (t *Test) Run(env *TestEnvironment) (err error) { if err == nil { t.finishAndEvaluate() + } else { + t.Result = TEST_RESULT_ABORT } return err diff --git a/run_test.go b/run_test.go index 9dc8865..d4e764e 100644 --- a/run_test.go +++ b/run_test.go @@ -3,6 +3,7 @@ package test161 import ( "fmt" "github.com/stretchr/testify/assert" + //"gopkg.in/mgo.v2" "math/rand" "os" "strings" @@ -283,3 +284,50 @@ func TestRunTT3(t *testing.T) { t.Log(test.OutputJSON()) t.Log(test.OutputString()) } + +/* +func TestRunBootMongo(t *testing.T) { + t.Parallel() + assert := assert.New(t) + + test, err := TestFromString("q") + assert.Nil(err) + assert.Nil(test.MergeConf(TEST_DEFAULTS)) + assert.Nil(test.Run(defaultEnv)) + + const ( + MongoDBHosts = "localhost:27017" + AuthDatabase = "test" + AuthUserName = "testUser" + AuthPassword = "test1234" + TestDatabase = "test" + ) + + // We need this object to establish a session to our MongoDB. + mongoDBDialInfo := &mgo.DialInfo{ + Addrs: []string{MongoDBHosts}, + Timeout: 60 * time.Second, + Database: AuthDatabase, + Username: AuthUserName, + Password: AuthPassword, + } + + // Create a session which maintains a pool of socket connections + // to our MongoDB. + mongoSession, err := mgo.DialWithInfo(mongoDBDialInfo) + if err != nil { + t.Logf("CreateSession: %s\n", err) + t.FailNow() + } + + defer mongoSession.Close() + + c := mongoSession.DB("test").C("tests") + err = c.Insert(test) + + if err != nil { + t.Logf("Insert: %s\n", err) + t.FailNow() + } +} +*/ diff --git a/stats.go b/stats.go index feb4c5d..b70dc2b 100644 --- a/stats.go +++ b/stats.go @@ -30,7 +30,7 @@ type Stat struct { WallLength TimeFixedPoint `json:"walllength"` // Read from stat line - Nsec uint64 `json:"-"` + Nsec uint64 `json:"-" bson:"-"` Kinsns uint32 `json:"kinsns"` Uinsns uint32 `json:"uinsns"` Udud uint32 `json:"udud"` diff --git a/submission.go b/submission.go new file mode 100644 index 0000000..3efc251 --- /dev/null +++ b/submission.go @@ -0,0 +1,86 @@ +package test161 + +import ( + "errors" + "gopkg.in/mgo.v2/bson" + "time" +) + +// SubmissionRequests are created by clients and used to generate Submissions. +// A SubmissionRequest represents the data required to run a test161 target +// for evaluation by the test161 server. +type SubmissionRequest struct { + Target string // Name of the target + Users []string // Email addresses of users + Repository string // Git repository to clone + CommitID string // Git commit id to checkout after cloning +} + +const ( + SUBMISSION_SUBMITTED = "submitted" // Submitted and queued + SUBMISSION_BUILDIND = "building" // Building the kernel + SUBMISSION_RUNNING = "running" // The tests started running + SUBMISSION_ABORTED = "aborted" // Aborted because one or more tests failed to error + SUBMISSION_COMPLETED = "completed" // Completed +) + +type Submission struct { + + // Configuration + ID bson.ObjectId `bson:"_id,omitempty"` + Users []string `bson:"users"` + Repository string `bson:"repository"` + CommitID string `bson:"commit_id"` + + // Target details + TargetID bson.ObjectId `bson:"target_id"` + TargetName string `bson:"target_name"` + TargetVersion uint `bson"target_version"` + PointsAvailable uint `bson:"max_score"` + TargetType string `bson:"target_type"` + + // Results + Status string `bson:"status"` + Score uint `bson:"score"` + Performance float64 `bson:"performance"` + TestIDs []bson.ObjectId `bson:"tests"` + Message string `bson:"message"` + + SubmissionTime time.Time `bson:"submission_time"` + CompletionTime time.Time `bson:"completion_time"` + + env *TestEnvironment +} + +func (req *SubmissionRequest) validate(env *TestEnvironment) error { + + if _, ok := env.Targets[req.Target]; !ok { + return errors.New("Invalid target: " + req.Target) + } + + // TODO: Check for closed targets + + if len(req.Users) == 0 { + return errors.New("No usernames specified") + } + + // TODO: Check users against users database + + if len(req.Repository) == 0 || len(req.CommitID) == 0 { + return errors.New("Must specify a Git repository and commit id") + } + + return nil +} + +// Create a new Submission that can be evaluated by the test161 server or client. +func NewSubmission(request *SubmissionRequest, defaultEnv *TestEnvironment) (*Submission, error) { + if err := request.validate(defaultEnv); err != nil { + return nil, err + } + + // TODO: Create build "test" + + //target := env.Targets[request.Target] + return nil, nil +} diff --git a/target.go b/target.go index 7a7cbeb..01af830 100644 --- a/target.go +++ b/target.go @@ -28,11 +28,15 @@ const ( ) type Target struct { - Name string `yaml:"name"` - Version uint `yaml:"version"` - Type string `yaml:"type"` - Points uint `yaml:"points"` // Total points for the target (asst) - Tests []*TargetTest `yaml:"tests"` + Name string `yaml:"name"` + Version uint `yaml:"version"` + Type string `yaml:"type"` + Points uint `yaml:"points"` // Total points for the target (asst) + Overlay string `yaml:"overlay"` + KernelConfig string `yaml:"kconfig"` + RequiredCommit string `yaml:"required_commit"` + RequiresUserland bool `yaml:"userland"` + Tests []*TargetTest `yaml:"tests"` } type TargetTest struct {