drmaa2os public

README.md

@@ -0,0 +1,67 @@
+# drmaa2os - A Go API for OS Processes, Docker Containers, Cloud Foundry Tasks, Kubernetes Jobs, Grid Engine and more...
+
+This is a Go API based on an open standard (Open Grid Forum DRMAA2) in order to submit and
+supervise workloads like OS processes, containers, PODs, tasks from a common interface.
+
+It allows to develop and run job workflows in OS processes, and later easily switch to 
+containers running as Cloud Foundry tasks, Docker containers, Grid Engine jobs, etc...
+
+Its main pupose is supporting application developers with an abstraction layer on top of 
+platforms, workload managers, and cluster schedulers, so that they don't require to deal
+with the underlaying details and differences when only simple operations (like starting 
+a container and waiting until it is finished) are required. 
+
+It can be easily integrated in applications which create and execute job workflows.
+
+If you are looking for a simple interface for creating job workflows without dealing
+with the DRMAA2 details, check out [*wfl*](https://github.com/dgruber/wfl).
+
+For details about the mapping of job operations please consult the platform specific READMEs:
+
+  * [OS Processes](pkg/jobtracker/simpletracker/README.md)
+  * [Cloud Foundry](pkg/jobtracker/cftracker/README.md)
+  * [Docker / Moby](pkg/jobtracker/dockertracker/README.md)
+  * [Kubernetes](pkg/jobtracker/kubernetestracker/README.md)
+
+For a DRMAA2 implementation based on C DRMAA2 (_libdrmaa2.so_) like for *Univa Grid Engine* please
+see [drmaa2](https://github.com/dgruber/drmaa2).
+
+Not yet implemented:
+
+  * [Mesos](pkg/jobtracker/mesostracker/README.md)
+  * [C DRMAA Version 1 (libdrmaa.so)](pkg/jobtracker/libdrmaa/README.md)
+
+## Basic Usage
+
+Following example demonstrates how a job running as OS process can be executed. More examples can be found in the _examples_ subdirectory. 
+
+```go
+	sm, _ := drmaa2os.NewDefaultSessionManager("testdb.db")
+
+	js, _ := sm.CreateJobSession("jobsession", "")
+
+	jt := drmaa2interface.JobTemplate{
+		JobName:       "job1",
+		RemoteCommand: "sleep",
+		Args:          []string{"2"},
+	}
+
+	job, _ := js.RunJob(jt)
+
+	job.WaitTerminated(drmaa2interface.InfiniteTime)
+
+	if job.GetState() == drmaa2interface.Done {
+		job2, _ := js.RunJob(jt)
+		job2.WaitTerminated(drmaa2interface.InfiniteTime)
+	} else {
+		fmt.Println("Failed to execute job1 successfully")
+	}
+
+	js.Close()
+	sm.DestroyJobSession("jobsession")
+```
+
+
+
+
+

ci/pipeline.yml

@@ -0,0 +1,25 @@
+---
+resources:
+  - name: drmaa2os 
+    type: git
+    source:
+      uri: {{uri}}
+
+jobs:
+  - name: build_all
+    public: true
+    plan:
+      - get: drmaa2os
+        trigger: true
+      - task: build_all
+        file: drmaa2os/ci/tasks/build-all/build_all.yml
+
+  - name: test_simpletracker
+    public: true
+    plan:
+      - get: drmaa2os
+        passed: [build_all]
+        trigger: true
+      - task: test
+        file: drmaa2os/ci/tasks/test-tracker/test_simpletracker.yml
+

ci/setPipeline.sh

@@ -0,0 +1,2 @@
+#!/bin/sh
+fly -t ci set-pipeline --config pipeline.yml --pipeline drmaa2os --load-vars-from params.yml 

ci/tasks/build-all/build_all.yml

@@ -0,0 +1,25 @@
+---
+platform: linux
+
+image_resource:
+  type: docker-image
+  source: {repository: golang, tag: "1.8"}
+
+inputs:
+- name: drmaa2os 
+
+run:
+  path: sh
+  args:
+  - -exc
+  - |
+    set -e -x
+    export GOPATH=$PWD
+    mkdir -p ./src/github.com/dgruber/drmaa2os
+    cp -r ./drmaa2os ./src/github.com/dgruber/drmaa2os
+    cd src/github.com/dgruber/drmaa2os
+    go version
+    go build ./pkg/jobtracker/simpletracker
+    go build ./pkg/jobtracker/dockertracker
+    go build ./examples/simple
+     

ci/tasks/test-tracker/test_simpletracker.yml

@@ -0,0 +1,20 @@
+---
+platform: linux
+
+image_resource:
+  type: docker-image
+  source: {repository: golang, tag: "1.8"}
+
+inputs:
+- name: drmaa2os 
+
+run:
+  path: sh
+  args:
+  - -exc
+  - |
+    mkdir -p $GOPATH/src/github.com/dgruber/drmaa2os
+    cp -r ./drmaa2os $GOPATH/src/github.com/dgruber/drmaa2os
+    cd $GOPATH/src/github.com/dgruber/drmaa2os/pkg/jobtracker/simpletracker
+    go test -v 
+     

drmaa2os.go

@@ -0,0 +1,34 @@
+// Package drmaa2os provides a DRMAA2 implementation based on the OS interface.
+package drmaa2os
+
+import (
+	_ "code.cloudfoundry.org/lager"
+	"github.com/dgruber/drmaa2interface"
+	_ "os"
+)
+
+type Config struct {
+	//l *lager.Logger
+}
+
+// Listen starts the DRMAA2 OS interface service.
+func (c *Config) Listen() error {
+	//c.l = lager.NewLogger("drmaa2os")
+	//c.l.RegisterSink(*lager.NewWriterSink(os.Stdout, lager.INFO))
+	return nil
+}
+
+// Stop stops the DRMAA2 OS interface service.
+func (c *Config) Stop() error {
+	return nil
+}
+
+func (c *Config) NewSessionManager() (drmaa2interface.SessionManager, error) {
+	/*
+		if c == nil || c.db == nil {
+			return nil, errors.New("DB not ready.")
+		}
+		sm := SessionManager{db: c.db, log: l}
+	*/
+	return &SessionManager{}, nil
+}

drmaa2os_suite_test.go

@@ -0,0 +1,13 @@
+package drmaa2os_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"testing"
+)
+
+func TestDrmaa2os(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Drmaa2os Suite")
+}

errors.go

@@ -0,0 +1,16 @@
+package drmaa2os
+
+type DRMAA2Error struct {
+	message string
+}
+
+func (d DRMAA2Error) Error() string {
+	return d.message
+}
+
+var (
+	ErrorUnsupportedOperation = DRMAA2Error{"This optional function is not suppported."}
+	ErrorJobNotExists         = DRMAA2Error{"The job does not exist."}
+	ErrorInvalidState         = DRMAA2Error{"Invalid state."}
+	ErrorInternal             = DRMAA2Error{"Internal error occured."}
+)

examples/counter/counter.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"github.com/dgruber/drmaa2interface"
+	"github.com/dgruber/drmaa2os"
+)
+
+func main() {
+	sm, err := drmaa2os.NewDefaultSessionManager("testdb.db")
+	if err != nil {
+		panic(err)
+	}
+
+	js, err := sm.CreateJobSession("jobsession1", "")
+	if err != nil {
+		js, err = sm.OpenJobSession("jobsession1")
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	jt := drmaa2interface.JobTemplate{
+		JobName:       "testjob",
+		RemoteCommand: "./plus.sh",
+		InputPath:     "in.txt",
+		OutputPath:    "out.txt",
+	}
+
+	job, err := js.RunJob(jt)
+	if err != nil {
+		panic(err)
+	}
+	job.WaitTerminated(drmaa2interface.InfiniteTime)
+
+	for i := 2; i < 100; i++ {
+		if i%2 == 0 {
+			jt.InputPath = "out.txt"
+			jt.OutputPath = "in.txt"
+		} else {
+			jt.InputPath = "in.txt"
+			jt.OutputPath = "out.txt"
+		}
+		job, err = js.RunJob(jt)
+		if err != nil {
+			panic(err)
+		}
+		err = job.WaitTerminated(drmaa2interface.InfiniteTime)
+		if err != nil {
+			panic(err)
+		}
+	}
+}

examples/counter/in.txt

@@ -0,0 +1 @@
+25747

examples/counter/out.txt

@@ -0,0 +1 @@
+25748

examples/counter/plus.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+read line
+echo $line+1  | bc 2>&1

examples/docker/dockerExample.go

@@ -0,0 +1,100 @@
+package main
+
+import (
+	"fmt"
+	"github.com/dgruber/drmaa2interface"
+	"github.com/dgruber/drmaa2os"
+)
+
+func removeJob(jobs []drmaa2interface.Job, job drmaa2interface.Job) (result []drmaa2interface.Job) {
+	if job == nil {
+		return jobs
+	}
+	for i := 0; i < len(jobs); i++ {
+		if job.GetID() != jobs[i].GetID() {
+			result = append(result, jobs[i])
+		}
+	}
+	return result
+}
+
+func main() {
+	sm, err := drmaa2os.NewDefaultSessionManager("testdb.db")
+	if err != nil {
+		panic(err)
+	}
+
+	js, err := sm.CreateJobSession("jobsession1", "docker")
+	if err != nil {
+		js, err = sm.OpenJobSession("jobsession1")
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	jt := drmaa2interface.JobTemplate{
+		JobName:       "job1",
+		RemoteCommand: "sleep",
+		JobCategory:   "dgruber/hello",
+		Args:          []string{"20"},
+	}
+
+	_, err = js.RunJob(jt)
+	if err != nil {
+		panic(err)
+	}
+
+	jt.JobName = "job2"
+	_, err = js.RunJob(jt)
+	if err != nil {
+		panic(err)
+	}
+
+	jobs, _ := js.GetJobs(drmaa2interface.CreateJobInfo())
+	for i := 0; i < 2; i++ {
+		j, err := js.WaitAnyTerminated(jobs, -1)
+		jobs = removeJob(jobs, j)
+
+		if err != nil {
+			fmt.Printf("Error while waiting for jobs to finish: %s\n", err.Error())
+			break
+		}
+		if j.GetState() == drmaa2interface.Done {
+			fmt.Printf("Job %s finished successfully\n", j.GetID())
+		} else {
+			fmt.Printf("Job %s finished with failure\n", j.GetID())
+		}
+	}
+
+	job1, err := js.RunJob(jt)
+	if err != nil {
+		panic(err)
+	}
+
+	job1.WaitTerminated(drmaa2interface.InfiniteTime)
+
+	jt.JobName = "job3"
+	job2, err := js.RunJob(jt)
+	if err != nil {
+		panic(err)
+	}
+
+	jt.JobName = "job4"
+	job3, err := js.RunJob(jt)
+	if err != nil {
+		panic(err)
+	}
+
+	job2.WaitTerminated(drmaa2interface.InfiniteTime)
+	if _, err := job2.GetJobInfo(); err != nil {
+		panic(err)
+	}
+
+	job3.WaitTerminated(drmaa2interface.InfiniteTime)
+
+	name, _ := js.GetSessionName()
+	fmt.Printf("Job session: %s\n", name)
+
+	js.Close()
+	sm.DestroyJobSession("jobsession1")
+}

examples/frontend/README.md

@@ -0,0 +1,3 @@
+# DRMAA2 OS Frontend
+
+Demonstrates DRMAA2