added default pipline variable, refactored example workflows

cli/helper.go

@@ -12,7 +12,7 @@ func GetPath(candidates *model.Activities, runner *Runner, fullPath bool) (strin
 
 	var activities = make([]*model.Activity, 0)
 	for i := 0; i < candidates.Len(); i++ {
-		if i > 0 && candidates.Get(i).Service == candidates.Get(i - 1).Service {
+		if i > 0 && candidates.Get(i).Service == candidates.Get(i-1).Service {
 			continue
 		}
 		activities = append(activities, candidates.Get(i))

cli/runner.go

@@ -20,7 +20,7 @@ var OnError = func(code int) {
 }
 
 const (
-	messageTypeAction         = iota + 10
+	messageTypeAction = iota + 10
 	messageTypeTagDescription
 )
 
@@ -534,7 +534,7 @@ func (r *Runner) reportTagSummary() {
 	for _, tag := range r.tags {
 		if (tag.FailedCount) > 0 {
 			var eventTag = tag.TagID
-			r.printMessage(r.ColorText(eventTag, "red"), len(eventTag), messageTypeTagDescription, tag.Description, msg.MessageStyleError, fmt.Sprintf("failed %v/%v", tag.FailedCount, (tag.FailedCount + tag.PassedCount)))
+			r.printMessage(r.ColorText(eventTag, "red"), len(eventTag), messageTypeTagDescription, tag.Description, msg.MessageStyleError, fmt.Sprintf("failed %v/%v", tag.FailedCount, (tag.FailedCount+tag.PassedCount)))
 			var minRange = 0
 			for i, event := range tag.Events {
 				validation := r.getValidation(event)
@@ -544,7 +544,7 @@ func (r *Runner) reportTagSummary() {
 				if validation.HasFailure() {
 					var beforeValidationEvents = []msg.Event{}
 					if i-minRange > 0 {
-						beforeValidationEvents = tag.Events[minRange: i-1]
+						beforeValidationEvents = tag.Events[minRange : i-1]
 					}
 					r.reportFailureWithMatchSource(tag, validation, beforeValidationEvents)
 					minRange = i + 1

context.go

@@ -34,8 +34,8 @@ type Context struct {
 	Source          *url.Resource
 	state           data.Map
 	toolbox.Context
-	cloned          []*Context
-	closed          int32
+	cloned []*Context
+	closed int32
 }
 
 //Publish publishes event to listeners, it updates current run details like activity workflow name etc ...

context_test.go

@@ -23,10 +23,6 @@ func TestNewDefaultState(t *testing.T) {
 
 }
 
-
-
-
-
 func TestContext_Expand_Resource(t *testing.T) {
 	manager := endly.New()
 	context := manager.NewContext(toolbox.NewContext())

deployment/vc/git.go

@@ -87,8 +87,6 @@ func (s *git) checkInfo(context *endly.Context, request *StatusRequest) (*Status
 		return result, nil
 	}
 
-
-
 	runRequest := exec.NewExtractRequest(request.Source, exec.DefaultOptions(),
 		exec.NewExtractCommand(fmt.Sprintf("git status"), "", nil, nil,
 			model.NewExtract("branch", "On branch[\\s\\t]+([^\\s]+)", true)),
@@ -144,7 +142,6 @@ func (s *git) checkout(context *endly.Context, request *CheckoutRequest) (*Info,
 		return nil, err
 	}
 
-
 	dest, err := context.ExpandResource(request.Dest)
 	if err != nil {
 		return nil, err
@@ -162,8 +159,6 @@ func (s *git) checkout(context *endly.Context, request *CheckoutRequest) (*Info,
 	var useParentDirectory = true
 	var _, originProjectName = path.Split(origin.DirectoryPath())
 
-
-
 	if originProjectName == projectName {
 		projectName = "."
 		if dest.DirectoryPath() != "/" {

deployment/vc/service.go

@@ -93,8 +93,6 @@ func (s *service) checkout(context *endly.Context, request *CheckoutRequest) (*C
 		return nil, err
 	}
 
-
-
 	var modules = request.Modules
 	var directory = target.DirectoryPath()
 	if len(modules) == 0 {
@@ -145,7 +143,6 @@ func (s *service) checkoutArtifact(context *endly.Context, versionControlType st
 		return nil, err
 	}
 
-
 	if exists {
 		var response *StatusResponse
 		response, err = s.checkInfo(context, &StatusRequest{Source: dest, Type: versionControlType})

example/etl/transformer/README.md

@@ -36,7 +36,7 @@ Run the following command:
 
 ```text
 git clone https://github.com/viant/endly
-cd endly/example/ws/reporter/endly/
+cd endly/example/etl/transformer/endly/
 
 
 ## run test with [manager](endly/manager.csv) workflow:

example/etl/transformer/endly/app.yaml

@@ -1,5 +1,5 @@
 tasks: $tasks
-params:
+defaults:
   app: $app
   sdk: $sdk
   target: $target
@@ -34,7 +34,7 @@ pipeline:
       - echo 'deployed'
 
   stop:
-    action: process:stop
+    action: process:stop-all
     input: ${app}
 
   start:

example/etl/transformer/endly/manager.csv

@@ -2,16 +2,16 @@ Workflow,,Name,Description,Tasks,Init,Sleep
 ,,manager,run system test,%Tasks,@var/init,
 []Tasks,,Name,Description,Actions,,
 ,,init,Initialise test,%Init,,
-[]Init,Service,Action,Description,Request,tasks,
-,workflow,run,initialise system services,@system,*,
+[]Init,Service,Action,Description,Request,,
+,workflow,run,initialise system services,@system,,
 ,workflow,run,initialise data,@datastore,,
-,workflow,run,deploy app,@app,*,
+,workflow,run,deploy app,@app,,
 []Tasks,,Name,Description,Actions,,
 ,,test,Run test,%Test,,
-[]Test,Service,Action,Description,Request,run.workflow,run.tasks
-,workflow,run,run test plan,@req/run|$run,regresion,*
+[]Test,Service,Action,Description,Request,workflow,tasks
+,workflow,run,run test plan, @req/run,regresion,*
 []Tasks,,Name,Description,Actions,,
 ,,destroy,destroy system,%Destroy,,
 []Destroy,Service,Action,Description,Request,tasks,
 ,workflow,run,stop app,@app,stop,
-,workflow,run,stop system services, @system,destory,
+,workflow,run,stop system services,@system,destory,

example/etl/transformer/endly/regresion/req/db3_register.json

@@ -2,7 +2,7 @@
   "Datastore": "db3",
   "Config": {
     "DriverName": "mysql",
-    "Descriptor": "[username]:[password]@tcp(127.0.0.1:3306)/[dbname]?parseTime=true",
+    "Descriptor": "[username]:[password]@tcp(${serviceHost}:3306)/[dbname]?parseTime=true",
     "Credentials": "$mysqlCredential"
   }
 }

example/etl/transformer/endly/regresion/req/db4_register.json

@@ -6,7 +6,7 @@
     "Parameters": {
       "dbname": "db4",
       "namespace": "db4",
-      "host": "$targetHost",
+      "host": "$serviceHost",
       "port": "3000",
       "dateFormat": "yyyy-MM-dd hh:mm:ss",
       "keyColumnName": "id"

example/etl/transformer/endly/req/run.yaml

@@ -0,0 +1,7 @@
+name: $workflow
+tasks: $tasks
+params:
+  app: $app
+  mysqlCredential: $mysqlCredential
+  targetHost: $targetHost
+  serviceHost: $serviceHost

example/etl/transformer/endly/run.yaml

@@ -1,5 +1,5 @@
 init: "@var/init"
-params:
+defaults:
   app: $app
   sdk: $sdk
   target: $target

example/etl/transformer/endly/system.yaml

@@ -1,5 +1,5 @@
 tasks: $tasks
-params:
+defaults:
   target: $serviceTarget
 pipeline:
   destory:

example/etl/transformer/endly/var/init.json

@@ -59,12 +59,10 @@
       "URL": "$Pwd(./../)"
     }
   },
-
   {
     "Name": "serviceHost",
     "Value": "$Hostname($serviceTarget.URL)"
   },
-
   {
     "Name": "targetHost",
     "Value": "$Hostname($target.URL)"

example/rt/elogger/README.md

@@ -8,59 +8,43 @@ Prerequisites:
 
 Enable ssh logic you your use on your machine (on osx System Preference / Sharing / Remote Login )
 
-Install [docker](https://docs.docker.com/engine/installation/) service
+Download [endly](https://github.com/viant/endly/releases/)
 
-[Download endly and secret for your platofrm](https://github.com/viant/endly/releases/)
+Provide a username and password to login to your box.
+```text
+endly -c=localhost
+```
+```
+Verify that secret file were created
+```text
+cat ~/.secret/localhost.json
+```
 
-Or alternatively build binary from scratch following above instruction:
 
-Install [go lang](https://golang.org/doc/install) version 1.8+
 
-after installing go run the following command
+#### Run E-logger app workflow
 
-```text
-mkdir -p ~/Projects/go
-export GOPATH=~/Projects/go
-go get -u github.com/viant/endly
-go get -u github.com/viant/endly/endly
-go get -u github.com/viant/toolbox/secret
+Run the following command:
 
-export PATH=$PATH:$GOPATH/bin
-```
+```text
+git clone https://github.com/viant/endly
+cd endly/example/rt/elogger/endly/
 
 
-Generate credentials file used by the workflow.
-```text
-endly -c=CREDENTIAL_FILENAME
-```
-Command endly -c generates a file that store blowfish encrypted password in $HOME/.secret/ directory.
+## run test with [manager](endly/manager.csv) workflow:
+endly -w=manager
 
-Provide a user name and password to login to your box.
-```text
-mkdir $HOME/.secret
-ssh-keygen -b 1024 -t rsa -f id_rsa -P "" -f $HOME/.secret/id_rsa
-cat $HOME/.secret/id_rsa.pub >  ~/.ssh/authorized_keys 
-chmod u+w authorized_keys
+## run test with inline pipeline tasks [run](endly/run.yaml) request
+endly -r=run
 
-endly -c=localhost -k=~/.secret/id_rsa.pub
-```
-```
 
-Verify that secret file were created
-```text
-cat ~/.secret/localhost.json
+## To check manager workflow tasks list
+endly -w=manager -t='?'
+ 
 ```
 
 
-Check that **'endly'** binary is created in $GOPATH/bin directory as result of 
-'go get -u github.com/viant/endly/endly'
-
 
-#### Run elogger workflow
+  
 
-Run the following command:
 
-```text
-cd $GOPATH/src/github.com/viant/endly/example/ws/rt/elogger/endly/
-endly
-```

example/rt/elogger/endly/app.yaml

@@ -1,5 +1,5 @@
 tasks: $tasks
-params:
+defaults:
   app: $app
   sdk: $sdk
   target: $target
@@ -34,7 +34,7 @@ pipeline:
       - echo 'deployed'
 
   stop:
-    action: process:stop
+    action: process:stop-all
     input: ${app}
 
   start:

example/rt/elogger/endly/manager.csv

@@ -6,8 +6,8 @@ Workflow,,Name,Description,Tasks,Init,
 ,workflow,run,"Build, deploy app", @app,,
 []Tasks,,Name,Description,Actions,,
 ,,test,Run test,%Test,,
-[]Test,Service,Action,Description,Request,run.workflow,run.tasks
-,workflow,run,Run test plan,@req/run|$run,regresion,*
+[]Test,Service,Action,Description,Request,workflow,tasks
+,workflow,run,Run test plan, @req/run,regresion,*
 []Tasks,,Name,Description,Actions,,
 ,,destroy,Destroy system,%Destroy,,
 []Destroy,Service,Action,Description,Request,tasks,

example/rt/elogger/endly/req/run.yaml

@@ -0,0 +1,6 @@
+name: $workflow
+tasks: $tasks
+params:
+  app: $app
+  targetHost: $targetHost
+  eLoggerHost: $eLoggerHost

example/rt/elogger/endly/run.yaml

@@ -1,5 +1,5 @@
 init: "@var/init"
-params:
+defaults:
   app: $app
   sdk: $sdk
   target: $target