Skip to content

Commit

Permalink
Compiler Requirements for Branch Nodes fix (flyteorg#214)
Browse files Browse the repository at this point in the history
  • Loading branch information
EngHabu authored Jan 6, 2021
1 parent f154b43 commit 8d854e8
Show file tree
Hide file tree
Showing 5 changed files with 285 additions and 18 deletions.
1 change: 0 additions & 1 deletion flytepropeller/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ require (
github.com/pkg/errors v0.9.1
github.com/prometheus/client_golang v1.5.0
github.com/prometheus/procfs v0.0.10 // indirect
github.com/robfig/cron v1.2.0 // indirect
github.com/sirupsen/logrus v1.4.2
github.com/spf13/cobra v0.0.6
github.com/spf13/pflag v1.0.5
Expand Down
14 changes: 0 additions & 14 deletions flytepropeller/go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -392,23 +392,11 @@ github.com/lyft/datacatalog v0.2.1/go.mod h1:ktrPvzTDUwHO5Lv0hLH38zLHnOJ++rGoAO0
github.com/lyft/flyteidl v0.17.0/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20=
github.com/lyft/flyteidl v0.18.9 h1:p9gLp92whTSSOeMGPtZ4tkgsVHNGuBuXXMQ447s0J9E=
github.com/lyft/flyteidl v0.18.9/go.mod h1:/zQXxuHO11u/saxTTZc8oYExIGEShXB+xCB1/F1Cu20=
github.com/lyft/flyteplugins v0.5.16 h1:/orpL0wlmU9yIhwvSge5mw4betcGoh/EalmJKCQDI8w=
github.com/lyft/flyteplugins v0.5.16/go.mod h1:1G2YORr5JZhfl4dyHGIO4/+ShNMul5DPnoBEdpJfuNc=
github.com/lyft/flyteplugins v0.5.17-0.20201106210712-9fd733e7b0e5 h1:M0hAB8SQ5wklBo39xzteq7fhrfTeRYcmD0AwJtnZ/1s=
github.com/lyft/flyteplugins v0.5.17-0.20201106210712-9fd733e7b0e5/go.mod h1:1G2YORr5JZhfl4dyHGIO4/+ShNMul5DPnoBEdpJfuNc=
github.com/lyft/flyteplugins v0.5.17 h1:+P6JE+AGxo+pvM6/FMDcE3pk6/fOjnILRQ4SFwLKlIA=
github.com/lyft/flyteplugins v0.5.17/go.mod h1:1G2YORr5JZhfl4dyHGIO4/+ShNMul5DPnoBEdpJfuNc=
github.com/lyft/flyteplugins v0.5.18-0.20201124013132-bb9c24dd58f7 h1:OX4KHjF7IFCECoj/Nc6rmHc/Y8vFEgYvSc5/w8dxaZk=
github.com/lyft/flyteplugins v0.5.18-0.20201124013132-bb9c24dd58f7/go.mod h1:1G2YORr5JZhfl4dyHGIO4/+ShNMul5DPnoBEdpJfuNc=
github.com/lyft/flyteplugins v0.5.18-0.20201130205551-ae0144e80968 h1:B/tmbddB48DEz30ev5f82i54k8k5iHXfjhJ9GKCMf2Y=
github.com/lyft/flyteplugins v0.5.18-0.20201130205551-ae0144e80968/go.mod h1:1G2YORr5JZhfl4dyHGIO4/+ShNMul5DPnoBEdpJfuNc=
github.com/lyft/flyteplugins v0.5.20 h1:+l4dakFCgbP/ePQY37SneMhlBGrujeXfUL+U1AtUMk4=
github.com/lyft/flyteplugins v0.5.20/go.mod h1:1G2YORr5JZhfl4dyHGIO4/+ShNMul5DPnoBEdpJfuNc=
github.com/lyft/flytestdlib v0.3.0/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU=
github.com/lyft/flytestdlib v0.3.9 h1:NaKp9xkeWWwhVvqTOcR/FqlASy1N2gu/kN7PVe4S7YI=
github.com/lyft/flytestdlib v0.3.9/go.mod h1:LJPPJlkFj+wwVWMrQT3K5JZgNhZi2mULsCG4ZYhinhU=
github.com/lyft/spark-on-k8s-operator v0.1.3 h1:rmke8lR2Oy8mvKXRhloKuEu7fgGuXepDxiBNiorVUFI=
github.com/lyft/spark-on-k8s-operator v0.1.3/go.mod h1:hkRqdqAsdNnxT/Zst6MNMRbTAoiCZ0JRw7svRgAYb0A=
github.com/lyft/spark-on-k8s-operator v0.1.4-0.20201027003055-c76b67e3b6d0 h1:1vSmc+Bo70X0JVYywQ9Hy/aet6p613ejacy9x5td0m4=
github.com/lyft/spark-on-k8s-operator v0.1.4-0.20201027003055-c76b67e3b6d0/go.mod h1:hkRqdqAsdNnxT/Zst6MNMRbTAoiCZ0JRw7svRgAYb0A=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
Expand Down Expand Up @@ -524,8 +512,6 @@ github.com/prometheus/procfs v0.0.10/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+G
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
github.com/remyoudompheng/bigfft v0.0.0-20170806203942-52369c62f446/go.mod h1:uYEyJGbgTkfkS4+E/PavXkNJcbFIpEtjt2B0KDQ5+9M=
github.com/robfig/cron v1.2.0 h1:ZjScXvvxeQ63Dbyxy76Fj3AT3Ut0aKsyd2/tl3DTMuQ=
github.com/robfig/cron v1.2.0/go.mod h1:JGuDeoQd7Z6yL4zQhZ3OPEVHB7fL6Ka6skscFHfmt2k=
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
Expand Down
4 changes: 4 additions & 0 deletions flytepropeller/pkg/compiler/requirements.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,5 +82,9 @@ func updateNodeRequirements(node *flyteNode, subWfs common.WorkflowIndex, taskId
for _, otherCase := range branchN.IfElse.Other {
updateNodeRequirements(otherCase.ThenNode, subWfs, taskIds, workflowIds, followSubworkflows, errs)
}

if elseNode := branchN.IfElse.GetElseNode(); elseNode != nil {
updateNodeRequirements(elseNode, subWfs, taskIds, workflowIds, followSubworkflows, errs)
}
}
}
10 changes: 7 additions & 3 deletions flytepropeller/pkg/compiler/test/compiler_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -109,21 +109,25 @@ func TestBranches(t *testing.T) {
}

t.Run(path, func(t *testing.T) {
//if !strings.HasSuffix(path, "success_5.json") {
//if !strings.HasSuffix(path, "success_6.json") {
// t.SkipNow()
//}

raw, err := ioutil.ReadFile(path)
assert.NoError(t, err)
wf := &core.WorkflowClosure{}
err = jsonpb.UnmarshalString(string(raw), wf)
assert.NoError(t, err)
if !assert.NoError(t, err) {
t.FailNow()
}

t.Log("Compiling Workflow")
compiledTasks := mustCompileTasks(t, wf.Tasks)
compiledWfc, err := compiler.CompileWorkflow(wf.Workflow, []*core.WorkflowTemplate{}, compiledTasks,
[]common.InterfaceProvider{})
assert.NoError(t, err)
if !assert.NoError(t, err) {
t.FailNow()
}

inputs := map[string]interface{}{}
for varName, v := range compiledWfc.Primary.Template.Interface.Inputs.Variables {
Expand Down
274 changes: 274 additions & 0 deletions flytepropeller/pkg/compiler/test/testdata/branch/success_6.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,274 @@
{
"workflow": {
"id": {
"resourceType": "WORKFLOW",
"project": "test_proj",
"domain": "test_domain",
"name": "recipes.02_intermediate.run_conditions.multiplier",
"version": "abc"
},
"metadata": {},
"interface": {
"inputs": {
"variables": {
"my_input": {
"type": {
"simple": "FLOAT"
},
"description": "my_input"
}
}
},
"outputs": {
"variables": {
"o0": {
"type": {
"simple": "FLOAT"
},
"description": "o0"
}
}
}
},
"nodes": [
{
"id": "node-0",
"metadata": {
"name": "fractions",
"retries": {},
"interruptible": false
},
"inputs": [
{
"var": ".my_input",
"binding": {
"promise": {
"var": "my_input"
}
}
}
],
"branchNode": {
"ifElse": {
"case": {
"condition": {
"conjunction": {
"leftExpression": {
"comparison": {
"operator": "GTE",
"leftValue": {
"var": ".my_input"
},
"rightValue": {
"primitive": {
"floatValue": 0.1
}
}
}
},
"rightExpression": {
"comparison": {
"operator": "LTE",
"leftValue": {
"var": ".my_input"
},
"rightValue": {
"primitive": {
"floatValue": 1.0
}
}
}
}
}
},
"thenNode": {
"id": "branchn0",
"metadata": {
"name": "flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.double",
"retries": {},
"interruptible": false
},
"inputs": [
{
"var": "n",
"binding": {
"promise": {
"var": "my_input"
}
}
}
],
"taskNode": {
"referenceId": {
"resourceType": "TASK",
"project": "test_proj",
"domain": "test_domain",
"name": "recipes.02_intermediate.run_conditions.double",
"version": "abc"
}
}
}
},
"elseNode": {
"id": "branchn1",
"metadata": {
"name": "flytekit.annotated.python_function_task.recipes.02_intermediate.run_conditions.square",
"retries": {},
"interruptible": false
},
"inputs": [
{
"var": "n",
"binding": {
"promise": {
"var": "my_input"
}
}
}
],
"taskNode": {
"referenceId": {
"resourceType": "TASK",
"project": "test_proj",
"domain": "test_domain",
"name": "recipes.02_intermediate.run_conditions.square",
"version": "abc"
}
}
}
}
}
}
],
"outputs": [
{
"var": "o0",
"binding": {
"promise": {
"nodeId": "node-0",
"var": "o0"
}
}
}
],
"metadataDefaults": {}
},
"tasks": [
{
"id": {
"resourceType": "TASK",
"project": "test_proj",
"domain": "test_domain",
"name": "recipes.02_intermediate.run_conditions.double",
"version": "abc"
},
"type": "python-task",
"metadata": {
"runtime": {
"type": "FLYTE_SDK",
"version": "0.16.0",
"flavor": "python"
},
"retries": {},
"interruptible": false
},
"interface": {
"inputs": {
"variables": {
"n": {
"type": {
"simple": "FLOAT"
},
"description": "n"
}
}
},
"outputs": {
"variables": {
"o0": {
"type": {
"simple": "FLOAT"
},
"description": "o0"
}
}
}
},
"container": {
"image": "image:name",
"args": [
"pyflyte-execute",
"--task-module",
"recipes.02_intermediate.run_conditions",
"--task-name",
"double",
"--inputs",
"{{.input}}",
"--output-prefix",
"{{.outputPrefix}}",
"--raw-output-data-prefix",
"{{.rawOutputDataPrefix}}"
],
"resources": {}
}
},
{
"id": {
"resourceType": "TASK",
"project": "test_proj",
"domain": "test_domain",
"name": "recipes.02_intermediate.run_conditions.square",
"version": "abc"
},
"type": "python-task",
"metadata": {
"runtime": {
"type": "FLYTE_SDK",
"version": "0.16.0",
"flavor": "python"
},
"retries": {},
"interruptible": false
},
"interface": {
"inputs": {
"variables": {
"n": {
"type": {
"simple": "FLOAT"
},
"description": "n"
}
}
},
"outputs": {
"variables": {
"o0": {
"type": {
"simple": "FLOAT"
},
"description": "o0"
}
}
}
},
"container": {
"image": "image:name",
"args": [
"pyflyte-execute",
"--task-module",
"recipes.02_intermediate.run_conditions",
"--task-name",
"square",
"--inputs",
"{{.input}}",
"--output-prefix",
"{{.outputPrefix}}",
"--raw-output-data-prefix",
"{{.rawOutputDataPrefix}}"
],
"resources": {}
}
}
]
}

0 comments on commit 8d854e8

Please sign in to comment.