From daf30cb443c91c5545c767a58886e20608a5479e Mon Sep 17 00:00:00 2001 From: Vladimir Vivien Date: Thu, 18 Jun 2020 15:41:44 -0400 Subject: [PATCH] Implemenation of host_list_provider function. See documentation for detail on the notion of providers. Signed-off-by: Vladimir Vivien --- starlark/crashd_config.go | 18 ++++--- starlark/crashd_config_test.go | 39 ++++++-------- starlark/hostlist_provider.go | 48 +++++++++++++++++ starlark/hostlist_provider_test.go | 84 ++++++++++++++++++++++++++++++ starlark/resources.go | 82 +++++++++++++++++++++++++++++ starlark/ssh_config.go | 16 +++--- starlark/starlark_exec.go | 28 ++++++---- starlark/support.go | 14 +++-- 8 files changed, 277 insertions(+), 52 deletions(-) create mode 100644 starlark/hostlist_provider.go create mode 100644 starlark/hostlist_provider_test.go create mode 100644 starlark/resources.go diff --git a/starlark/crashd_config.go b/starlark/crashd_config.go index fc5b82df..6c7ead53 100644 --- a/starlark/crashd_config.go +++ b/starlark/crashd_config.go @@ -5,6 +5,7 @@ package starlark import ( "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" ) // addDefaultCrashdConf initalizes a Starlark Dict with default @@ -25,20 +26,23 @@ func addDefaultCrashdConf(thread *starlark.Thread) error { return nil } -// crashConfig is built-in starlark function that wraps the kwargs into a dictionary value. -// The result is also added to the thread for other built-in to access. +// crashConfig is built-in starlark function that saves and returns the kwargs as a struct value. +// Starlark format: crashd_config(conf0=val0, ..., confN=ValN) func crashdConfigFn(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { - var dictionary *starlark.Dict + var dictionary starlark.StringDict if kwargs != nil { - dict, err := tupleSliceToDict(kwargs) + dict, err := kwargsToStringDict(kwargs) if err != nil { return starlark.None, err } dictionary = dict } - // save dict to be used as default - thread.SetLocal(identifiers.crashdCfg, dictionary) + structVal := starlarkstruct.FromStringDict(starlarkstruct.Default, dictionary) - return dictionary, nil + // save values to be used as default + thread.SetLocal(identifiers.crashdCfg, structVal) + + // return values as a struct (i.e. config.arg0, ... , config.argN) + return starlark.None, nil } diff --git a/starlark/crashd_config_test.go b/starlark/crashd_config_test.go index f68a7b31..4e902795 100644 --- a/starlark/crashd_config_test.go +++ b/starlark/crashd_config_test.go @@ -7,7 +7,7 @@ import ( "strings" "testing" - "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" ) func TestCrashdConfigNew(t *testing.T) { @@ -39,19 +39,16 @@ func TestCrashdConfigFunc(t *testing.T) { if data == nil { t.Fatal("crashd_config not saved in thread local") } - cfg, ok := data.(*starlark.Dict) + cfg, ok := data.(*starlarkstruct.Struct) if !ok { t.Fatalf("unexpected type for thread local key configs.crashd: %T", data) } - if cfg.Len() != 2 { - t.Fatalf("unexpected item count in configs.crashd: %d", cfg.Len()) + if len(cfg.AttrNames()) != 2 { + t.Fatalf("unexpected item count in configs.crashd: %d", len(cfg.AttrNames())) } - val, found, err := cfg.Get(starlark.String("foo")) + val, err := cfg.Attr("foo") if err != nil { - t.Fatal(err) - } - if !found { - t.Fatalf("key 'foo' not found in configs.crashd") + t.Fatalf("key 'foo' not found in crashd_config: %s", err) } if trimQuotes(val.String()) != "fooval" { t.Fatalf("unexpected value for key 'foo': %s", val.String()) @@ -71,20 +68,17 @@ func TestCrashdConfigFunc(t *testing.T) { if data == nil { t.Fatal("crashd_config function not returning value") } - cfg, ok := data.(*starlark.Dict) + cfg, ok := data.(*starlarkstruct.Struct) if !ok { t.Fatalf("unexpected type for thread local key configs.crashd: %T", data) } - if cfg.Len() != 2 { - t.Fatalf("unexpected item count in configs.crashd: %d", cfg.Len()) + if len(cfg.AttrNames()) != 2 { + t.Fatalf("unexpected item count in configs.crashd: %d", len(cfg.AttrNames())) } - val, found, err := cfg.Get(starlark.String("foo")) + val, err := cfg.Attr("foo") if err != nil { t.Fatal(err) } - if !found { - t.Fatalf("key 'foo' not found in configs.crashd") - } if trimQuotes(val.String()) != "fooval" { t.Fatalf("unexpected value for key %s in configs.crashd", val.String()) } @@ -104,19 +98,16 @@ func TestCrashdConfigFunc(t *testing.T) { t.Fatal("default crashd_config not saved in thread local") } - cfg, ok := data.(*starlark.Dict) + cfg, ok := data.(*starlarkstruct.Struct) if !ok { t.Fatalf("unexpected type for thread local key crashd_config: %T", data) } - if cfg.Len() != 4 { - t.Fatalf("unexpected item count in configs.crashd: %d", cfg.Len()) + if len(cfg.AttrNames()) != 4 { + t.Fatalf("unexpected item count in configs.crashd: %d", len(cfg.AttrNames())) } - val, found, err := cfg.Get(starlark.String("uid")) + val, err := cfg.Attr("uid") if err != nil { - t.Fatal(err) - } - if !found { - t.Fatalf("key 'foo' not found in configs.crashd") + t.Fatalf("key 'foo' not found in configs.crashd: %s", err) } if trimQuotes(val.String()) != getUid() { t.Fatalf("unexpected value for key %s in configs.crashd", val.String()) diff --git a/starlark/hostlist_provider.go b/starlark/hostlist_provider.go new file mode 100644 index 00000000..4c1e9482 --- /dev/null +++ b/starlark/hostlist_provider.go @@ -0,0 +1,48 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package starlark + +import ( + "fmt" + + "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" +) + +// hostListProvider is a built-in starlark function that collects compute resources as a list of host IPs +// Starlark format: host_list_provider(hosts= [, ssh_config=ssh_config()]) +func hostListProvider(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var dictionary starlark.StringDict + if kwargs != nil { + dict, err := kwargsToStringDict(kwargs) + if err != nil { + return starlark.None, err + } + dictionary = dict + } + + return newHostListProvider(thread, dictionary) +} + +// newHostListProvider returns a struct with host list provider info +func newHostListProvider(thread *starlark.Thread, dictionary starlark.StringDict) (*starlarkstruct.Struct, error) { + // validate args + if _, ok := dictionary["hosts"]; !ok { + return nil, fmt.Errorf("%s: missing hosts argument", identifiers.hostListProvider) + } + + // augment args + dictionary["kind"] = starlark.String(identifiers.hostListProvider) + dictionary["transport"] = starlark.String("ssh") + if _, ok := dictionary[identifiers.sshCfg]; !ok { + data := thread.Local(identifiers.sshCfg) + sshcfg, ok := data.(starlark.StringDict) + if !ok { + return nil, fmt.Errorf("%s: default ssh_config not found", identifiers.hostListProvider) + } + dictionary[identifiers.sshCfg] = starlarkstruct.FromStringDict(starlarkstruct.Default, sshcfg) + } + + return starlarkstruct.FromStringDict(starlarkstruct.Default, dictionary), nil +} \ No newline at end of file diff --git a/starlark/hostlist_provider_test.go b/starlark/hostlist_provider_test.go new file mode 100644 index 00000000..1f1afff5 --- /dev/null +++ b/starlark/hostlist_provider_test.go @@ -0,0 +1,84 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package starlark + +import ( + "strings" + "testing" + + "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" +) + +func TestHostListProvider(t *testing.T) { + tests := []struct { + name string + script string + eval func(t *testing.T, script string) + }{ + { + name: "single host", + script: `provider = host_list_provider(hosts="foo.host")`, + eval: func(t *testing.T, script string) { + exe := New() + if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { + t.Fatal(err) + } + data := exe.result["provider"] + if data == nil { + t.Fatalf("%s function not returning value", identifiers.hostListProvider) + } + provider, ok := data.(*starlarkstruct.Struct) + if !ok { + t.Fatalf("expecting *starlark.Struct, got %T", data) + } + if len(provider.AttrNames()) != 1 { + t.Fatalf("unexpected item count in configs.crashd: %d", len(provider.AttrNames())) + } + val, err := provider.Attr("hosts") + if err != nil { + t.Fatal(err) + } + if trimQuotes(val.String()) != "foo.host" { + t.Fatalf("unexpected value for key %s in configs.crashd", val.String()) + } + }, + }, + { + name: "multiple hosts", + script: `provider = host_list_provider(hosts=["foo.host.1", "foo.host.2"])`, + eval: func(t *testing.T, script string) { + exe := New() + if err := exe.Exec("test.star", strings.NewReader(script)); err != nil { + t.Fatal(err) + } + data := exe.result["provider"] + if data == nil { + t.Fatalf("%s function not returning value", identifiers.hostListProvider) + } + provider, ok := data.(*starlarkstruct.Struct) + if !ok { + t.Fatalf("expecting *starlark.Struct, got %T", data) + } + if len(provider.AttrNames()) != 1 { + t.Fatalf("unexpected item %s: %d", identifiers.hostListProvider, len(provider.AttrNames())) + } + val, err := provider.Attr("hosts") + if err != nil { + t.Fatal(err) + } + list := val.(*starlark.List) + if list.Len() != 2 { + t.Fatalf("expecting %d items for argument 'hosts', got %d", 2, list.Len()) + } + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + test.eval(t, test.script) + }) + } +} diff --git a/starlark/resources.go b/starlark/resources.go new file mode 100644 index 00000000..6e4dbf06 --- /dev/null +++ b/starlark/resources.go @@ -0,0 +1,82 @@ +// Copyright (c) 2020 VMware, Inc. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +package starlark + +import ( + "fmt" + + "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" +) + +// resourcesFunc is a built-in starlark function that prepares returns compute resources as a struct. +// Starlark format: resources(provider=) +func resourcesFunc(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { + var dictionary starlark.StringDict + if kwargs != nil { + dict, err := kwargsToStringDict(kwargs) + if err != nil { + return starlark.None, err + } + dictionary = dict + } + + var provider *starlarkstruct.Struct + if hosts, ok := dictionary["hosts"]; ok { + prov, err := newHostListProvider(thread, starlark.StringDict{"hosts": hosts}) + if err != nil { + return starlark.None, err + } + provider = prov + } else if prov, ok := dictionary["provider"]; ok { + provider = prov.(*starlarkstruct.Struct) + } + + // enumerates resources + return enum(provider) +} + +// enum returns a struct containing the fully enumerated compute resource +// info needed to execute commands. +func enum(provider *starlarkstruct.Struct) (*starlarkstruct.Struct, error) { + if provider == nil { + fmt.Errorf("missing provider") + } + + var resStruct *starlarkstruct.Struct + + kindVal, err := provider.Attr("kind") + if err != nil { + return nil, fmt.Errorf("provider missing field kind") + } + + kind := trimQuotes(kindVal.String()) + + switch kind { + case identifiers.hostListProvider: + names, err := provider.Attr("hosts") + if err != nil { + return nil, fmt.Errorf("hosts not found in %s", identifiers.hostListProvider) + } + transport, err := provider.Attr("transport") + if err != nil { + return nil, fmt.Errorf("transport not found in %s", identifiers.hostListProvider) + } + + sshCfg, err := provider.Attr(identifiers.sshCfg) + if err != nil { + return nil, fmt.Errorf("ssh_config not found in %s", identifiers.hostListProvider) + } + + dict := starlark.StringDict{ + "kind": starlark.String("host_list_resources"), + "names": names, + "ip_addresses": names, + "transport": transport, + "ssh_config": sshCfg, + } + resStruct = starlarkstruct.FromStringDict(starlarkstruct.Default, dict) + } + return resStruct, nil +} diff --git a/starlark/ssh_config.go b/starlark/ssh_config.go index 0c37862e..fc98055e 100644 --- a/starlark/ssh_config.go +++ b/starlark/ssh_config.go @@ -5,6 +5,7 @@ package starlark import ( "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" ) // addDefaultSshConf initalizes a Starlark Dict with default @@ -25,21 +26,22 @@ func addDefaultSSHConf(thread *starlark.Thread) error { return nil } -// sshConfigFn is the backing built-in function for the `ssh_config` configuration function. -// It creates and returns a dictionary from collected configs (as kwargs) -// It also saves the dict into the thread as the last known ssh config to be used as default. +// sshConfigFn is the backing built-in fn that saves and returns its argument as struct value. +// Starlark format: ssh_config(conf0=val0, ..., confN=valN) func sshConfigFn(thread *starlark.Thread, b *starlark.Builtin, args starlark.Tuple, kwargs []starlark.Tuple) (starlark.Value, error) { - var dictionary *starlark.Dict + var dictionary starlark.StringDict if kwargs != nil { - dict, err := tupleSliceToDict(kwargs) + dict, err := kwargsToStringDict(kwargs) if err != nil { return starlark.None, err } dictionary = dict } + structVal := starlarkstruct.FromStringDict(starlarkstruct.Default, dictionary) + // save to be used as default when needed - thread.SetLocal(identifiers.sshCfg, dictionary) + thread.SetLocal(identifiers.sshCfg, structVal) - return dictionary, nil + return structVal, nil } diff --git a/starlark/starlark_exec.go b/starlark/starlark_exec.go index 213248fb..694d58ae 100644 --- a/starlark/starlark_exec.go +++ b/starlark/starlark_exec.go @@ -9,6 +9,7 @@ import ( "github.com/vladimirvivien/echo" "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" ) type Executor struct { @@ -50,29 +51,36 @@ func newThreadLocal() *starlark.Thread { // runing script. func newPredeclareds() starlark.StringDict { return starlark.StringDict{ - identifiers.crashdCfg: starlark.NewBuiltin(identifiers.crashdCfg, crashdConfigFn), - identifiers.sshCfg: starlark.NewBuiltin(identifiers.sshCfg, sshConfigFn), + identifiers.crashdCfg: starlark.NewBuiltin(identifiers.crashdCfg, crashdConfigFn), + identifiers.sshCfg: starlark.NewBuiltin(identifiers.sshCfg, sshConfigFn), + identifiers.hostListProvider: starlark.NewBuiltin(identifiers.hostListProvider, hostListProvider), } } -func tupleSliceToDict(tuples []starlark.Tuple) (*starlark.Dict, error) { - if len(tuples) == 0 { - return &starlark.Dict{}, nil +func kwargsToStringDict(kwargs []starlark.Tuple) (starlark.StringDict, error) { + if len(kwargs) == 0 { + return starlark.StringDict{}, nil } - dictionary := starlark.NewDict(len(tuples)) e := echo.New() + dictionary := make(starlark.StringDict) - for _, tup := range tuples { + for _, tup := range kwargs { key, value := tup[0], tup[1] if value.Type() == "string" { unquoted := trimQuotes(value.String()) value = starlark.String(e.Eval(unquoted)) } - if err := dictionary.SetKey(key, value); err != nil { - return nil, err - } + dictionary[trimQuotes(key.String())] = value } return dictionary, nil } + +func kwargsToStruct(kwargs []starlark.Tuple) (*starlarkstruct.Struct, error) { + dict, err := kwargsToStringDict(kwargs) + if err != nil { + return &starlarkstruct.Struct{}, err + } + return starlarkstruct.FromStringDict(starlarkstruct.Default, dict), nil +} diff --git a/starlark/support.go b/starlark/support.go index 501fd565..762133cf 100644 --- a/starlark/support.go +++ b/starlark/support.go @@ -10,11 +10,17 @@ import ( var ( identifiers = struct { - crashdCfg string - sshCfg string + crashdCfg string + sshCfg string + hostListProvider string + hostListResources string + resources string }{ - crashdCfg: "crashd_config", - sshCfg: "ssh_config", + crashdCfg: "crashd_config", + sshCfg: "ssh_config", + hostListProvider: "host_list_provider", + hostListResources: "host_list_resources", + resources: "resources", } defaults = struct {