diff --git a/ds/dataset.go b/ds/dataset.go index ec8c7e9..b14ac51 100644 --- a/ds/dataset.go +++ b/ds/dataset.go @@ -139,16 +139,24 @@ func (d *Dataset) SetStructure(thread *starlark.Thread, _ *starlark.Builtin, arg return nil, err } - if err := d.checkField("structure", "schema"); err != nil { + if err := d.checkField("structure"); err != nil { return starlark.None, err } d.ds.Structure = &dataset.Structure{} - if err := json.Unmarshal([]byte(valx.String()), d.ds.Structure); err != nil { + + val, err := util.Unmarshal(valx) + if err != nil { return starlark.None, err } - return starlark.None, nil + data, err := json.Marshal(val) + if err != nil { + return starlark.None, err + } + + err = json.Unmarshal(data, d.ds.Structure) + return starlark.None, err } // GetBody returns the body of the dataset we're transforming diff --git a/qri/qri.go b/qri/qri.go index 970871b..0389811 100644 --- a/qri/qri.go +++ b/qri/qri.go @@ -141,7 +141,7 @@ func (m *Module) loadDsHead(refstr string) (*dataset.Dataset, error) { if err := repo.CanonicalizeDatasetRef(m.node.Repo, &ref); err != nil { return nil, err } - m.node.LocalStreams.Out.Write([]byte(fmt.Sprintf("loading dataset: %s", ref.String()))) + m.node.LocalStreams.PrintErr(fmt.Sprintf("load: %s\n", ref.String())) ds, err := dsfs.LoadDataset(m.node.Repo.Store(), ref.Path) if err != nil { diff --git a/startf.go b/startf.go index 1901d0d..345736b 100644 --- a/startf.go +++ b/startf.go @@ -2,4 +2,4 @@ package startf // Version is the current version of this startf, this version number will be written // with each transformation exectution -const Version = "0.3.0" +const Version = "0.3.1" diff --git a/transform.go b/transform.go index f71a4bb..0ae3db0 100644 --- a/transform.go +++ b/transform.go @@ -46,7 +46,7 @@ func AddMutateFieldCheck(check func(path ...string) error) func(o *ExecOpts) { } } -// SetOutWriter provides a writer to record the "stdout" of the transform script to +// SetOutWriter provides a writer to record the "stderr" diagnostic output of the transform script func SetOutWriter(w io.Writer) func(o *ExecOpts) { return func(o *ExecOpts) { if w != nil { @@ -71,7 +71,7 @@ type transform struct { checkFunc func(path ...string) error globals starlark.StringDict bodyFile qfs.File - stdout io.Writer + stderr io.Writer download starlark.Iterable } @@ -121,12 +121,12 @@ func ExecScript(ds *dataset.Dataset, opts ...func(o *ExecOpts)) error { ds: ds, skyqri: skyqri.NewModule(o.Node, ds), checkFunc: o.MutateFieldCheck, - stdout: o.OutWriter, + stderr: o.OutWriter, } if o.Node != nil { // if node localstreams exists, write to both localstreams and output buffer - t.stdout = io.MultiWriter(o.OutWriter, o.Node.LocalStreams.Out) + t.stderr = io.MultiWriter(o.OutWriter, o.Node.LocalStreams.ErrOut) } ctx := skyctx.NewContext(ds.Transform.Config, o.Secrets) @@ -135,7 +135,7 @@ func ExecScript(ds *dataset.Dataset, opts ...func(o *ExecOpts)) error { Load: t.Loader, Print: func(thread *starlark.Thread, msg string) { // note we're ignoring a returned error here - _, _ = t.stdout.Write([]byte(msg)) + _, _ = t.stderr.Write([]byte(msg)) }, } @@ -167,6 +167,9 @@ func ExecScript(ds *dataset.Dataset, opts ...func(o *ExecOpts)) error { } err = callTransformFunc(t, thread, ctx) + if evalErr, ok := err.(*starlark.EvalError); ok { + return fmt.Errorf(evalErr.Backtrace()) + } // restore consumed script file ds.Transform.SetScriptFile(qfs.NewMemfileBytes("transform.star", buf.Bytes())) @@ -271,7 +274,7 @@ func (t *transform) setSpinnerMsg(msg string) { // print writes output only if a node is specified func (t *transform) print(msg string) { - t.stdout.Write([]byte(msg)) + t.stderr.Write([]byte(msg)) } func (t *transform) Loader(thread *starlark.Thread, module string) (dict starlark.StringDict, err error) { diff --git a/transform_test.go b/transform_test.go index 327dbfb..a9a0d6a 100644 --- a/transform_test.go +++ b/transform_test.go @@ -28,8 +28,8 @@ func TestExecScript(t *testing.T) { } ds.Transform.SetScriptFile(scriptFile(t, "testdata/tf.star")) - stdout := &bytes.Buffer{} - err := ExecScript(ds, SetOutWriter(stdout)) + stderr := &bytes.Buffer{} + err := ExecScript(ds, SetOutWriter(stderr)) if err != nil { t.Error(err.Error()) return @@ -38,14 +38,14 @@ func TestExecScript(t *testing.T) { t.Error("expected transform") } - output, err := ioutil.ReadAll(stdout) + output, err := ioutil.ReadAll(stderr) if err != nil { t.Fatal(err) } expect := `🤖 running transform... hello world!` if string(output) != expect { - t.Errorf("stdout mismatch. expected: '%s', got: '%s'", expect, string(output)) + t.Errorf("stderr mismatch. expected: '%s', got: '%s'", expect, string(output)) } entryReader, err := dsio.NewEntryReader(ds.Structure, ds.BodyFile())