From ca00916e886c8e46b81a1f92ebc9af4318272db9 Mon Sep 17 00:00:00 2001 From: "Moises P. Sena" Date: Wed, 6 Mar 2024 10:37:22 -0300 Subject: [PATCH] core!: 1) add context to Importable API; 2) more iterators changes. --- builtins.go | 6 ++ builtins_funcs.go | 10 +-- builtins_types.go | 28 ++++++- compiler.go | 2 + compiler_nodes.go | 2 +- encoder/encoder_test.go | 6 +- errors.go | 3 + helper_foreach.go | 7 +- importers/importers.go | 3 +- iterator.go | 182 +++++++++++++++++++++++++++++++++++----- modules.go | 7 +- objects.go | 12 ++- objects_namedargs.go | 34 +++++++- vm_loop.go | 4 +- vm_test.go | 21 ++++- 15 files changed, 276 insertions(+), 51 deletions(-) diff --git a/builtins.go b/builtins.go index b1a56b5..0a87ee1 100644 --- a/builtins.go +++ b/builtins.go @@ -82,6 +82,7 @@ const ( BuiltinValues BuiltinItems BuiltinCollect + BuiltinEnumerate BuiltinIteratorInput BuiltinVMPushWriter BuiltinVMPopWriter @@ -236,6 +237,7 @@ var BuiltinsMap = map[string]BuiltinType{ "values": BuiltinValues, "items": BuiltinItems, "collect": BuiltinCollect, + "enumerate": BuiltinEnumerate, "iterator": BuiltinIterator, "iteratorInput": BuiltinIteratorInput, "keyValue": BuiltinKeyValue, @@ -649,6 +651,10 @@ func init() { Name: "collect", Value: BuiltinCollectFunc, } + BuiltinObjects[BuiltinEnumerate] = &BuiltinFunction{ + Name: "enumerate", + Value: BuiltinEnumerateFunc, + } BuiltinObjects[BuiltinIterator] = TIterator BuiltinObjects[BuiltinIteratorInput] = &BuiltinFunction{ Name: "iteratorInput", diff --git a/builtins_funcs.go b/builtins_funcs.go index 112a4d1..e1cab6f 100644 --- a/builtins_funcs.go +++ b/builtins_funcs.go @@ -454,12 +454,12 @@ func BuiltinEnumerateFunc(c Call) (_ Object, err error) { if _, it, err = ToIterator(c.VM, v, &c.NamedArgs); err != nil { return } - return TypedIteratorObject(TEnumerateIterator, WrapIterator(it, func(e *IteratorEntry) (Object, error) { - kv := e.KeyValue - e.K = i - e.V = &kv + return TypedIteratorObject(TEnumerateIterator, WrapIterator(it, func(state *IteratorState) error { + kv := state.Entry.KeyValue + state.Entry.K = i + state.Entry.V = &kv i++ - return e, nil + return nil })), nil } diff --git a/builtins_types.go b/builtins_types.go index 430a64c..402e2b4 100644 --- a/builtins_types.go +++ b/builtins_types.go @@ -13,6 +13,7 @@ var ( TIterator = &Type{Parent: TAny, TypeName: "Iterator"} TIterabler = &Type{Parent: TAny, TypeName: "Iterabler"} TNilIterator = &Type{Parent: TIterator, TypeName: "NilIterator"} + TStateIterator = &Type{Parent: TIterator, TypeName: "StateIterator"} TStrIterator = &Type{Parent: TIterator, TypeName: "StrIterator"} TRawStrIterator = &Type{Parent: TIterator, TypeName: "RawStrIterator"} TArrayIterator = &Type{Parent: TIterator, TypeName: "ArrayIterator"} @@ -32,7 +33,6 @@ var ( TEachIterator = &Type{Parent: TIterator, TypeName: "EachIterator"} TMapIterator = &Type{Parent: TIterator, TypeName: "MapIterator"} TFilterIterator = &Type{Parent: TIterator, TypeName: "FilterIterator"} - TReduceIterator = &Type{Parent: TIterator, TypeName: "ReduceIterator"} TZipIterator = &Type{Parent: TIterator, TypeName: "ZipIterator"} TPipedInvokeIterator = &Type{Parent: TIterator, TypeName: "PipedInvokeIterator"} ) @@ -60,6 +60,17 @@ func (t *Type) AddCallerMethod(vm *VM, types MultipleObjectTypes, handler Caller }, override) } +func (t *Type) WithMethod(types MultipleObjectTypes, handler CallerObject, override bool) *Type { + if len(types) == 0 { + // overrides default constructor. uses Type.new to instantiate. + override = true + } + t.calllerMethods.Add(types, &CallerMethod{ + CallerObject: handler, + }, override) + return t +} + func (t *Type) HasCallerMethods() bool { return !t.calllerMethods.IsZero() } @@ -199,3 +210,18 @@ func TypesOf(obj Object) (types []ObjectType) { } return types } + +func init() { + TIterator.WithMethod( + MultipleObjectTypes{{nil}}, + &Function{ + Value: func(c Call) (o Object, err error) { + if err = c.Args.CheckLen(1); err != nil { + return + } + _, o, err = ToStateIterator(c.VM, c.Args.GetOnly(0), &c.NamedArgs) + return + }, + }, + true) +} diff --git a/compiler.go b/compiler.go index cff2cb4..2960945 100644 --- a/compiler.go +++ b/compiler.go @@ -5,6 +5,7 @@ package gad import ( + "context" "errors" "fmt" "io" @@ -75,6 +76,7 @@ type ( // CompilerOptions represents customizable options for Compile(). CompilerOptions struct { + Context context.Context ModuleMap *ModuleMap ModulePath string Constants []Object diff --git a/compiler_nodes.go b/compiler_nodes.go index 700a08b..da965f2 100644 --- a/compiler_nodes.go +++ b/compiler_nodes.go @@ -1444,7 +1444,7 @@ func (c *Compiler) compileImportExpr(nd *node.ImportExpr) error { module, exists := c.getModule(moduleName) if !exists { - mod, err := importer.Import(moduleName) + mod, err := importer.Import(c.opts.Context, moduleName) if err != nil { return c.error(nd, err) } diff --git a/encoder/encoder_test.go b/encoder/encoder_test.go index 85e24c7..7c83a1f 100644 --- a/encoder/encoder_test.go +++ b/encoder/encoder_test.go @@ -611,8 +611,8 @@ func testBytecodeConstants(t *testing.T, vm *gad.VM, expected, decoded []gad.Obj _, decIt, err := gad.ToStateIterator(vm, decoded[i], gad.NewNamedArgs()) require.NoError(t, err) - for next(decIt.Next()) { - require.True(t, next(it.Next())) + for next(decIt.Read()) { + require.True(t, next(it.Read())) key := decIt.Key() v1, err := gad.Val(expected[i].(gad.IndexGetter).IndexGet(vm, key)) require.NoError(t, err) @@ -633,7 +633,7 @@ func testBytecodeConstants(t *testing.T, vm *gad.VM, expected, decoded []gad.Obj require.Equal(t, v1, v2) } } - require.False(t, next(it.Next())) + require.False(t, next(it.Read())) continue } require.Equalf(t, expected[i], decoded[i], diff --git a/errors.go b/errors.go index db3a447..88d7b45 100644 --- a/errors.go +++ b/errors.go @@ -80,6 +80,9 @@ var ( // ErrNotInitializable represents a not initializable type error. ErrNotInitializable = &Error{Name: "ErrNotInitializable"} + + // ErrNotWriteable represents a not writeable type error. + ErrNotWriteable = &Error{Name: "ErrNotWriteable"} ) // NewOperandTypeError creates a new Error from ErrType. diff --git a/helper_foreach.go b/helper_foreach.go index ec8345f..110a728 100644 --- a/helper_foreach.go +++ b/helper_foreach.go @@ -81,14 +81,9 @@ func (f *PipedInvokeIterator) SetHandler(handler func(state *IteratorState) erro func (f *PipedInvokeIterator) checkNext(vm *VM, state *IteratorState) (err error) { try: - if state.Mode == IteratorStateModeDone { + if err = IteratorStateCheck(vm, f.it, state); err != nil || state.Mode == IteratorStateModeDone { return } - for state.Mode == IteratorStateModeContinue { - if err = f.it.Next(vm, state); err != nil || state.Mode == IteratorStateModeDone { - return - } - } if err = f.handler(state); err == nil { if err = f.Call(state); state.Mode != IteratorStateModeEntry { goto try diff --git a/importers/importers.go b/importers/importers.go index 2da1ffa..844c459 100644 --- a/importers/importers.go +++ b/importers/importers.go @@ -1,6 +1,7 @@ package importers import ( + "context" "errors" "io/ioutil" "path/filepath" @@ -45,7 +46,7 @@ func (m *FileImporter) Name() string { // Import returns the content of the path determined by Name call. Empty name // will return an error. -func (m *FileImporter) Import(moduleName string) (any, error) { +func (m *FileImporter) Import(_ context.Context, moduleName string) (any, error) { // Note that; moduleName == Literal() if m.name == "" || moduleName == "" { return nil, errors.New("invalid import call") diff --git a/iterator.go b/iterator.go index c02b814..c75f87d 100644 --- a/iterator.go +++ b/iterator.go @@ -5,6 +5,7 @@ package gad import ( + "fmt" "reflect" "sort" "strconv" @@ -55,6 +56,18 @@ const ( IteratorStateCollectModeValues ) +func (m IteratorStateCollectMode) String() string { + switch m { + case IteratorStateCollectModePair: + return "pair" + case IteratorStateCollectModeValues: + return "values" + case IteratorStateCollectModeKeys: + return "keys" + } + return fmt.Sprint(uint8(m)) +} + type IteratorState struct { Mode IteratorStateMode CollectMode IteratorStateCollectMode @@ -62,6 +75,18 @@ type IteratorState struct { Value Object } +func (s IteratorState) Get() Object { + switch s.CollectMode { + case IteratorStateCollectModeKeys: + return s.Entry.K + case IteratorStateCollectModePair: + kv := s.Entry.KeyValue + return &kv + default: + return s.Entry.V + } +} + // Iterator wraps the methods required to iterate Objects in VM. type Iterator interface { Representer @@ -405,13 +430,122 @@ func (o *iteratorObject) ToString() string { } type StateIteratorObject struct { - ObjectImpl - Iterator Iterator - State *IteratorState - VM *VM + Iterator + State *IteratorState + VM *VM +} + +func (s *StateIteratorObject) IndexGet(vm *VM, index Object) (value Object, err error) { + switch index.ToString() { + case "entry": + if s.State == nil { + return Nil, err + } + return &s.State.Entry.KeyValue, nil + case "k": + if s.State == nil { + return Nil, err + } + return s.State.Entry.K, nil + case "v": + if s.State == nil { + return Nil, err + } + return s.State.Entry.V, nil + case "started": + if s.State == nil { + return False, err + } + return True, nil + case "done": + if s.State == nil { + return False, err + } + if s.State.Mode == IteratorStateModeDone { + return True, nil + } + return False, nil + case "next": + var hasNext bool + if hasNext, err = s.Read(); err != nil { + return + } + if hasNext { + return s.State.Get(), nil + } + return Nil, err + } + return nil, ErrInvalidIndex +} + +func (s *StateIteratorObject) IsFalsy() bool { + if s.State == nil { + return false + } + return s.State.Mode == IteratorStateModeDone +} + +func (s *StateIteratorObject) ToString() string { + return "StateIterator: " + s.Info().ToString() +} + +func (s *StateIteratorObject) Info() Dict { + status := "wait" + if s.State != nil { + if s.State.Mode == IteratorStateModeDone { + status = "done" + } + } + d := Dict{ + "Status": Str(status), + } + if s.State != nil { + d["Value"] = s.State.Value + d["Entry"] = &s.State.Entry.KeyValue + d["CollectMode"] = Str(s.State.CollectMode.String()) + } + return d +} + +func (s *StateIteratorObject) Equal(right Object) bool { + if o, _ := right.(*StateIteratorObject); o != nil { + return o == s + } + return false +} + +func (s *StateIteratorObject) Type() ObjectType { + return TStateIterator +} + +func (s *StateIteratorObject) Repr(vm *VM) (r string, err error) { + if r, err = s.Iterator.Repr(vm); err != nil { + return + } + return "StateIterator:" + s.Info().ToString() + " of " + r, nil +} + +func (s *StateIteratorObject) Start(vm *VM) (state *IteratorState, err error) { + if state, err = s.Iterator.Start(vm); err != nil { + return + } + s.State = state + err = IteratorStateCheck(s.VM, s.Iterator, s.State) + return +} + +func (s *StateIteratorObject) Next(vm *VM, state *IteratorState) (err error) { + s.State = state + if err = s.Iterator.Next(vm, state); err == nil { + err = IteratorStateCheck(s.VM, s.Iterator, s.State) + } + return } func NewStateIteratorObject(vm *VM, it Iterator) *StateIteratorObject { + if si, _ := it.(*StateIteratorObject); si != nil { + return si + } return &StateIteratorObject{Iterator: it, VM: vm} } @@ -419,20 +553,15 @@ func (s *StateIteratorObject) GetIterator() Iterator { return s.Iterator } -func (s *StateIteratorObject) Next() (_ bool, err error) { +func (s *StateIteratorObject) Read() (_ bool, err error) { if s.State == nil { - if s.State, err = s.Iterator.Start(s.VM); err != nil { + if s.State, err = s.Start(s.VM); err != nil { return } - if s.State.Mode == IteratorStateModeDone { - return false, nil - } - } else if err = s.Iterator.Next(s.VM, s.State); err != nil { + } else if err = s.Next(s.VM, s.State); err != nil { return - } else if s.State.Mode == IteratorStateModeDone { - return false, nil } - return true, nil + return s.State.Mode != IteratorStateModeDone, nil } func (s *StateIteratorObject) Key() Object { @@ -526,7 +655,7 @@ func (o Dict) Iterate(_ *VM, na *NamedArgs) Iterator { for k := range o { keys = append(keys, k) } - if !na.GetValue("sorted").IsFalsy() { + if !na.GetValue("sorted").IsFalsy() || !na.MustGetValue("reversed").IsFalsy() { sort.Strings(keys) } return SliceEntryIteration(TDictIterator, o, keys, func(v string) (_, _ Object, _ error) { @@ -606,24 +735,19 @@ func (s *ReflectStruct) Iterate(vm *VM, na *NamedArgs) Iterator { type wrapIterator struct { Iterator - Wrap func(e *IteratorEntry) (Object, error) + Wrap func(state *IteratorState) error } -func WrapIterator(iterator Iterator, wrap func(e *IteratorEntry) (Object, error)) *wrapIterator { +func WrapIterator(iterator Iterator, wrap func(state *IteratorState) error) *wrapIterator { return &wrapIterator{Iterator: iterator, Wrap: wrap} } func (f *wrapIterator) checkNext(vm *VM, state *IteratorState) (err error) { try: - if state.Mode == IteratorStateModeDone { + if err = IteratorStateCheck(vm, f.Iterator, state); err != nil || state.Mode == IteratorStateModeDone { return } - for state.Mode == IteratorStateModeContinue { - if err = f.Iterator.Next(vm, state); err != nil || state.Mode == IteratorStateModeDone { - return - } - } - if state.Value, err = f.Wrap(&state.Entry); err == nil { + if err = f.Wrap(state); err == nil { if state.Mode != IteratorStateModeEntry { goto try } @@ -686,3 +810,15 @@ func (it *itemsIterator) Collect(vm *VM) (_ Object, err error) { }) return ret, err } + +func IteratorStateCheck(vm *VM, it Iterator, state *IteratorState) (err error) { + if state.Mode == IteratorStateModeDone { + return + } + for state.Mode == IteratorStateModeContinue { + if err = it.Next(vm, state); err != nil || state.Mode == IteratorStateModeDone { + return + } + } + return +} diff --git a/modules.go b/modules.go index f7d74cd..ae31eff 100644 --- a/modules.go +++ b/modules.go @@ -5,13 +5,14 @@ package gad import ( + "context" "errors" ) // Importable interface represents importable module instance. type Importable interface { // Import should return either an Object or module source code ([]byte). - Import(moduleName string) (any, error) + Import(ctx context.Context, moduleName string) (any, error) } // ExtImporter wraps methods for a module which will be impored dynamically like @@ -118,7 +119,7 @@ type SourceModule struct { } // Import returns a module source code. -func (m *SourceModule) Import(_ string) (any, error) { +func (m *SourceModule) Import(context.Context, string) (any, error) { return m.Src, nil } @@ -128,7 +129,7 @@ type BuiltinModule struct { } // Import returns an immutable map for the module. -func (m *BuiltinModule) Import(moduleName string) (any, error) { +func (m *BuiltinModule) Import(_ context.Context, moduleName string) (any, error) { if m.Attrs == nil { return nil, errors.New("module attributes not set") } diff --git a/objects.go b/objects.go index c722a61..926cc04 100644 --- a/objects.go +++ b/objects.go @@ -213,7 +213,11 @@ func (o RawStr) ToString() string { } func (o RawStr) Repr(*VM) (string, error) { - return repr.Quote("rawstr:" + utils.Quote(string(o), '`')), nil + return repr.Quote("rawstr:" + o.Quoted()), nil +} + +func (o RawStr) Quoted() string { + return utils.Quote(string(o), '`') } func (o RawStr) IsFalsy() bool { @@ -342,7 +346,11 @@ func (o Str) ToString() string { } func (o Str) Repr(*VM) (string, error) { - return repr.Quote("str:" + strconv.Quote(string(o))), nil + return repr.Quote("str:" + o.Quoted()), nil +} + +func (o Str) Quoted() string { + return strconv.Quote(string(o)) } // IndexGet represents string values and implements Object interface. diff --git a/objects_namedargs.go b/objects_namedargs.go index e0db295..14de33d 100644 --- a/objects_namedargs.go +++ b/objects_namedargs.go @@ -128,7 +128,9 @@ func (o *KeyValue) ToString() string { sb.WriteString("=") switch t := o.V.(type) { case Str: - sb.WriteString(strconv.Quote(string(t))) + sb.WriteString(t.Quoted()) + case RawStr: + sb.WriteString(t.Quoted()) case *KeyValue: sb.WriteByte('[') sb.WriteString(t.ToString()) @@ -881,7 +883,10 @@ func (o KeyValueArrays) CallName(name string, c Call) (Object, error) { } } +var EmptyNamedArgs = &NamedArgs{ro: true} + type NamedArgs struct { + ro bool sources KeyValueArrays m Dict ready Dict @@ -901,6 +906,9 @@ func (o *NamedArgs) Contains(key string) bool { } func (o *NamedArgs) Add(obj Object) error { + if o.ro { + return ErrNotWriteable + } arr, err := KeyValueArray{}.AppendObject(obj) if err != nil { return err @@ -1074,6 +1082,14 @@ func (o *NamedArgs) GetValue(key string) (val Object) { return } +// MustGetValue Must return value from key but not takes as read +func (o *NamedArgs) MustGetValue(key string) (val Object) { + if val = o.MustGetValueOrNil(key); val == nil { + val = Nil + } + return +} + // GetPassedValue Get passed value func (o *NamedArgs) GetPassedValue(key string) (val Object) { o.Walk(func(na *KeyValue) error { @@ -1091,8 +1107,20 @@ func (o *NamedArgs) GetValueOrNil(key string) (val Object) { o.check() if val = o.m[key]; val != nil { - delete(o.m, key) - o.ready[key] = nil + if !o.ro { + delete(o.m, key) + o.ready[key] = nil + } + return + } + return nil +} + +// MustGetValueOrNil Must return value from key nut not takes as read +func (o *NamedArgs) MustGetValueOrNil(key string) (val Object) { + o.check() + + if val = o.m[key]; val != nil { return } return nil diff --git a/vm_loop.go b/vm_loop.go index 50a5531..f38297b 100644 --- a/vm_loop.go +++ b/vm_loop.go @@ -507,7 +507,7 @@ VMLoop: } case OpIterNext: iterator := vm.stack[vm.sp-1].(*StateIteratorObject) - hasMore, err := iterator.Next() + hasMore, err := iterator.Read() if err != nil { if err = vm.throwGenErr(err); err != nil { vm.err = err @@ -521,7 +521,7 @@ VMLoop: falsePos := int(vm.curInsts[vm.ip+4]) | int(vm.curInsts[vm.ip+3])<<8 vm.ip += 4 - hasMore, err := iterator.Next() + hasMore, err := iterator.Read() if err != nil { if err = vm.throwGenErr(err); err != nil { vm.err = err diff --git a/vm_test.go b/vm_test.go index 0cad2e2..0f40d4f 100644 --- a/vm_test.go +++ b/vm_test.go @@ -725,13 +725,13 @@ func TestVMIterator(t *testing.T) { return str(ret) `, nil, Str(`[[0, "a"], [1, "b"], [2, "c"]]`)) + expectRun(t, rg+` func iterator(r Range) => [r.start, str('a' + r.start)] func iterator(r Range, state) => state >= r.end ? nil : [state+1, str('a' + state+1)] return str(collect(values(Range()))) `, nil, Str(`["a", "b", "c"]`)) - expectRun(t, rg+` func iterator(r Range) => [r.start, str('a' + r.start)] func iterator(r Range, state) => state >= r.end ? nil : [state+1, str('a' + state+1)] @@ -832,6 +832,25 @@ func TestVMIterator(t *testing.T) { expectRun(t, `return reduce([1,2], (cur, v, k) => cur + v)`, nil, Int(4)) expectRun(t, `return str(reduce([1,2,3], ((cur, v, k) => {cur.tot += v; cur[str(k+'a')] ??= v; cur}), {tot:100}))`, nil, Str("{a: 1, b: 2, c: 3, tot: 106}")) + + expectRun(t, `a := []; it := iterator({a:"A",b:"B"};reversed); it.next; for k, v in it {a += keyValue(k,v)}; return str(a)`, + nil, Str(`[a="A"]`)) + expectRun(t, `a := []; it := iterator({a:"A",b:"B"};sorted); it.next; for k, v in it {a += keyValue(k,v)}; return str(a)`, + nil, Str(`[b="B"]`)) + expectRun(t, `a := []; it := iterator({a:"A",b:"B"};sorted); it.next; for {v := it.next; if v {a += v;} else {break;} }; return str(a)`, + nil, Str(`[b="B"]`)) + expectRun(t, `a := []; it := iterator({a:"A",b:"B"};sorted); for {v := it.next; if v {a += v;} else {break;} }; return str(a)`, + nil, Str(`[a="A", b="B"]`)) + expectRun(t, `a := []; for k, v in iterator({a:"A",b:"B"};reversed) {a += keyValue(k,v)}; return str(a)`, + nil, Str(`[b="B", a="A"]`)) + expectRun(t, `a := []; for k, v in iterator({a:"A",b:"B"};sorted) {a += keyValue(k,v)}; return str(a)`, + nil, Str(`[a="A", b="B"]`)) + expectRun(t, `a := []; for k, v in (;a="A",b="B") {a += keyValue(k,v)}; return str(a)`, + nil, Str(`[a="A", b="B"]`)) + expectRun(t, `return str(collect(enumerate(iterator({a:"A",b:"B"};sorted))))`, + nil, Str(`[0=[a="A"], 1=[b="B"]]`)) + expectRun(t, `return str(collect(enumerate({a:"A",b:"B"};sorted)))`, + nil, Str(`[0=[a="A"], 1=[b="B"]]`)) } func TestVMBuiltinFunction(t *testing.T) {