diff --git a/antlrdoc.go b/antlrdoc.go index 3bb4fd7..48bd362 100644 --- a/antlrdoc.go +++ b/antlrdoc.go @@ -17,9 +17,9 @@ ANTLR4 that it is compatible with (I.E. uses the /v4 path). However, this was found to be problematic, as it meant that with the runtime embedded so far underneath the root of the repo, the `go get` and related commands could not properly resolve the location of the go runtime source code. This meant that the reference to the runtime in your `go.mod` file would refer to the correct source code, but would not -list the release tag such as @4.12.0 - this was confusing, to say the least. +list the release tag such as @4.13.1 - this was confusing, to say the least. -As of 4.12.1, the runtime is now available as a go module in its own repo, and can be imported as `github.com/antlr4-go/antlr` +As of 4.13.0, the runtime is now available as a go module in its own repo, and can be imported as `github.com/antlr4-go/antlr` (the go get command should also be used with this path). See the main documentation for the ANTLR4 project for more information, which is available at [ANTLR docs]. The documentation for using the Go runtime is available at [Go runtime docs]. @@ -49,7 +49,7 @@ Here is a general/recommended template for an ANTLR based recognizer in Go: . ├── parser │ ├── mygrammar.g4 - │ ├── antlr-4.12.1-complete.jar + │ ├── antlr-4.13.1-complete.jar │ ├── generate.go │ └── generate.sh ├── parsing - generated code goes here @@ -71,7 +71,7 @@ And the generate.sh file will look similar to this: #!/bin/sh - alias antlr4='java -Xmx500M -cp "./antlr4-4.12.1-complete.jar:$CLASSPATH" org.antlr.v4.Tool' + alias antlr4='java -Xmx500M -cp "./antlr4-4.13.1-complete.jar:$CLASSPATH" org.antlr.v4.Tool' antlr4 -Dlanguage=Go -no-visitor -package parsing *.g4 depending on whether you want visitors or listeners or any other ANTLR options. Not that another option here diff --git a/atn.go b/atn.go index cdeefed..e749ebd 100644 --- a/atn.go +++ b/atn.go @@ -4,8 +4,6 @@ package antlr -import "sync" - // ATNInvalidAltNumber is used to represent an ALT number that has yet to be calculated or // which is invalid for a particular struct such as [*antlr.BaseRuleContext] var ATNInvalidAltNumber int @@ -56,9 +54,9 @@ type ATN struct { // states []ATNState - mu sync.Mutex - stateMu sync.RWMutex - edgeMu sync.RWMutex + mu Mutex + stateMu RWMutex + edgeMu RWMutex } // NewATN returns a new ATN struct representing the given grammarType and is used diff --git a/atn_config.go b/atn_config.go index a83f25d..267308b 100644 --- a/atn_config.go +++ b/atn_config.go @@ -73,9 +73,6 @@ func NewATNConfig1(c *ATNConfig, state ATNState, context *PredictionContext) *AT // NewATNConfig creates a new ATNConfig instance given an existing config, a state, a context and a semantic context, other 'constructors' // are just wrappers around this one. func NewATNConfig(c *ATNConfig, state ATNState, context *PredictionContext, semanticContext SemanticContext) *ATNConfig { - if semanticContext == nil { - panic("semanticContext cannot be nil") // TODO: Remove this - probably put here for some bug that is now fixed - } b := &ATNConfig{} b.InitATNConfig(c, state, c.GetAlt(), context, semanticContext) b.cType = parserConfig diff --git a/go.mod b/go.mod index cc695e0..54baab8 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,5 @@ module github.com/antlr4-go/antlr/v4 -go 1.20 +go 1.22 -require golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc +require golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 diff --git a/go.sum b/go.sum index 5bdfc81..d023c2f 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,2 @@ -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e h1:+WEEuIdZHnUeJJmEUjyYC2gfUMj69yZXw17EnHg/otA= -golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842 h1:vr/HnozRka3pE4EsMEg1lgkXJkTFJCVUX+S/ZT6wYzM= +golang.org/x/exp v0.0.0-20240506185415-9bf2ced13842/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= diff --git a/input_stream.go b/input_stream.go index b737fe8..ab4e96b 100644 --- a/input_stream.go +++ b/input_stream.go @@ -148,7 +148,7 @@ func (is *InputStream) GetTextFromInterval(i Interval) string { } func (*InputStream) GetSourceName() string { - return "" + return "Obtained from string" } // String returns the entire input stream as a string diff --git a/jcollect.go b/jcollect.go index ceccd96..6d668f7 100644 --- a/jcollect.go +++ b/jcollect.go @@ -8,7 +8,6 @@ import ( "container/list" "runtime/debug" "sort" - "sync" ) // Collectable is an interface that a struct should implement if it is to be @@ -587,12 +586,12 @@ type VisitRecord struct { type VisitList struct { cache *list.List - lock sync.RWMutex + lock RWMutex } var visitListPool = VisitList{ cache: list.New(), - lock: sync.RWMutex{}, + lock: RWMutex{}, } // NewVisitRecord returns a new VisitRecord instance from the pool if available. diff --git a/lexer.go b/lexer.go index 3c7896a..e5594b2 100644 --- a/lexer.go +++ b/lexer.go @@ -207,7 +207,7 @@ func (b *BaseLexer) NextToken() Token { for { b.thetype = TokenInvalidType - ttype := b.safeMatch() + ttype := b.safeMatch() // Defaults to LexerSkip if b.input.LA(1) == TokenEOF { b.hitEOF = true diff --git a/ll1_analyzer.go b/ll1_analyzer.go index 4955ac8..dfdff00 100644 --- a/ll1_analyzer.go +++ b/ll1_analyzer.go @@ -40,6 +40,7 @@ func (la *LL1Analyzer) getDecisionLookahead(s ATNState) []*IntervalSet { for alt := 0; alt < count; alt++ { look[alt] = NewIntervalSet() + // TODO: This is one of the reasons that ATNConfigs are allocated and freed all the time - fix this tomorrow jim! lookBusy := NewJStore[*ATNConfig, Comparator[*ATNConfig]](aConfEqInst, ClosureBusyCollection, "LL1Analyzer.getDecisionLookahead for lookBusy") la.look1(s.GetTransitions()[alt].getTarget(), nil, BasePredictionContextEMPTY, look[alt], lookBusy, NewBitSet(), false, false) diff --git a/mutex.go b/mutex.go new file mode 100644 index 0000000..2b0cda4 --- /dev/null +++ b/mutex.go @@ -0,0 +1,41 @@ +//go:build !antlr.nomutex +// +build !antlr.nomutex + +package antlr + +import "sync" + +// Mutex is a simple mutex implementation which just delegates to sync.Mutex, it +// is used to provide a mutex implementation for the antlr package, which users +// can turn off with the build tag -tags antlr.nomutex +type Mutex struct { + mu sync.Mutex +} + +func (m *Mutex) Lock() { + m.mu.Lock() +} + +func (m *Mutex) Unlock() { + m.mu.Unlock() +} + +type RWMutex struct { + mu sync.RWMutex +} + +func (m *RWMutex) Lock() { + m.mu.Lock() +} + +func (m *RWMutex) Unlock() { + m.mu.Unlock() +} + +func (m *RWMutex) RLock() { + m.mu.RLock() +} + +func (m *RWMutex) RUnlock() { + m.mu.RUnlock() +} diff --git a/mutex_nomutex.go b/mutex_nomutex.go new file mode 100644 index 0000000..35ce435 --- /dev/null +++ b/mutex_nomutex.go @@ -0,0 +1,32 @@ +//go:build antlr.nomutex +// +build antlr.nomutex + +package antlr + +type Mutex struct{} + +func (m *Mutex) Lock() { + // No-op +} + +func (m *Mutex) Unlock() { + // No-op +} + +type RWMutex struct{} + +func (m *RWMutex) Lock() { + // No-op +} + +func (m *RWMutex) Unlock() { + // No-op +} + +func (m *RWMutex) RLock() { + // No-op +} + +func (m *RWMutex) RUnlock() { + // No-op +} diff --git a/parser_atn_simulator.go b/parser_atn_simulator.go index ae28696..724fa17 100644 --- a/parser_atn_simulator.go +++ b/parser_atn_simulator.go @@ -10,8 +10,6 @@ import ( "strings" ) -var () - // ClosureBusy is a store of ATNConfigs and is a tiny abstraction layer over // a standard JStore so that we can use Lazy instantiation of the JStore, mostly // to avoid polluting the stats module with a ton of JStore instances with nothing in them. @@ -883,7 +881,7 @@ func (p *ParserATNSimulator) getPredicatePredictions(ambigAlts *BitSet, altToPre // the ERROR state was reached, outerContext as the initial parser context from the paper // or the parser stack at the instant before prediction commences. // -// Teh func returns the value to return from [AdaptivePredict], or +// The func returns the value to return from [AdaptivePredict], or // [ATNInvalidAltNumber] if a suitable alternative was not // identified and [AdaptivePredict] should report an error instead. func (p *ParserATNSimulator) getSynValidOrSemInvalidAltThatFinishedDecisionEntryRule(configs *ATNConfigSet, outerContext ParserRuleContext) int { diff --git a/prediction_context.go b/prediction_context.go index c1b80cc..a1d5186 100644 --- a/prediction_context.go +++ b/prediction_context.go @@ -6,7 +6,6 @@ package antlr import ( "fmt" - "golang.org/x/exp/slices" "strconv" ) @@ -101,7 +100,7 @@ func NewArrayPredictionContext(parents []*PredictionContext, returnStates []int) hash = murmurUpdate(hash, returnState) } hash = murmurFinish(hash, len(parents)<<1) - + nec := &PredictionContext{} nec.cachedHash = hash nec.pcType = PredictionContextArray @@ -115,6 +114,9 @@ func (p *PredictionContext) Hash() int { } func (p *PredictionContext) Equals(other Collectable[*PredictionContext]) bool { + if p == other { + return true + } switch p.pcType { case PredictionContextEmpty: otherP := other.(*PredictionContext) @@ -138,13 +140,11 @@ func (p *PredictionContext) ArrayEquals(o Collectable[*PredictionContext]) bool if p.cachedHash != other.Hash() { return false // can't be same if hash is different } - + // Must compare the actual array elements and not just the array address // - return slices.Equal(p.returnStates, other.returnStates) && - slices.EqualFunc(p.parents, other.parents, func(x, y *PredictionContext) bool { - return x.Equals(y) - }) + return intSlicesEqual(p.returnStates, other.returnStates) && + pcSliceEqual(p.parents, other.parents) } func (p *PredictionContext) SingletonEquals(other Collectable[*PredictionContext]) bool { @@ -152,23 +152,23 @@ func (p *PredictionContext) SingletonEquals(other Collectable[*PredictionContext return false } otherP := other.(*PredictionContext) - if otherP == nil { + if otherP == nil || otherP.pcType != PredictionContextSingleton { return false } - + if p.cachedHash != otherP.Hash() { return false // Can't be same if hash is different } - + if p.returnState != otherP.getReturnState(0) { return false } - + // Both parents must be nil if one is if p.parentCtx == nil { return otherP.parentCtx == nil } - + return p.parentCtx.Equals(otherP.parentCtx) } @@ -225,27 +225,27 @@ func (p *PredictionContext) String() string { return "$" case PredictionContextSingleton: var up string - + if p.parentCtx == nil { up = "" } else { up = p.parentCtx.String() } - + if len(up) == 0 { if p.returnState == BasePredictionContextEmptyReturnState { return "$" } - + return strconv.Itoa(p.returnState) } - + return strconv.Itoa(p.returnState) + " " + up case PredictionContextArray: if p.isEmpty() { return "[]" } - + s := "[" for i := 0; i < len(p.returnStates); i++ { if i > 0 { @@ -263,7 +263,7 @@ func (p *PredictionContext) String() string { } } return s + "]" - + default: return "unknown" } @@ -309,18 +309,18 @@ func predictionContextFromRuleContext(a *ATN, outerContext RuleContext) *Predict parent := predictionContextFromRuleContext(a, outerContext.GetParent().(RuleContext)) state := a.states[outerContext.GetInvokingState()] transition := state.GetTransitions()[0] - + return SingletonBasePredictionContextCreate(parent, transition.(*RuleTransition).followState.GetStateNumber()) } func merge(a, b *PredictionContext, rootIsWildcard bool, mergeCache *JPCMap) *PredictionContext { - + // Share same graph if both same // if a == b || a.Equals(b) { return a } - + if a.pcType == PredictionContextSingleton && b.pcType == PredictionContextSingleton { return mergeSingletons(a, b, rootIsWildcard, mergeCache) } @@ -334,7 +334,7 @@ func merge(a, b *PredictionContext, rootIsWildcard bool, mergeCache *JPCMap) *Pr return b } } - + // Convert either Singleton or Empty to arrays, so that we can merge them // ara := convertToArray(a) @@ -395,7 +395,7 @@ func mergeSingletons(a, b *PredictionContext, rootIsWildcard bool, mergeCache *J return previous } } - + rootMerge := mergeRoot(a, b, rootIsWildcard) if rootMerge != nil { if mergeCache != nil { @@ -564,7 +564,7 @@ func mergeArrays(a, b *PredictionContext, rootIsWildcard bool, mergeCache *JPCMa i := 0 // walks a j := 0 // walks b k := 0 // walks target M array - + mergedReturnStates := make([]int, len(a.returnStates)+len(b.returnStates)) mergedParents := make([]*PredictionContext, len(a.returnStates)+len(b.returnStates)) // walk and merge to yield mergedParents, mergedReturnStates @@ -626,9 +626,9 @@ func mergeArrays(a, b *PredictionContext, rootIsWildcard bool, mergeCache *JPCMa mergedParents = mergedParents[0:k] mergedReturnStates = mergedReturnStates[0:k] } - + M := NewArrayPredictionContext(mergedParents, mergedReturnStates) - + // if we created same array as a or b, return that instead // TODO: JI track whether this is possible above during merge sort for speed and possibly avoid an allocation if M.Equals(a) { @@ -650,7 +650,7 @@ func mergeArrays(a, b *PredictionContext, rootIsWildcard bool, mergeCache *JPCMa return b } combineCommonParents(&mergedParents) - + if mergeCache != nil { mergeCache.Put(a, b, M) } @@ -666,7 +666,7 @@ func mergeArrays(a, b *PredictionContext, rootIsWildcard bool, mergeCache *JPCMa //goland:noinspection GoUnusedFunction func combineCommonParents(parents *[]*PredictionContext) { uniqueParents := NewJStore[*PredictionContext, Comparator[*PredictionContext]](pContextEqInst, PredictionContextCollection, "combineCommonParents for PredictionContext") - + for p := 0; p < len(*parents); p++ { parent := (*parents)[p] _, _ = uniqueParents.Put(parent) @@ -685,7 +685,7 @@ func getCachedBasePredictionContext(context *PredictionContext, contextCache *Pr if present { return existing } - + existing, present = contextCache.Get(context) if present { visited.Put(context, existing) @@ -722,6 +722,6 @@ func getCachedBasePredictionContext(context *PredictionContext, contextCache *Pr contextCache.add(updated) visited.Put(updated, updated) visited.Put(context, updated) - + return updated } diff --git a/recognizer.go b/recognizer.go index 2e0b504..dcb8548 100644 --- a/recognizer.go +++ b/recognizer.go @@ -56,7 +56,7 @@ var tokenTypeMapCache = make(map[string]int) var ruleIndexMapCache = make(map[string]int) func (b *BaseRecognizer) checkVersion(toolVersion string) { - runtimeVersion := "4.12.0" + runtimeVersion := "4.13.1" if runtimeVersion != toolVersion { fmt.Println("ANTLR runtime and generated code versions disagree: " + runtimeVersion + "!=" + toolVersion) } diff --git a/statistics.go b/statistics.go index 70c0673..8cb5f3e 100644 --- a/statistics.go +++ b/statistics.go @@ -9,7 +9,6 @@ import ( "path/filepath" "sort" "strconv" - "sync" ) // This file allows the user to collect statistics about the runtime of the ANTLR runtime. It is not enabled by default @@ -30,7 +29,7 @@ type goRunStats struct { // within this package. // jStats []*JStatRec - jStatsLock sync.RWMutex + jStatsLock RWMutex topN int topNByMax []*JStatRec topNByUsed []*JStatRec diff --git a/token.go b/token.go index 9670efb..f5bc342 100644 --- a/token.go +++ b/token.go @@ -104,6 +104,25 @@ func (b *BaseToken) GetSource() *TokenSourceCharStreamPair { return b.source } +func (b *BaseToken) GetText() string { + if b.text != "" { + return b.text + } + input := b.GetInputStream() + if input == nil { + return "" + } + n := input.Size() + if b.GetStart() < n && b.GetStop() < n { + return input.GetTextFromInterval(NewInterval(b.GetStart(), b.GetStop())) + } + return "" +} + +func (b *BaseToken) SetText(text string) { + b.text = text +} + func (b *BaseToken) GetTokenIndex() int { return b.tokenIndex } @@ -120,6 +139,28 @@ func (b *BaseToken) GetInputStream() CharStream { return b.source.charStream } +func (b *BaseToken) String() string { + txt := b.GetText() + if txt != "" { + txt = strings.Replace(txt, "\n", "\\n", -1) + txt = strings.Replace(txt, "\r", "\\r", -1) + txt = strings.Replace(txt, "\t", "\\t", -1) + } else { + txt = "" + } + + var ch string + if b.GetChannel() > 0 { + ch = ",channel=" + strconv.Itoa(b.GetChannel()) + } else { + ch = "" + } + + return "[@" + strconv.Itoa(b.GetTokenIndex()) + "," + strconv.Itoa(b.GetStart()) + ":" + strconv.Itoa(b.GetStop()) + "='" + + txt + "',<" + strconv.Itoa(b.GetTokenType()) + ">" + + ch + "," + strconv.Itoa(b.GetLine()) + ":" + strconv.Itoa(b.GetColumn()) + "]" +} + type CommonToken struct { BaseToken } @@ -170,44 +211,3 @@ func (c *CommonToken) clone() *CommonToken { t.text = c.GetText() return t } - -func (c *CommonToken) GetText() string { - if c.text != "" { - return c.text - } - input := c.GetInputStream() - if input == nil { - return "" - } - n := input.Size() - if c.start < n && c.stop < n { - return input.GetTextFromInterval(NewInterval(c.start, c.stop)) - } - return "" -} - -func (c *CommonToken) SetText(text string) { - c.text = text -} - -func (c *CommonToken) String() string { - txt := c.GetText() - if txt != "" { - txt = strings.Replace(txt, "\n", "\\n", -1) - txt = strings.Replace(txt, "\r", "\\r", -1) - txt = strings.Replace(txt, "\t", "\\t", -1) - } else { - txt = "" - } - - var ch string - if c.channel > 0 { - ch = ",channel=" + strconv.Itoa(c.channel) - } else { - ch = "" - } - - return "[@" + strconv.Itoa(c.tokenIndex) + "," + strconv.Itoa(c.start) + ":" + strconv.Itoa(c.stop) + "='" + - txt + "',<" + strconv.Itoa(c.tokenType) + ">" + - ch + "," + strconv.Itoa(c.line) + ":" + strconv.Itoa(c.column) + "]" -} diff --git a/utils.go b/utils.go index 733d7df..36a37f2 100644 --- a/utils.go +++ b/utils.go @@ -326,3 +326,56 @@ func isDirectory(dir string) (bool, error) { } return fileInfo.IsDir(), err } + +// intSlicesEqual returns true if the two slices of ints are equal, and is a little +// faster than slices.Equal. +func intSlicesEqual(s1, s2 []int) bool { + if s1 == nil && s2 == nil { + return true + } + if s1 == nil || s2 == nil { + return false + } + if len(s1) == 0 && len(s2) == 0 { + return true + } + + if len(s1) == 0 || len(s2) == 0 || len(s1) != len(s2) { + return false + } + // If the slices are using the same memory, then they are the same slice + if &s1[0] == &s2[0] { + return true + } + for i, v := range s1 { + if v != s2[i] { + return false + } + } + return true +} + +func pcSliceEqual(s1, s2 []*PredictionContext) bool { + if s1 == nil && s2 == nil { + return true + } + if s1 == nil || s2 == nil { + return false + } + if len(s1) == 0 && len(s2) == 0 { + return true + } + if len(s1) == 0 || len(s2) == 0 || len(s1) != len(s2) { + return false + } + // If the slices are using the same memory, then they are the same slice + if &s1[0] == &s2[0] { + return true + } + for i, v := range s1 { + if !v.Equals(s2[i]) { + return false + } + } + return true +}