textsplitter: remove wrapper

tmc · Feb 20, 2024 · 3f4ee00 · 3f4ee00
1 parent ba73dea
commit 3f4ee00
Showing 1 changed file with 1 addition and 5 deletions.
diff --git a/textsplitter/options.go b/textsplitter/options.go
@@ -23,7 +23,7 @@ func DefaultOptions() Options {
 		ChunkSize:    _defaultTokenChunkSize,
 		ChunkOverlap: _defaultTokenChunkOverlap,
 		Separators:   []string{"\n\n", "\n", " ", ""},
-		LenFunc:      defaultLenFunc,
+		LenFunc:      utf8.RuneCountInString,
 
 		ModelName:         _defaultTokenModelName,
 		EncodingName:      _defaultTokenEncoding,
@@ -118,7 +118,3 @@ func WithReferenceLinks(referenceLinks bool) Option {
 		o.ReferenceLinks = referenceLinks
 	}
 }
-
-func defaultLenFunc(s string) int {
-	return utf8.RuneCountInString(s)
-}