version 2.0.0

README.md

@@ -13,7 +13,7 @@ iterator based tokenizer for writing parsers
  · <a href="https://github.com/stagas/tokenizer-next/issues">   🖐️ <strong>Help</strong></a>
 </p>
 
-***
+---
 
 ## Install
 
@@ -37,34 +37,34 @@ See [TokenizerFactory](#tokenizerfactory).
 
 #### Table of Contents
 
-*   [createTokenizer](#createtokenizer)
-    *   [Parameters](#parameters)
-*   [TokenizerFactory](#tokenizerfactory)
-    *   [Parameters](#parameters-1)
-*   [TokenizerCallableIterable](#tokenizercallableiterable)
+- [createTokenizer](#createtokenizer)
+  - [Parameters](#parameters)
+- [TokenizerFactory](#tokenizerfactory)
+  - [Parameters](#parameters-1)
+- [TokenizerCallableIterable](#tokenizercallableiterable)
 
 ### createTokenizer
 
-[src/index.ts:19-37](https://github.com/stagas/tokenizer-next/blob/3e0b365963ea3d146d78a76492d12c6b256721c2/src/index.ts#L19-L37 "Source code on GitHub")
+[src/index.ts:19-37](https://github.com/stagas/tokenizer-next/blob/c23a53a4be8e46b64d92477d01dea8f86d9c4eea/src/index.ts#L19-L37 'Source code on GitHub')
 
 Create a [TokenizerFactory](#tokenizerfactory) for the given RegExps.
 
 To capture, RegExps must use a [named group](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Groups_and_Ranges#using_named_groups).
 
 ```ts
 const tokenize = createTokenizer(
-  /(?<ident>[a-z]+)/,
+  /(?<ident>[a-z]+)/, // named groups determine token `group`
   /(?<number>[0-9]+)/
 )
 ```
 
 #### Parameters
 
-*   `regexps` **...[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[RegExp](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/RegExp)>** RegExps to match.
+- `regexps` **...[Array](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/Array)<[RegExp](https://developer.mozilla.org/docs/Web/JavaScript/Reference/Global_Objects/RegExp)>** RegExps to match.
 
 ### TokenizerFactory
 
-[src/index.ts:67-67](https://github.com/stagas/tokenizer-next/blob/3e0b365963ea3d146d78a76492d12c6b256721c2/src/index.ts#L39-L66 "Source code on GitHub")
+[src/index.ts:67-67](https://github.com/stagas/tokenizer-next/blob/c23a53a4be8e46b64d92477d01dea8f86d9c4eea/src/index.ts#L39-L66 'Source code on GitHub')
 
 Create a [TokenizerCallableIterable](#tokenizercallableiterable) for given input string.
 
@@ -95,11 +95,11 @@ Type: function (input: [string](https://developer.mozilla.org/docs/Web/JavaScrip
 
 #### Parameters
 
-*   `input`  The string to tokenize.
+- `input` The string to tokenize.
 
 ### TokenizerCallableIterable
 
-[src/index.ts:74-75](https://github.com/stagas/tokenizer-next/blob/3e0b365963ea3d146d78a76492d12c6b256721c2/src/index.ts#L69-L73 "Source code on GitHub")
+[src/index.ts:74-75](https://github.com/stagas/tokenizer-next/blob/c23a53a4be8e46b64d92477d01dea8f86d9c4eea/src/index.ts#L69-L73 'Source code on GitHub')
 
 Can be called to return next <a href="https://github.com/stagas/match-to-token#token">Token</a> or can be used as an
 [Iterable](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols#the_iterable_protocol)

examples/basic.ts

@@ -1,27 +1,27 @@
 import { createTokenizer } from '../src'
 
 const tokenize = createTokenizer(
-  /(?<ident>[a-z]+)/, // named groups determine token `kind`
+  /(?<ident>[a-z]+)/, // named groups determine token `group`
   /(?<number>[0-9]+)/
 )
 
 // using next()
 const next = tokenize('hello 123')
-console.log(next()) // => {kind: 'ident', value: 'hello', index: 0}
-console.log(next()) // => {kind: 'number', value: '123', index: 6}
+console.log(next()) // => {group: 'ident', value: 'hello', index: 0}
+console.log(next()) // => {group: 'number', value: '123', index: 6}
 console.log(next()) // => undefined
 
 // using for of
 for (const token of tokenize('hello 123')) {
   console.log(token)
-  // => {kind: 'ident', value: 'hello', index: 0}
-  // => {kind: 'number', value: '123', index: 6}
+  // => {group: 'ident', value: 'hello', index: 0}
+  // => {group: 'number', value: '123', index: 6}
 }
 
 // using spread
 const tokens = [...tokenize('hello 123')]
 console.log(tokens)
 // => [
-//   {kind: 'ident', value: 'hello', index: 0},
-//   {kind: 'number', value: '123', index: 6}
+//   {group: 'ident', value: 'hello', index: 0},
+//   {group: 'number', value: '123', index: 6}
 // ]

package.json

@@ -3,7 +3,7 @@
   "author": "stagas",
   "short": "stagas/tokenizer-next",
   "description": "iterator based tokenizer for writing parsers",
-  "version": "1.0.0",
+  "version": "2.0.0",
   "license": "MIT",
   "private": true,
   "repository": {

src/index.ts

@@ -9,7 +9,7 @@ import { joinRegExps } from './util'
  *
  * ```ts
  * const tokenize = createTokenizer(
- *   /(?<ident>[a-z]+)/,
+ *   /(?<ident>[a-z]+)/, // named groups determine token `group`
  *   /(?<number>[0-9]+)/
  * )
  * ```