version 1.0.0

.eslintrc.json

@@ -0,0 +1,19 @@
+{
+  "env": {
+    "browser": true,
+    "es2021": true
+  },
+  "parser": "@typescript-eslint/parser",
+  "parserOptions": {
+    "ecmaVersion": 2021,
+    "sourceType": "module"
+  },
+  "extends": [
+    "plugin:@typescript-eslint/recommended",
+    "plugin:prettier/recommended"
+  ],
+  "plugins": [],
+  "rules": {
+    "@typescript-eslint/no-non-null-assertion": "off"
+  }
+}

.gitignore

@@ -0,0 +1,5 @@
+coverage
+dist
+node_modules
+.husky
+.parcel-cache

.parcelrc

@@ -0,0 +1,8 @@
+{
+  "extends": "@parcel/config-default",
+  "validators": {
+    "*.{ts,tsx}": [
+      "@parcel/validator-typescript"
+    ]
+  }
+}

.prettierrc

@@ -0,0 +1,9 @@
+{
+  "tabWidth": 2,
+  "useTabs": false,
+  "semi": false,
+  "singleQuote": true,
+  "printWidth": 80,
+  "trailingComma": "none",
+  "arrowParens": "avoid"
+}

.vscode/extensions.json

@@ -0,0 +1,13 @@
+{
+	"recommendations": [
+		"stagas.vscode-web-test-runner-explorer",
+		"esbenp.prettier-vscode",
+		// "tenninebt.vscode-koverage",
+		"dbaeumer.vscode-eslint",
+		"usernamehw.errorlens",
+		"ryanluker.vscode-coverage-gutters",
+		"salbert.comment-ts",
+		"gabrielgrinberg.auto-run-command",
+		"gruntfuggly.todo-tree"
+	]
+}

.vscode/settings.json

@@ -0,0 +1,12 @@
+{
+  "auto-run-command.rules": [
+    {
+      "condition": [
+        "hasFile: coverage/lcov.info"
+      ],
+      "command": "coverage-gutters.watchCoverageAndVisibleEditors",
+      "message": "Coverage Watch"
+    }
+  ],
+  "todo-tree.tree.scanMode": "workspace only"
+}

.vscode/tasks.json

@@ -0,0 +1,25 @@
+{
+  "version": "2.0.0",
+  "tasks": [
+    {
+      "type": "npm",
+      "script": "cov:watch",
+      "group": "build",
+      "label": "Begin Test Coverage",
+      "isBackground": true
+    },
+    {
+      "type": "npm",
+      "script": "docs",
+      "group": "build",
+      "label": "Generate Docs"
+    },
+    {
+      "type": "npm",
+      "script": "examples",
+      "label": "Browse Examples",
+      "isBackground": true,
+      "problemMatcher": []
+    }
+  ]
+}

README.md

@@ -1 +1,66 @@
-# tokenizer-next
+<h1 align="center">tokenizer-next</h1>
+
+<p align="center">
+iterator based tokenizer for writing parsers
+</p>
+
+<p align="center">
+   <a href="#Install">    🔧 <strong>Install</strong></a>
+ · <a href="examples">    🧩 <strong>Examples</strong></a>
+ · <a href="docs">        📜 <strong>API docs</strong></a>
+ · <a href="https://github.com/stagas/tokenizer-next/releases"> 🔥 <strong>Releases</strong></a>
+ · <a href="#Contribute"> 💪🏼 <strong>Contribute</strong></a>
+ · <a href="https://github.com/stagas/tokenizer-next/issues">   🖐️ <strong>Help</strong></a>
+</p>
+
+---
+
+## Install
+
+```sh
+$ npm i tokenizer-next
+```
+
+## Usage
+
+```js
+import { createTokenizer } from 'tokenizer-next'
+
+const tokenize = createTokenizer(
+  /(?<ident>[a-z]+)/, // named groups determine token `kind`
+  /(?<number>[0-9]+)/
+)
+
+// using next()
+const next = tokenize('hello 123')
+console.log(next()) // => {kind: 'ident', value: 'hello', index: 0}
+console.log(next()) // => {kind: 'number', value: '123', index: 6}
+console.log(next()) // => undefined
+
+// using for of
+for (const token of tokenize('hello 123')) {
+  console.log(token)
+  // => {kind: 'ident', value: 'hello', index: 0}
+  // => {kind: 'number', value: '123', index: 6}
+}
+
+// using spread
+const tokens = [...tokenize('hello 123')]
+console.log(tokens)
+// => [
+//   {kind: 'ident', value: 'hello', index: 0},
+//   {kind: 'number', value: '123', index: 6}
+// ]
+```
+
+## Contribute
+
+[Fork](https://github.com/stagas/tokenizer-next/fork) or
+[edit](https://github.dev/stagas/tokenizer-next) and submit a PR.
+
+All contributions are welcome!
+
+## License
+
+MIT &copy; 2021
+[stagas](https://github.com/stagas)

TODO.md

@@ -0,0 +1 @@
+- [ ] Better/more examples

docs/README.md

@@ -0,0 +1,119 @@
+# tokenizer-next
+
+## Table of contents
+
+### References
+
+- [default](README.md#default)
+
+### Interfaces
+
+- [Token](interfaces/Token.md)
+
+### Type aliases
+
+- [TokenizerCallableIterable](README.md#tokenizercallableiterable)
+- [TokenizerFactory](README.md#tokenizerfactory)
+
+### Functions
+
+- [createTokenizer](README.md#createtokenizer)
+
+## References
+
+### default
+
+Renames and re-exports [createTokenizer](README.md#createtokenizer)
+
+## Type aliases
+
+### TokenizerCallableIterable
+
+Ƭ **TokenizerCallableIterable**: () => [`Token`](interfaces/Token.md) \| `void` & `Iterable`<[`Token`](interfaces/Token.md)\>
+
+Can be called to return next [Token](interfaces/Token.md) or can be used as an [Iterable](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Iteration_protocols#the_iterable_protocol).
+
+#### Defined in
+
+[index.ts:71](https://github.com/stagas/tokenizer-next/blob/main/src/index.ts#L71)
+
+___
+
+### TokenizerFactory
+
+Ƭ **TokenizerFactory**: (`input`: `string`) => [`TokenizerCallableIterable`](README.md#tokenizercallableiterable)
+
+#### Type declaration
+
+▸ (`input`): [`TokenizerCallableIterable`](README.md#tokenizercallableiterable)
+
+Create a [TokenizerCallableIterable](README.md#tokenizercallableiterable) for given input string.
+
+```js
+// using next()
+const next = tokenize('hello 123')
+console.log(next()) // => {kind: 'ident', value: 'hello', index: 0}
+console.log(next()) // => {kind: 'number', value: '123', index: 6}
+console.log(next()) // => undefined
+
+// using for of
+for (const token of tokenize('hello 123')) {
+  console.log(token)
+  // => {kind: 'ident', value: 'hello', index: 0}
+  // => {kind: 'number', value: '123', index: 6}
+}
+
+// using spread
+const tokens = [...tokenize('hello 123')]
+console.log(tokens)
+// => [
+//   {kind: 'ident', value: 'hello', index: 0},
+//   {kind: 'number', value: '123', index: 6}
+// ]
+```
+
+##### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `input` | `string` | The string to tokenize. |
+
+##### Returns
+
+[`TokenizerCallableIterable`](README.md#tokenizercallableiterable)
+
+#### Defined in
+
+[index.ts:66](https://github.com/stagas/tokenizer-next/blob/main/src/index.ts#L66)
+
+## Functions
+
+### createTokenizer
+
+▸ `Const` **createTokenizer**(...`regexps`): [`TokenizerFactory`](README.md#tokenizerfactory)
+
+Create a [TokenizerFactory](README.md#tokenizerfactory) for the given RegExps.
+
+RegExps must use a [named group](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Guide/Regular_Expressions/Groups_and_Ranges#using_named_groups)
+to capture and must not have any flags set:
+
+```js
+const tokenize = createTokenizer(
+  /(?<ident>[a-z]+)/,
+  /(?<number>[0-9]+)/
+)
+```
+
+#### Parameters
+
+| Name | Type | Description |
+| :------ | :------ | :------ |
+| `...regexps` | `RegExp`[] | RegExps to match. |
+
+#### Returns
+
+[`TokenizerFactory`](README.md#tokenizerfactory)
+
+#### Defined in
+
+[index.ts:18](https://github.com/stagas/tokenizer-next/blob/main/src/index.ts#L18)

docs/interfaces/Token.md

@@ -0,0 +1,47 @@
+# Interface: Token
+
+A single Token as returned by the [TokenizerCallableIterable](../README.md#tokenizercallableiterable).
+
+## Table of contents
+
+### Properties
+
+- [index](Token.md#index)
+- [kind](Token.md#kind)
+- [value](Token.md#value)
+
+## Properties
+
+### index
+
+• **index**: `number`
+
+Index position of captured string.
+
+#### Defined in
+
+[index.ts:88](https://github.com/stagas/tokenizer-next/blob/main/src/index.ts#L88)
+
+___
+
+### kind
+
+• **kind**: `string`
+
+RegExp's group name.
+
+#### Defined in
+
+[index.ts:80](https://github.com/stagas/tokenizer-next/blob/main/src/index.ts#L80)
+
+___
+
+### value
+
+• **value**: `string`
+
+Captured string value.
+
+#### Defined in
+
+[index.ts:84](https://github.com/stagas/tokenizer-next/blob/main/src/index.ts#L84)

examples/basic/index.html

@@ -0,0 +1,10 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>Examples: Basic</title>
+  </head>
+  <body>
+    <script src="./script.ts" type="module"></script>
+  </body>
+</html>

examples/basic/script.ts

@@ -0,0 +1,27 @@
+import { createTokenizer } from '../../src/index'
+
+const tokenize = createTokenizer(
+  /(?<ident>[a-z]+)/, // named groups determine token `kind`
+  /(?<number>[0-9]+)/
+)
+
+// using next()
+const next = tokenize('hello 123')
+console.log(next()) // => {kind: 'ident', value: 'hello', index: 0}
+console.log(next()) // => {kind: 'number', value: '123', index: 6}
+console.log(next()) // => undefined
+
+// using for of
+for (const token of tokenize('hello 123')) {
+  console.log(token)
+  // => {kind: 'ident', value: 'hello', index: 0}
+  // => {kind: 'number', value: '123', index: 6}
+}
+
+// using spread
+const tokens = [...tokenize('hello 123')]
+console.log(tokens)
+// => [
+//   {kind: 'ident', value: 'hello', index: 0},
+//   {kind: 'number', value: '123', index: 6}
+// ]

examples/index.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html>
+  <head>
+    <meta charset="utf-8" />
+    <title>Examples</title>
+  </head>
+  <body>
+    <h1>Examples</h1>
+    <ul>
+      <li><a href="./basic/index.html">Basic</a></li>
+    </ul>
+  </body>
+</html>