--wip-- [skip ci]

projects/ng-dynamic-component/src/lib/template/tokeniser.spec.ts

@@ -0,0 +1,164 @@
+import {
+  TemplateTokenAssignment,
+  TemplateTokenString,
+  TemplateTokenInputPropBindingClose,
+  TemplateTokenInputPropBindingOpen,
+  TemplateTokeniser,
+  TemplateTokenOutputBindingClose,
+  TemplateTokenOutputBindingOpen,
+  TemplateTokenQuote,
+  TemplateToken,
+} from './tokeniser';
+
+describe('TemplateTokeniser', () => {
+  it('should produce no tokens without template', async () => {
+    const tokeniser = new TemplateTokeniser();
+
+    await expect(tokeniser.getAll()).resolves.toEqual([]);
+  });
+
+  it('should produce no tokens from empty template', async () => {
+    const tokeniser = new TemplateTokeniser();
+
+    tokeniser.feed('');
+
+    await expect(tokeniser.getAll()).resolves.toEqual([]);
+  });
+
+  it('should produce tokens from template', async () => {
+    const tokeniser = new TemplateTokeniser();
+
+    tokeniser.feed('[input]="prop" (out');
+    tokeniser.feed('put)=handler()');
+
+    await expect(tokeniser.getAll()).resolves.toEqual([
+      new TemplateTokenInputPropBindingOpen(),
+      new TemplateTokenString('input'),
+      new TemplateTokenInputPropBindingClose(),
+      new TemplateTokenAssignment(),
+      new TemplateTokenQuote(),
+      new TemplateTokenString('prop'),
+      new TemplateTokenQuote(),
+      new TemplateTokenOutputBindingOpen(),
+      new TemplateTokenString('output'),
+      new TemplateTokenOutputBindingClose(),
+      new TemplateTokenAssignment(),
+      new TemplateTokenString('handler'),
+      new TemplateTokenOutputBindingOpen(),
+      new TemplateTokenOutputBindingClose(),
+    ]);
+  });
+
+  it('should produce tokens from template stream', async () => {
+    const tokeniser = new TemplateTokeniser();
+    const stream = new ControlledStream<string>();
+
+    tokeniser.feed(stream);
+
+    const tokenStream = tokeniser.getStream();
+
+    let actualTokens: Promise<IteratorResult<TemplateToken>>[] = [];
+    let expectedTokens: IteratorResult<TemplateToken>[] = [];
+
+    function collectNextToken(expectedToken: TemplateToken | null) {
+      expectedTokens.push({
+        value: expectedToken ?? undefined,
+        done: !expectedToken,
+      } as IteratorResult<TemplateToken>);
+      actualTokens.push(tokenStream.next());
+    }
+
+    collectNextToken(new TemplateTokenInputPropBindingOpen());
+    collectNextToken(new TemplateTokenString('input'));
+    collectNextToken(new TemplateTokenInputPropBindingClose());
+    collectNextToken(new TemplateTokenAssignment());
+    collectNextToken(new TemplateTokenQuote());
+    collectNextToken(new TemplateTokenString('prop'));
+    collectNextToken(new TemplateTokenQuote());
+    collectNextToken(new TemplateTokenOutputBindingOpen());
+
+    await stream.flushBuffer(['[input]="prop"', ' (out']);
+
+    await expect(Promise.all(actualTokens)).resolves.toEqual(expectedTokens);
+
+    actualTokens = [];
+    expectedTokens = [];
+
+    collectNextToken(new TemplateTokenString('output'));
+    collectNextToken(new TemplateTokenOutputBindingClose());
+    collectNextToken(new TemplateTokenAssignment());
+    collectNextToken(new TemplateTokenString('handler'));
+    collectNextToken(new TemplateTokenOutputBindingOpen());
+    collectNextToken(new TemplateTokenOutputBindingClose());
+    collectNextToken(null);
+
+    await stream.flushBuffer(['put)=handler()', null]);
+
+    await expect(Promise.all(actualTokens)).resolves.toEqual(expectedTokens);
+  });
+});
+
+class ControlledStream<T> implements AsyncIterable<T> {
+  protected finished = false;
+  protected bufferPromise?: Promise<(T | null)[]>;
+  protected bufferFlushedPromise?: Promise<void>;
+  protected _flushBuffer = (buffer: (T | null)[]) => Promise.resolve();
+  protected bufferFlushed = () => {};
+
+  async *[Symbol.asyncIterator](): AsyncIterableIterator<T> {
+    yield* this.getStream();
+  }
+
+  /**
+   * Flushes the buffer and resolves once buffer has been drained
+   * by the tokenizer and controls are ready for next setup
+   * `null` indicates the end of the stream
+   */
+  flushBuffer(buffer: (T | null)[]): Promise<void> {
+    return this._flushBuffer(buffer);
+  }
+
+  async *getStream(): AsyncGenerator<T> {
+    this.resetControls();
+
+    while (!this.finished) {
+      const buf = await this.bufferPromise!;
+      let i = 0;
+
+      for (const template of buf) {
+        // Final yield will block this function
+        // so we need to schedule `bufferFlushed` call
+        // when we are on the last item in current buffer
+        // and reset controls before `bufferFlushed` call
+        // so the tests can prepare next buffer once call is done
+        if (++i >= buf.length) {
+          setTimeout(() => {
+            const _bufferFlushed = this.bufferFlushed;
+            this.resetControls();
+            _bufferFlushed();
+          });
+        }
+
+        if (template) {
+          yield template;
+        } else {
+          this.finished = true;
+          break;
+        }
+      }
+    }
+  }
+
+  protected resetControls() {
+    this.bufferFlushedPromise = new Promise<void>(
+      (res) => (this.bufferFlushed = res),
+    );
+    this.bufferPromise = new Promise<(T | null)[]>(
+      (res) =>
+        (this._flushBuffer = (buffer) => {
+          res(buffer);
+          return this.bufferFlushedPromise!;
+        }),
+    );
+  }
+}

projects/ng-dynamic-component/src/lib/template/tokeniser.ts

@@ -0,0 +1,182 @@
+export class TemplateTokenString {
+  constructor(public string: string) {}
+}
+export class TemplateTokenAssignment {}
+export class TemplateTokenQuote {}
+export class TemplateTokenInputPropBindingOpen {}
+export class TemplateTokenInputPropBindingClose {}
+export class TemplateTokenOutputBindingOpen {}
+export class TemplateTokenOutputBindingClose {}
+
+export type TemplateToken =
+  | TemplateTokenString
+  | TemplateTokenAssignment
+  | TemplateTokenQuote
+  | TemplateTokenInputPropBindingOpen
+  | TemplateTokenInputPropBindingClose
+  | TemplateTokenOutputBindingOpen
+  | TemplateTokenOutputBindingClose;
+
+export enum TemplateTokenMap {
+  Space = ' ',
+  Assignment = '=',
+  QuoteSingle = "'",
+  QuoteDouble = '"',
+  InputPropBindingOpen = '[',
+  InputPropBindingClose = ']',
+  OutputBindingOpen = '(',
+  OutputBindingClose = ')',
+}
+
+export class TemplateTokeniser implements AsyncIterable<TemplateToken> {
+  protected templatesIters: (Iterator<string> | AsyncIterator<string>)[] = [];
+  protected templatesQueue: string[] = [];
+
+  protected currentTemplate?: string;
+  protected currentPos = 0;
+  protected nextToken?: TemplateToken;
+  protected lastToken?: TemplateToken;
+
+  constructor(protected tokenMap = TemplateTokenMap) {}
+
+  async *[Symbol.asyncIterator](): AsyncIterableIterator<TemplateToken> {
+    yield* this.getStream();
+  }
+
+  feed(template: string | Iterable<string> | AsyncIterable<string>) {
+    if (typeof template === 'string') {
+      this.templatesQueue.push(template);
+    } else if (this.isIterable(template)) {
+      this.templatesIters.push(template[Symbol.iterator]());
+    } else {
+      this.templatesIters.push(template[Symbol.asyncIterator]());
+    }
+  }
+
+  async getAll(): Promise<TemplateToken[]> {
+    const array: TemplateToken[] = [];
+    for await (const item of this) {
+      array.push(item);
+    }
+    return array;
+  }
+
+  async *getStream(): AsyncIterableIterator<TemplateToken> {
+    while (await this.nextTemplate()) {
+      if (this.nextToken) {
+        yield this.consumeNextToken()!;
+      }
+
+      const token = this.consumeToken() ?? this.consumeNextToken();
+
+      if (token) {
+        yield token;
+      }
+    }
+
+    if (this.nextToken) {
+      yield this.consumeNextToken()!;
+    }
+  }
+
+  protected consumeToken() {
+    let token = this.consumeLastToken();
+    let i = this.currentPos;
+    let tokenEnded = false;
+    let lastCharIdx = this.currentTemplate!.length - 1;
+
+    for (i; i <= lastCharIdx; i++) {
+      const char = this.currentTemplate![i];
+
+      switch (char) {
+        case this.tokenMap.Space:
+          tokenEnded = true;
+          break;
+        case this.tokenMap.Assignment:
+          this.nextToken = new TemplateTokenAssignment();
+          break;
+        case this.tokenMap.QuoteSingle:
+        case this.tokenMap.QuoteDouble:
+          this.nextToken = new TemplateTokenQuote();
+          break;
+        case this.tokenMap.InputPropBindingOpen:
+          this.nextToken = new TemplateTokenInputPropBindingOpen();
+          break;
+        case this.tokenMap.InputPropBindingClose:
+          this.nextToken = new TemplateTokenInputPropBindingClose();
+          break;
+        case this.tokenMap.OutputBindingOpen:
+          this.nextToken = new TemplateTokenOutputBindingOpen();
+          break;
+        case this.tokenMap.OutputBindingClose:
+          this.nextToken = new TemplateTokenOutputBindingClose();
+          break;
+        default:
+          if (!token || token instanceof TemplateTokenString === false) {
+            token = new TemplateTokenString(char);
+          } else {
+            (token as TemplateTokenString).string += char;
+          }
+          if (i >= lastCharIdx) {
+            this.lastToken = token;
+            token = undefined;
+          }
+          break;
+      }
+
+      if (this.nextToken || (tokenEnded && (token || this.nextToken))) {
+        i++;
+        break;
+      }
+    }
+
+    this.currentPos = i;
+
+    return token;
+  }
+
+  protected consumeNextToken() {
+    const token = this.nextToken;
+    this.nextToken = undefined;
+    return token;
+  }
+
+  protected consumeLastToken() {
+    const token = this.lastToken;
+    this.lastToken = undefined;
+    return token;
+  }
+
+  protected async nextTemplate() {
+    if (
+      !this.currentTemplate ||
+      this.currentPos >= this.currentTemplate.length
+    ) {
+      if (!this.templatesQueue.length) {
+        await this.drainTemplateIters();
+      }
+
+      this.currentTemplate = this.templatesQueue.shift();
+      this.currentPos = 0;
+    }
+
+    return this.currentTemplate;
+  }
+
+  protected async drainTemplateIters() {
+    for (const iter of this.templatesIters) {
+      const result = await iter.next();
+
+      if (!result.done) {
+        this.templatesQueue.push(result.value);
+        break;
+      } else {
+        this.templatesIters.shift();
+      }
+    }
+  }
+
+  protected isIterable<T>(val: unknown | Iterable<T>): val is Iterable<T> {
+    return typeof val === 'object' && !!val && Symbol.iterator in val;
+  }
+}