feat(core): add built-in component embed (#218)

docs/guides/basic/built-in-components.md

@@ -75,7 +75,12 @@ order: 50
 
 ## Embed
 
-WIP.
+Embed 组件可以在一个 Markdown 文档中嵌入另一个 Markdown 文档的内容：
+```html
+<embed src="./foo.md"></embed>
+```
+展示效果如下：
+<embed src="./foo.md"></embed>
 
 ## 自定义内置组件
 在默认的`.docgeni/components`文件夹下创建自定义内置组件，比如如下结构：

docs/guides/basic/foo.md

@@ -0,0 +1,4 @@
+---
+hidden: true
+---
+This is foo

package.json

@@ -65,7 +65,7 @@
     "chokidar": "^3.3.1",
     "cosmiconfig": "^6.0.0",
     "fancy-log": "^1.3.3",
-    "marked": "^0.8.1",
+    "marked": "^3.0.2",
     "rxjs": "~6.5.4",
     "tslib": "^2.0.0",
     "zone.js": "~0.10.2"
@@ -83,7 +83,7 @@
     "@types/chai": "^4.2.11",
     "@types/jasmine": "^3.6.6",
     "@types/jasminewd2": "~2.0.3",
-    "@types/marked": "^0.7.3",
+    "@types/marked": "^3.0.0",
     "@types/node": "^12.12.30",
     "@types/yargs": "^15.0.4",
     "@typescript-eslint/eslint-plugin": "4.9.1",

packages/cli/package.json

@@ -55,7 +55,6 @@
     "front-matter": "^3.1.0",
     "fs-extra": "^9.0.0",
     "glob": "^7.1.6",
-    "marked": "^0.8.1",
     "stringify-object": "^3.3.0",
     "tapable": "^1.1.3",
     "yargs": "15.3.1",

packages/core/package.json

@@ -37,7 +37,7 @@
     "front-matter": "^3.1.0",
     "fs-extra": "^9.0.0",
     "glob": "^7.1.6",
-    "marked": "^2.0.0",
+    "marked": "^3.0.2",
     "node-prismjs": "0.1.2",
     "prismjs": "^1.20.0",
     "semver": "7.3.2",

packages/core/src/builders/doc-file.ts

@@ -87,7 +87,9 @@ export class DocSourceFile<TMeta = DocMeta> {
         const content = await this.read();
         const result = Markdown.parse<TMeta>(content);
         this.meta = result.attributes;
-        this.output = Markdown.toHTML(result.body);
+        this.output = Markdown.toHTML(result.body, {
+            absFilePath: this.path
+        });
     }
 
     public async emit(destRootPath: string) {

packages/core/src/fs/node-host.spec.ts

@@ -55,7 +55,7 @@ describe('DocgeniNodeJsAsyncHost', () => {
             .then(() => host.write(normalize('/sub1/file2'), content2).toPromise())
             .then(() => host.delete(normalize('/sub1/file1')).toPromise())
             // eslint-disable-next-line no-restricted-globals
-            .then(() => new Promise(resolve => setTimeout(resolve, 1000)))
+            .then(() => new Promise(resolve => setTimeout(resolve, 2000)))
             .then(() => {
                 expect(allEvents.length).toBe(3);
                 subscription.unsubscribe();

packages/core/src/markdown/embed.ts

@@ -0,0 +1,54 @@
+import { toolkit } from '@docgeni/toolkit';
+import * as path from 'path';
+import fm from 'front-matter';
+import { RendererExtension, TokenizerExtension } from 'marked';
+import { compatibleNormalize } from './utils';
+
+export function getEmbedBody(input: string, url: string) {
+    // TODO: add hash
+    return fm(compatibleNormalize(input)).body;
+}
+
+export interface EmbedToken {
+    type: 'embed';
+    raw: string;
+    src: string;
+    tokens: [];
+    message?: string;
+}
+
+export const embed: TokenizerExtension & RendererExtension = {
+    name: 'embed',
+    level: 'block',
+    start(src: string) {
+        return src.match(/<embed/)?.index;
+    },
+    tokenizer(src: string, tokens: any[]) {
+        const rule = /^<embed\W*src=['"]([^"']*)\W*\/?>(<\/embed>?)/gi; // Regex for the complete token
+        const match = rule.exec(src);
+        if (match) {
+            const token: EmbedToken = {
+                // Token to generate
+                type: 'embed',
+                raw: match[0],
+                src: match[1].trim(),
+                tokens: []
+            };
+
+            // eslint-disable-next-line dot-notation
+            const absFilePath: string = this.lexer.options['absFilePath'];
+            const absDirPath = path.dirname(absFilePath);
+            const nodeAbsPath = path.resolve(absDirPath, token.src);
+            if (nodeAbsPath !== absFilePath && toolkit.fs.pathExistsSync(nodeAbsPath)) {
+                const content = toolkit.fs.readFileSync(nodeAbsPath).toString();
+                this.lexer.blockTokens(getEmbedBody(content, token.src), token.tokens);
+            } else {
+                token.message = `can't resolve path ${token.src}`;
+            }
+            return token;
+        }
+    },
+    renderer(token: EmbedToken) {
+        return `<div embed src="${token.src}">${token.message ? token.message : this.parser.parse(token.tokens)}</div>`;
+    }
+};

packages/core/src/markdown/index.ts

@@ -1 +1,2 @@
 export * from './markdown';
+export * from './utils';

packages/core/src/markdown/markdown.spec.ts

@@ -1,9 +1,73 @@
+import { toolkit } from '@docgeni/toolkit';
+import { EOL } from 'os';
+import { loadFixture, FixtureResult } from '../testing/fixture-loader';
 import { Markdown } from './markdown';
+import path from 'path';
+import marked from 'marked';
+import { compatibleNormalize } from './utils';
+
 describe('markdown', () => {
-    const data = ['```bash', 'ng add alib', '```'].join('\n');
-    it('run', () => {
+    it('should get correct result for base and ng command', () => {
+        const data = ['```bash', 'ng add alib', '```'].join('\n');
         const result = Markdown.toHTML(data);
         expect(result).not.toContain('%20');
         expect(result).toContain(`<span class="token function">ng</span>`);
     });
+
+    describe('full', () => {
+        let fixture: FixtureResult;
+
+        beforeAll(async () => {
+            fixture = await loadFixture('markdown-full');
+        });
+
+        it('should transform full marked success', () => {
+            const output = Markdown.toHTML(fixture.src['hello.md'], {
+                absFilePath: fixture.getSrcPath('hello.md')
+            });
+            expect(output).toContain(`This is an <h1> tag`);
+            expect(output).toContain(`This is an <h2> tag`);
+            expect(output).toContain(`<example name="my-example" ></example>`);
+        });
+
+        it('should transform custom tag success', () => {
+            const output = Markdown.toHTML('<label><div>content</div></label>', {});
+            expect(output).toContain(`<div class="dg-paragraph"><label><div>content</div></label></div>`);
+        });
+    });
+
+    describe('link', () => {
+        it('should renderer link success', () => {
+            const output = Markdown.toHTML('[book](./book)', {});
+            expect(output).toContain(`<a href="./book">book</a>`);
+        });
+
+        it('should add target="_blank" for external link', () => {
+            const output = Markdown.toHTML('[book](http://pingcode.com/book)', {});
+            expect(output).toContain(`<a target="_blank" href="http://pingcode.com/book">book</a>`);
+        });
+    });
+
+    describe('embed', () => {
+        let fixture: FixtureResult;
+
+        beforeAll(async () => {
+            fixture = await loadFixture('markdown-embed');
+        });
+
+        it('should transform embed success', () => {
+            const output = Markdown.toHTML(fixture.src['hello.md'], { absFilePath: fixture.getSrcPath('hello.md') });
+            expect(compatibleNormalize(output).trim()).toEqual(fixture.getOutputContent('hello.html'));
+        });
+
+        xit('should throw error when embed ref self', () => {
+            const output = Markdown.toHTML(fixture.src['embed-self.md'], { absFilePath: fixture.getSrcPath('embed-self.md') });
+            expect(output).toContain(`<div embed src="./embed-self.md">can't resolve path ./embed-self.md</div>`);
+        });
+
+        xit('should throw error when embed ref-not-found.md', () => {
+            const output = Markdown.toHTML(fixture.src['not-found.md'], { absFilePath: fixture.getSrcPath('not-found.md') });
+            expect(output).toContain(`<div embed src="./ref-not-found.md">can't resolve path ./ref-not-found.md</div>`);
+        });
+    });
 });

packages/core/src/markdown/markdown.ts

@@ -1,8 +1,19 @@
 import marked from 'marked';
-import { DocsMarkdownRenderer } from './renderer';
+import { DocsMarkdownRenderer, MarkdownRendererOptions } from './renderer';
 import fm from 'front-matter';
-const renderer = new DocsMarkdownRenderer();
 import { highlight } from '../utils';
+import { embed } from './embed';
+
+marked.use({
+    pedantic: false,
+    gfm: true,
+    breaks: false,
+    sanitize: false,
+    smartLists: true,
+    smartypants: false,
+    xhtml: false,
+    extensions: [embed]
+});
 
 export interface MarkdownParseResult<TAttributes = unknown> {
     attributes: TAttributes;
@@ -12,12 +23,15 @@ export interface MarkdownParseResult<TAttributes = unknown> {
 }
 
 export class Markdown {
-    static toHTML(src: string) {
-        return marked(src, {
+    static toHTML(src: string, options?: MarkdownRendererOptions) {
+        const renderer = new DocsMarkdownRenderer();
+        const content = marked(src, {
             renderer,
             highlight,
-            gfm: true
+            gfm: true,
+            ...options
         });
+        return content;
     }
 
     static parse<TAttributes>(content: string): MarkdownParseResult<TAttributes> {

packages/core/src/markdown/renderer.ts

@@ -8,11 +8,20 @@ const whitespaceRegex = /( |\.|\?)/g;
 /** Regular expression that matches example comments. */
 const exampleCommentRegex = /<!--\W*example\(([^)]+)\)\W*-->/g;
 const exampleRegex = /<example\W*name=['"]([^"']+)\W*(inline)?\W*\/>/g;
+
+export type MarkdownRendererOptions = marked.MarkedOptions & {
+    absFilePath?: string;
+};
+
 /**
  * Custom renderer for marked that will be used to transform markdown files to HTML
  * files that can be used in the Angular Material docs.
  */
-export class DocsMarkdownRenderer extends Renderer {
+export class DocsMarkdownRenderer extends Renderer<any> {
+    constructor(options?: MarkdownRendererOptions) {
+        super(options);
+    }
+
     /**
      * Transforms a markdown heading into the corresponding HTML output. In our case, we
      * want to create a header-link for each H3 and H4 heading. This allows users to jump to
@@ -34,13 +43,22 @@ export class DocsMarkdownRenderer extends Renderer {
 
     /** Transforms markdown links into the corresponding HTML output. */
     link(href: string, title: string, text: string) {
-        let output = super.link(href, title, text);
+        let output = super.link.call(this, href, title, text);
         if (href.startsWith('http')) {
-            output = output.replace('<a ', `<a target="_blank"`);
+            output = output.replace('<a ', `<a target="_blank" `);
         }
         return output;
     }
 
+    paragraph(text: string) {
+        // for custom tag e.g. <label></label> to contains <div>
+        if (text.startsWith('<')) {
+            return '<div class="dg-paragraph">' + text + '</div>\n';
+        } else {
+            return '<p>' + text + '</p>\n';
+        }
+    }
+
     /**
      * Method that will be called whenever inline HTML is processed by marked. In that case,
      * we can easily transform the example comments into real HTML elements. For example:
@@ -49,12 +67,11 @@ export class DocsMarkdownRenderer extends Renderer {
      *  `<example name="name" inline />` turns into `<example name="name" inline></example>`
      */
     html(html: string) {
-        // html = html.replace(exampleCommentRegex, (_match: string, name: string) => `<div docgeni-docs-example="${name}"></div>`);
         html = html.replace(exampleRegex, (_match: string, name: string, inline: string) => {
             return `<example name="${name}" ${inline || ''}></example>`;
         });
 
-        return super.html(html);
+        return super.html.call(this, html);
     }
 
     /**

packages/core/src/markdown/utils.ts

@@ -0,0 +1,3 @@
+export function compatibleNormalize(input: string) {
+    return input.replace(/\r\n|\r/g, '\n').replace(/\t/g, '    ');
+}

packages/core/src/testing/fixture-loader.ts

@@ -1,20 +1,30 @@
 import * as path from 'path';
 import { toolkit } from '@docgeni/toolkit';
 import { relative, resolve } from '../fs';
+import { compatibleNormalize } from '../markdown';
 
-export const fixturesPath = path.resolve(__dirname, '../../test/fixtures');
+export const FIXTURES_PATH = path.resolve(__dirname, '../../test/fixtures');
 export const basicFixturePath = path.resolve(__dirname, '../../test/fixtures/basic');
 
 export class FixtureResult {
-    constructor(public src: Record<string, string>, public output: Record<string, string>) {}
+    constructor(public rootPath: string, public src: Record<string, string>, public output: Record<string, string>) {}
+
+    getSrcPath(relativePath: string) {
+        return path.resolve(this.rootPath, relativePath ? `src/${relativePath}` : 'src');
+    }
+
+    getOutputContent(relativePath: string) {
+        const output = this.output[relativePath];
+        return compatibleNormalize(output).trim();
+    }
 }
 
 async function internalLoadFixture(name: string, rootName: 'src' | 'output'): Promise<Record<string, string>> {
-    const allDirAndFiles = toolkit.fs.globSync(path.resolve(fixturesPath, `./${name}/${rootName}/**`));
+    const allDirAndFiles = toolkit.fs.globSync(path.resolve(FIXTURES_PATH, `./${name}/${rootName}/**`));
     const files: Record<string, string> = {};
     for (const dirOrFile of allDirAndFiles) {
         if (!toolkit.fs.isDirectory(dirOrFile)) {
-            const basePath = resolve(fixturesPath, `./${name}/${rootName}`);
+            const basePath = resolve(FIXTURES_PATH, `./${name}/${rootName}`);
             const relativePath = relative(basePath, dirOrFile);
             files[relativePath] = await toolkit.fs.readFileContent(dirOrFile);
         }
@@ -25,5 +35,6 @@ async function internalLoadFixture(name: string, rootName: 'src' | 'output'): Pr
 export async function loadFixture(name: string): Promise<FixtureResult> {
     const src = await internalLoadFixture(name, 'src');
     const output = await internalLoadFixture(name, 'output');
-    return new FixtureResult(src, output);
+    const rootPath = path.resolve(FIXTURES_PATH, `./${name}`);
+    return new FixtureResult(rootPath, src, output);
 }

packages/core/test/fixtures/markdown-embed/output/hello.html

@@ -0,0 +1,16 @@
+<p>Foo:</p>
+<div embed src="./foo.md">
+        <h2 id="foo" class="docs-header-link">
+          <span header-link="foo"></span>
+          Foo
+        </h2>
+      <p>This is foo</p>
+</div><p>this is newline--
+Bar:</p>
+<div embed src="./bar.md">
+        <h2 id="bar" class="docs-header-link">
+          <span header-link="bar"></span>
+          Bar
+        </h2>
+      <p>This is <em>bar</em></p>
+</div>

packages/core/test/fixtures/markdown-embed/output/hello.md

@@ -0,0 +1,11 @@
+Foo:
+## Foo
+
+This is foo
+
+this is newline--
+Bar:
+## Bar
+
+This is bar
+

packages/core/test/fixtures/markdown-embed/src/bar.md

@@ -0,0 +1,3 @@
+## Bar
+
+This is *bar*

packages/core/test/fixtures/markdown-embed/src/embed-self.md

@@ -0,0 +1 @@
+<embed src="./embed-self.md"></embed>

packages/core/test/fixtures/markdown-embed/src/foo.md

@@ -0,0 +1,3 @@
+## Foo
+
+This is foo