feat: include renderer revamp

README.md

@@ -15,23 +15,20 @@ This project was motivated by past experiences dealing with outdated or faulty c
 
 Coldsnip can be used as a library, as a CLI or through direct integrations with other platforms. Check the [getting started guide](https://roxlabs.github.io/coldsnip/getting-started/) in order to determine the best option for your needs.
 
-### CLI Commands
-
-<!-- commands -->
-<!-- commandsstop -->
 
 ### Library
 
-<!--- @snippet:include(readme.lib) --->
+<!-- @include:start("readme.lib") -->
 ```ts
 const snippets = await extractSnippets([
   { path: "src/__tests__", pattern: "snippets/twoSnippets.js" },
 ]);
 ```
+<!-- @include:end -->
 
 The return type is an map between the key and the snippet information, as detailed bellow:
 
-<!--- @snippet:include(readme.types) --->
+<!-- @include:start("readme.types") -->
 ```ts
 /**
  * Represents a code snippet extracted from a source file. The field
@@ -42,10 +39,14 @@ export interface Snippet {
   language: string;
   /** The file path relative to the working directory. */
   sourcePath: string;
+  /** The name of the file, derived from `sourcePath`. */
+  filename: string;
   /** The start line of the snippet. */
   startLine: number;
   /** The end line of the snippet. */
   endLine: number;
+  /** The lines to be highlighted, if any. */
+  highlightedLines: number[];
   /** The snippet content. Leading spaces are trimmed. */
   content: string;
   /** The link to the file on the remote Git repo when available. */
@@ -65,6 +66,7 @@ export interface Snippet {
  */
 export type Snippets = { [key: string]: Snippet[] };
 ```
+<!-- @include:end -->
 
 ## Roadmap
 
@@ -77,9 +79,9 @@ Contributions are what make the open source community such an amazing place to b
 1. Make sure you read our [Code of Conduct](https://github.com/roxlabs/coldsnip/blob/main/CODE_OF_CONDUCT.md)
 1. Fork the project and clone your fork
 1. Setup the local environment with `npm install`
-1. Create a feature branch (`git checkout -b feature/AmazingFeature`) or a bugfix branch (`git checkout -b fix/BoringBug`)
-1. Commit the changes (`git commit -m 'Some meaningful message'`)
-1. Push to the branch (`git push origin feature/AmazingFeature`)
+1. Create a feature branch (`git checkout -b feature/cool-thing`) or a bugfix branch (`git checkout -b fix/bad-bug`)
+1. Commit the changes (`git commit -m 'feat: some meaningful message'`)
+1. Push to the branch (`git push origin feature/cool-thing`)
 1. Open a Pull Request
 
 

package.json

@@ -29,7 +29,7 @@
     "build:cli": "tsc --project tsconfig.cli.json",
     "build": "npm run build:lib && npm run build:cli",
     "docs:readme": "cp README.md ./docs/_includes/home.md",
-    "docs:coldsnip": "./bin/run generate --out docs/snippets.json",
+    "docs:coldsnip": "./bin/run generate --source=./src --pattern=**/*.ts --format=include",
     "docs:typedoc": "typedoc --tsconfig tsconfig.lib.json",
     "docs": "npm run docs:readme & npm run docs:typedoc",
     "format:imports": "organize-imports-cli tsconfig.json",

src/__tests__/extractSnippets.spec.ts

@@ -17,7 +17,7 @@ describe("the extractSnippets public API test suite", () => {
   });
 
   it("should extract two snippets", async () => {
-    // @snippet:start(readme.lib)
+    // @snippet:start("readme.lib")
     const snippets = await extractSnippets([
       { path: "src/__tests__", pattern: "snippets/twoSnippets.js" },
     ]);

src/__tests__/patterns.spec.ts

@@ -1,11 +1,9 @@
-import exp from "constants";
 import {
   isSnippetStartTag,
   matchesEndTag,
   matchesStartTag,
   parseValue,
 } from "../patterns";
-import { parse } from "path";
 
 describe("the pattern matching test suite", () => {
   it("should match the start tag with a key", () => {
@@ -47,13 +45,13 @@ describe("the pattern matching test suite", () => {
   it("should match the snippet end tag", () => {
     // SQL style comments
     const comment = "-- @snippet:end";
-    expect(matchesEndTag(comment)).toEqual({ name: "snippet" });
+    expect(matchesEndTag(comment)).toEqual({ name: "snippet", indent: "" });
   });
 
   it("should match the highlight end tag", () => {
     // Python style comments
-    const comment = "# @highlight:end";
-    expect(matchesEndTag(comment)).toEqual({ name: "highlight" });
+    const comment = "  # @highlight:end";
+    expect(matchesEndTag(comment)).toEqual({ name: "highlight", indent: "  " });
   });
 
   it("should parse values correctly", () => {

src/extractSnippets.ts

@@ -119,15 +119,8 @@ async function extractSnippetFromFile(
           key = openTag.args.id;
           qualifier = openTag.args.qualifier;
           startLine = lineNumber + 1;
-        } else if (
-          lineContent.includes("@highlight") ||
-          lineContent.includes("@highlight:start")
-        ) {
-          // TODO improve error state detection
-          throw new Error(
-            `Invalid state: '@highlight' or '@highlight:start' found outside a snippet at line ${lineNumber}`,
-          );
         }
+        // TODO improve edge case handling and invalid tag handling
         break;
 
       case "INSIDE_SNIPPET":

src/patterns.ts

@@ -1,18 +1,20 @@
-export const COMMENT_TOKEN = /(\/\/|\/\*|#|--|\(\*|<!--|{-|')/;
-export const TAG_TOKEN = /@(snippet|highlight)/;
+export const COMMENT_TOKEN = /(\/\/|\/\*|#|--|\(\*|<!---?|{-|'|\{\/\*})/;
+export const TAG_TOKEN = /@(snippet|highlight|include)/;
 
 export const START = new RegExp(
-  `${COMMENT_TOKEN.source}\\s*(?:${TAG_TOKEN.source}):(?:start)\\s*(?:\\(([\\S ]+)\\))?`,
+  `(\\s*)${COMMENT_TOKEN.source}\\s*(?:${TAG_TOKEN.source}):(?:start)\\s*(?:\\(([\\S ]+)\\))?`,
 );
-const START_TAG_GROUP = 2;
-const START_ARG_GROUP = 3;
+const START_INDENT_GROUP = 1;
+const START_TAG_GROUP = 3;
+const START_ARG_GROUP = 4;
 
 export const END = new RegExp(
-  `${COMMENT_TOKEN.source}\\s*(?:${TAG_TOKEN.source}):(end)`,
+  `(.*)${COMMENT_TOKEN.source}\\s*(?:${TAG_TOKEN.source}):(end)`,
 );
 
 type Tag = {
-  name: "snippet" | "highlight";
+  name: "snippet" | "highlight" | "include";
+  indent: string;
 };
 
 export type OpenTag = Tag & {
@@ -75,13 +77,15 @@ function parseArguments(args: string): TagArgs {
 export function matchesStartTag(value: string): OpenTag | undefined {
   const match = START.exec(value);
   if (match && match[START_TAG_GROUP]) {
+    const indent = match[START_INDENT_GROUP] || "";
     const name = match[START_TAG_GROUP];
     const argsStr = match[START_ARG_GROUP];
     let args;
     if (argsStr) {
       args = parseArguments(argsStr);
     }
     return {
+      indent,
       name: name as OpenTag["name"],
       args,
     };
@@ -93,7 +97,8 @@ export function matchesEndTag(value: string): CloseTag | undefined {
   const match = END.exec(value);
   if (match && match[START_TAG_GROUP]) {
     const name = match[START_TAG_GROUP];
-    return { name: name as CloseTag["name"] };
+    const indent = match[START_INDENT_GROUP] || "";
+    return { indent, name: name as CloseTag["name"] };
   }
   return undefined;
 }

src/renderer/include.ts

@@ -1,42 +1,47 @@
 import { asyncify, queue } from "async";
 import glob from "fast-glob";
 import { writeFile } from "fs/promises";
+import { EOL } from "os";
 import process from "process";
 import forEachLine from "../forEachLine";
 import lookupSnippet from "../lookupSnippet";
+import { matchesEndTag, matchesStartTag, OpenTag } from "../patterns";
 import type { Snippets } from "../types";
+import { codeBlock, CodeBlockOptions } from "./markdown";
 import type { Renderer } from "./types";
 
 type Options = {
   pattern: string | string[];
 };
 
 const DEFAULT_OPTIONS: Options = {
-  pattern: "**/*.md",
+  pattern: "**/*.{md,mdx}",
 };
 
 const TAG_PATTERN = /^(\s*)<!---?\s*@snippet:include\((.*)\)\s*-?-->/;
 const CODE_FENCE_PATTERN = /^(\s*)```/;
 const CODE_FENCE = "```";
 
-type IncludeTag = {
-  key: string;
-  language?: string;
-  qualifier?: string;
+type IncludeStartTag = OpenTag & {
+  args: {
+    id: string;
+    language?: string;
+    qualifier?: string;
+  } & CodeBlockOptions;
 };
 
-type FileRenderStep = "content" | "tag" | "codeblock";
-
-function parseIncludeTag(value: string): IncludeTag {
-  const [key, language, qualifier] = value
-    .split(",")
-    .map((item) => item.trim());
-  if (!key || key.length === 0) {
-    throw Error("");
-  }
-  return { key, language, qualifier };
+export function isIncludeStartTag(
+  tag: OpenTag | undefined,
+): tag is IncludeStartTag {
+  return (
+    tag !== undefined &&
+    tag.name === "include" &&
+    typeof tag.args?.id === "string"
+  );
 }
 
+type FileRenderStep = "content" | "tag";
+
 type ParseInput = {
   file: string;
   snippets: Snippets;
@@ -51,41 +56,38 @@ async function parseFile(input: ParseInput) {
   await forEachLine(file, (line) => {
     const { lineContent } = line;
 
-    const includeTagMatch = TAG_PATTERN.exec(lineContent);
-    const codeFenceMatch = CODE_FENCE_PATTERN.exec(lineContent);
-
+    const includeTag = matchesStartTag(lineContent);
     switch (step) {
       case "content":
         content.push(lineContent);
-        if (includeTagMatch) {
+        if (isIncludeStartTag(includeTag)) {
           shouldRewrite = true;
-          const [, indent, args] = includeTagMatch;
-          const tag = parseIncludeTag(args);
           step = "tag";
+          // const indent = includeTag.indent;
 
-          const snippet = lookupSnippet(snippets, tag);
+          const snippet = lookupSnippet(snippets, {
+            key: includeTag.args.id,
+            language: includeTag.args.language,
+            qualifier: includeTag.args.qualifier,
+          });
           if (snippet) {
-            content.push(indent + CODE_FENCE + snippet.language);
-            content.push(
-              ...snippet.content.split("\n").map((value) => indent + value),
-            );
-            content.push(indent + CODE_FENCE);
+            content.push(codeBlock(snippet, { ...includeTag.args }));
           } else {
-            console.warn(`Snippet with key "${tag.key}" not found`);
+            console.warn(`Snippet with key "${includeTag.args.id}" not found`);
           }
         }
         break;
       case "tag":
-        step = codeFenceMatch ? "codeblock" : "content";
-        break;
-      case "codeblock":
-        step = codeFenceMatch ? "content" : "codeblock";
+        if (matchesEndTag(lineContent)?.name === "include") {
+          step = "content";
+          content.push(lineContent);
+        }
         break;
     }
   });
 
   if (shouldRewrite) {
-    await writeFile(file, content.join("\n"), { encoding: "utf-8" });
+    await writeFile(file, content.join("\n") + EOL, { encoding: "utf-8" });
   }
 }
 
@@ -98,7 +100,11 @@ export default class IncludeRenderer implements Renderer {
 
   async render(snippets: Snippets) {
     const { pattern } = this.options;
-    const files = await glob(pattern, { cwd: process.cwd(), absolute: true });
+    const files = await glob(pattern, {
+      cwd: process.cwd(),
+      absolute: true,
+      ignore: ["./node_modules/**"],
+    });
 
     const renderingQueue = queue<ParseInput>(asyncify(parseFile));
     for (const file of files) {