feat: pass buffer and parse (#70)

content/10-module-core.md

@@ -12,7 +12,7 @@ Where the parsing happens, it accepts a source string and returns a `{ content,
 ```typescript
 export function parse(source: string): {
   content: string;
-  metadata: FrontMatter & {
+  metadata: Record<string, any> & {
     readonly estimate: number;
     readonly table: MarquaTable[];
   };

content/13-module-fs.md

@@ -7,8 +7,8 @@ Marqua provides a couple of functions coupled with the FileSystem module to `com
 ```typescript
 interface HydrateChunk {
   breadcrumb: string[];
-  content: string;
-  frontMatter: FrontMatter;
+  buffer: Buffer;
+  parse: typeof parse;
 }
 
 export function compile(
@@ -19,7 +19,7 @@ export function compile(
 export function traverse(
   options: {
     entry: string;
-    extensions?: string[];
+    compile?: RegExp[];
     depth?: number;
   },
   hydrate?: (chunk: HydrateChunk) => undefined | Output,
@@ -52,8 +52,8 @@ content
 ```typescript
 interface HydrateChunk {
   breadcrumb: string[];
-  content: string;
-  frontMatter: FrontMatter;
+  buffer: Buffer;
+  parse: typeof parse;
 }
 
 export function compile(
@@ -70,15 +70,17 @@ The first argument of `compile` is the source entry point.
 export function traverse(
   options: {
     entry: string;
-    extensions?: string[];
+    compile?(path: string): boolean;
     depth?: number;
   },
   hydrate?: (chunk: HydrateChunk) => undefined | Output,
   transform?: (items: Output[]) => Transformed
 ): Transformed;
 ```
 
-The first argument of `traverse` is the `TraverseOptions` and the second argument is an optional `hydrate` callback that can return an object with `content` property and all properties of `frontMatter`.
+The first argument of `traverse` is its `typeof options` and the second argument is an optional `hydrate` callback function. The third argument is an optional `transform` callback function.
+
+The `compile` property of the `options` object is an optional function that takes the full path of a file from the `entry` point and returns a boolean. If the function returns `true`, the file will be processed by the `compile` function, else it will be passed over to the `hydrate` function if it exists.
 
 An example usage from the *hypothetical* content folder structure above should look like
 
@@ -88,27 +90,30 @@ import { compile, traverse } from 'marqua/fs';
 /* compile - parse a single source file */
 const body = compile(
   'content/posts/2021-04-01.my-first-post.md',
-  ({ breadcrumb: [filename], content, frontMatter }) => {
+  ({ breadcrumb: [filename], buffer, parse }) => {
     const [date, slug] = filename.split('.');
-    return { slug, date, ...frontMatter, content };
+    const { content, metadata } = parse(buffer.toString('utf-8'));
+    return { ...metadata, slug, date, content };
   }
 ); // {'posts/2021-04-01.my-first-post.md'}
 
 /* traverse - scans a directory for sources */
 const data = traverse(
   { entry: 'content/posts'},
-  ({ breadcrumb: [filename], content, frontMatter }) => {
+  ({ breadcrumb: [filename], buffer, parse }) => {
     if (filename.startsWith('draft')) return;
     const [date, slug] = filename.split('.');
-    return { slug, date, ...frontMatter, content };
+    const { content, metadata } = parse(buffer.toString('utf-8'));
+    return { ...metadata, slug, date, content };
   }
 ); // [{'posts/3'}, {'posts/4'}]
 
 /* traverse - nested directories infinite recursive traversal */
 const data = traverse(
   { entry: 'content/reviews', depth: -1 },
-  ({ breadcrumb: [slug, category], content, frontMatter }) => {
-    return { slug, category, ...frontMatter, content };
+  ({ breadcrumb: [slug, category], buffer, parse }) => {
+    const { content, metadata } = parse(buffer.toString('utf-8'));
+    return { ...metadata, slug, category, content };
   }
 ); // [{'game/0'}, {'book/0'}, {'book/1'}, {'movie/0'}, {'movie/1'}]
 ```

workspace/documentation/src/routes/+page.server.ts

@@ -7,11 +7,11 @@ export const prerender = true;
 export const load: import('./$types').PageServerLoad = async () => {
 	const docs: ComponentProps<Docs>['sections'] = traverse(
 		{ entry: '../../content' },
-		({ breadcrumb: [filename], content, frontMatter: { title, ...rest } }) => {
-			if (filename.includes('draft')) return;
+		({ breadcrumb: [filename], buffer, parse }) => {
 			const path = `content/${filename}`;
-			const [, index, slug] = /^(\d{2})-(.+).md$/.exec(filename) || [];
-			return { index, slug, title, ...rest, content, path };
+			const [, index, slug] = /^(\d{2})-(.+).md$/.exec(filename)!;
+			const { content, metadata } = parse(buffer.toString('utf-8'));
+			return { index, slug, title: metadata.title, path, content };
 		}
 	);
 

workspace/marqua/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 ## Unreleased
 
+- [#70](https://github.com/ignatiusmb/marqua/pull/70): pass `buffer` and `parse`
 - [#69](https://github.com/ignatiusmb/marqua/pull/69): fix traversal of `depth` level
 - [#68](https://github.com/ignatiusmb/marqua/pull/68): add language style for markdown
 - [#67](https://github.com/ignatiusmb/marqua/pull/67): drop Node v14 support
@@ -10,6 +11,12 @@
 
 ### Breaking Changes
 
+- [#70](https://github.com/ignatiusmb/marqua/pull/70) | Pass `buffer` and `parse`
+  - Removed `frontMatter` in favor of `parse` for more granular control
+  - Replaced `content: string` with `buffer: Buffer` in `HydrateChunk`
+  - Use `parse(buffer.toString('utf-8'))` to get `content` and `metadata` (`"frontMatter"`)
+  - Replaced `extensions: string[]` with `compile: RegExp[]` in `traverse` options
+  - Calling `traverse` now reads everything and only processes files that match `compile`
 - [#67](https://github.com/ignatiusmb/marqua/pull/67) | Drop support for Node v14
 - [#65](https://github.com/ignatiusmb/marqua/pull/65) | Minimal YAML syntax support
   - Colons in values now require quotes

workspace/marqua/src/core/index.ts

@@ -4,7 +4,7 @@ import { generate } from '../utils.js';
 export function parse(source: string) {
 	const match = /---\r?\n([\s\S]+?)\r?\n---/.exec(source);
 	const crude = source.slice(match ? match.index + match[0].length + 1 : 0);
-	const memory = construct((match && match[1].trim()) || '') as FrontMatter;
+	const memory = construct((match && match[1].trim()) || '') as Record<string, any>;
 
 	return {
 		content: inject(crude, memory),

workspace/marqua/src/fs/index.ts

@@ -1,22 +1,23 @@
 import type { MarquaTable } from '../types.js';
 import * as fs from 'fs';
+import { scope } from 'mauss';
 import { marker } from '../artisan/index.js';
 import { parse } from '../core/index.js';
 
-interface FrontMatter {
+interface Metadata {
 	//----> computed properties
 	estimate: number;
 	table: MarquaTable[];
 }
 
-interface Compiled extends FrontMatter {
+interface Compiled extends Metadata {
 	content: string;
 }
 
 interface HydrateChunk {
 	breadcrumb: string[];
-	content: string;
-	frontMatter: FrontMatter & Record<string, any>;
+	buffer: Buffer;
+	parse: typeof parse;
 }
 
 export function compile(entry: string): Compiled;
@@ -28,12 +29,13 @@ export function compile<Output extends object>(
 	entry: string,
 	hydrate?: (chunk: HydrateChunk) => undefined | Output
 ) {
-	const crude = fs.readFileSync(entry, 'utf-8').trim();
-	const { content: source, metadata } = parse(crude);
-	const breadcrumb = entry.split(/[/\\]/).reverse();
-	const result = !hydrate
-		? ({ ...metadata, content: source } as Compiled)
-		: hydrate({ breadcrumb, content: source, frontMatter: metadata as any });
+	const buffer = fs.readFileSync(entry);
+	const result = scope(() => {
+		const breadcrumb = entry.split(/[/\\]/).reverse();
+		if (hydrate) return hydrate({ breadcrumb, buffer, parse });
+		const { content, metadata } = parse(buffer.toString('utf-8'));
+		return { ...metadata, content } as Compiled;
+	});
 
 	if (!result /* hydrate returns nothing */) return;
 	if ('content' in result && typeof result.content === 'string') {
@@ -44,13 +46,17 @@ export function compile<Output extends object>(
 }
 
 export function traverse<
-	Options extends { entry: string; extensions?: string[]; depth?: number },
+	Options extends {
+		entry: string;
+		compile?(path: string): boolean;
+		depth?: number;
+	},
 	Output extends object,
-	Transformed = Array<Output & FrontMatter>
+	Transformed = Array<Output & Metadata>
 >(
-	{ entry, extensions = ['.md'], depth: level = 0 }: Options,
+	{ entry, compile: fn = (v) => v.endsWith('.md'), depth: level = 0 }: Options,
 	hydrate?: (chunk: HydrateChunk) => undefined | Output,
-	transform?: (items: Array<Output & FrontMatter>) => Transformed
+	transform?: (items: Array<Output & Metadata>) => Transformed
 ): Transformed {
 	if (!fs.existsSync(entry)) {
 		console.warn(`Skipping "${entry}", path does not exists`);
@@ -61,18 +67,22 @@ export function traverse<
 		const pathname = join(entry, name);
 		if (level !== 0 && fs.lstatSync(pathname).isDirectory()) {
 			const depth = level < 0 ? level : level - 1;
-			return traverse({ entry: pathname, extensions, depth }, hydrate);
+			const options = { entry: pathname, depth, compile: fn };
+			return traverse(options, hydrate);
 		}
-		if (extensions.some((e) => name.endsWith(e))) {
-			const data = compile(pathname, hydrate);
-			const keys = Object.keys(data || {});
-			return data && keys.length ? [data] : [];
-		}
-		return [];
+
+		const data = scope(() => {
+			if (fn(pathname)) return compile(pathname, hydrate);
+			if (!hydrate) return; // no need to do anything else
+			const breadcrumb = pathname.split(/[/\\]/).reverse();
+			const buffer = fs.readFileSync(pathname);
+			return hydrate({ breadcrumb, buffer, parse });
+		});
+		return data && Object.keys(data).length ? [data] : [];
 	});
 
 	if (!transform) return backpack as Transformed;
-	return transform(backpack as Array<Output & FrontMatter>);
+	return transform(backpack as Array<Output & Metadata>);
 
 	// adapted from https://github.com/alchemauss/mauss/pull/153
 	function join(...paths: string[]) {