Skip to content

replace gray-matter and deprecated remark plugins #984

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Mar 11, 2025
Merged

replace gray-matter and deprecated remark plugins #984

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
4084821
replace gray-matter and deprecated remark plugins
cometkim Mar 11, 2025
c9eca0a
fix
cometkim Mar 11, 2025
9faee32
fix scripts
cometkim Mar 11, 2025
b3cf8e0
fix compability issues
cometkim Mar 11, 2025
4561432
fix headers
cometkim Mar 11, 2025
dcb4518
use interface file
cometkim Mar 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30,118 changes: 18,520 additions & 11,598 deletions package-lock.json
View file
Open in desktop

Large diffs are not rendered by default.

25 changes: 13 additions & 12 deletions package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
"@babel/traverse": "^7.24.7",
"@docsearch/react": "^3.5.2",
"@headlessui/react": "^1.2.0",
"@mdx-js/loader": "^2.3.0",
"@mdx-js/loader": "^3.1.0",
"@rescript/core": "^1.4.0",
"@rescript/react": "^0.12.0-alpha.3",
"@rescript/tools": "^0.5.0",
Expand All @@ -33,28 +33,29 @@
"fuse.js": "^6.4.3",
"gentype": "^3.44.0",
"glob": "^7.1.4",
"gray-matter": "^4.0.3",
"highlight.js": "^11.9.0",
"highlightjs-rescript": "^0.2.2",
"lz-string": "^1.4.4",
"next": "^14.2.21",
"next-mdx-remote": "^4.4.1",
"next-mdx-remote": "^5.0.0",
"prettier": "^1.18.2",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-markdown": "^9.0.1",
"rehype-slug": "^5.1.0",
"react-markdown": "^10.1.0",
"rehype-slug": "^6.0.0",
"rehype-stringify": "^10.0.1",
"remark-comment": "^1.0.0",
"remark-frontmatter": "^4.0.1",
"remark-gfm": "^3.0.1",
"remark-mdx-frontmatter": "^3.0.0",
"remark-parse": "^10.0.2",
"remark-slug": "^5.1.2",
"remark-stringify": "^7.0.3",
"remark-frontmatter": "^5.0.0",
"remark-gfm": "^4.0.1",
"remark-mdx-frontmatter": "^5.0.0",
"remark-parse": "^11.0.0",
"remark-rehype": "^11.1.1",
"remark-stringify": "^11.0.0",
"request": "^2.88.0",
"rescript": "^11.1.0",
"stringify-object": "^3.3.0",
"unified": "^8.4.0"
"unified": "^11.0.5",
"vfile-matter": "^5.0.0"
},
"scripts": {
"dev": "next",
Expand Down
64 changes: 3 additions & 61 deletions scripts/extract-indices.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,77 +5,19 @@
* - Module names (h1)
* - function names (```res sig)
*/
import unified from "unified";
import markdown from "remark-parse";
import stringify from "remark-stringify";
import slug from "remark-slug";
import glob from "glob";
import path from "path";
import fs from "fs";
import { URL } from 'url';

import { defaultProcessor } from "./markdown.js";

const pathname = new URL('.', import.meta.url).pathname;
const __dirname = process.platform !== 'win32' ? pathname : pathname.substring(1)

const headers = options => (tree, file) => {
const headers = [];
let mainHeader;
tree.children.forEach(child => {
if (child.type === "heading" && child.depth === 1) {
if (child.children.length > 0) {
mainHeader = child.children.map(element => element.value).join("");
}
}
if (child.type === "heading" && child.depth === 2) {
if (child.children.length > 0) {
const id = child.data.id || "";
const name = child.children.map(element => element.value).join("");
headers.push({ name, href: id });
}
}
});

file.data = Object.assign({}, file.data, { headers, mainHeader });
};

const codeblocks = options => (tree, file) => {
const { children } = tree;
const codeblocks = {};

const formatter = value => {
// Strip newlines and weird spacing
return value
.replace(/\n/g, " ")
.replace(/\s+/g, " ")
.replace(/\(\s+/g, "(")
.replace(/\s+\)/g, ")");
};

children.forEach(child => {
if (child.type === "code" && child.value) {
const { meta, lang } = child;
if (meta === "sig" && lang === "re") {
if (codeblocks[lang] == null) {
codeblocks[lang] = [];
}
codeblocks[lang].push(formatter(child.value));
}
}
});

file.data = Object.assign({}, file.data, { codeblocks });
};

const processor = unified()
.use(markdown, { gfm: true })
.use(slug)
.use(stringify)
.use(headers)
.use(codeblocks);

const processFile = filepath => {
const content = fs.readFileSync(filepath, "utf8");
const result = processor.processSync(content);
const result = defaultProcessor.processSync(content);

const pagesPath = path.resolve("./pages");
const relFilepath = path.relative(pagesPath, filepath);
Expand Down
19 changes: 10 additions & 9 deletions scripts/extract-syntax.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,28 +1,29 @@
import matter from "gray-matter";
import glob from "glob";
import path from "path";
import fs from "fs";
import { URL } from 'url';

import { defaultProcessor } from "./markdown.js";

const pathname = new URL('.', import.meta.url).pathname;
const __dirname = process.platform !== 'win32' ? pathname : pathname.substring(1)

const processFile = filepath => {
const raw = fs.readFileSync(filepath, "utf8");
const { data } = matter(raw);
const content = fs.readFileSync(filepath, "utf8");
const { data: { matter } } = defaultProcessor.processSync(content);

const syntaxPath = path.resolve("./misc_docs/syntax");
const relFilePath = path.relative(syntaxPath, filepath);
const parsedPath = path.parse(relFilePath);

if (data.id && data.keywords && data.name && data.summary && data.category) {
if (matter.id && matter.keywords && matter.name && matter.summary && matter.category) {
return {
file: parsedPath.name,
id: data.id,
keywords: data.keywords,
name: data.name,
summary: data.summary,
category: data.category
id: matter.id,
keywords: matter.keywords,
name: matter.name,
summary: matter.summary,
category: matter.category
}
}

Expand Down
42 changes: 6 additions & 36 deletions scripts/extract-tocs.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,16 +2,14 @@
* This script is used for generating the table of contents for prose
* text documents
*/
import unified from "unified";
import markdown from "remark-parse";
import matter from "gray-matter";
import stringify from "remark-stringify";
import slug from "remark-slug";
import { unified } from "unified";
import glob from "glob";
import path from "path";
import fs from "fs";
import { URL } from "url";

import { defaultProcessor } from "./markdown.js";

const pathname = new URL(".", import.meta.url).pathname;
const __dirname =
process.platform !== "win32" ? pathname : pathname.substring(1);
Expand Down Expand Up @@ -56,39 +54,11 @@ const collapseHeaderChildren = (children) => {
}, "");
};

const headers = (options) => (tree, file) => {
const headers = [];
let mainHeader;
tree.children.forEach((child) => {
if (child.type === "heading" && child.depth === 1) {
if (child.children.length > 0) {
mainHeader = collapseHeaderChildren(child.children);
}
}
if (child.type === "heading" && child.depth === 2) {
if (child.children.length > 0) {
// Take the id generated from remark-slug
const headerValue = collapseHeaderChildren(child.children);
const id = child.data.id || "";
headers.push({ name: headerValue, href: id });
}
}
});

file.data = Object.assign({}, file.data, { headers, mainHeader });
};

const processor = unified()
.use(markdown, { gfm: true })
.use(slug)
.use(stringify)
.use(headers);

// sidebarJson: { [category: string]: array<plain_filename_without_ext> }
const processFile = (filepath, sidebarJson = {}) => {
const raw = fs.readFileSync(filepath, "utf8");
const { content, data } = matter(raw);
const result = processor.processSync(content);
const content = fs.readFileSync(filepath, "utf8");
const result = defaultProcessor.processSync(content);
const data = result.data.matter;

const pagesPath = path.resolve("./pages");
const relFilepath = path.relative(pagesPath, filepath);
Expand Down
72 changes: 72 additions & 0 deletions scripts/markdown.js
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,72 @@
import { unified } from "unified";
import remarkParse from "remark-parse";
import remarkGfm from "remark-gfm";
import remarkFrontmatter from "remark-frontmatter";
import remarkRehype from "remark-rehype";
import rehypeSlug from "rehype-slug";
import rehypeStringify from "rehype-stringify";
import { matter } from "vfile-matter";

const remarkVfileMatter = options => (tree, file) => {
matter(file);
};

const remarkCodeblocks = options => (tree, file) => {
const { children } = tree;
const codeblocks = {};

const formatter = value => {
// Strip newlines and weird spacing
return value
.replace(/\n/g, " ")
.replace(/\s+/g, " ")
.replace(/\(\s+/g, "(")
.replace(/\s+\)/g, ")");
};

children.forEach(child => {
if (child.type === "code" && child.value) {
const { meta, lang } = child;
if (meta === "sig" && lang === "re") {
if (codeblocks[lang] == null) {
codeblocks[lang] = [];
}
codeblocks[lang].push(formatter(child.value));
}
}
});

Object.assign(file.data, { codeblocks });
};

const rehypeHeaders = options => (tree, file) => {
const headers = [];
let mainHeader;
tree.children.forEach(child => {
if (child.tagName === "h1") {
if (child.children.length > 0) {
mainHeader = child.children.map(element => element.value).join("");
}
}
if (child.tagName === "h2") {
if (child.children.length > 0) {
const id = child.properties.id || "";
const name = child.children.map(element => element.value).join("");
headers.push({ name, href: id });
}
}
});

Object.assign(file.data, { headers, mainHeader });
};

export const defaultProcessor = unified()
.use(remarkParse)
.use(remarkGfm)
.use(remarkFrontmatter, [{ type: 'yaml', marker: '-' }])
.use(remarkVfileMatter)
.use(remarkCodeblocks)
.use(remarkRehype)
.use(rehypeSlug)
.use(rehypeHeaders)
.use(rehypeStringify);
Loading