fix sas syntax bug

.gitignore

@@ -1,4 +1,3 @@
-yarn-error.log
 node_modules
 .RData
 .RDataTmp

compiler/package.json

@@ -21,29 +21,29 @@
     "@mapbox/remark-lint-link-text": "^0.6.0",
     "ansicolor": "^1.1.100",
     "base64-arraybuffer": "^1.0.2",
-    "chalk": "^5.0.0",
+    "chalk": "^5.3.0",
     "dargs": "^8.1.0",
     "dictionary-en-gb": "^2.4.0",
     "escape-latex": "^1.2.0",
     "figures": "^4.0.0",
     "hash-sum": "^2.0.0",
-    "highlight.js": "^11.4.0",
-    "image-size": "^1.0.1",
+    "highlight.js": "^11.9.0",
+    "image-size": "^1.0.2",
     "js-yaml": "^4.0.0",
     "latex": "^0.0.1",
     "lodash": "^4.17.20",
     "lowlight": "^2.5.0",
-    "markdown-table": "^3.0.2",
-    "mathjax-full": "^3.1.2",
+    "markdown-table": "^3.0.3",
+    "mathjax-full": "^3.2.2",
     "mdast-normalize-headings": "3.1.0",
     "mdast-util-toc": "^6.1.0",
     "mime": "^3.0.0",
-    "node-fetch": "^3.2.0",
+    "node-fetch": "^3.3.2",
     "parse-english": "^5.0.0",
     "pdfjs-dist": "^2.12.313",
     "puppeteer": "^13.1.1",
     "puppeteer-core": "^13.1.1",
-    "refractor": "^4.7.0",
+    "refractor": "^4.8.1",
     "rehype-document": "^6.0.1",
     "rehype-format": "^4.0.1",
     "rehype-highlight": "^5.0.2",
@@ -57,10 +57,10 @@
     "remark-frontmatter": "^4.0.1",
     "remark-gfm": "^3.0.1",
     "remark-highlight.js": "^7.0.1",
-    "remark-lint": "^9.1.1",
+    "remark-lint": "^9.1.2",
     "remark-math": "^5.1.1",
     "remark-parse": "^10.0.1",
-    "remark-preset-lint-markdown-style-guide": "^5.1.2",
+    "remark-preset-lint-markdown-style-guide": "^5.1.3",
     "remark-rehype": "^10.1.0",
     "remark-retext": "^5.0.1",
     "remark-sectionize": "^1.1.1",
@@ -69,39 +69,39 @@
     "retext-english": "^4.1.0",
     "retext-spell": "^5.1.0",
     "sandboxed-module": "^2.0.4",
-    "speech-rule-engine": "^4.0.0",
+    "speech-rule-engine": "^4.0.7",
     "svgo": "^2.4.0",
     "temp-dir": "^2.0.0",
     "to-vfile": "^7.2.3",
     "unist-util-visit": "4.1.0",
     "vfile-reporter": "^7.0.3",
     "vfile-reporter-pretty": "^6.1.0",
-    "yargs": "^17.3.1",
+    "yargs": "^17.7.2",
     "yup": "^0.32.8"
   },
   "devDependencies": {
-    "@types/hash-sum": "^1.0.0",
-    "@types/html-pdf": "^3.0.0",
-    "@types/js-yaml": "^4.0.5",
+    "@types/hash-sum": "^1.0.2",
+    "@types/html-pdf": "^3.0.3",
+    "@types/js-yaml": "^4.0.9",
     "@types/marked": "^4.0.1",
     "@types/mime": "^2.0.3",
     "@types/node-fetch": "^3.0.3",
-    "@types/pretty": "^2.0.1",
-    "@types/sandboxed-module": "^2.0.29",
+    "@types/pretty": "^2.0.3",
+    "@types/sandboxed-module": "^2.0.33",
     "@types/svgo": "^2.6.1",
     "@types/tmp": "^0.2.3",
     "babel-loader": "^8.2.2",
     "babel-plugin-inline-import": "^3.0.0",
     "clean-webpack-plugin": "^4.0.0",
     "copy-webpack-plugin": "^10.2.1",
-    "css-loader": "^6.5.1",
+    "css-loader": "^6.8.1",
     "generate-package-json-webpack-plugin": "^2.5.1",
     "inline-environment-variables-webpack-plugin": "^1.2.1",
     "raw-loader": "^4.0.2",
-    "style-loader": "^3.3.1",
-    "webpack": "^5.74.0",
+    "style-loader": "^3.3.3",
+    "webpack": "^5.89.0",
     "webpack-cli": "^4.4.0",
     "webpack-node-externals": "^3.0.0",
-    "webpack-shebang-plugin": "^1.1.4"
+    "webpack-shebang-plugin": "^1.1.8"
   }
 }

compiler/src/build-unit.ts

@@ -16,6 +16,7 @@ import { mdastPhase } from './mdast';
 import { combinedMdastPhase } from './mdast/combined';
 import { convertToPdf } from './pdf';
 import { preParsePhase } from './pre-parse';
+import { codeToAliasDirective } from './code/code-to-alias-directive';
 
 export type BuiltUnit = {
   unit: Unit;
@@ -53,7 +54,7 @@ export async function buildUnit(unit: Unit, ctx: Context) {
       mdast,
       unifiedFile,
       unit,
-      ctx
+      ctx,
     );
   }
 
@@ -63,7 +64,7 @@ export async function buildUnit(unit: Unit, ctx: Context) {
       unifiedFile,
       unit,
       ctx,
-      true
+      true,
     );
     result.pdf = {
       ...transformed,
@@ -81,6 +82,7 @@ export async function buildUnit(unit: Unit, ctx: Context) {
 async function inSituTransforms(file: VFile, ctx: Context) {
   assertNoImageAttributes(file);
   preParsePhase(file);
+  await codeToAliasDirective(file, ctx);
   texToAliasDirective(file, ctx);
   return mdastPhase(file, ctx);
 }
@@ -92,7 +94,7 @@ function combineMdFiles(file: VFile) {
 function removeDirectoryLines(md: string) {
   return md
     .split(EOL)
-    .filter((line) => !/^:directory\[.+\]$/.test(line))
+    .filter((line) => !/^::directory\[.+\]$/.test(line))
     .join(EOL);
 }
 
@@ -101,14 +103,14 @@ async function syntaxTreeTransforms(
   file: VFile,
   unit: Unit,
   ctx: Context,
-  targetPdf?: boolean
+  targetPdf?: boolean,
 ) {
   const mdast = await combinedMdastPhase(_mdast, ctx, file, targetPdf);
   const hast = (await hastPhase(
     mdast,
     ctx,
     file,
-    targetPdf
+    targetPdf,
   )) as HastParent;
   const html = await htmlPhase(hast, mdast, file, unit, ctx, targetPdf);
   return { mdast, hast, html };

compiler/src/code/__test__/code.test.ts

@@ -0,0 +1,43 @@
+import {
+  ignoreWhitespace,
+  testProcessor,
+} from '../../test-utils/test-processor';
+
+it('should NOT try to transform SAS code', async () => {
+  const { md, html } = await testProcessor(`
+    Let's look at a simple example.
+
+    \`\`\`sas
+    %macro sortid(dsn);
+      proc sort data = &dsn;
+        by employee_id;
+      run;
+    %mend;
+    \`\`\`
+  `);
+
+  const expectedMd = ignoreWhitespace(`
+    Let's look at a simple example.
+
+    ::codeBlock[0]
+  `);
+
+  expect(ignoreWhitespace(md)).toBe(expectedMd);
+
+  const expectedHtml = ignoreWhitespace(`
+    <p>Let's look at a simple example.</p>
+    <div class="code-wrapper">
+      <pre>
+        <code>
+          %macro sortid(dsn);
+          proc sort data = &#x26;dsn;
+            by employee_id;
+          run;
+          %mend;
+        </code>
+      </pre>
+    </div>
+  `);
+
+  expect(ignoreWhitespace(html)).toBe(expectedHtml);
+});

compiler/src/code/alias-directive-to-code.ts

@@ -0,0 +1,36 @@
+import { visit } from 'unist-util-visit';
+import { Literal } from 'hast';
+import { Parent, Root } from 'mdast';
+
+import { CodeBlock, Context } from '../context';
+
+export function aliasDirectiveToCode(ctx: Context) {
+  return (tree: Root) => {
+    visit(tree, (node) => {
+      if (node.type === 'leafDirective' && node.name === 'codeBlock') {
+        const idx = getStoreIdx(node);
+        if (ctx.codeStore === undefined) {
+          return;
+        }
+        const stored = ctx.codeStore[idx] as CodeBlock;
+        if (!ctx.codeStore[idx]) {
+          return;
+        }
+
+        Object.assign(node, {
+          type: 'code',
+          name: undefined,
+          lang: stored.lang,
+          meta: stored.meta,
+          value: stored.value,
+          children: [],
+        });
+      }
+    });
+  };
+}
+
+function getStoreIdx(node: Parent) {
+  const firstChild = node.children[0] as Literal;
+  return Number(firstChild.value || 0);
+}

compiler/src/code/code-to-alias-directive.ts

@@ -0,0 +1,59 @@
+import { VFile } from 'vfile';
+import { unified } from 'unified';
+import { Code } from 'mdast';
+import { Node } from 'unist';
+import markdown from 'remark-parse';
+import directive from 'remark-directive';
+import stringify from 'remark-stringify';
+import { visit } from 'unist-util-visit';
+
+import { Context, CodeBlock } from '../context';
+
+// The reason for replacing all fenced code blocks with aliases
+// temporarily is because of MathJax.  MathJax is designed to look
+// for TeX code inside HTML files, and in our case we need to make it
+// look inside a Markdown file. This leads to MathJax looking for TeX
+// inside code blocks, which can causes problems (especially with SAS
+// code syntax).  So this function replaces code blocks with an alias,
+// allows MathJax to do it's thing, then adds it back in with
+// `aliasDirectiveToCode`.
+
+export async function codeToAliasDirective(file: VFile, ctx: Context) {
+  const store: CodeBlock[] = [];
+
+  const processed = await unified()
+    .use(markdown)
+    .use(stringify)
+    .use(codeBlocks, store)
+    .use(directive)
+    .process(file);
+
+  file.value = String(processed);
+  ctx.codeStore = store;
+  return file;
+}
+
+function codeBlocks(store: CodeBlock[]) {
+  return (tree: Node) => {
+    visit(tree, 'code', (node: Code) => {
+      store.push({
+        lang: String(node.lang),
+        meta: String(node.meta),
+        value: node.value,
+      });
+      Object.assign(node, {
+        type: 'leafDirective',
+        name: 'codeBlock',
+        lang: undefined,
+        meta: undefined,
+        value: undefined,
+        children: [
+          {
+            type: 'text',
+            value: String(store.length - 1),
+          },
+        ],
+      });
+    });
+  };
+}

compiler/src/context.ts

@@ -29,13 +29,20 @@ export type Options = {
   output?: 'md' | 'html';
 };
 
+export type CodeBlock = {
+  lang: string,
+  meta: string,
+  value: string
+}
+
 export type Context = {
   dirPath: string;
   buildDir: string;
   cacheDir: string;
   course: Course;
   options: Options;
   mmlStore?: string[];
+  codeStore?: CodeBlock[];
   refStore: Record<string, string>;
   figureCounter: number;
 };