feat(vscode): share body parsing and external ref logic with core (#160)

* feat: initial working vscode web extension

* chore: update sveltekit and vite

This fixes the underlying issue that required a `skipLibCheck` in the first place.

* refactor: fold shared logic back in to core package

* fix: correct link in markdown hover preview

* fix: hover markdown links in codespaces

* docs: brain dump next steps

* chore: fix build-time warnings

* refactor: push Ref logic down to core

* chore: clean up imports

* refactor: simplify citation rendering

defining it as a function means we can simplify the reactivity

* refactor: push SerializedTheorem down to core

* feat: unify entity description parsing logic

and update the VSCode extension to use the body parsing logic in
hover previews. Still a few polishing items to do here:

- expand internal references to linked names
- fix katex styling oddities

but this is a reasonable enough start to at least publish as a
prerelease version.

.github/workflows/test.yaml

@@ -17,7 +17,7 @@ jobs:
 
       # https://github.com/pnpm/action-setup
       - name: Install pnpm
-        uses: pnpm/action-setup@v2
+        uses: pnpm/action-setup@v4
         with:
           version: 8.x.x
 
@@ -27,13 +27,13 @@ jobs:
         run: |
           echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT
 
-      - uses: actions/cache@v3
+      - uses: actions/cache@v4
         name: Setup pnpm cache
         with:
           path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
-          key: ${{ runner.os }}-pnpm-store-v1-${{ hashFiles('**/pnpm-lock.yaml') }}
+          key: ${{ runner.os }}-pnpm-store-v2-${{ hashFiles('**/pnpm-lock.yaml') }}
           restore-keys: |
-            ${{ runner.os }}-pnpm-store-v1-
+            ${{ runner.os }}-pnpm-store-v2-
 
       - name: Install dependencies
         run: pnpm install

.gitignore

@@ -2,3 +2,6 @@ coverage/
 dist/
 node_modules/
 data/
+out
+.vscode-test/
+*.vsix

.prettierignore

@@ -1,4 +1,6 @@
 coverage
 dist
+out
 pnpm-lock.yaml
 .svelte-kit
+.vscode-test

.tool-versions

@@ -1,3 +1,3 @@
 # Pinned here and in devcontainer to avoid OOM issues stemming from
 # https://github.com/TypeStrong/ts-node/issues/1995
-nodejs 20.11.0
+nodejs 20.14.0

LICENSE.md

@@ -1,4 +1,4 @@
-Copyright (c) 2014-2023 James Dabbs
+Copyright (c) 2014-2024 James Dabbs
 
 Permission is hereby granted, free of charge, to any person obtaining a copy
 of this software and associated documentation files (the "Software"), to deal

package.json

@@ -9,15 +9,15 @@
     "test:cov": "pnpm run --recursive test:cov"
   },
   "devDependencies": {
-    "@types/node": "^20.11.3",
-    "@vitest/coverage-v8": "^0.34.6",
+    "@types/node": "^20.14.0",
+    "@vitest/coverage-v8": "^1.3.0",
     "nodemon": "^2.0.22",
     "npm-check-updates": "^16.14.12",
     "prettier": "^2.8.8",
     "prettier-plugin-svelte": "^2.10.1",
     "ts-node": "^10.9.2",
-    "typescript": "^5.3.3",
-    "vite": "^4.5.1",
-    "vitest": "^0.34.6"
+    "typescript": "^5.4.5",
+    "vite": "^5.0.0",
+    "vitest": "^1.3.0"
   }
 }

packages/compile/package.json

@@ -37,7 +37,7 @@
     "glob": "^8.1.0",
     "js-yaml": "^4.1.0",
     "yaml-front-matter": "^4.1.1",
-    "zod": "^3.22.4"
+    "zod": "^3.23.8"
   },
   "devDependencies": {
     "@types/cors": "^2.8.17",

packages/compile/tsconfig.json

@@ -4,7 +4,7 @@
     "declaration": true,
     "declarationDir": "dist/types",
     "module": "es2022",
-    "moduleResolution": "node",
+    "moduleResolution": "bundler",
     "noImplicitAny": true,
     "outDir": "dist/esm",
     "preserveConstEnums": true,

packages/core/bin/build

@@ -0,0 +1,12 @@
+#!/bin/bash
+set -exo pipefail
+
+# VSCode extensions can only use CommonJS modules (for now), but we want to 
+# continue using our standard Vite/ESM build process elsewhere. This builds
+# both versions, corresponding to `package.json`'s `exports` field.
+
+tsc --module es2022 --outDir dist/esm/
+echo '{"type": "module"}' > dist/esm/package.json
+
+tsc --module commonjs --outDir dist/cjs/
+echo '{"type": "commonjs"}' > dist/cjs/package.json

packages/core/package.json

@@ -14,23 +14,27 @@
   },
   "license": "MIT",
   "author": "James Dabbs <james.dabbs@gmail.com> (https://jdabbs.com)",
-  "main": "./dist/esm/index.js",
   "types": "./dist/types/index.d.ts",
+  "exports": {
+    "types": "./dist/types/index.d.ts",
+    "require": "./dist/cjs/index.js",
+    "import": "./dist/esm/index.js"
+  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/pi-base/web.git"
   },
   "scripts": {
     "build:peg": "peggy --plugin ./node_modules/ts-pegjs/dist/tspegjs -o src/Formula/Grammar.ts --cache src/Formula/Grammar.pegjs",
-    "build": "pnpm build:peg && tsc",
+    "build": "pnpm build:peg && ./bin/build",
     "dev": "pnpm build:peg && tsc --watch",
     "test": "vitest run",
     "test:cov": "vitest run --coverage",
     "test:watch": "vitest"
   },
   "dependencies": {
     "debug": "^4.3.4",
-    "micromark-util-types": "1.0.2",
+    "js-yaml": "^4.1.0",
     "rehype-katex": "^6.0.3",
     "rehype-stringify": "^9.0.4",
     "remark": "^14.0.3",
@@ -40,15 +44,17 @@
     "unified": "^10.1.2",
     "unist-util-is": "^5.2.1",
     "unist-util-visit": "^4.1.2",
-    "zod": "^3.22.4"
+    "zod": "^3.23.8"
   },
   "devDependencies": {
     "@types/debug": "^4.1.12",
     "@types/hast": "^2.3.9",
+    "@types/js-yaml": "^4.0.9",
     "@types/mdast": "^3.0.15",
     "@types/unist": "^2.0.10",
     "hast-util-to-html": "^8.0.4",
     "mdast-util-from-markdown": "1.3.0",
+    "micromark-util-types": "1.0.2",
     "peggy": "^3.0.2",
     "ts-pegjs": "^4.2.1"
   }

packages/core/src/Bundle.ts

@@ -1,16 +1,11 @@
-/// <reference types="vite/client" />
 import { z } from 'zod'
 import { Id, traitId } from './Id.js'
 import { Property, propertySchema } from './Property.js'
 import { Space, spaceSchema } from './Space.js'
 import { Theorem, theoremSchema } from './Theorem.js'
 import { Trait, traitSchema } from './Trait.js'
 
-export const defaultHost = import.meta.env?.VITE_PUBLIC_DATA_URL
-  ? import.meta.env.VITE_PUBLIC_DATA_URL
-  : import.meta.env?.DEV
-  ? 'http://localhost:3141'
-  : 'https://pi-base-bundles.s3.us-east-2.amazonaws.com'
+export const defaultHost = 'https://pi-base-bundles.s3.us-east-2.amazonaws.com'
 
 export type Version = {
   ref: string

packages/core/src/Document.ts

@@ -0,0 +1,19 @@
+import * as z from 'zod'
+import * as yaml from 'js-yaml'
+
+// TODO: unify with logic in packages/compiler
+export function parseDocument<T>(schema: z.ZodSchema<T>, contents: string) {
+  const match = contents.match(
+    /^(---)?\s*(?<frontmatter>[\s\S]*?)\s*---(?<body>[\s\S]*)/,
+  )
+  if (!match?.groups) {
+    return
+  }
+
+  const { frontmatter, body } = match.groups
+  const meta = yaml.load(frontmatter)
+  const data = { description: body.trim() }
+  const raw = typeof meta === 'object' ? { ...meta, ...data } : data
+
+  return schema.safeParse(raw)
+}

packages/core/src/Id.ts

@@ -54,3 +54,39 @@ export function toInt(id: string): number {
 
   return tagged.id
 }
+
+// TODO: these were extracted from other parallel-but-divergent implementations
+// and should be unified.
+
+type Pad = '' | '0' | '00' | '000' | '0000' | '00000' | '00000'
+type XId<Prefix extends string> = `${Prefix}${Pad}${number}`
+
+export type SId = XId<'S'>
+export type PId = XId<'P'>
+export type TId = XId<'T'>
+export type SPId = [SId, PId]
+export type EntityId = SId | PId | TId | SPId
+
+export function isSpaceId(token: string): token is SId {
+  return token.match(/^S\d{1,6}$/) !== null
+}
+
+export function isPropertyId(token: string): token is PId {
+  return token.match(/^P\d{1,6}$/) !== null
+}
+
+export function isTheoremId(token: string): token is SId {
+  return token.match(/^T\d{1,6}$/) !== null
+}
+
+export function isTraitId(pair: [string, string]): pair is SPId {
+  return isSpaceId(pair[0]) && isPropertyId(pair[1])
+}
+
+export const idExp = /[PST]\d{1,6}/g
+
+export function normalizeId(id: SId): SId
+export function normalizeId(id: PId): PId
+export function normalizeId(id: string) {
+  return `${id[0]}${id.slice(1).padStart(6, '0')}`
+}

packages/core/src/Parser.ts

@@ -13,7 +13,7 @@ import { truncate as truncator } from './Parser/truncate.js'
 import { unnest } from './Parser/unnest.js'
 
 export type Options = {
-  link: Linkers
+  link: Partial<Linkers>
   truncate?: boolean
 }
 

packages/core/src/Parser/references.ts

@@ -10,12 +10,30 @@ import { ExternalLinkNode, InternalLinkNode, Linkers } from './types'
  *
  * See https://github.com/syntax-tree/mdast-util-to-hast#fields-on-nodes
  */
-export function references({ internal, external }: Linkers) {
+export function references({ internal, external }: Partial<Linkers>) {
   return (): Transformer<Root, Root> => {
     return function transformer(tree: Root) {
       visit(tree, 'internalLink', (node: InternalLinkNode) => {
         const { kind, id } = node
 
+        if (!internal) {
+          Object.assign(node, {
+            data: {
+              hName: 'span',
+              hProperties: {
+                className: 'internal-link',
+              },
+              hChildren: [
+                {
+                  type: 'text',
+                  value: `${kind}${id}`,
+                },
+              ],
+            },
+          })
+          return
+        }
+
         const { href, title } = internal([kind, id])
 
         Object.assign(node, {
@@ -36,26 +54,27 @@ export function references({ internal, external }: Linkers) {
         })
       })
 
-      visit(tree, 'externalLink', (node: ExternalLinkNode) => {
-        const { href, title } = external([node.kind, node.id])
+      external &&
+        visit(tree, 'externalLink', (node: ExternalLinkNode) => {
+          const { href, title } = external([node.kind, node.id])
 
-        Object.assign(node, {
-          data: {
-            hName: 'a',
-            hProperties: {
-              href,
-              title,
-              className: 'external-link',
-            },
-            hChildren: [
-              {
-                type: 'text',
-                value: title,
+          Object.assign(node, {
+            data: {
+              hName: 'a',
+              hProperties: {
+                href,
+                title,
+                className: 'external-link',
               },
-            ],
-          },
+              hChildren: [
+                {
+                  type: 'text',
+                  value: title,
+                },
+              ],
+            },
+          })
         })
-      })
     }
   }
 }

packages/core/src/Property.ts

@@ -1,5 +1,15 @@
 import { z } from 'zod'
 import { recordSchema } from './Record.js'
+import { refSchema } from './Ref.js'
+
+export const propertyPageSchema = z.object({
+  uid: z.string(),
+  name: z.string(),
+  aliases: z.array(z.string()).optional(),
+  counterexamples_id: z.number().nullable().optional(),
+  refs: z.array(refSchema).optional(),
+  description: z.string(),
+})
 
 export const propertySchema = z.intersection(
   z.object({
@@ -10,3 +20,4 @@ export const propertySchema = z.intersection(
 )
 
 export type Property = z.infer<typeof propertySchema>
+export type PropertyPage = z.infer<typeof propertyPageSchema>

packages/core/src/Ref.ts

@@ -24,6 +24,8 @@ export type TaggedRef =
   | { kind: 'mo'; id: string; name?: string }
   | { kind: 'zb'; id: string; name?: string }
 
+export type Kind = TaggedRef['kind']
+
 export function tag(ref: Ref): TaggedRef {
   const { name } = ref
 
@@ -41,3 +43,42 @@ export function tag(ref: Ref): TaggedRef {
     return { kind: 'zb', id: String(ref.zb), name }
   }
 }
+
+type FormatInput = { kind: Kind; id: string; name?: string } | Ref
+
+export function format(input: FormatInput) {
+  if ('kind' in input) {
+    const { kind, id, name } = input
+    switch (kind) {
+      case 'doi':
+        return { href: `https://doi.org/${id}`, title: name || `DOI ${id}` }
+      case 'mr':
+        return {
+          href: `https://mathscinet.ams.org/mathscinet-getitem?mr=${id}`,
+          title: name || `MR ${id}`,
+        }
+      case 'wikipedia':
+        return {
+          href: `https://en.wikipedia.org/wiki/${id}`,
+          title: name || `Wikipedia ${id}`,
+        }
+      case 'mathse':
+        return {
+          href: `https://math.stackexchange.com/questions/${id}`,
+          title: name || `Math StackExchange ${id}`,
+        }
+      case 'mo':
+        return {
+          href: `https://mathoverflow.net/questions/${id}`,
+          title: name || `MathOverflow ${id}`,
+        }
+      case 'zb':
+        return {
+          href: `https://zbmath.org/${id}`,
+          title: name || `zbMATH ${id}`,
+        }
+    }
+  } else {
+    return format(tag(input))
+  }
+}