Ship *.js and *.d.ts together under build/js/lib (#12635)

* Ship *.js and *.d.ts together under build/js/lib

* Add missing 'override' modifier

* Normalize tsconfig.json in tests

* Allow to rewrite imports in import("...")

* Use `skipLibCheck: true` consistently for now

* Run `npm pack` as part of bokehjs build

* Fix implementation of latex_label test

* Respect lib dir if typescript is in devDependencies

* Redesign MANIFEST.in

* Make all core/kinds imports relative

* Use locally packaged bokehjs

* Use locally installed typescript in gears

* Run advanced/extensions/gears (no_js)

* Eliminate the need for using bokeh.json

* Fail examples' tests if extensions fail to build

* Upload npm package artifact in bokehjs-ci

* Minor refactoring of compiler's code

* Make bokehjs package available in test jobs in CI

* Use the same layout for bokehjs' package artifact

.github/workflows/bokeh-ci.yml

@@ -62,6 +62,12 @@ jobs:
           name: wheel-package
           path: dist/bokeh-*-py3-none-any.whl
 
+      - name: Upload bokehjs package
+        uses: actions/upload-artifact@v3
+        with:
+          name: bokehjs-package
+          path: bokehjs/build/dist/bokeh-bokehjs-*.tgz
+
   codebase:
     needs: build
     runs-on: ${{ matrix.os }}

.github/workflows/bokehjs-ci.yml

@@ -97,6 +97,13 @@ jobs:
             cp ${SRC}/report.{json,out} ${CHANGED} ${DST}
           fi
 
+      - name: Upload package
+        if: runner.os == 'Linux' && (success() || failure())
+        uses: actions/upload-artifact@v3
+        with:
+          name: bokehjs-package
+          path: bokehjs/build/dist/bokeh-bokehjs-*.tgz
+
       - name: Upload report
         if: runner.os == 'Linux' && (success() || failure())
         uses: actions/upload-artifact@v3

.github/workflows/composite/test-setup/action.yml

@@ -37,6 +37,17 @@ runs:
       shell: bash -l {0}
       run: pip install dist/bokeh-*-py3-none-any.whl
 
+    - name: Download bokehjs package
+      uses: actions/download-artifact@v3
+      with:
+        name: bokehjs-package
+        path: bokehjs/build/dist/
+
+    - name: Symlink bokehjs package
+      shell: bash -l {0}
+      working-directory: ./bokehjs/build/dist/
+      run: ln -s $(ls -t bokeh-bokehjs-*.tgz | head -n 1) bokeh-bokehjs.tgz
+
     - name: Cache node modules
       if: ${{ inputs.source-tree == 'keep' }}
       uses: actions/cache@v3

MANIFEST.in

@@ -2,14 +2,27 @@
 graft src/bokeh/core/_templates
 graft src/bokeh/sampledata/_data
 graft src/bokeh/sphinxext/_templates
-graft src/bokeh/server/static
-exclude src/bokeh/server/static/js/*.json
-global-exclude *.js.map
-global-exclude *.ts.map
-include src/bokeh/server/static/js/bokeh.json
 include src/bokeh/server/views/*.html
 include src/bokeh/server/views/*.ico
 include src/bokeh/util/sampledata.json
 include src/bokeh/_sri.json
 include src/bokeh/LICENSE.txt
 include src/bokeh/py.typed
+
+# start with an empty static/ dir
+prune src/bokeh/server/static
+
+# include bundles
+include src/bokeh/server/static/js/bokeh*.js
+include src/bokeh/server/static/js/bokeh*.min.js
+
+# don't include ESM bundles yet
+exclude src/bokeh/server/static/js/bokeh*.esm.js
+exclude src/bokeh/server/static/js/bokeh*.esm.min.js
+
+# include extensions' compiler
+include src/bokeh/server/static/js/compiler.js
+
+# include bokeh's and typescript's type declarations
+recursive-include src/bokeh/server/static/js/lib *.d.ts
+recursive-include src/bokeh/server/static/lib *.d.ts

bokehjs/examples/tsconfig.json

@@ -32,7 +32,7 @@
     "types": [],
     "baseUrl": ".",
     "paths": {
-      "*": ["../build/js/lib/*", "../build/js/types/*"]
+      "*": ["../build/js/lib/*"]
     },
     "outDir": "../build/examples"
   },

bokehjs/make/paths.ts

@@ -14,7 +14,6 @@ export const build_dir = {
   js: JS_BUILD_DIR,
   css: CSS_BUILD_DIR,
   test: join(BUILD_DIR, "test"),
-  types: join(JS_BUILD_DIR, "types"),
   lib: join(JS_BUILD_DIR, "lib"),
   compiler: join(JS_BUILD_DIR, "compiler"),
   server: join(JS_BUILD_DIR, "server"),

bokehjs/make/tasks/build.ts

@@ -1,6 +1,6 @@
 import {task} from "../task"
 
-task("build", ["scripts:build", "compiler:build", "examples:build"])
+task("build", ["scripts:build", "compiler:build", "examples:build", "pack"])
 task("build:all", ["build", "test:build"])
 
 task("dev", ["lib:build"])

bokehjs/make/tasks/index.ts

@@ -5,6 +5,7 @@ import "./scripts"
 import "./lint"
 import "./test"
 import "./examples"
+import "./pack"
 import "./build"
 import "./all"
 import "./default"

bokehjs/make/tasks/pack.ts

@@ -0,0 +1,35 @@
+import cp from "child_process"
+import fs from "fs"
+import path from "path"
+import {task, BuildError} from "../task"
+
+function npm_pack() {
+  const dist = "build/dist"
+  if (!fs.existsSync(dist)) {
+    fs.mkdirSync(dist, {recursive: true})
+  }
+
+  const npm = process.platform != "win32" ? "npm" : "npm.cmd"
+  const {status, stdout, stderr} = cp.spawnSync(npm, ["pack", `--pack-destination=${dist}`], {stdio: "pipe", encoding: "utf-8"})
+  if (status !== 0) {
+    console.error(stdout)
+    console.error(stderr)
+    throw new BuildError("pack", `failed to run '${npm} pack'`)
+  }
+
+  const tgz = stdout.trim()
+  if (!tgz.endsWith(".tgz")) {
+    throw new BuildError("pack", "can't find tgz archive name in the output")
+  }
+
+  const tgz_latest = path.join(dist, "bokeh-bokehjs.tgz")
+  if (fs.existsSync(tgz_latest)) {
+    fs.unlinkSync(tgz_latest)
+  }
+
+  fs.symlinkSync(tgz, tgz_latest)
+}
+
+task("pack", async () => {
+  npm_pack()
+})

bokehjs/make/tasks/scripts.ts

@@ -17,13 +17,13 @@ task("scripts:version", async () => {
   const dts = "export declare const version: string;\n"
 
   write(join(paths.build_dir.lib, "version.js"), js)
-  write(join(paths.build_dir.types, "version.d.ts"), dts)
+  write(join(paths.build_dir.lib, "version.d.ts"), dts)
 })
 
 task("scripts:styles", ["styles:compile"], async () => {
   const css_dir = paths.build_dir.css
   const js_dir = paths.build_dir.lib
-  const dts_dir = paths.build_dir.types
+  const dts_dir = paths.build_dir.lib
   const dts_internal_dir = join(paths.build_dir.all, "dts")
 
   wrap_css_modules(css_dir, js_dir, dts_dir, dts_internal_dir)
@@ -33,7 +33,7 @@ task("scripts:glsl", async () => {
   const lib_base = paths.src_dir.lib
 
   const js_base = paths.build_dir.lib
-  const dts_base = paths.build_dir.types
+  const dts_base = paths.build_dir.lib
 
   for (const glsl_path of scan(lib_base, [".vert", ".frag"])) {
     const sub_path = relative(lib_base, glsl_path)

bokehjs/package.json

@@ -23,7 +23,7 @@
   "files": [
     "build/js/bokeh*.min.js",
     "build/js/lib/**/*.js",
-    "build/js/types/**/*.d.ts"
+    "build/js/lib/**/*.d.ts"
   ],
   "workspaces": [
     "./make",

bokehjs/src/compiler/build.ts

@@ -1,11 +1,13 @@
 import * as ts from "typescript"
 
 import * as cp from "child_process"
-import {join, basename, relative} from "path"
+import {join, dirname, basename, relative} from "path"
 
 import {read, read_json, write, rename, file_exists, directory_exists, hash, hash_file, Path} from "./sys"
-import {compile_files, read_tsconfig, parse_tsconfig, is_failed,
-        default_transformers, compiler_host, report_diagnostics} from "./compiler"
+import {
+  compile_files, read_tsconfig, parse_tsconfig, is_failed,
+  default_transformers, compiler_host, report_diagnostics,
+} from "./compiler"
 import {Linker} from "./linker"
 import {compile_styles, wrap_css_modules} from "./styles"
 import * as preludes from "./prelude"
@@ -22,6 +24,17 @@ import "@typescript-eslint/parser"
 
 import * as readline from "readline"
 
+type Package = {
+  name: string
+  version: string
+  description: string
+  license: string
+  keywords: string[]
+  repository: {type: string, url: string}
+  dependencies: {[key: string]: string}
+  devDependencies: {[key: string]: string}
+}
+
 const toString = Object.prototype.toString
 export function isString(obj: unknown): obj is string {
   return toString.call(obj) === "[object String]"
@@ -266,14 +279,31 @@ export async function build(base_dir: Path, bokehjs_dir: Path, base_setup: Build
     }
   }
 
+  const package_json = (is_package ? read_json(package_json_path) : {}) as Partial<Package>
+
+  const is_static_dir = basename(bokehjs_dir) == "static"
+
+  const tslib_dir = (() => {
+    if (package_json.devDependencies?.typescript != null) {
+      return join(base_dir, "node_modules", "typescript", "lib")
+    } else {
+      // bokeh/server/static or bokehjs/build
+      if (is_static_dir)
+        return join(bokehjs_dir, "lib")
+      else
+        return join(dirname(bokehjs_dir), "node_modules", "typescript", "lib")
+    }
+  })()
+
+  print(`TypeScript lib: ${cyan(tslib_dir)}`)
+
   const tsconfig_path = join(base_dir, "tsconfig.json")
   const tsconfig = (() => {
     const preconfigure: ts.CompilerOptions = {
       baseUrl: base_dir,
       paths: {
         "@bokehjs/*": [
           join(bokehjs_dir, "js/lib/*"),
-          join(bokehjs_dir, "js/types/*"),
         ],
       },
     }
@@ -294,7 +324,7 @@ export async function build(base_dir: Path, bokehjs_dir: Path, base_setup: Build
   const {files, options} = tsconfig
 
   const dist_dir = join(base_dir, "dist")
-  const lib_dir = options.outDir ?? dist_dir
+  const lib_dir = options.outDir ?? join(dist_dir, "lib")
   const dts_dir = options.declarationDir ?? lib_dir
   const dts_internal_dir = join(dist_dir, "dts")
 
@@ -308,7 +338,7 @@ export async function build(base_dir: Path, bokehjs_dir: Path, base_setup: Build
   wrap_css_modules(css_dir, lib_dir, dts_dir, dts_internal_dir)
 
   const transformers = default_transformers(options)
-  const host = compiler_host(new Map(), options, bokehjs_dir)
+  const host = compiler_host(new Map(), options, tslib_dir)
 
   print(`Compiling TypeScript (${magenta(`${files.length} files`)})`)
   const tsoutput = compile_files(files, options, transformers, host)

bokehjs/src/compiler/compile.ts

@@ -1,4 +1,4 @@
-import * as path from "path"
+import {join, basename, dirname} from "path"
 import ts from "typescript"
 import lesscss from "less"
 
@@ -15,13 +15,14 @@ function parse_patched_tsconfig(base_dir: string, preconfigure: ts.CompilerOptio
 }
 
 export function compile_typescript(base_dir: string, inputs: Inputs, bokehjs_dir: string): {outputs?: Outputs} & TSOutput {
+  const is_static_dir = basename(bokehjs_dir) == "static"
+
   const preconfigure: ts.CompilerOptions = {
     module: ts.ModuleKind.CommonJS,
     target: ts.ScriptTarget.ES2017,
     paths: {
       "*": [
-        path.join(bokehjs_dir, "js/lib/*"),
-        path.join(bokehjs_dir, "js/types/*"),
+        join(bokehjs_dir, "js/lib/*"),
       ],
     },
     outDir: undefined,
@@ -31,7 +32,15 @@ export function compile_typescript(base_dir: string, inputs: Inputs, bokehjs_dir
   if (tsconfig.diagnostics != null)
     return {diagnostics: tsconfig.diagnostics}
 
-  const host = compiler_host(inputs, tsconfig.options, bokehjs_dir)
+  const tslib_dir = (() => {
+    // bokeh/server/static or bokehjs/build
+    if (is_static_dir)
+      return join(bokehjs_dir, "lib")
+    else
+      return join(dirname(bokehjs_dir), "node_modules", "typescript", "lib")
+  })()
+
+  const host = compiler_host(inputs, tsconfig.options, tslib_dir)
   const transformers = default_transformers(tsconfig.options)
 
   const outputs: Outputs = new Map()

bokehjs/src/compiler/compiler.ts

@@ -1,14 +1,14 @@
 import chalk from "chalk"
 import ts from "typescript"
 
-import {basename, dirname, join, relative} from "path"
+import {dirname, join, relative} from "path"
 
 import * as transforms from "./transforms"
 import {Path} from "./sys"
 import {BuildError} from "./error"
 
 export type CompileConfig = {
-  bokehjs_dir?: Path
+  tslib_dir?: Path
   inputs?(files: Path[]): Inputs
 }
 
@@ -52,7 +52,7 @@ export function report_diagnostics(diagnostics: Diagnostics): {count: number, te
   return {count: errors.length, text}
 }
 
-export function compiler_host(inputs: Inputs, options: ts.CompilerOptions, bokehjs_dir?: Path): ts.CompilerHost {
+export function compiler_host(inputs: Inputs, options: ts.CompilerOptions, tslib_dir?: Path): ts.CompilerHost {
   const default_host = ts.createIncrementalCompilerHost(options)
 
   const host = {
@@ -74,14 +74,8 @@ export function compiler_host(inputs: Inputs, options: ts.CompilerOptions, bokeh
     },
   }
 
-  if (bokehjs_dir != null) {
-    host.getDefaultLibLocation = () => {
-      // bokeh/server/static or bokehjs/build
-      if (basename(bokehjs_dir) == "static")
-        return join(bokehjs_dir, "lib")
-      else
-        return join(dirname(bokehjs_dir), "node_modules/typescript/lib")
-    }
+  if (tslib_dir != null) {
+    host.getDefaultLibLocation = () => tslib_dir
   }
 
   return host
@@ -172,7 +166,7 @@ function compile_project(tsconfig_path: Path, config: CompileConfig): TSOutput {
 
   const transformers = default_transformers(tsconfig.options)
   const inputs = config.inputs?.(files) ?? new Map()
-  const host = compiler_host(inputs, options, config.bokehjs_dir)
+  const host = compiler_host(inputs, options, config.tslib_dir)
 
   const input_files = [...inputs.keys(), ...files]
   return compile_files(input_files, options, transformers, host)

bokehjs/src/compiler/main.ts

@@ -25,18 +25,19 @@ function reply(data: unknown): void {
 }
 
 async function compile() {
-  if (argv.file != null) {
-    const input = {
-      code: argv.code != null ? argv.code as string : read(argv.file as string)!,
-      lang: (argv.lang as string | undefined) ?? "typescript",
-      file: argv.file as string,
-      bokehjs_dir: (argv.bokehjsDir as string | undefined) ?? "./build", // this is what bokeh.settings defaults to
+  const input = await (async () => {
+    if (argv.file != null) {
+      return {
+        code: argv.code != null ? argv.code as string : read(argv.file as string)!,
+        lang: (argv.lang as string | undefined) ?? "typescript",
+        file: argv.file as string,
+        bokehjs_dir: (argv.bokehjsDir as string | undefined) ?? "./build", // this is what bokeh.settings defaults to
+      }
+    } else {
+      return JSON.parse(await read_stdin())
     }
-    return await compile_and_resolve_deps(input)
-  } else {
-    const input = JSON.parse(await read_stdin())
-    return await compile_and_resolve_deps(input)
-  }
+  })()
+  return await compile_and_resolve_deps(input)
 }
 
 async function main() {