Lazy-load compression filters in H5WasmProvider

apps/demo/package.json

@@ -16,6 +16,7 @@
     "@h5web/h5wasm": "workspace:*",
     "axios": "1.5.0",
     "axios-hooks": "4.0.0",
+    "h5wasm-plugins": "0.0.1",
     "normalize.css": "8.0.1",
     "react": "18.2.0",
     "react-dom": "18.2.0",

apps/demo/src/h5wasm/H5WasmApp.tsx

@@ -6,6 +6,7 @@ import { useLocation } from 'react-router-dom';
 import { getFeedbackURL } from '../utils';
 import DropZone from './DropZone';
 import type { H5File } from './models';
+import { getPlugin } from './plugin-utils';
 
 function H5WasmApp() {
   const query = new URLSearchParams(useLocation().search);
@@ -16,7 +17,7 @@ function H5WasmApp() {
   }
 
   return (
-    <H5WasmProvider {...h5File}>
+    <H5WasmProvider {...h5File} getPlugin={getPlugin}>
       <App sidebarOpen={!query.has('wide')} getFeedbackURL={getFeedbackURL} />
     </H5WasmProvider>
   );

apps/demo/src/h5wasm/plugin-utils.ts

@@ -0,0 +1,30 @@
+// Import compression plugins as static assets (i.e. as URLs)
+// cf. `vite.config.ts` and `src/vite-env.d.ts
+import blosc from 'h5wasm-plugins/plugins/libH5Zblosc.so';
+import bz2 from 'h5wasm-plugins/plugins/libH5Zbz2.so';
+import lz4 from 'h5wasm-plugins/plugins/libH5Zlz4.so';
+import lzf from 'h5wasm-plugins/plugins/libH5Zlzf.so';
+import szf from 'h5wasm-plugins/plugins/libH5Zszf.so';
+import zfp from 'h5wasm-plugins/plugins/libH5Zzfp.so';
+import zstd from 'h5wasm-plugins/plugins/libH5Zzstd.so';
+
+const PLUGINS: Record<string, string> = {
+  blosc,
+  bz2,
+  lz4,
+  lzf,
+  szf,
+  zfp,
+  zstd,
+};
+
+export async function getPlugin(
+  name: string,
+): Promise<ArrayBuffer | undefined> {
+  if (!PLUGINS[name]) {
+    return undefined;
+  }
+
+  const response = await fetch(PLUGINS[name]);
+  return response.arrayBuffer();
+}

apps/demo/src/vite-env.d.ts

@@ -1,3 +1,9 @@
 /* eslint-disable spaced-comment */
 
 /// <reference types="vite/client" />
+
+// HDF5 compression plugins
+declare module '*.so' {
+  const src: string;
+  export default src;
+}

apps/demo/vite.config.js

@@ -10,6 +10,9 @@ export default defineConfig({
     { ...checker({ typescript: true }), apply: 'serve' }, // dev only to reduce build time
   ],
 
+  // Import HDF5 compression plugins as static assets
+  assetsInclude: ['**/*.so'],
+
   // `es2020` required by @h5web/h5wasm for BigInt `123n` notation support
   optimizeDeps: { esbuildOptions: { target: 'es2020' } },
   build: {

packages/h5wasm/README.md

@@ -120,7 +120,7 @@ means that:
 
 - your build tool must understand the BigInt notation (e.g.
   [babel-plugin-syntax-bigint](https://babeljs.io/docs/en/babel-plugin-syntax-bigint))
-- your application will only run in browsers that
+- your application will run only in browsers that
   [support BigInt](https://caniuse.com/bigint).
 
 ### External links are not resolved
@@ -165,3 +165,42 @@ See
 `H5WasmProvider` does not provide a fallback implementation of `getExportURL` at
 this time, so if you don't provide your own, the export menu will remain
 disabled in the toolbar.
+
+#### `getPlugin?: (name: string) => Promise<ArrayBuffer | undefined>`
+
+If provided, this aysnchronous function is invoked when loading a compressed
+dataset. It receives the name of a compression plugin as parameter and should
+return:
+
+- the compression plugin's source file as `ArrayBuffer`,
+- or `undefined` if the plugin is not available.
+
+`@h5web/h5wasm` is capable of identifying and requesting the plugins supported
+by the
+[`h5wasm-plugins@0.0.1`](https://github.com/h5wasm/h5wasm-plugins/tree/v0.0.1)
+package: `blosc`, `bz2`, `lz4`, `lzf`, `szf`, `zfp`, `zstd`.
+
+A typical implementation of `getPlugin` in a bundled front-end application might
+look like this:
+
+```ts
+/*
+ * Import the plugins' source files as static assets (i.e. as URLs).
+ * The exact syntax may vary depending on your bundler (Vite, webpack ...)
+ * and may require extra configuration/typing.
+ */
+import blosc from 'h5wasm-plugins/plugins/libH5Zblosc.so';
+import bz2 from 'h5wasm-plugins/plugins/libH5Zbz2.so';
+// ...
+
+const PLUGNS = { blosc, bz2 /* ... */ };
+
+async function getPlugin(name: string): Promise<ArrayBuffer | undefined> {
+  if (!PLUGINS[name]) {
+    return undefined;
+  }
+
+  const response = await fetch(PLUGINS[name]);
+  return response.arrayBuffer();
+}
+```

packages/h5wasm/package.json

@@ -45,7 +45,7 @@
     }
   },
   "dependencies": {
-    "h5wasm": "0.6.2",
+    "h5wasm": "0.6.8",
     "nanoid": "5.0.1"
   },
   "devDependencies": {

packages/h5wasm/src/H5WasmProvider.tsx

@@ -9,19 +9,20 @@ interface Props {
   filename: string;
   buffer: ArrayBuffer;
   getExportURL?: DataProviderApi['getExportURL'];
+  getPlugin?: (name: string) => Promise<ArrayBuffer | undefined>;
 }
 
 function H5WasmProvider(props: PropsWithChildren<Props>) {
-  const { filename, buffer, getExportURL, children } = props;
+  const { filename, buffer, getExportURL, getPlugin, children } = props;
 
   const [api, setApi] = useState<H5WasmApi>();
 
   useEffect(() => {
-    const h5wasmApi = new H5WasmApi(filename, buffer, getExportURL);
+    const h5wasmApi = new H5WasmApi(filename, buffer, getExportURL, getPlugin);
     setApi(h5wasmApi);
 
     return () => void h5wasmApi.cleanUp();
-  }, [filename, buffer, getExportURL]);
+  }, [filename, buffer, getExportURL, getPlugin]);
 
   if (!api) {
     return null;

packages/h5wasm/src/h5wasm-api.ts

@@ -22,8 +22,8 @@ import {
   EntityKind,
   hasArrayShape,
 } from '@h5web/shared';
-import type { Attribute as H5WasmAttribute } from 'h5wasm';
-import { File as H5WasmFile, Module, ready as h5wasmReady } from 'h5wasm';
+import type { Attribute as H5WasmAttribute, Filter, Module } from 'h5wasm';
+import { File as H5WasmFile, ready as h5wasmReady } from 'h5wasm';
 import { nanoid } from 'nanoid';
 
 import {
@@ -37,17 +37,29 @@ import {
   isH5WasmSoftLink,
 } from './guards';
 import type { H5WasmAttributes, H5WasmEntity } from './models';
-import { convertMetadataToDType, convertSelectionToRanges } from './utils';
+import {
+  convertMetadataToDType,
+  convertSelectionToRanges,
+  PLUGINS_BY_FILTER_ID,
+} from './utils';
+
+const PLUGINS_PATH = '/plugins'; // path to plugins on EMScripten virtual file system
 
 export class H5WasmApi extends DataProviderApi {
+  private readonly h5wasm: Promise<typeof Module>;
   private readonly file: Promise<H5WasmFile>;
 
   public constructor(
     filename: string,
     buffer: ArrayBuffer,
     private readonly _getExportURL?: DataProviderApi['getExportURL'],
+    private readonly getPlugin?: (
+      name: string,
+    ) => Promise<ArrayBuffer | undefined>,
   ) {
     super(filename);
+
+    this.h5wasm = this.initH5Wasm();
     this.file = this.initFile(buffer);
   }
 
@@ -62,9 +74,8 @@ export class H5WasmApi extends DataProviderApi {
     const h5wDataset = await this.getH5WasmEntity(dataset.path);
     assertH5WasmDataset(h5wDataset);
 
-    if (h5wDataset.filters?.some((f) => f.id >= Module.H5Z_FILTER_RESERVED)) {
-      throw new Error('Compression filter not supported');
-    }
+    // Ensure all filters are supported and loaded (if available)
+    await this.processFilters(h5wDataset.filters);
 
     /* h5wasm returns bigints for (u)int64 dtypes, so we use `to_array` to get numbers instead.
      * We do this only for datasets that are supported by at least one visualization (other than `RawVis`),
@@ -135,21 +146,59 @@ export class H5WasmApi extends DataProviderApi {
     return [];
   }
 
-  private async initFile(buffer: ArrayBuffer): Promise<H5WasmFile> {
-    const { FS } = await h5wasmReady;
+  private async initH5Wasm(): Promise<typeof Module> {
+    const module = await h5wasmReady;
+
+    // Replace default plugins path
+    module.remove_plugin_search_path(0);
+    module.insert_plugin_search_path(PLUGINS_PATH, 0);
 
-    // `FS` is guaranteed to be defined once H5Wasm is ready
-    // https://github.com/silx-kit/h5web/pull/1082#discussion_r858613242
-    assertNonNull(FS);
+    // Create plugins folder on Emscripten virtual file system
+    // @ts-expect-error
+    module.FS.mkdirTree(PLUGINS_PATH);
 
-    // Write file to Emscripten virtual file system
+    return module;
+  }
+
+  private async initFile(buffer: ArrayBuffer): Promise<H5WasmFile> {
+    const h5Module = await this.h5wasm;
+
+    // Write HDF5 file to Emscripten virtual file system
     // https://emscripten.org/docs/api_reference/Filesystem-API.html#FS.writeFile
     const id = nanoid(); // use unique ID instead of `this.filepath` to avoid slashes and other unsupported characters
-    FS.writeFile(id, new Uint8Array(buffer), { flags: 'w+' });
+    h5Module.FS.writeFile(id, new Uint8Array(buffer), { flags: 'w+' });
 
     return new H5WasmFile(id, 'r');
   }
 
+  private async processFilters(filters: Filter[]): Promise<void> {
+    const h5Module = await this.h5wasm;
+
+    for await (const filter of filters) {
+      if (filter.id < h5Module.H5Z_FILTER_RESERVED) {
+        continue; // filter supported out of the box
+      }
+
+      const plugin = PLUGINS_BY_FILTER_ID[filter.id];
+      if (!plugin) {
+        throw new Error(
+          `Compression filter ${filter.id} not supported (${filter.name})`,
+        );
+      }
+
+      const pluginPath = `${PLUGINS_PATH}/libH5Z${plugin}.so`;
+
+      if (h5Module.FS.analyzePath(pluginPath).exists) {
+        continue; // plugin already loaded
+      }
+
+      const buffer = await this.getPlugin?.(plugin);
+      if (buffer) {
+        h5Module.FS.writeFile(pluginPath, new Uint8Array(buffer));
+      }
+    }
+  }
+
   private async getH5WasmEntity(
     path: string,
   ): Promise<NonNullable<H5WasmEntity>> {

packages/h5wasm/src/utils.ts

@@ -14,6 +14,18 @@ import {
 } from './guards';
 import type { NumericMetadata } from './models';
 
+// https://github.com/h5wasm/h5wasm-plugins#included-plugins
+// https://support.hdfgroup.org/services/contributions.html
+export const PLUGINS_BY_FILTER_ID: Record<number, string> = {
+  307: 'bz2',
+  32_000: 'lzf',
+  32_001: 'blosc',
+  32_004: 'lz4',
+  32_013: 'zfp',
+  32_015: 'zstd',
+  32_017: 'szf',
+};
+
 export function convertNumericMetadataToDType(
   metadata: NumericMetadata,
 ): NumericType {
@@ -131,3 +143,8 @@ export function convertSelectionToRanges(
     return [Number(member), Number(member) + 1];
   });
 }
+
+export async function fetchFile(url: string): Promise<ArrayBuffer> {
+  const response = await fetch(url);
+  return response.arrayBuffer();
+}