chore(loader-utils): Preparing to migrate from FileProviders to Reada…

…bleFiles (#3000)

examples/experimental/slpk-in-browser/src/browser-file.ts

@@ -1,13 +1,13 @@
 // loaders.gl
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
-import {FileProvider} from '@loaders.gl/loader-utils';
+import {FileProviderInterface} from '@loaders.gl/loader-utils';
 
 /**
  * Provides file data using node fs library
  * @deprecated - will be replaced with ReadableFile
  */
-export class BrowserFile implements FileProvider {
+export class BrowserFile implements FileProviderInterface {
   /** The File object from which data is provided */
   private file: File;
 

modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-archive.ts

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright vis.gl contributors
 
-import {FileProvider} from '@loaders.gl/loader-utils';
+import {FileProviderInterface} from '@loaders.gl/loader-utils';
 import {MD5Hash} from '@loaders.gl/crypto';
 import {DeflateCompression, NoCompression} from '@loaders.gl/compression';
 import {IndexedArchive, parseZipLocalFileHeader} from '@loaders.gl/zip';
@@ -31,7 +31,11 @@ export class Tiles3DArchive extends IndexedArchive {
    * @param fileProvider - FileProvider with the whole file
    * @param hashTable - hash info
    */
-  constructor(fileProvider: FileProvider, hashTable?: Record<string, bigint>, fileName?: string) {
+  constructor(
+    fileProvider: FileProviderInterface,
+    hashTable?: Record<string, bigint>,
+    fileName?: string
+  ) {
     super(fileProvider, hashTable, fileName);
     this.hashTable = hashTable;
   }

modules/3d-tiles/src/3d-tiles-archive/3d-tiles-archive-parser.ts

@@ -2,7 +2,7 @@
 // SPDX-License-Identifier: MIT
 // Copyright vis.gl contributors
 
-import {FileProvider} from '@loaders.gl/loader-utils';
+import {FileProviderInterface} from '@loaders.gl/loader-utils';
 import {
   CD_HEADER_SIGNATURE,
   makeHashTableFromZipHeaders,
@@ -20,7 +20,7 @@ import {Tiles3DArchive} from './3d-tiles-archive-archive';
  * @returns 3tz file handler
  */
 export const parse3DTilesArchive = async (
-  fileProvider: FileProvider,
+  fileProvider: FileProviderInterface,
   cb?: (msg: string) => void
 ): Promise<Tiles3DArchive> => {
   const hashCDOffset = await searchFromTheEnd(fileProvider, CD_HEADER_SIGNATURE);

modules/i3s/src/lib/parsers/parse-slpk/parse-slpk.ts

@@ -1,4 +1,4 @@
-import {FileProvider} from '@loaders.gl/loader-utils';
+import {FileProviderInterface} from '@loaders.gl/loader-utils';
 import {
   parseZipCDFileHeader,
   CD_HEADER_SIGNATURE,
@@ -16,7 +16,7 @@ import {SLPKArchive} from './slpk-archieve';
  * @returns slpk file handler
  */
 export async function parseSLPKArchive(
-  fileProvider: FileProvider,
+  fileProvider: FileProviderInterface,
   cb?: (msg: string) => void,
   fileName?: string
 ): Promise<SLPKArchive> {

modules/i3s/src/lib/parsers/parse-slpk/slpk-archieve.ts

@@ -1,5 +1,5 @@
 import {MD5Hash} from '@loaders.gl/crypto';
-import {FileProvider} from '@loaders.gl/loader-utils';
+import {FileProviderInterface} from '@loaders.gl/loader-utils';
 import {IndexedArchive, parseZipLocalFileHeader} from '@loaders.gl/zip';
 import {GZipCompression} from '@loaders.gl/compression';
 
@@ -56,7 +56,11 @@ export class SLPKArchive extends IndexedArchive {
    * @param hashTable - pre-loaded hashTable. If presented, getFile will skip reading the hash file
    * @param fileName - name of the archive. It is used to add to an URL of a loader context
    */
-  constructor(fileProvider: FileProvider, hashTable?: Record<string, bigint>, fileName?: string) {
+  constructor(
+    fileProvider: FileProviderInterface,
+    hashTable?: Record<string, bigint>,
+    fileName?: string
+  ) {
     super(fileProvider, hashTable, fileName);
     this.hashTable = hashTable;
   }

modules/loader-utils/src/index.ts

@@ -133,8 +133,9 @@ export type {FileSystem, RandomAccessFileSystem} from './lib/filesystems/filesys
 export {NodeFileSystemFacade as NodeFilesystem} from './lib/filesystems/node-filesystem-facade';
 
 // TODO - replace with ReadableFile
-export type {FileProvider} from './lib/file-provider/file-provider';
-export {isFileProvider} from './lib/file-provider/file-provider';
+export type {FileProviderInterface} from './lib/file-provider/file-provider-interface';
+export {isFileProvider} from './lib/file-provider/file-provider-interface';
+export {FileProvider} from './lib/file-provider/file-provider';
 export {FileHandleFile} from './lib/file-provider/file-handle-file';
 export {DataViewFile} from './lib/file-provider/data-view-file';
 

modules/loader-utils/src/lib/file-provider/data-view-file.ts

@@ -1,4 +1,4 @@
-import {FileProvider} from './file-provider';
+import {FileProviderInterface} from './file-provider-interface';
 
 /**
  * Checks if bigint can be converted to number and convert it if possible
@@ -16,7 +16,7 @@ const toNumber = (bigint: bigint) => {
  * Provides file data using DataView
  * @deprecated - will be replaced with ReadableFile
  */
-export class DataViewFile implements FileProvider {
+export class DataViewFile implements FileProviderInterface {
   /** The DataView from which data is provided */
   private file: DataView;
 

modules/loader-utils/src/lib/file-provider/file-handle-file.ts

@@ -2,14 +2,14 @@
 // SPDX-License-Identifier: MIT
 // Copyright (c) vis.gl contributors
 
-import {FileProvider} from './file-provider';
+import {FileProviderInterface} from './file-provider-interface';
 import {NodeFileFacade as NodeFile} from '../files/node-file-facade';
 
 /**
  * Provides file data using node fs library
  * @deprecated - will be replaced with ReadableFile
  */
-export class FileHandleFile implements FileProvider {
+export class FileHandleFile implements FileProviderInterface {
   /** The FileHandle from which data is provided */
   private file: NodeFile;
 

modules/loader-utils/src/lib/file-provider/file-provider-interface.ts

@@ -0,0 +1,56 @@
+/**
+ * Interface for providing file data
+ */
+export interface FileProviderInterface {
+  /**
+   * Cleanup class data
+   */
+  destroy(): Promise<void>;
+  /**
+   * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint8(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
+   */
+  getUint16(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in bytes, from the file of the view where to read the data.
+   */
+  getUint32(offset: bigint): Promise<number>;
+
+  /**
+   * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
+   * @param offset The offset, in byte, from the file of the view where to read the data.
+   */
+  getBigUint64(offset: bigint): Promise<bigint>;
+
+  /**
+   * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
+   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
+   */
+  slice(startOffset: bigint, endOffset: bigint): Promise<ArrayBuffer>;
+
+  /**
+   * the length (in bytes) of the data.
+   */
+  length: bigint;
+}
+
+/**
+ * Check is the object has FileProvider members
+ * @param fileProvider - tested object
+ */
+export const isFileProvider = (fileProvider: unknown) => {
+  return (
+    (fileProvider as FileProviderInterface)?.getUint8 &&
+    (fileProvider as FileProviderInterface)?.slice &&
+    (fileProvider as FileProviderInterface)?.length
+  );
+};

modules/loader-utils/src/lib/file-provider/file-provider.ts

@@ -1,57 +1,127 @@
+// loaders.gl
+// SPDX-License-Identifier: MIT
+// Copyright (c) vis.gl contributors
+import {ReadableFile} from '../files/file';
+import {FileProviderInterface} from './file-provider-interface';
+
 /**
- * Interface for providing file data
+ * Provides file data using range requests to the server
  * @deprecated - will be replaced with ReadableFile
  */
-export interface FileProvider {
+export class FileProvider implements FileProviderInterface {
+  /** The File object from which data is provided */
+  private file: ReadableFile;
+  private size: bigint;
+
+  /** Create a new BrowserFile */
+  private constructor(file: ReadableFile, size: bigint | number) {
+    this.file = file;
+    this.size = BigInt(size);
+  }
+
+  static async create(file: ReadableFile): Promise<FileProvider> {
+    return new FileProvider(
+      file,
+      file.bigsize > 0n
+        ? file.bigsize
+        : file.size > 0n
+        ? file.size
+        : (await file.stat?.())?.bigsize ?? 0n
+    );
+  }
+
   /**
-   * Cleanup class data
+   * Truncates the file descriptor.
+   * @param length desired file lenght
    */
-  destroy(): Promise<void>;
+  async truncate(length: number): Promise<void> {
+    throw new Error('file loaded via range requests cannot be changed');
+  }
+
+  /**
+   * Append data to a file.
+   * @param buffer data to append
+   */
+  async append(buffer: Uint8Array): Promise<void> {
+    throw new Error('file loaded via range requests cannot be changed');
+  }
+
+  /** Close file */
+  async destroy(): Promise<void> {
+    throw new Error('file loaded via range requests cannot be changed');
+  }
+
   /**
    * Gets an unsigned 8-bit integer at the specified byte offset from the start of the file.
    * @param offset The offset, in bytes, from the start of the file where to read the data.
    */
-  getUint8(offset: bigint): Promise<number>;
+  async getUint8(offset: number | bigint): Promise<number> {
+    const arrayBuffer = await this.file.read(offset, 1);
+    const val = new Uint8Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
 
   /**
    * Gets an unsigned 16-bit integer at the specified byte offset from the start of the file.
    * @param offset The offset, in bytes, from the start of the file where to read the data.
    */
-  getUint16(offset: bigint): Promise<number>;
+  async getUint16(offset: number | bigint): Promise<number> {
+    const arrayBuffer = await this.file.read(offset, 2);
+    const val = new Uint16Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
 
   /**
    * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
-   * @param offset The offset, in bytes, from the file of the view where to read the data.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
    */
-  getUint32(offset: bigint): Promise<number>;
+  async getUint32(offset: number | bigint): Promise<number> {
+    const arrayBuffer = await this.file.read(offset, 4);
+    const val = new Uint32Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
 
   /**
    * Gets an unsigned 32-bit integer at the specified byte offset from the start of the file.
-   * @param offset The offset, in byte, from the file of the view where to read the data.
+   * @param offset The offset, in bytes, from the start of the file where to read the data.
    */
-  getBigUint64(offset: bigint): Promise<bigint>;
+  async getBigUint64(offset: number | bigint): Promise<bigint> {
+    const arrayBuffer = await this.file.read(offset, 8);
+    const val = new BigInt64Array(arrayBuffer).at(0);
+    if (val === undefined) {
+      throw new Error('something went wrong');
+    }
+    return val;
+  }
 
   /**
    * returns an ArrayBuffer whose contents are a copy of this file bytes from startOffset, inclusive, up to endOffset, exclusive.
-   * @param startOffset The offset, in bytes, from the start of the file where to start reading the data.
+   * @param startOffset The offset, in byte, from the start of the file where to start reading the data.
    * @param endOffset The offset, in bytes, from the start of the file where to end reading the data.
    */
-  slice(startOffset: bigint, endOffset: bigint): Promise<ArrayBuffer>;
+  async slice(startOffset: bigint | number, endOffset: bigint | number): Promise<ArrayBuffer> {
+    const bigLength = BigInt(endOffset) - BigInt(startOffset);
+    if (bigLength > Number.MAX_SAFE_INTEGER) {
+      throw new Error('too big slice');
+    }
+    const length = Number(bigLength);
+
+    return await this.file.read(startOffset, length);
+  }
 
   /**
    * the length (in bytes) of the data.
    */
-  length: bigint;
+  get length(): bigint {
+    return this.size;
+  }
 }
-
-/**
- * Check is the object has FileProvider members
- * @param fileProvider - tested object
- */
-export const isFileProvider = (fileProvider: unknown) => {
-  return (
-    (fileProvider as FileProvider)?.getUint8 &&
-    (fileProvider as FileProvider)?.slice &&
-    (fileProvider as FileProvider)?.length
-  );
-};

modules/loader-utils/test/index.ts

@@ -22,5 +22,6 @@ import './lib/request-utils/request-scheduler.spec';
 
 import './lib/file-provider/data-view-file.spec';
 import './lib/file-provider/file-handle-file.spec';
+import './lib/file-provider/file-provider.spec';
 
 import './lib/worker-loader-utils/parse-with-worker.spec';

modules/loader-utils/test/lib/file-provider/file-provider.spec.ts

@@ -0,0 +1,40 @@
+import test from 'tape-promise/tape';
+import {DATA_ARRAY} from '@loaders.gl/i3s/test/data/test.zip';
+import {BlobFile, FileProvider} from '@loaders.gl/loader-utils';
+
+export const getSignature = () => new Uint8Array([0x50, 0x4b, 0x03, 0x04]);
+
+const getProvider = () => {
+  return FileProvider.create(new BlobFile(DATA_ARRAY.buffer));
+};
+
+test('FileProvider#slice', async (t) => {
+  const provider = await getProvider();
+  const slice = await provider.slice(0, 4);
+  t.deepEqual(new Uint8Array(slice), getSignature());
+  t.end();
+});
+
+test('FileProvider#getUint8', async (t) => {
+  const provider = await getProvider();
+  t.equals(await provider.getUint8(0), 80);
+  t.end();
+});
+
+test('FileProvider#getUint16', async (t) => {
+  const provider = await getProvider();
+  t.equals(await provider.getUint16(0), 19280);
+  t.end();
+});
+
+test('FileProvider#getUint32', async (t) => {
+  const provider = await getProvider();
+  t.equals(await provider.getUint32(0), 67324752);
+  t.end();
+});
+
+test('FileProvider#getBigUint64', async (t) => {
+  const provider = await getProvider();
+  t.equals(await provider.getBigUint64(0), 563035920091984n);
+  t.end();
+});