Merge remote-tracking branch 'origin/tools-ts' into tools-ts-implicit

src/ToolsMain.ts

@@ -1,4 +1,5 @@
 import fs from "fs";
+import path from "path";
 
 import { Paths } from "./base/Paths";
 import { DeveloperError } from "./base/DeveloperError";
@@ -14,6 +15,8 @@ import { ContentDataTypes } from "./contentTypes/ContentDataTypes";
 
 import { PipelineExecutor } from "./pipelines/PipelineExecutor";
 import { Pipelines } from "./pipelines/Pipelines";
+import { Buffers } from "./base/Buffers";
+import { TileDataLayouts } from "./tileFormats/TileDataLayouts";
 
 /**
  * Functions that directly correspond to the command line functionality.
@@ -109,6 +112,142 @@ export class ToolsMain {
     fs.writeFileSync(output, outputBuffer);
   }
 
+  static analyze(
+    inputFileName: string,
+    outputDirectoryName: string,
+    force: boolean
+  ) {
+    console.log(`Analyzing ${inputFileName}`);
+    console.log(`writing results to ${outputDirectoryName}`);
+
+    const inputBaseName = path.basename(inputFileName);
+    const inputBuffer = fs.readFileSync(inputFileName);
+    ToolsMain.analyzeInternal(
+      inputBaseName,
+      inputBuffer,
+      outputDirectoryName,
+      force
+    );
+  }
+  static analyzeInternal(
+    inputBaseName: string,
+    inputBuffer: Buffer,
+    outputDirectoryName: string,
+    force: boolean
+  ) {
+    // A function to create a JSON string from an
+    // object. The formatting will be controlled
+    // via a command line flag in the future.
+    const doFormatJson = true;
+    const stringify = (input: any) => {
+      if (doFormatJson) {
+        return JSON.stringify(input, null, 2);
+      }
+      return JSON.stringify(input);
+    };
+
+    // A function to write a JSON string to a file, if
+    // the JSON string does not represent an empty
+    // object, and if the file can be written.
+    const writeJsonFileOptional = (jsonString: string, fileName: string) => {
+      if (jsonString === "{}") {
+        return;
+      }
+      if (!ToolsMain.canWrite(fileName, force)) {
+        console.log(`Cannot write ${fileName}`);
+        return;
+      }
+      console.log(`Writing ${fileName}`);
+      fs.writeFileSync(fileName, Buffer.from(jsonString));
+    };
+
+    // A function to write a buffer to a file, if
+    // the buffer is not empty, and if the file
+    // can be written.
+    const writeFileOptional = (buffer: Buffer, fileName: string) => {
+      if (buffer.length === 0) {
+        return;
+      }
+      if (!ToolsMain.canWrite(fileName, force)) {
+        console.log(`Cannot write ${fileName}`);
+        return;
+      }
+      console.log(`Writing ${fileName}`);
+      fs.writeFileSync(fileName, buffer);
+    };
+
+    // Read the buffer and its magic header
+    const magic = Buffers.getMagic(inputBuffer, 0);
+
+    if (magic === "b3dm" || magic === "i3dm" || magic === "pnts") {
+      // Handle the basic legacy tile formats
+      const tileDataLayout = TileDataLayouts.create(inputBuffer);
+      const tileData = TileFormats.extractTileData(inputBuffer, tileDataLayout);
+
+      // Create the JSON strings for the layout information,
+      // feature table, batch table, and the GLB JSON
+      const layoutJsonString = stringify(tileDataLayout);
+      const featureTableJsonString = stringify(tileData.featureTable.json);
+      const batchTableJsonString = stringify(tileData.batchTable.json);
+      let glbJsonString = "{}";
+      if (tileData.payload.length !== 0) {
+        const glbJsonBuffer = GltfUtilities.extractJsonFromGlb(
+          tileData.payload
+        );
+        glbJsonString = glbJsonBuffer.toString();
+      }
+      if (doFormatJson) {
+        const glbJson = JSON.parse(glbJsonString);
+        glbJsonString = stringify(glbJson);
+      }
+
+      // Determine the output file names. They are files in the
+      // output directory, prefixed with the name of the input
+      // file, and with suffixes that indicate the actual contents
+      const outputBaseName = Paths.resolve(outputDirectoryName, inputBaseName);
+      const layoutFileName = outputBaseName + ".layout.json";
+      const featureTableJsonFileName = outputBaseName + ".featureTable.json";
+      const batchTableJsonFileName = outputBaseName + ".batchTable.json";
+      const glbFileName = outputBaseName + ".glb";
+      const glbJsonFileName = outputBaseName + ".glb.json";
+
+      // Write all output files
+      Paths.ensureDirectoryExists(outputDirectoryName);
+      writeJsonFileOptional(layoutJsonString, layoutFileName);
+      writeFileOptional(tileData.payload, glbFileName);
+      writeJsonFileOptional(featureTableJsonString, featureTableJsonFileName);
+      writeJsonFileOptional(batchTableJsonString, batchTableJsonFileName);
+      writeJsonFileOptional(glbJsonString, glbJsonFileName);
+    } else if (magic === "cmpt") {
+      // Handle composite tiles
+      const compositeTileData = TileFormats.readCompositeTileData(inputBuffer);
+      const n = compositeTileData.innerTileBuffers.length;
+      for (let i = 0; i < n; i++) {
+        const innerTileDataBuffer = compositeTileData.innerTileBuffers[i];
+        const innerTileBaseName = inputBaseName + ".inner[" + i + "]";
+        ToolsMain.analyzeInternal(
+          innerTileBaseName,
+          innerTileDataBuffer,
+          outputDirectoryName,
+          force
+        );
+      }
+    } else if (magic === "glTF") {
+      // Handle GLB files
+      let glbJsonString = "{}";
+      const glbJsonBuffer = GltfUtilities.extractJsonFromGlb(inputBuffer);
+      glbJsonString = glbJsonBuffer.toString();
+      if (doFormatJson) {
+        const glbJson = JSON.parse(glbJsonString);
+        glbJsonString = stringify(glbJson);
+      }
+      const outputBaseName = Paths.resolve(outputDirectoryName, inputBaseName);
+      const glbJsonFileName = outputBaseName + ".glb.json";
+      Paths.ensureDirectoryExists(outputDirectoryName);
+      writeJsonFileOptional(glbJsonString, glbJsonFileName);
+    }
+  }
+
   private static createGzipPipelineJson(
     input: string,
     output: string,
@@ -266,21 +405,38 @@ export class ToolsMain {
   }
 
   /**
-   * Ensures that the specified file can be written, and throws
-   * a `DeveloperError` otherwise.
+   * Returns whether the specified file can be written.
+   *
+   * This is the case when `force` is `true`, or when it does not
+   * exist yet.
    *
    * @param fileName - The file name
    * @param force The 'force' flag state from the command line
    * @returns Whether the file can be written
-   * @throws DeveloperError When the file exists and `force` was `false`.
    */
-  static ensureCanWrite(fileName: string, force: boolean): true {
+  static canWrite(fileName: string, force: boolean): boolean {
     if (force) {
       return true;
     }
     if (!fs.existsSync(fileName)) {
       return true;
     }
+    return false;
+  }
+
+  /**
+   * Ensures that the specified file can be written, and throws
+   * a `DeveloperError` otherwise.
+   *
+   * @param fileName - The file name
+   * @param force The 'force' flag state from the command line
+   * @returns Whether the file can be written
+   * @throws DeveloperError When the file exists and `force` was `false`.
+   */
+  static ensureCanWrite(fileName: string, force: boolean): true {
+    if (ToolsMain.canWrite(fileName, force)) {
+      return true;
+    }
     throw new DeveloperError(
       `File ${fileName} already exists. Specify -f or --force to overwrite existing files.`
     );

src/contentProcessing/GtlfUtilities.ts

@@ -1,4 +1,6 @@
 import GltfPipeline from "gltf-pipeline";
+import { Buffers } from "../base/Buffers";
+import { TileFormatError } from "../tileFormats/TileFormatError";
 import { GltfPipelineLegacy } from "./GltfPipelineLegacy";
 
 /**
@@ -16,11 +18,93 @@ export class GltfUtilities {
    * @returns A promise that resolves with the upgraded GLB.
    */
   static async upgradeGlb(glbBuffer: Buffer): Promise<Buffer> {
-    const processGlb = GltfPipeline.processGlb;
-    const result = await processGlb(glbBuffer);
+    const result = await GltfPipeline.processGlb(glbBuffer);
     return result.glb;
   }
 
+  /**
+   * Extract the JSON part from the given GLB buffer and return it
+   * as a buffer.
+   *
+   * The given buffer may contain glTF 2.0 binary data, or glTF 1.0
+   * binary data.
+   *
+   * Note that this does NOT convert the input data. It only extracts
+   * the data, as-it-is.
+   *
+   * @param glbBuffer - The buffer containing the GLB
+   * @returns The JSON buffer
+   * @throws TileFormatError If the input does not contain valid GLB data.
+   */
+  static extractJsonFromGlb(glbBuffer: Buffer): Buffer {
+    const magic = Buffers.getMagic(glbBuffer);
+    if (magic !== "glTF") {
+      throw new TileFormatError(
+        `Expected magic header to be 'gltf', but found ${magic}`
+      );
+    }
+    if (glbBuffer.length < 12) {
+      throw new TileFormatError(
+        `Expected at least 12 bytes, but only got ${glbBuffer.length}`
+      );
+    }
+    const version = glbBuffer.readUInt32LE(4);
+    const length = glbBuffer.readUInt32LE(8);
+    if (length > glbBuffer.length) {
+      throw new TileFormatError(
+        `Header indicates ${length} bytes, but input has ${glbBuffer.length} bytes`
+      );
+    }
+    if (version === 1) {
+      if (glbBuffer.length < 20) {
+        throw new TileFormatError(
+          `Expected at least 20 bytes, but only got ${glbBuffer.length}`
+        );
+      }
+      const contentLength = glbBuffer.readUint32LE(12);
+      const contentFormat = glbBuffer.readUint32LE(16);
+      if (contentFormat !== 0) {
+        throw new TileFormatError(
+          `Expected content format to be 0, but found ${contentFormat}`
+        );
+      }
+      const contentStart = 20;
+      const contentEnd = contentStart + contentLength;
+      if (glbBuffer.length < contentEnd) {
+        throw new TileFormatError(
+          `Expected at least ${contentEnd} bytes, but only got ${glbBuffer.length}`
+        );
+      }
+      const contentData = glbBuffer.subarray(contentStart, contentEnd);
+      return contentData;
+    } else if (version === 2) {
+      if (glbBuffer.length < 20) {
+        throw new TileFormatError(
+          `Expected at least 20 bytes, but only got ${glbBuffer.length}`
+        );
+      }
+      const chunkLength = glbBuffer.readUint32LE(12);
+      const chunkType = glbBuffer.readUint32LE(16);
+      const jsonChunkType = 0x4e4f534a; // ASCII string for "JSON"
+      if (chunkType !== jsonChunkType) {
+        throw new TileFormatError(
+          `Expected chunk type to be ${jsonChunkType}, but found ${chunkType}`
+        );
+      }
+      const chunkStart = 20;
+      const chunkEnd = chunkStart + chunkLength;
+      if (glbBuffer.length < chunkEnd) {
+        throw new TileFormatError(
+          `Expected at least ${chunkEnd} bytes, but only got ${glbBuffer.length}`
+        );
+      }
+      const chunkData = glbBuffer.subarray(chunkStart, chunkEnd);
+      return chunkData;
+    } else {
+      throw new TileFormatError(`Expected version 1 or 2, but got ${version}`);
+    }
+  }
+
   /**
    * Given an input buffer containing a binary glTF asset, optimize it
    * using gltf-pipeline with the provided options.