Skip to content

Commit

Permalink
Move gltf handling internal to @xviz/io (#605)
Browse files Browse the repository at this point in the history
Latest loaders.gl version is incompatible with the version we depend
upon and causes problems when trying to interop with the latest deck.gl

For this reason just bring the parsing and encoding internal for now and
investigate effort to move to latest version.

Additional changes:
- fix image encoding for GLTF based xviz data
- upgrade ocular-dev-tools
- Update docker and deps
  • Loading branch information
twojtasz committed Jul 22, 2020
1 parent 6d1f3a3 commit 885a682
Show file tree
Hide file tree
Showing 32 changed files with 3,258 additions and 1,266 deletions.
8 changes: 4 additions & 4 deletions .buildkite/pipeline.yml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
steps:
- name: ':docker:'
plugins:
docker-compose#v1.5.2:
docker-compose#v3.5.0:
build: xviz
image-repository: 027047743804.dkr.ecr.us-east-2.amazonaws.com/uber
agents:
Expand All @@ -12,23 +12,23 @@ steps:
- name: ':eslint:'
command: 'yarn lint'
plugins:
docker-compose#v1.5.2:
docker-compose#v3.5.0:
run: xviz
agents:
queue: workers

- name: ':node:'
command: 'yarn test-ci'
plugins:
docker-compose#v1.5.2:
docker-compose#v3.5.0:
run: xviz
agents:
queue: workers

- name: ':chrome:'
command: 'yarn test browser-headless'
plugins:
docker-compose#v1.5.2:
docker-compose#v3.5.0:
run: xviz
agents:
queue: workers
5 changes: 3 additions & 2 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,12 @@ RUN apt-get -y install jq
RUN apt-get update \
# Install latest chrome dev package, which installs the necessary libs to
# make the bundled version of Chromium that Puppeteer installs work.
&& apt-get install -y wget --no-install-recommends \
&& apt-get install -y wget gnupg --no-install-recommends \
&& wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
&& sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list' \
&& apt-get update \
&& apt-get install -y google-chrome-unstable --no-install-recommends \
&& apt-get install -y google-chrome-unstable fonts-ipafont-gothic fonts-wqy-zenhei fonts-thai-tlwg fonts-kacst fonts-freefont-ttf libxss1 \
--no-install-recommends \
&& rm -rf /var/lib/apt/lists/* \
&& wget --quiet https://raw.githubusercontent.com/vishnubob/wait-for-it/master/wait-for-it.sh -O /usr/sbin/wait-for-it.sh \
&& chmod +x /usr/sbin/wait-for-it.sh
Expand Down
3 changes: 2 additions & 1 deletion examples/converters/kitti/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
"version": "1.1.0",
"description": "Provides a minimal set of tools for working with the KITTI dataset in Javascript",
"scripts": {
"start": "node ./src/index.js"
"start": "node ./src/index.js",
"start-debug": "node --inspect-brk ./src/index.js"
},
"dependencies": {
"argparse": "^1.0.10",
Expand Down
2 changes: 0 additions & 2 deletions modules/builder/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@
"dist"
],
"dependencies": {
"@loaders.gl/gltf": "~1.1.7",
"@loaders.gl/polyfills": "~1.1.7",
"@turf/turf": "^5.1.6",
"@xviz/io": "1.0.5",
"base64-js": "^1.3.0",
Expand Down
5 changes: 1 addition & 4 deletions modules/builder/src/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,13 +15,10 @@
// UTILS
export {loadUri} from './utils/load-uri.js';
export {flattenToTypedArray} from './utils/flatten.js';
// This is used in @xviz/io and will eventually be moved there. Prefix with underscore
// to mark this as a private export
export {packBinaryJson as _packBinaryJson} from './writers/xviz-writer/xviz-pack-binary';

// WRITERS
export {default as XVIZWriter} from './writers/xviz-writer/xviz-writer';
export {encodeBinaryXVIZ} from './writers/xviz-writer/xviz-binary-writer';
export {encodeBinaryXVIZ} from '@xviz/io';

// BUILDERS
export {default as XVIZBuilder} from './builders/xviz-builder';
Expand Down
25 changes: 1 addition & 24 deletions modules/builder/src/writers/xviz-writer/xviz-binary-writer.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,30 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.

// TODO: remove this code, it duplicates io/src/writers/xviz-binary-writer.js

export const XVIZ_GLTF_EXTENSION = 'AVS_xviz'; // copied from @xviz/parser

import '@loaders.gl/polyfills';
import {GLTFBuilder} from '@loaders.gl/gltf';
import {packBinaryJson} from './xviz-pack-binary';

export function encodeBinaryXVIZ(xvizJson, options) {
const gltfBuilder = new GLTFBuilder(options);

// Pack appropriate large data elements (point clouds and images) in binary
const packedData = packBinaryJson(xvizJson, gltfBuilder, null, options);

// As permitted by glTF, we put all XVIZ data in a top-level subfield.
const {useAVSXVIZExtension} = options;
if (useAVSXVIZExtension === true) {
gltfBuilder.addExtension(XVIZ_GLTF_EXTENSION, packedData, {nopack: true});
} else {
gltfBuilder.addApplicationData('xviz', packedData, {nopack: true});
}

return gltfBuilder.encodeAsGLB(options);
}
import {encodeBinaryXVIZ} from '@xviz/io';

export function writeBinaryXVIZtoFile(sink, directory, name, json, options) {
const glbFileBuffer = encodeBinaryXVIZ(json, options);
Expand Down
90 changes: 0 additions & 90 deletions modules/builder/src/writers/xviz-writer/xviz-pack-binary.js

This file was deleted.

6 changes: 2 additions & 4 deletions modules/io/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -15,20 +15,18 @@
"node.js"
],
"dependencies": {
"@loaders.gl/gltf": "^1.3.3",
"@loaders.gl/polyfills": "^1.3.3",
"@turf/turf": "^5.1.6",
"@xviz/schema": "1.0.5",
"base64-js": "^1.3.0",
"math.gl": "^3.0.0",
"protobufjs": "https://github.com/jlisee/protobuf.js#ed8e699671da6f9a169074aa0f2a160cefe0844a",
"text-encoding": "^0.6.4"
"fast-text-encoding": "^1.0.3"
},
"scripts": {},
"browser": {
"./src/io/fs-sink.js": false,
"./src/io/fs-source.js": false,
"text-encoding": false
"fast-text-encoding": false
},
"engines": {
"node": ">= 10.16",
Expand Down
2 changes: 1 addition & 1 deletion modules/io/src/common/loaders.js
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
// See the License for the specific language governing permissions and
// limitations under the License.
/* global Buffer */
import {GLTFParser} from '@loaders.gl/gltf';
import {GLTFParser} from '../gltf/gltf-parser';

import {XVIZ_GLTF_EXTENSION} from './constants';
import {TextDecoder} from './text-encoding';
Expand Down
6 changes: 3 additions & 3 deletions modules/io/src/common/text-encoding.js
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
/* global TextEncoder,TextDecoder */

if (typeof TextDecoder === 'undefined') {
module.exports = require('text-encoding');
} else {
module.exports = {TextEncoder, TextDecoder};
require('fast-text-encoding');
}

module.exports = {TextEncoder, TextDecoder};
21 changes: 21 additions & 0 deletions modules/io/src/gltf/assert.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
// Copyright (c) 2019 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Replacement for the external assert method to reduce bundle size
// Note: We don't use the second "message" argument in calling code,
// so no need to support it here
export function assert(condition, message) {
if (!condition) {
throw new Error(message || 'gltf/glb assertion failed.');
}
}
88 changes: 88 additions & 0 deletions modules/io/src/gltf/encode-glb.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,88 @@
// Copyright (c) 2019 Uber Technologies, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
/* eslint-disable camelcase, max-statements */
import {
copyPaddedStringToDataView,
copyPaddedArrayBufferToDataView
} from '@loaders.gl/loader-utils';

const MAGIC_glTF = 0x46546c67; // glTF in ASCII
const MAGIC_JSON = 0x4e4f534a; // JSON in ASCII
const MAGIC_BIN = 0x004e4942; // BIN\0 in ASCII

const LE = true; // Binary GLTF is little endian.

// Encode the full GLB buffer with header etc
// https://github.com/KhronosGroup/glTF/tree/master/specification/2.0#
// glb-file-format-specification
export default function encodeGLBSync(glb, dataView, byteOffset = 0, options = {}) {
const {magic = MAGIC_glTF, version = 2, json = {}, binary} = glb;

const byteOffsetStart = byteOffset;

// Write GLB Header
if (dataView) {
dataView.setUint32(byteOffset + 0, magic, LE); // Magic number (the ASCII string 'glTF').
dataView.setUint32(byteOffset + 4, version, LE); // Version 2 of binary glTF container format uint32
dataView.setUint32(byteOffset + 8, 0, LE); // Total byte length of generated file (uint32), will be set last
}
const byteOffsetFileLength = byteOffset + 8;
byteOffset += 12; // GLB_FILE_HEADER_SIZE

// Write the JSON chunk header
const byteOffsetJsonHeader = byteOffset;
if (dataView) {
dataView.setUint32(byteOffset + 0, 0, LE); // Byte length of json chunk (will be written later)
dataView.setUint32(byteOffset + 4, MAGIC_JSON, LE); // Chunk type
}
byteOffset += 8; // GLB_CHUNK_HEADER_SIZE

// Write the JSON chunk
const jsonString = JSON.stringify(json);
byteOffset = copyPaddedStringToDataView(dataView, byteOffset, jsonString, 4);

// Now we know the JSON chunk length so we can write it.
if (dataView) {
const jsonByteLength = byteOffset - byteOffsetJsonHeader - 8; // GLB_CHUNK_HEADER_SIZE
dataView.setUint32(byteOffsetJsonHeader + 0, jsonByteLength, LE); // Byte length of json chunk (uint32)
}

// Write the BIN chunk if present. The BIN chunk is optional.
if (binary) {
const byteOffsetBinHeader = byteOffset;

// Write the BIN chunk header
if (dataView) {
dataView.setUint32(byteOffset + 0, 0, LE); // Byte length BIN (uint32)
dataView.setUint32(byteOffset + 4, MAGIC_BIN, LE); // Chunk type
}
byteOffset += 8; // GLB_CHUNK_HEADER_SIZE

byteOffset = copyPaddedArrayBufferToDataView(dataView, byteOffset, binary, 4);

// Now we know the BIN chunk length so we can write it.
if (dataView) {
const binByteLength = byteOffset - byteOffsetBinHeader - 8; // GLB_CHUNK_HEADER_SIZE
dataView.setUint32(byteOffsetBinHeader + 0, binByteLength, LE); // Byte length BIN (uint32)
}
}

// Now we know the glb file length so we can write it.
if (dataView) {
const fileByteLength = byteOffset - byteOffsetStart;
dataView.setUint32(byteOffsetFileLength, fileByteLength, LE); // Total byte length of generated file (uint32)
}

return byteOffset;
}

0 comments on commit 885a682

Please sign in to comment.