Merge pull request #483 from CesiumGS/duplicate-resources

Don't duplicate resources that are referenced multiple times

CHANGES.md

@@ -1,6 +1,10 @@
 Change Log
 ==========
 
+### 2.?.? - 2020-??-??
+
+* Fixed writing duplicate resource that are referenced multiple times. [#483](https://github.com/AnalyticalGraphicsInc/gltf-pipeline/pull/483)
+
 ### 2.1.8 - 2020-03-04
 
 * Fixed a bug in `processGltf` and `gltfToGlb` where the user's `options` object was getting modified with internal options. [#528](https://github.com/CesiumGS/gltf-pipeline/pull/528)

lib/readResources.js

@@ -64,27 +64,27 @@ function readResources(gltf, options) {
 }
 
 function readBuffer(gltf, buffer, options) {
-    return readResource(gltf, buffer, options)
+    return readResource(gltf, buffer, false, options)
         .then(function(data) {
             buffer.extras._pipeline.source = data;
         });
 }
 
 function readImage(gltf, image, options) {
-    return readResource(gltf, image, options)
+    return readResource(gltf, image, true, options)
         .then(function(data) {
             image.extras._pipeline.source = data;
         });
 }
 
 function readShader(gltf, shader, options) {
-    return readResource(gltf, shader, options)
+    return readResource(gltf, shader, true, options)
         .then(function(data) {
             shader.extras._pipeline.source = data.toString();
         });
 }
 
-function readResource(gltf, object, options) {
+function readResource(gltf, object, saveResourceId, options) {
     const uri = object.uri;
     delete object.uri; // Don't hold onto the uri, its contents will be stored in extras._pipeline.source
 
@@ -98,27 +98,30 @@ function readResource(gltf, object, options) {
     if (defined(extensions)) {
         const khrBinaryGltf = extensions.KHR_binary_glTF;
         if (defined(khrBinaryGltf)) {
-            return Promise.resolve(readBufferView(gltf, khrBinaryGltf.bufferView));
+            return Promise.resolve(readBufferView(gltf, khrBinaryGltf.bufferView, object, saveResourceId));
         }
     }
     if (defined(object.bufferView)) {
-        return Promise.resolve(readBufferView(gltf, object.bufferView));
+        return Promise.resolve(readBufferView(gltf, object.bufferView, object, saveResourceId));
     }
     if (isDataUri(uri)) {
         return Promise.resolve(dataUriToBuffer(uri));
     }
-    return readFile(object, uri, options);
+    return readFile(object, uri, saveResourceId, options);
 }
 
-function readBufferView(gltf, bufferViewId) {
+function readBufferView(gltf, bufferViewId, object, saveResourceId) {
+    if (saveResourceId) {
+        object.extras._pipeline.resourceId = bufferViewId;
+    }
     const bufferView = gltf.bufferViews[bufferViewId];
     const buffer = gltf.buffers[bufferView.buffer];
     const source = buffer.extras._pipeline.source;
     const byteOffset = defaultValue(bufferView.byteOffset, 0);
     return source.slice(byteOffset, byteOffset + bufferView.byteLength);
 }
 
-function readFile(object, uri, options) {
+function readFile(object, uri, saveResourceId, options) {
     const resourceDirectory = options.resourceDirectory;
     const hasResourceDirectory = defined(resourceDirectory);
 
@@ -140,6 +143,10 @@ function readFile(object, uri, options) {
         object.name = path.basename(relativePath, extension);
     }
 
+    if (saveResourceId) {
+        object.extras._pipeline.resourceId = absolutePath;
+    }
+
     object.extras._pipeline.absolutePath = absolutePath;
     object.extras._pipeline.relativePath = relativePath;
     return fsExtra.readFile(absolutePath);

lib/writeResources.js

@@ -46,32 +46,35 @@ function writeResources(gltf, options) {
     options.separateShaders = defaultValue(options.separateShaders, false);
     options.dataUris = defaultValue(options.dataUris, false);
 
+    // Remember which of the resources have been written, so we can re-use them.
+    const writtenResourceMap = {};
+
     ForEach.image(gltf, function(image, i) {
-        writeImage(gltf, image, i, options);
+        writeImage(gltf, image, i, writtenResourceMap, options);
         ForEach.compressedImage(image, function(compressedImage) {
-            writeImage(gltf, compressedImage, i, options);
+            writeImage(gltf, compressedImage, i, writtenResourceMap, options);
         });
     });
 
     ForEach.shader(gltf, function(shader, i) {
-        writeShader(gltf, shader, i, options);
+        writeShader(gltf, shader, i, writtenResourceMap, options);
     });
 
     // Buffers need to be written last because images and shaders may write to new buffers
-    removeUnusedElements(gltf, ['acessor', 'bufferView', 'buffer']);
+    removeUnusedElements(gltf, ['accessor', 'bufferView', 'buffer']);
     mergeBuffers(gltf, options.name);
 
     ForEach.buffer(gltf, function(buffer, bufferId) {
-        writeBuffer(gltf, buffer, bufferId, options);
+        writeBuffer(gltf, buffer, bufferId, writtenResourceMap, options);
     });
     return gltf;
 }
 
-function writeBuffer(gltf, buffer, i, options) {
+function writeBuffer(gltf, buffer, i, writtenResourceMap, options) {
     if (defined(options.bufferStorage) && !options.separateBuffers) {
         writeBufferStorage(buffer, options);
     } else {
-        writeResource(gltf, buffer, i, options.separateBuffers, true, '.bin', options);
+        writeResource(gltf, buffer, i, options.separateBuffers, true, '.bin', writtenResourceMap, options);
     }
 }
 
@@ -82,26 +85,26 @@ function writeBufferStorage(buffer, options) {
     options.bufferStorage.buffer = combinedBuffer;
 }
 
-function writeImage(gltf, image, i, options) {
+function writeImage(gltf, image, i, writtenResourceMap, options) {
     const extension = getImageExtension(image.extras._pipeline.source);
-    writeResource(gltf, image, i, options.separateTextures, options.dataUris, extension, options);
+    writeResource(gltf, image, i, options.separateTextures, options.dataUris, extension, writtenResourceMap, options);
     if (defined(image.bufferView)) {
         // Preserve the image mime type when writing to a buffer view
         image.mimeType = mime.getType(extension);
     }
 }
 
-function writeShader(gltf, shader, i, options) {
-    writeResource(gltf, shader, i, options.separateShaders, options.dataUris, '.glsl', options);
+function writeShader(gltf, shader, i, writtenResourceMap, options) {
+    writeResource(gltf, shader, i, options.separateShaders, options.dataUris, '.glsl', writtenResourceMap, options);
 }
 
-function writeResource(gltf, object, index, separate, dataUris, extension, options) {
+function writeResource(gltf, object, index, separate, dataUris, extension, writtenResourceMap, options) {
     if (separate) {
-        writeFile(gltf, object, index, extension, options);
+        writeFile(gltf, object, index, extension, writtenResourceMap, options);
     } else if (dataUris) {
         writeDataUri(object, extension);
     } else {
-        writeBufferView(gltf, object);
+        writeBufferView(gltf, object, writtenResourceMap);
     }
 }
 
@@ -112,13 +115,26 @@ function writeDataUri(object, extension) {
     object.uri = 'data:' + mimeType + ';base64,' + source.toString('base64');
 }
 
-function writeBufferView(gltf, object) {
+function writeBufferView(gltf, object, writtenResourceMap) {
     delete object.uri;
+
+    // If we've written this resource before, re-use the bufferView
+    const resourceId = object.extras._pipeline.resourceId;
+    if (defined(resourceId) && defined(writtenResourceMap[resourceId])) {
+        object.bufferView = writtenResourceMap[resourceId];
+        return;
+    }
+
     let source = object.extras._pipeline.source;
     if (typeof source === 'string') {
         source = Buffer.from(source);
     }
     object.bufferView = addBuffer(gltf, source);
+
+    // Save the bufferView so we can re-use it later
+    if (defined(resourceId)) {
+        writtenResourceMap[resourceId] = object.bufferView;
+    }
 }
 
 function getProgram(gltf, shaderIndex) {
@@ -183,12 +199,25 @@ function getRelativePath(gltf, object, index, extension, options) {
     return relativePath;
 }
 
-function writeFile(gltf, object, index, extension, options) {
+function writeFile(gltf, object, index, extension, writtenResourceMap, options) {
     delete object.bufferView;
+
+    // If we've written this resource before, re-use the uri
+    const resourceId = object.extras._pipeline.resourceId;
+    if (defined(resourceId) && defined(writtenResourceMap[resourceId])) {
+        object.uri = writtenResourceMap[resourceId];
+        return;
+    }
+
     const source = object.extras._pipeline.source;
     const relativePath = getRelativePath(gltf, object, index, extension, options);
     object.uri = relativePath;
     if (defined(options.separateResources)) {
         options.separateResources[relativePath] = source;
     }
+
+    // Save the uri so we can re-use it later
+    if (defined(resourceId)) {
+        writtenResourceMap[resourceId] = object.uri;
+    }
 }

specs/data/2.0/box-shared-image-references-separate/CesiumTexturedBoxTest0FS.glsl

@@ -0,0 +1,18 @@
+precision highp float;
+varying vec3 v_normal;
+varying vec2 v_texcoord0;
+uniform sampler2D u_diffuse;
+uniform vec4 u_specular;
+uniform float u_shininess;
+void main(void) {
+vec3 normal = normalize(v_normal);
+vec4 color = vec4(0., 0., 0., 0.);
+vec4 diffuse = vec4(0., 0., 0., 1.);
+vec4 specular;
+diffuse = texture2D(u_diffuse, v_texcoord0);
+specular = u_specular;
+diffuse.xyz *= max(dot(normal,vec3(0.,0.,1.)), 0.);
+color.xyz += diffuse.xyz;
+color = vec4(color.rgb * diffuse.a, diffuse.a);
+gl_FragColor = color;
+}

specs/data/2.0/box-shared-image-references-separate/CesiumTexturedBoxTest0VS.glsl

@@ -0,0 +1,15 @@
+precision highp float;
+attribute vec3 a_position;
+attribute vec3 a_normal;
+varying vec3 v_normal;
+uniform mat3 u_normalMatrix;
+uniform mat4 u_modelViewMatrix;
+uniform mat4 u_projectionMatrix;
+attribute vec2 a_texcoord0;
+varying vec2 v_texcoord0;
+void main(void) {
+vec4 pos = u_modelViewMatrix * vec4(a_position,1.0);
+v_normal = u_normalMatrix * a_normal;
+v_texcoord0 = a_texcoord0;
+gl_Position = u_projectionMatrix * pos;
+}