From c9855c7eae003a9fb591d92d16e5c1e40cd9f1fd Mon Sep 17 00:00:00 2001 From: daumann Date: Tue, 13 Jun 2023 12:04:09 +0200 Subject: [PATCH 1/4] add write support for various typed arrays --- src/geotiffwriter.js | 75 +++++++++++++++++++++++++++++++------ src/utils.js | 37 ++++++++++++++++++ test/geotiff.spec.js | 89 ++++++++++++++++++++++++++++++++++---------- 3 files changed, 170 insertions(+), 31 deletions(-) diff --git a/src/geotiffwriter.js b/src/geotiffwriter.js index 8cf2aba9..38ad2cab 100644 --- a/src/geotiffwriter.js +++ b/src/geotiffwriter.js @@ -5,7 +5,8 @@ https://github.com/photopea/UTIF.js/blob/master/LICENSE */ import { fieldTagNames, fieldTagTypes, fieldTypeNames, geoKeyNames } from './globals.js'; -import { assign, endsWith, forEach, invert, times } from './utils.js'; +import { assign, endsWith, forEach, invert, times, typeMap, + isTypedUintArray, isTypedIntArray, isTypedFloatArray } from './utils.js'; const tagName2Code = invert(fieldTagNames); const geoKeyName2Code = invert(geoKeyNames); @@ -251,17 +252,47 @@ const encodeImage = (values, width, height, metadata) => { } const prfx = new Uint8Array(encodeIfds([ifd])); + const samplesPerPixel = ifd[277]; - const img = new Uint8Array(values); + const dataType = values.constructor.name; + const TypedArray = typeMap[dataType]; - const samplesPerPixel = ifd[277]; + let elementSize = 4; + if (TypedArray) { + elementSize = TypedArray.BYTES_PER_ELEMENT; + } + + const data = new Uint8Array(numBytesInIfd + (values.length * elementSize * samplesPerPixel)); - const data = new Uint8Array(numBytesInIfd + (width * height * samplesPerPixel)); times(prfx.length, (i) => { data[i] = prfx[i]; }); - forEach(img, (value, i) => { - data[numBytesInIfd + i] = value; + + forEach(values, (value, i) => { + if (!TypedArray) { + data[numBytesInIfd + i] = value; + return; + } + + const buffer = new ArrayBuffer(elementSize); + const view = new DataView(buffer); + + if (dataType === 'Float32Array') { + view.setFloat32(0, value, false); + } else if (dataType === 'Uint32Array') { + view.setUint32(0, value, false); + } else if (dataType === 'Uint16Array') { + view.setUint16(0, value, false); + } else if (dataType === 'Uint8Array') { + view.setUint8(0, value); + } + + const typedArray = new Uint8Array(view.buffer); + const idx = numBytesInIfd + (i * elementSize); + + for (let j = 0; j < elementSize; j++) { + data[idx + j] = typedArray[j]; + } }); return data.buffer; @@ -328,7 +359,11 @@ export function writeGeotiff(data, metadata) { // consult https://www.loc.gov/preservation/digital/formats/content/tiff_tags.shtml if (!metadata.BitsPerSample) { - metadata.BitsPerSample = times(numBands, () => 8); + let bitsPerSample = 8; + if (ArrayBuffer.isView(flattenedValues)) { + bitsPerSample = 8 * flattenedValues.BYTES_PER_ELEMENT; + } + metadata.BitsPerSample = times(numBands, () => bitsPerSample); } metadataDefaults.forEach((tag) => { @@ -352,7 +387,15 @@ export function writeGeotiff(data, metadata) { if (!metadata.StripByteCounts) { // we are only writing one strip - metadata.StripByteCounts = [numBands * height * width]; + + // default for Uint8 + let elementSize = 1; + + if (ArrayBuffer.isView(flattenedValues)) { + elementSize = flattenedValues.BYTES_PER_ELEMENT; + } + + metadata.StripByteCounts = [numBands * elementSize * height * width]; } if (!metadata.ModelPixelScale) { @@ -361,7 +404,17 @@ export function writeGeotiff(data, metadata) { } if (!metadata.SampleFormat) { - metadata.SampleFormat = times(numBands, () => 1); + let sampleFormat = 1; + if (isTypedFloatArray(flattenedValues)) { + sampleFormat = 3; + } + if (isTypedIntArray(flattenedValues)) { + sampleFormat = 2; + } + if (isTypedUintArray(flattenedValues)) { + sampleFormat = 1; + } + metadata.SampleFormat = times(numBands, () => sampleFormat); } // if didn't pass in projection information, assume the popular 4326 "geographic projection" @@ -373,8 +426,8 @@ export function writeGeotiff(data, metadata) { } const geoKeys = Object.keys(metadata) - .filter((key) => endsWith(key, 'GeoKey')) - .sort((a, b) => name2code[a] - name2code[b]); + .filter((key) => endsWith(key, 'GeoKey')) + .sort((a, b) => name2code[a] - name2code[b]); if (!metadata.GeoAsciiParams) { let geoAsciiParams = ''; diff --git a/src/utils.js b/src/utils.js index 834531a2..6a78a10e 100644 --- a/src/utils.js +++ b/src/utils.js @@ -156,3 +156,40 @@ export class CustomAggregateError extends Error { } export const AggregateError = CustomAggregateError; + +export function isTypedFloatArray(input) { + if (ArrayBuffer.isView(input)) { + const ctr = input.constructor; + if (ctr === Float32Array || ctr === Float64Array) { + return true; + } + } + return false; +} + +export function isTypedIntArray(input) { + if (ArrayBuffer.isView(input)) { + const ctr = input.constructor; + if (ctr === Int8Array || ctr === Int16Array || ctr === Int32Array) { + return true; + } + } + return false; +} + +export function isTypedUintArray(input) { + if (ArrayBuffer.isView(input)) { + const ctr = input.constructor; + if (ctr === Uint8Array || ctr === Uint16Array || ctr === Uint32Array || ctr === Uint8ClampedArray) { + return true; + } + } + return false; +} + +export const typeMap = { + Float32Array, + Uint32Array, + Uint16Array, + Uint8Array, +}; diff --git a/test/geotiff.spec.js b/test/geotiff.spec.js index 641bf2b1..090f07b9 100644 --- a/test/geotiff.spec.js +++ b/test/geotiff.spec.js @@ -78,6 +78,20 @@ function normalize(input) { return JSON.stringify(toArrayRecursively(input)); } +function generateTestDataArray(min, max, length, onlyWholeNumbers) { + const data = []; + + for (let i = 0; i < length; i++) { + let randomValue = (Math.random() * (max - min + 1)) + min; + if (onlyWholeNumbers) { + randomValue = Math.floor(randomValue); + } + data.push(randomValue); + } + + return data; +} + function getMockMetaData(height, width) { return { ImageWidth: width, // only necessary if values aren't multi-dimensional @@ -103,11 +117,46 @@ function getMockMetaData(height, width) { }; } +describe('writeTypedArrays', () => { + const dataLength = 512 * 512 * 4; + + const variousDataTypeExamples = [ + generateTestDataArray(0, 255, dataLength, true), + new Uint8Array(generateTestDataArray(0, 255, dataLength, true)), + new Uint16Array(generateTestDataArray(0, 65535, dataLength, true)), + new Uint32Array(generateTestDataArray(0, 4294967295, dataLength, true)), + new Float32Array(generateTestDataArray(-3.4e+38, 3.4e+38, dataLength, false)), + ]; + + const height = Math.sqrt(dataLength); + const width = Math.sqrt(dataLength); + + for (let s = 0; s < variousDataTypeExamples.length; ++s) { + const originalValues = variousDataTypeExamples[s]; + const dataType = originalValues.constructor.name; + + it(`should write ${dataType}`, async () => { + const metadata = { + height, + width, + }; + + const newGeoTiffAsBinaryData = await writeArrayBuffer((originalValues), metadata); + const newGeoTiff = await fromArrayBuffer(newGeoTiffAsBinaryData); + const image = await newGeoTiff.getImage(); + const newValues = await image.readRasters(); + const valueArray = toArrayRecursively(newValues)[0]; + const originalValueArray = Array.from(originalValues); + expect(valueArray).to.be.deep.equal(originalValueArray); + }); + } +}); + describe('GeoTIFF - external overviews', () => { it('Can load', async () => { const tiff = await fromUrls( - 'http://localhost:3000/data/overviews_external.tiff', - ['http://localhost:3000/data/overviews_external.tiff.ovr'], + 'http://localhost:3000/data/overviews_external.tiff', + ['http://localhost:3000/data/overviews_external.tiff.ovr'], ); const count = await tiff.getImageCount(); expect(count).to.equal(5); @@ -422,7 +471,7 @@ describe('ifdRequestTests', () => { await tiff.getImage(index + 1); // first image slot is empty so we filter out the Promises, of which there are two expect( - tiff.ifdRequests.filter((ifdRequest) => ifdRequest instanceof Promise).length, + tiff.ifdRequests.filter((ifdRequest) => ifdRequest instanceof Promise).length, ).to.equal(2); }); @@ -550,7 +599,7 @@ describe('Geo metadata tests', async () => { describe('GDAL_METADATA tests', async () => { it('should parse stats for specific sample', async () => { const tiff = await GeoTIFF.fromSource( - createSource('abetow-ERD2018-EBIRD_SCIENCE-20191109-a5cf4cb2_hr_2018_abundance_median.tiff'), + createSource('abetow-ERD2018-EBIRD_SCIENCE-20191109-a5cf4cb2_hr_2018_abundance_median.tiff'), ); const image = await tiff.getImage(); const metadata = await image.getGDALMetadata(10); @@ -841,10 +890,10 @@ describe('64 bit tests', () => { 0x00, ]); const littleEndianSlice = new DataSlice( - littleEndianBytes.buffer, - 0, - true, - true, + littleEndianBytes.buffer, + 0, + true, + true, ); const bigEndianBytes = new Uint8Array([ // ((2 ** 52) - 1) @@ -881,10 +930,10 @@ describe('64 bit tests', () => { 0xff, ]); const littleEndianSlice = new DataSlice( - littleEndianBytes.buffer, - 0, - true, - true, + littleEndianBytes.buffer, + 0, + true, + true, ); const bigEndianBytes = new Uint8Array([ // -(2 ** 32 - 1) @@ -932,10 +981,10 @@ describe('64 bit tests', () => { 0xff, ]); const littleEndianSlice = new DataSlice( - littleEndianBytes.buffer, - 0, - true, - true, + littleEndianBytes.buffer, + 0, + true, + true, ); const bigEndianBytes = new Uint8Array([ // ((2 ** 53) - 1) @@ -988,7 +1037,7 @@ describe('writeTests', () => { const rasters = await image.readRasters(); const newValues = toArrayRecursively(rasters[0]); expect( - JSON.stringify(newValues.slice(0, -1)), + JSON.stringify(newValues.slice(0, -1)), ).to.equal(JSON.stringify(originalValues.slice(0, -1))); const geoKeys = image.getGeoKeys(); @@ -1082,7 +1131,7 @@ describe('writeTests', () => { const rasters = await image.readRasters(); const newValues = toArrayRecursively(rasters[0]); expect( - JSON.stringify(newValues.slice(0, -1)), + JSON.stringify(newValues.slice(0, -1)), ).to.equal(JSON.stringify(originalValues.slice(0, -1))); }); @@ -1106,7 +1155,7 @@ describe('writeTests', () => { return chunk(band, width); }); expect( - JSON.stringify(newValuesReshaped.slice(0, -1)), + JSON.stringify(newValuesReshaped.slice(0, -1)), ).to.equal(JSON.stringify(originalValues.slice(0, -1))); }); @@ -1122,7 +1171,7 @@ describe('writeTests', () => { const rasters = await image.readRasters(); const newValues = toArrayRecursively(rasters[0]); expect( - JSON.stringify(newValues.slice(0, -1)), + JSON.stringify(newValues.slice(0, -1)), ).to.equal(JSON.stringify(originalValues.slice(0, -1))); const { fileDirectory } = image; From 4caed19a47ba89457f275ca40313738376920c6e Mon Sep 17 00:00:00 2001 From: daumann Date: Tue, 13 Jun 2023 12:10:00 +0200 Subject: [PATCH 2/4] fix indentation --- test/geotiff.spec.js | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/test/geotiff.spec.js b/test/geotiff.spec.js index 090f07b9..bb9bd869 100644 --- a/test/geotiff.spec.js +++ b/test/geotiff.spec.js @@ -155,8 +155,8 @@ describe('writeTypedArrays', () => { describe('GeoTIFF - external overviews', () => { it('Can load', async () => { const tiff = await fromUrls( - 'http://localhost:3000/data/overviews_external.tiff', - ['http://localhost:3000/data/overviews_external.tiff.ovr'], + 'http://localhost:3000/data/overviews_external.tiff', + ['http://localhost:3000/data/overviews_external.tiff.ovr'], ); const count = await tiff.getImageCount(); expect(count).to.equal(5); @@ -471,7 +471,7 @@ describe('ifdRequestTests', () => { await tiff.getImage(index + 1); // first image slot is empty so we filter out the Promises, of which there are two expect( - tiff.ifdRequests.filter((ifdRequest) => ifdRequest instanceof Promise).length, + tiff.ifdRequests.filter((ifdRequest) => ifdRequest instanceof Promise).length, ).to.equal(2); }); @@ -599,7 +599,7 @@ describe('Geo metadata tests', async () => { describe('GDAL_METADATA tests', async () => { it('should parse stats for specific sample', async () => { const tiff = await GeoTIFF.fromSource( - createSource('abetow-ERD2018-EBIRD_SCIENCE-20191109-a5cf4cb2_hr_2018_abundance_median.tiff'), + createSource('abetow-ERD2018-EBIRD_SCIENCE-20191109-a5cf4cb2_hr_2018_abundance_median.tiff'), ); const image = await tiff.getImage(); const metadata = await image.getGDALMetadata(10); @@ -890,10 +890,10 @@ describe('64 bit tests', () => { 0x00, ]); const littleEndianSlice = new DataSlice( - littleEndianBytes.buffer, - 0, - true, - true, + littleEndianBytes.buffer, + 0, + true, + true, ); const bigEndianBytes = new Uint8Array([ // ((2 ** 52) - 1) @@ -930,10 +930,10 @@ describe('64 bit tests', () => { 0xff, ]); const littleEndianSlice = new DataSlice( - littleEndianBytes.buffer, - 0, - true, - true, + littleEndianBytes.buffer, + 0, + true, + true, ); const bigEndianBytes = new Uint8Array([ // -(2 ** 32 - 1) @@ -1037,7 +1037,7 @@ describe('writeTests', () => { const rasters = await image.readRasters(); const newValues = toArrayRecursively(rasters[0]); expect( - JSON.stringify(newValues.slice(0, -1)), + JSON.stringify(newValues.slice(0, -1)), ).to.equal(JSON.stringify(originalValues.slice(0, -1))); const geoKeys = image.getGeoKeys(); @@ -1131,7 +1131,7 @@ describe('writeTests', () => { const rasters = await image.readRasters(); const newValues = toArrayRecursively(rasters[0]); expect( - JSON.stringify(newValues.slice(0, -1)), + JSON.stringify(newValues.slice(0, -1)), ).to.equal(JSON.stringify(originalValues.slice(0, -1))); }); @@ -1155,7 +1155,7 @@ describe('writeTests', () => { return chunk(band, width); }); expect( - JSON.stringify(newValuesReshaped.slice(0, -1)), + JSON.stringify(newValuesReshaped.slice(0, -1)), ).to.equal(JSON.stringify(originalValues.slice(0, -1))); }); @@ -1171,7 +1171,7 @@ describe('writeTests', () => { const rasters = await image.readRasters(); const newValues = toArrayRecursively(rasters[0]); expect( - JSON.stringify(newValues.slice(0, -1)), + JSON.stringify(newValues.slice(0, -1)), ).to.equal(JSON.stringify(originalValues.slice(0, -1))); const { fileDirectory } = image; From 711ec5c9ea191cbb8e571b4d59738e162d23695d Mon Sep 17 00:00:00 2001 From: daumann Date: Tue, 13 Jun 2023 12:10:43 +0200 Subject: [PATCH 3/4] fix indentation p2 --- test/geotiff.spec.js | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/geotiff.spec.js b/test/geotiff.spec.js index bb9bd869..0c923b57 100644 --- a/test/geotiff.spec.js +++ b/test/geotiff.spec.js @@ -981,10 +981,10 @@ describe('64 bit tests', () => { 0xff, ]); const littleEndianSlice = new DataSlice( - littleEndianBytes.buffer, - 0, - true, - true, + littleEndianBytes.buffer, + 0, + true, + true, ); const bigEndianBytes = new Uint8Array([ // ((2 ** 53) - 1) From 5072abff1c23ff13199b89fdf6b102b175188143 Mon Sep 17 00:00:00 2001 From: daumann Date: Tue, 13 Jun 2023 12:12:22 +0200 Subject: [PATCH 4/4] fix indentation p3 --- src/geotiffwriter.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/geotiffwriter.js b/src/geotiffwriter.js index 38ad2cab..453c61f7 100644 --- a/src/geotiffwriter.js +++ b/src/geotiffwriter.js @@ -426,8 +426,8 @@ export function writeGeotiff(data, metadata) { } const geoKeys = Object.keys(metadata) - .filter((key) => endsWith(key, 'GeoKey')) - .sort((a, b) => name2code[a] - name2code[b]); + .filter((key) => endsWith(key, 'GeoKey')) + .sort((a, b) => name2code[a] - name2code[b]); if (!metadata.GeoAsciiParams) { let geoAsciiParams = '';