Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Feature write uint8 uint16 uint32 float32 typedarrays #366

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

Feature write uint8 uint16 uint32 float32 typedarrays #366

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
c9855c7
add write support for various typed arrays
daumann Jun 13, 2023
4caed19
fix indentation
daumann Jun 13, 2023
711ec5c
fix indentation p2
daumann Jun 13, 2023
5072abf
fix indentation p3
daumann Jun 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
71 changes: 62 additions & 9 deletions src/geotiffwriter.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,8 @@
https://github.com/photopea/UTIF.js/blob/master/LICENSE
*/
import { fieldTagNames, fieldTagTypes, fieldTypeNames, geoKeyNames } from './globals.js';
import { assign, endsWith, forEach, invert, times } from './utils.js';
import { assign, endsWith, forEach, invert, times, typeMap,
isTypedUintArray, isTypedIntArray, isTypedFloatArray } from './utils.js';

const tagName2Code = invert(fieldTagNames);
const geoKeyName2Code = invert(geoKeyNames);
Expand Down Expand Up @@ -251,17 +252,47 @@ const encodeImage = (values, width, height, metadata) => {
}

const prfx = new Uint8Array(encodeIfds([ifd]));
const samplesPerPixel = ifd[277];

const img = new Uint8Array(values);
const dataType = values.constructor.name;
const TypedArray = typeMap[dataType];

const samplesPerPixel = ifd[277];
let elementSize = 4;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could you add a comment describing why you set the default to 4?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

could we consider defaulting elementSize to 8, so we can support all floating point numbers by default?

if (TypedArray) {
elementSize = TypedArray.BYTES_PER_ELEMENT;
}

const data = new Uint8Array(numBytesInIfd + (values.length * elementSize * samplesPerPixel));

const data = new Uint8Array(numBytesInIfd + (width * height * samplesPerPixel));
times(prfx.length, (i) => {
data[i] = prfx[i];
});
forEach(img, (value, i) => {
data[numBytesInIfd + i] = value;

forEach(values, (value, i) => {
if (!TypedArray) {
data[numBytesInIfd + i] = value;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think this would only work if the Array of values is within the 0-255 range. Is there any way we could update this to support Float64 by default? Could we do view.setFloat64 to support any floating point number?

Open to your thoughts!

return;
}

const buffer = new ArrayBuffer(elementSize);
const view = new DataView(buffer);

if (dataType === 'Float32Array') {
view.setFloat32(0, value, false);
} else if (dataType === 'Uint32Array') {
view.setUint32(0, value, false);
} else if (dataType === 'Uint16Array') {
view.setUint16(0, value, false);
} else if (dataType === 'Uint8Array') {
view.setUint8(0, value);
}

const typedArray = new Uint8Array(view.buffer);
const idx = numBytesInIfd + (i * elementSize);

for (let j = 0; j < elementSize; j++) {
data[idx + j] = typedArray[j];
}
});

return data.buffer;
Expand Down Expand Up @@ -328,7 +359,11 @@ export function writeGeotiff(data, metadata) {
// consult https://www.loc.gov/preservation/digital/formats/content/tiff_tags.shtml

if (!metadata.BitsPerSample) {
metadata.BitsPerSample = times(numBands, () => 8);
let bitsPerSample = 8;
if (ArrayBuffer.isView(flattenedValues)) {
bitsPerSample = 8 * flattenedValues.BYTES_PER_ELEMENT;
}
metadata.BitsPerSample = times(numBands, () => bitsPerSample);
}

metadataDefaults.forEach((tag) => {
Expand All @@ -352,7 +387,15 @@ export function writeGeotiff(data, metadata) {

if (!metadata.StripByteCounts) {
// we are only writing one strip
metadata.StripByteCounts = [numBands * height * width];

// default for Uint8
let elementSize = 1;
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm wondering if we want to change the default to 8 for Float64 instead of Uint8? Ideally, the writer will work for the greatest number of use cases by default. If someone wants Uint8, they could pass in a Uint8Array of values instead of an Array.


if (ArrayBuffer.isView(flattenedValues)) {
elementSize = flattenedValues.BYTES_PER_ELEMENT;
}

metadata.StripByteCounts = [numBands * elementSize * height * width];
}

if (!metadata.ModelPixelScale) {
Expand All @@ -361,7 +404,17 @@ export function writeGeotiff(data, metadata) {
}

if (!metadata.SampleFormat) {
metadata.SampleFormat = times(numBands, () => 1);
let sampleFormat = 1;
if (isTypedFloatArray(flattenedValues)) {
sampleFormat = 3;
}
if (isTypedIntArray(flattenedValues)) {
sampleFormat = 2;
}
if (isTypedUintArray(flattenedValues)) {
sampleFormat = 1;
}
metadata.SampleFormat = times(numBands, () => sampleFormat);
}

// if didn't pass in projection information, assume the popular 4326 "geographic projection"
Expand Down
37 changes: 37 additions & 0 deletions src/utils.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,3 +156,40 @@ export class CustomAggregateError extends Error {
}

export const AggregateError = CustomAggregateError;

export function isTypedFloatArray(input) {
if (ArrayBuffer.isView(input)) {
const ctr = input.constructor;
if (ctr === Float32Array || ctr === Float64Array) {
return true;
}
}
return false;
}

export function isTypedIntArray(input) {
if (ArrayBuffer.isView(input)) {
const ctr = input.constructor;
if (ctr === Int8Array || ctr === Int16Array || ctr === Int32Array) {
return true;
}
}
return false;
}

export function isTypedUintArray(input) {
if (ArrayBuffer.isView(input)) {
const ctr = input.constructor;
if (ctr === Uint8Array || ctr === Uint16Array || ctr === Uint32Array || ctr === Uint8ClampedArray) {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

good catch also including the clamped version of Uint8!

return true;
}
}
return false;
}

export const typeMap = {
Float32Array,
Uint32Array,
Uint16Array,
Uint8Array,
};
49 changes: 49 additions & 0 deletions test/geotiff.spec.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -78,6 +78,20 @@ function normalize(input) {
return JSON.stringify(toArrayRecursively(input));
}

function generateTestDataArray(min, max, length, onlyWholeNumbers) {
const data = [];

for (let i = 0; i < length; i++) {
let randomValue = (Math.random() * (max - min + 1)) + min;
if (onlyWholeNumbers) {
randomValue = Math.floor(randomValue);
}
data.push(randomValue);
}

return data;
}

function getMockMetaData(height, width) {
return {
ImageWidth: width, // only necessary if values aren't multi-dimensional
Expand All @@ -103,6 +117,41 @@ function getMockMetaData(height, width) {
};
}

describe('writeTypedArrays', () => {
const dataLength = 512 * 512 * 4;

const variousDataTypeExamples = [
generateTestDataArray(0, 255, dataLength, true),
new Uint8Array(generateTestDataArray(0, 255, dataLength, true)),
new Uint16Array(generateTestDataArray(0, 65535, dataLength, true)),
new Uint32Array(generateTestDataArray(0, 4294967295, dataLength, true)),
new Float32Array(generateTestDataArray(-3.4e+38, 3.4e+38, dataLength, false)),
];

const height = Math.sqrt(dataLength);
const width = Math.sqrt(dataLength);

for (let s = 0; s < variousDataTypeExamples.length; ++s) {
const originalValues = variousDataTypeExamples[s];
const dataType = originalValues.constructor.name;

it(`should write ${dataType}`, async () => {
const metadata = {
height,
width,
};

const newGeoTiffAsBinaryData = await writeArrayBuffer((originalValues), metadata);
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you have an extra parens wrapping originalValues

const newGeoTiff = await fromArrayBuffer(newGeoTiffAsBinaryData);
const image = await newGeoTiff.getImage();
const newValues = await image.readRasters();
const valueArray = toArrayRecursively(newValues)[0];
const originalValueArray = Array.from(originalValues);
expect(valueArray).to.be.deep.equal(originalValueArray);
});
}
});

describe('GeoTIFF - external overviews', () => {
it('Can load', async () => {
const tiff = await fromUrls(
Expand Down