Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Check datasets shape and dtype #16

Merged
merged 5 commits into from
Oct 31, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion src/App.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import Modal from "svelte-simple-modal";

import { getJson } from "./utils";
import CheckMark from "./CheckMark.svelte";

const searchParams = new URLSearchParams(window.location.search);
let source = searchParams.get("source");
Expand Down Expand Up @@ -43,7 +44,8 @@
{/if}
</div>
{:catch error}
<p style="color: red">{error.message}</p>
<CheckMark valid={false}/>
<p style="color: red; margin: 20px 0">{error.message}</p>
{/await}
{:else}
<article>
Expand Down
33 changes: 33 additions & 0 deletions src/CheckMark.svelte
Original file line number Diff line number Diff line change
@@ -0,0 +1,33 @@
<script>
export let valid;
</script>

{#if valid}
<div class="valid">✓</div>
{:else}
<div class="invalid">×</div>
{/if}

<style>
.invalid,
.valid {
border-radius: 50%;
padding: 10px;
margin: 10px auto;
color: white;
width: 100px;
height: 100px;
font-size: 50px;
text-align: center;
padding: 15px;
}
.invalid {
background-color: red;
border: solid red 1px;
}

.valid {
background-color: green;
border: solid green 1px;
}
</style>
4 changes: 2 additions & 2 deletions src/JsonBrowser/JsonKeyLink.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -33,9 +33,9 @@
}
}

const version_urls = all_urls[version];
const version_urls = all_urls[version] || {};

let url;
let url = "";

if (version_urls[name]) {
url = `https://ngff.openmicroscopy.org/${version}/index.html#${version_urls[name]}`;
Expand Down
84 changes: 84 additions & 0 deletions src/JsonValidator/MultiscaleArrays/Multiscale.svelte
Original file line number Diff line number Diff line change
@@ -0,0 +1,84 @@
<script>
import { getJson } from "../../utils";
import CheckMark from "../../CheckMark.svelte";

export let source;
export let multiscale;

// We check that all multiscale Datasets have same dtype and
// shape.length (number of dimensions)

// If multiscale.axes (version > 0.3) check it matches shape
const {axes, datasets, version} = multiscale;

// TODO: add "0.4" to this list once tested!
const checkDtypes = !["0.1", "0.2", "0.3"].includes(version);
const checkDimSeparator = ["0.2", "0.3", "0.4"].includes(version);

function allEqual(items) {
return items.every((value) => value == items[0]);
}

async function loadAndValidate() {
let dtypes = [];
let dimCounts = [];
let shapes = [];
let dimSeparators = [];

for (let i = 0; i < datasets.length; i++) {
let dataset = datasets[i];
let zarray = await getJson(source + dataset.path + "/.zarray");
dimCounts.push(zarray.shape.length);
dtypes.push(zarray.dtype);
shapes.push(zarray.shape);
dimSeparators.push(zarray.dimension_separator);
}

let errors = [];
if (dtypes.length === 0) {
errors.push("No multiscale datasets")
}

if (checkDtypes && !allEqual(dtypes)) {
errors.push(`dtypes mismatch: ${dtypes.join(", ")}`)
}
if (!allEqual(dimCounts)) {
errors.push(`number of dimensions mismatch: ${dimCounts.join(", ")}`)
}
if (axes) {
shapes.forEach((shape) => {
if (shape.length != axes.length) {
errors.push(`Shape (${shape.join(", ")}) doesn't match axes length: ${axes.length}`)
}
});
}
if (checkDimSeparator) {
dimSeparators.forEach((sep) => {
if (sep != "/") {
errors.push(`Dimension separator must be / for version ${version}`)
}
});
}
return errors;
}

const promise = loadAndValidate();
</script>

{#await promise}
<p>loading...</p>
{:then errors}
{#if errors.length > 0}
<!-- only show X if not valid - no tick if valid -->
<CheckMark valid={false} />
{#each errors as error}
<p style="color: red">Error: {error}</p>
{/each}
{:else}
<p title="dtypes match and shapes are consistent">
{datasets.length} Datasets checked <span style="color:green">✓</span>
</p>
{/if}
{:catch error}
<p style="color: red">{error.message}</p>
{/await}
4 changes: 3 additions & 1 deletion src/JsonValidator/MultiscaleArrays/index.svelte
Original file line number Diff line number Diff line change
@@ -1,13 +1,15 @@
<script>
import ZarrArray from "./ZarrArray/index.svelte";
import Multiscale from "./Multiscale.svelte";

export let source;
export let rootAttrs;
</script>

{#each rootAttrs.multiscales as multiscale, idx}
<article>
<h2>Dataset {idx}</h2>
<h2>Multiscale {idx}</h2>
<Multiscale {source} {multiscale} />
{#each multiscale.datasets as dataset}
<ZarrArray {source} path={dataset.path} />
{/each}
Expand Down
29 changes: 3 additions & 26 deletions src/JsonValidator/index.svelte
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import Plate from "./Plate/index.svelte";
import Well from "./Well/index.svelte"
import JsonBrowser from "../JsonBrowser/index.svelte";
import CheckMark from "../CheckMark.svelte";
import {
CURRENT_VERSION,
getSchemaUrlForJson,
Expand Down Expand Up @@ -43,18 +44,17 @@
{#await promise}
<div>loading schema...</div>
{:then errors}
<CheckMark valid={errors.length == 0} />
{#if errors.length > 0}
<div class="invalid">×</div>
<div class="error">
Errors:
{#each errors as error}
<pre><code>{JSON.stringify(error, null, 2)}</code></pre>
{/each}
</div>
{:else}
<div class="valid">✓</div>
{/if}
{:catch error}
<CheckMark valid={false}/>
<p style="color: red">{error.message}</p>
{/await}

Expand Down Expand Up @@ -109,29 +109,6 @@
font-size: 14px;
border-radius: 10px;
}

.invalid,
.valid {
border-radius: 50%;
padding: 10px;
margin: 10px auto;
color: white;
width: 100px;
height: 100px;
font-size: 50px;
text-align: center;
padding: 15px;
}
.invalid {
background-color: red;
border: solid red 1px;
}

.valid {
background-color: green;
border: solid green 1px;
}

.error {
text-align: left;
overflow: auto;
Expand Down
12 changes: 8 additions & 4 deletions src/utils.js
Original file line number Diff line number Diff line change
Expand Up @@ -47,10 +47,14 @@ export async function getSchema(version, schemaName = "image") {
if (!schemas[cacheKey]) {
const schema_url = getSchemaUrl(schemaName, version);
console.log("Loading schema... " + schema_url);
const schema = await getJson(schema_url);
// delete to avoid invalid: $schema: "https://json-schema.org/draft/2020-12/schema" not found
delete schema["$schema"];
schemas[cacheKey] = schema;
try {
const schema = await getJson(schema_url);
// delete to avoid invalid: $schema: "https://json-schema.org/draft/2020-12/schema" not found
delete schema["$schema"];
schemas[cacheKey] = schema;
} catch (error) {
throw new Error(`No schema at ${schema_url}. Version ${version} may be invalid.`);
}
}
return schemas[cacheKey];
}
Expand Down