Skip to content

Add a flow to use Zarr datasets through the Hub #83

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 6 commits into from
Mar 26, 2024
Merged

Add a flow to use Zarr datasets through the Hub #83

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e2cbae9
Changed pointer paths to be relative and save Zarr archive to dataset
cwognum Mar 25, 2024
8d018a0
Update tutorials
cwognum Mar 25, 2024
e640001
Added the basic flow for uploading Zarr datasets to the Hub
cwognum Mar 25, 2024
0b7a244
New flow to use Zarr Datasets through the Hub
cwognum Mar 26, 2024
73385d4
renamed zarr_archive to zarr_root_path
cwognum Mar 26, 2024
046eb83
Fixed error in docs
cwognum Mar 26, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 14 additions & 31 deletions docs/tutorials/dataset_factory.ipynb
View file
Open in desktop

Large diffs are not rendered by default.

40 changes: 18 additions & 22 deletions docs/tutorials/dataset_zarr.ipynb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,8 +129,8 @@
"source": [
"# For performance reasons, Polaris expects all data related to a column to be saved in a single Zarr array. \n",
"# To index a specific element in that array, the pointer path can have a suffix to specify the index. \n",
"train_path = f\"{base_path}/{inp_col_name}#0\"\n",
"test_path = f\"{base_path}/{inp_col_name}#1\""
"train_path = f\"{inp_col_name}#0\"\n",
"test_path = f\"{inp_col_name}#1\""
]
},
{
Expand Down Expand Up @@ -176,6 +176,8 @@
" # To indicate that we are dealing with a pointer column here,\n",
" # we need to annotate the column.\n",
" annotations={\"images\": ColumnAnnotation(is_pointer=True)},\n",
" # We also need to specify the path to the root of the Zarr archive\n",
" zarr_root_path=base_path,\n",
")"
]
},
Expand All @@ -190,7 +192,7 @@
"tags": []
},
"source": [
"Note how the table does not contain the image data, but rather stores a path. "
"Note how the table does not contain the image data, but rather stores a path relative to the root of the Zarr. "
]
},
{
Expand All @@ -208,7 +210,7 @@
{
"data": {
"text/plain": [
"'/home/cas/.cache/polaris-tutorials/002/data.zarr/images#0'"
"'images#0'"
]
},
"execution_count": 7,
Expand Down Expand Up @@ -360,12 +362,12 @@
"\n",
"Which will get parsed into a table like: \n",
"\n",
"| column_a |\n",
"| ------------------------------------ |\n",
"| /path/to/root.zarr/column_a/array#1 |\n",
"| /path/to/root.zarr/column_a/array#2 |\n",
"| ... |\n",
"| /path/to/root.zarr/column_a/array#N |\n",
"| column_a |\n",
"| ----------------- |\n",
"| column_a/array#1 |\n",
"| column_a/array#2 |\n",
"| ... |\n",
"| column_a/array#N |\n",
"\n",
"<div class=\"admonition info highlight\">\n",
" <p class=\"admonition-title\">Note</p>\n",
Expand Down Expand Up @@ -411,17 +413,10 @@
"id": "3c7c11ac",
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"A\n"
]
},
{
"data": {
"text/plain": [
"'/home/cas/.cache/polaris-tutorials/002/zarr/data.zarr//images#0'"
"'images#0'"
]
},
"execution_count": 13,
Expand Down Expand Up @@ -531,7 +526,7 @@
{
"data": {
"text/html": [
"<table border=\"1\"><tr><th>name</th><td>None</td></tr><tr><th>description</th><td></td></tr><tr><th>tags</th><td></td></tr><tr><th>user_attributes</th><td></td></tr><tr><th>owner</th><td>None</td></tr><tr><th>default_adapters</th><td>None</td></tr><tr><th>md5sum</th><td>6ef8d23737aafcbf82c421e7f99e1d95</td></tr><tr><th>readme</th><td></td></tr><tr><th>annotations</th><td><table border=\"1\"><tr><th>images</th><td><table border=\"1\"><tr><th>is_pointer</th><td>True</td></tr><tr><th>modality</th><td>UNKNOWN</td></tr><tr><th>description</th><td>None</td></tr><tr><th>user_attributes</th><td></td></tr><tr><th>dtype</th><td>object</td></tr></table></td></tr></table></td></tr><tr><th>source</th><td>None</td></tr><tr><th>license</th><td>None</td></tr><tr><th>curation_reference</th><td>None</td></tr><tr><th>cache_dir</th><td>/home/cas/.cache/polaris/datasets/None/6ef8d23737aafcbf82c421e7f99e1d95</td></tr><tr><th>artifact_id</th><td>None</td></tr><tr><th>n_rows</th><td>1000</td></tr><tr><th>n_columns</th><td>1</td></tr></table>"
"<table border=\"1\"><tr><th>name</th><td>None</td></tr><tr><th>description</th><td></td></tr><tr><th>tags</th><td></td></tr><tr><th>user_attributes</th><td></td></tr><tr><th>owner</th><td>None</td></tr><tr><th>default_adapters</th><td></td></tr><tr><th>zarr_root_path</th><td>/home/cas/.cache/polaris-tutorials/002/json/data.zarr</td></tr><tr><th>md5sum</th><td>5488b4909fd67d3208624288e720e1b8</td></tr><tr><th>readme</th><td></td></tr><tr><th>annotations</th><td><table border=\"1\"><tr><th>images</th><td><table border=\"1\"><tr><th>is_pointer</th><td>True</td></tr><tr><th>modality</th><td>UNKNOWN</td></tr><tr><th>description</th><td>None</td></tr><tr><th>user_attributes</th><td></td></tr><tr><th>dtype</th><td>object</td></tr></table></td></tr></table></td></tr><tr><th>source</th><td>None</td></tr><tr><th>license</th><td>None</td></tr><tr><th>curation_reference</th><td>None</td></tr><tr><th>cache_dir</th><td>/home/cas/.cache/polaris/datasets/None/5488b4909fd67d3208624288e720e1b8</td></tr><tr><th>artifact_id</th><td>None</td></tr><tr><th>n_rows</th><td>1000</td></tr><tr><th>n_columns</th><td>1</td></tr></table>"
],
"text/plain": [
"{\n",
Expand All @@ -540,8 +535,9 @@
" \"tags\": [],\n",
" \"user_attributes\": {},\n",
" \"owner\": null,\n",
" \"default_adapters\": null,\n",
" \"md5sum\": \"6ef8d23737aafcbf82c421e7f99e1d95\",\n",
" \"default_adapters\": {},\n",
" \"zarr_root_path\": \"/home/cas/.cache/polaris-tutorials/002/json/data.zarr\",\n",
" \"md5sum\": \"5488b4909fd67d3208624288e720e1b8\",\n",
" \"readme\": \"\",\n",
" \"annotations\": {\n",
" \"images\": {\n",
Expand All @@ -555,7 +551,7 @@
" \"source\": null,\n",
" \"license\": null,\n",
" \"curation_reference\": null,\n",
" \"cache_dir\": \"/home/cas/.cache/polaris/datasets/None/6ef8d23737aafcbf82c421e7f99e1d95\",\n",
" \"cache_dir\": \"/home/cas/.cache/polaris/datasets/None/5488b4909fd67d3208624288e720e1b8\",\n",
" \"artifact_id\": null,\n",
" \"n_rows\": 1000,\n",
" \"n_columns\": 1\n",
Expand Down
2 changes: 1 addition & 1 deletion polaris/benchmark/_base.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
import fsspec
import numpy as np
import pandas as pd
from datamol.utils import fs
from pydantic import (
Field,
FieldValidationInfo,
Expand All @@ -20,7 +21,6 @@
from polaris.dataset import Dataset, Subset
from polaris.evaluate import BenchmarkResults, Metric, ResultsType
from polaris.hub.settings import PolarisHubSettings
from polaris.utils import fs
from polaris.utils.context import tmp_attribute_change
from polaris.utils.dict2html import dict2html
from polaris.utils.errors import InvalidBenchmarkError, PolarisChecksumError
Expand Down
Loading