Skip to content

Commit

Permalink
WIP: Add zip driver support
Browse files Browse the repository at this point in the history
  • Loading branch information
@dzenanz
dzenanz committed Mar 15, 2023
1 parent 5619617 commit 8c121c4
Show file tree
Hide file tree
Showing 3 changed files with 38 additions and 17 deletions.
2 changes: 1 addition & 1 deletion include/itkOMEZarrNGFFImageIO.h
View file
Original file line number Diff line number Diff line change
Expand Up @@ -126,7 +126,7 @@ class IOOMEZarrNGFF_EXPORT OMEZarrNGFFImageIO : public ImageIOBase

/** Read a single array and set relevant metadata. */
void
ReadArrayMetadata(std::string path);
ReadArrayMetadata(std::string path, std::string driver);

/** Sets the requested dimension, and initializes spatial metadata to identity. */
void
Expand Down
4 changes: 2 additions & 2 deletions src/CMakeLists.txt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,8 @@ set(CMAKE_FOLDER TensorStore)
include(FetchContent)
FetchContent_Declare(
tensorstore
GIT_REPOSITORY https://github.com/google/tensorstore
GIT_TAG b040186346f8543f865e4205d9045b42514c9f92
GIT_REPOSITORY https://github.com/dzenanz/tensorstore
GIT_TAG master
)
FetchContent_MakeAvailable(tensorstore)
unset(CMAKE_FOLDER)
Expand Down
49 changes: 35 additions & 14 deletions src/itkOMEZarrNGFFImageIO.cxx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -167,13 +167,28 @@ itkToTensorstoreComponentType(const IOComponentEnum itkComponentType)
}
}

// Returns TensorStore KvStore driver name appropriate for this path.
// Options are file, zip. TODO: http, gcs (GoogleCouldStorage), etc.
std::string
getKVstoreDriver(std::string path)
{
if (path.size() < 4)
{
return "file";
}
if (path.substr(path.size() - 4) == ".zip")
{
return "zip";
}
return "file";
}

// JSON file path, e.g. "C:/Dev/ITKIOOMEZarrNGFF/v0.4/cyx.ome.zarr/.zgroup"
void
writeJson(nlohmann::json json, std::string path)
writeJson(nlohmann::json json, std::string path, std::string driver)
{
auto attrs_store = tensorstore::Open<nlohmann::json, 0>(
{ { "driver", "json" }, { "kvstore", { { "driver", "file" }, { "path", path } } } },
{ { "driver", "json" }, { "kvstore", { { "driver", driver }, { "path", path } } } },
tsContext,
tensorstore::OpenMode::create | tensorstore::OpenMode::delete_existing,
tensorstore::ReadWriteMode::read_write)
Expand All @@ -191,11 +206,11 @@ writeJson(nlohmann::json json, std::string path)

// JSON file path, e.g. "C:/Dev/ITKIOOMEZarrNGFF/v0.4/cyx.ome.zarr/.zattrs"
bool
jsonRead(std::string path, nlohmann::json & result)
jsonRead(std::string path, nlohmann::json & result, std::string driver)
{
// Reading JSON via TensorStore allows it to be in the cloud
auto attrs_store = tensorstore::Open<nlohmann::json, 0>(
{ { "driver", "json" }, { "kvstore", { { "driver", "file" }, { "path", path } } } })
{ { "driver", "json" }, { "kvstore", { { "driver", driver }, { "path", path } } } })
.result()
.value();

Expand Down Expand Up @@ -224,16 +239,17 @@ OMEZarrNGFFImageIO::CanReadFile(const char * filename)
{
try
{
std::string driver = getKVstoreDriver(filename);
nlohmann::json json;
if (!jsonRead(std::string(filename) + "/.zgroup", json))
if (!jsonRead(std::string(filename) + "/.zgroup", json, driver))
{
return false;
}
if (json.at("zarr_format").get<int>() != 2)
{
return false; // unsupported zarr format
}
if (!jsonRead(std::string(filename) + "/.zattrs", json))
if (!jsonRead(std::string(filename) + "/.zattrs", json, driver))
{
return false;
}
Expand Down Expand Up @@ -265,10 +281,10 @@ OMEZarrNGFFImageIO::CanReadFile(const char * filename)
thread_local tensorstore::TensorStore<> store; // initialized by ReadImageInformation/ReadArrayMetadata

void
OMEZarrNGFFImageIO::ReadArrayMetadata(std::string path)
OMEZarrNGFFImageIO::ReadArrayMetadata(std::string path, std::string driver)
{
auto openFuture =
tensorstore::Open({ { "driver", "zarr" }, { "kvstore", { { "driver", "file" }, { "path", path } } } },
tensorstore::Open({ { "driver", "zarr" }, { "kvstore", { { "driver", "zip" }, { "path", path } } } },
tsContext,
tensorstore::OpenMode::open,
tensorstore::RecheckCached{ false },
Expand Down Expand Up @@ -345,10 +361,11 @@ void
OMEZarrNGFFImageIO::ReadImageInformation()
{
nlohmann::json json;
std::string driver = getKVstoreDriver(this->GetFileName());

bool status = jsonRead(std::string(this->GetFileName()) + "/.zgroup", json);
bool status = jsonRead(std::string(this->GetFileName()) + "/.zgroup", json, driver);
assert(json.at("zarr_format").get<int>() == 2); // only v2 for now
status = jsonRead(std::string(this->GetFileName()) + "/.zattrs", json);
status = jsonRead(std::string(this->GetFileName()) + "/.zattrs", json, driver);
json = json.at("multiscales")[0]; // multiscales must be present in OME-NGFF
auto version = json.at("version").get<std::string>();
if (version == "0.4" || version == "0.3" || version == "0.2" || version == "0.1")
Expand Down Expand Up @@ -400,7 +417,7 @@ OMEZarrNGFFImageIO::ReadImageInformation()

// TODO: parse stuff from "metadata" object into metadata dictionary

ReadArrayMetadata(std::string(this->GetFileName()) + "/" + path);
ReadArrayMetadata(std::string(this->GetFileName()) + "/" + path, driver);
}

// We call tensorstoreToITKComponentType for each type.
Expand Down Expand Up @@ -454,9 +471,11 @@ OMEZarrNGFFImageIO::CanWriteFile(const char * name)
void
OMEZarrNGFFImageIO::WriteImageInformation()
{
std::string driver = getKVstoreDriver(this->GetFileName());

nlohmann::json group;
group["zarr_format"] = 2;
writeJson(group, std::string(this->GetFileName()) + "/.zgroup");
writeJson(group, std::string(this->GetFileName()) + "/.zgroup", driver);

unsigned dim = this->GetNumberOfDimensions();

Expand Down Expand Up @@ -490,7 +509,7 @@ OMEZarrNGFFImageIO::WriteImageInformation()

nlohmann::json zattrs;
zattrs["multiscales"] = multiscales;
writeJson(zattrs, std::string(this->GetFileName()) + "/.zattrs");
writeJson(zattrs, std::string(this->GetFileName()) + "/.zattrs", driver);
}


Expand Down Expand Up @@ -522,7 +541,7 @@ OMEZarrNGFFImageIO::WriteImageInformation()
auto openFuture = tensorstore::Open( \
{ \
{ "driver", "zarr" }, \
{ "kvstore", { { "driver", "file" }, { "path", this->m_FileName + "/" + path } } }, \
{ "kvstore", { { "driver", driver }, { "path", this->m_FileName + "/" + path } } }, \
{ "metadata", \
{ \
{ "compressor", { { "id", "blosc" } } }, \
Expand Down Expand Up @@ -575,6 +594,8 @@ OMEZarrNGFFImageIO::Write(const void * buffer)
dtype = "<";
}

std::string driver = getKVstoreDriver(this->GetFileName());

if (false) // start with a plain "if"
{} // so element statements can all be "else if"
ELEMENT_WRITE(int8_t)
Expand Down

0 comments on commit 8c121c4

Please sign in to comment.