From 9899ac6772b6c22804c99791adc1c7ee4f40afc3 Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Thu, 23 Dec 2021 15:09:37 -0600 Subject: [PATCH 1/2] write zarr documentation --- .../sidebars/netcdfJavaTutorial_sidebar.yml | 4 + docs/src/site/pages/netcdfJava/ReadingZarr.md | 100 ++++++++++++ docs/src/site/pages/netcdfJava/Upgrade.md | 1 + .../pages/netcdfJava/developer/FileTypes.md | 3 +- .../runtime/runtimeloading.md | 13 ++ docs/src/test/java/examples/ZarrExamples.java | 143 ++++++++++++++++++ .../writingiosp/LightningExampleTutorial.java | 2 +- .../src/test/java/tests/TestZarrExamples.java | 25 +++ 8 files changed, 289 insertions(+), 2 deletions(-) create mode 100644 docs/src/site/pages/netcdfJava/ReadingZarr.md create mode 100644 docs/src/test/java/examples/ZarrExamples.java create mode 100644 docs/src/test/java/tests/TestZarrExamples.java diff --git a/docs/src/site/_data/sidebars/netcdfJavaTutorial_sidebar.yml b/docs/src/site/_data/sidebars/netcdfJavaTutorial_sidebar.yml index f52890697a..c390af02d4 100644 --- a/docs/src/site/_data/sidebars/netcdfJavaTutorial_sidebar.yml +++ b/docs/src/site/_data/sidebars/netcdfJavaTutorial_sidebar.yml @@ -267,6 +267,10 @@ entries: url: /read_over_http.html output: web, pdf + - title: Reading Zarr + url: /reading_zarr.html + output: web, pdf + - title: Standard Horizontal Coordinate Transforms url: /std_horizonal_coord_transforms.html output: web, pdf diff --git a/docs/src/site/pages/netcdfJava/ReadingZarr.md b/docs/src/site/pages/netcdfJava/ReadingZarr.md new file mode 100644 index 0000000000..fe85cc1527 --- /dev/null +++ b/docs/src/site/pages/netcdfJava/ReadingZarr.md @@ -0,0 +1,100 @@ +--- +title: Reading the Zarr data model +last_updated: 2021-12-22 +sidebar: netcdfJavaTutorial_sidebar +toc: false +permalink: reading_zarr.html +--- +## Zarr + +As of version 5.5.1, the netCDF-Java library provides read-only support for the [Zarr v2 data model](https://zarr.readthedocs.io/en/stable/spec/v2.html#){:target="_blank"}. +Any dataset that adheres to the v2 spec can be read into a `NetcdfFile` object, as long as the following is true: + +* all filters and compressors used by the dataset must be known to the netCDF-Java library (see [Filters](reading_zarr.html#filters)) +* the underlying storage of the dataset must be a directory store, zip store, or object store + +### Enabling Zarr support + +To use Zarr in the netCDF-Java library, you must include the `cdm-zarr` module in your netCDF-Java build. +See [here](using_netcdf_java_artifacts.html) for more information on including optional modules. + +### How to read a Zarr dataset + +Reading a Zarr dataset is syntactically the same as reading a netCDF file: + +{% capture rmd %} +{% includecodeblock netcdf-java&docs/src/test/java/examples/ZarrExamples.java&readZarrStores %} +{% endcapture %} +{{ rmd | markdownify }} + +If the file is a legal Zarr dataset, the library will map it to a `NetcdfFile` object for reading. +See [reading CDM files](reading_cdm.html) for more examples on accessing data once the `NetcdfFile` object is returned. + +## Filters + +As of netCDF-Java version 5.5.1, a `ucar.nc2.filter` package is included, that provides a suite of implemented filters and compressors, +as well as a mechanism for user-supplied filters. This package is used by both the Zarr and HDF5 IOSPs, and is available for public use. + +The current list of filters included natively in the netCDF-Java library is: + +* Deflate (zlib) +* Shuffle +* 32-bit Checksum (CRC, Fletcher, and Adler) +* ScaleOffset + +This list is still expanding, but if the filter you are looking for is not provided at this time, you are able to provide it yourself +(See [Implementing a Filter](#implementing-a-filter)) for details.) + +### Implementing a Filter + +To add a user-supplied `Filter` to the netDF-Java library, you will have to provide two classes: + +* A class that extends the `ucar.nc2.filter.Filter` abstract class +* A class that implements the `ucar.nc2.filter.FilterProvider` interface + +Once implemented, you will need to include these classes as JAR files in your classpath. See [here](runtime_loading.html) for more information. + +#### `Filter` implementation + +To implement a user-supplied filter, you will need to extend the abstract `ucar.nc2.filter.Filter` class, and provide implementations for the following methods: +* `encode` takes a `byte[]` of unfiltered data and returns a `byte[]` of filtered data +* `decode` takes a `byte[]` of filtered data and returns a `byte[]` of unfiltered data + +Your `Filter` class should look something like this: + +{% capture rmd %} +{% includecodeblock netcdf-java&docs/src/test/java/examples/ZarrExamples.java&implementFilter %} +{% endcapture %} +{{ rmd | markdownify }} + +#### `FilterProvider` implementation + +For the netCDF-Java library to find your `Filter` implementation, you will need to provide a `FilterProvider` as well. + +{% capture rmd %} +{% includecodeblock netcdf-java&docs/src/test/java/examples/ZarrExamples.java&implementFilterProvider %} +{% endcapture %} +{{ rmd | markdownify }} + +{%include note.html content=" +`getName` and `getId` +When reading data, [IOSPs](writing_iosp.html) can look up filters by either a `String` or `int` (name or id). Currently, the `ucar.nc2.filter` +package is shared by the Zarr and HDF5 IOSPs. The Zarr IOSP looks up filters by name; if you plan to use your third party filter with Zarr data, +the string returned by `getName` should match that specified by the [NumCodecs](https://numcodecs.readthedocs.io/en/stable/) library. +The HDF5 IOSP looks up filters by id; if you plan to use your third party filter with HDF5 data, the int returned by `getId` should adhere to the +[guidelines](https://portal.hdfgroup.org/display/support/Registered+Filter+Plugins) set by the HDF group. +If your filter is registered with the HDF group, your `getId` method should return the HDF id. If your filter is not registered with the HDF group, +" %} + +There are two more methods in the `FilterProvider` interface: the `canProvide` methods. By default, these methods work as follows: +* `boolean canProvide(String name)` returns true if the string returned by `getName()` matches the string passed to the method +* `boolean canProvide(int id)` returns true if the int returned by `getId()` matches the int passed to the method + +It is unlikely that you would want to override these methods, as the netCDF-Java [IOSPs](writing_iosp.html) look for the correct filter implementations +for a dataset by either name or numeric id. However, it is possible to write your own implementation of `canProvide`; for example, the +following `FilterProvider` returns an instance of a `DefaultFilter` regardless of the name or id provided. + +{% capture rmd %} +{% includecodeblock netcdf-java&docs/src/test/java/examples/ZarrExamples.java&implementFilterProvider2 %} +{% endcapture %} +{{ rmd | markdownify }} diff --git a/docs/src/site/pages/netcdfJava/Upgrade.md b/docs/src/site/pages/netcdfJava/Upgrade.md index c394e02cab..2048d84180 100644 --- a/docs/src/site/pages/netcdfJava/Upgrade.md +++ b/docs/src/site/pages/netcdfJava/Upgrade.md @@ -37,6 +37,7 @@ Point release notes: 5.5.x is the first release that includes read support for [Zarr](https://zarr.readthedocs.io/en/stable/index.html){:target="_blank"}. There is also a new package for filters, `ucar.ma2.filters`, which allows user-provided compressors and filters to be supplied through a [Service Provider Interface](https://www.baeldung.com/java-spi){:target="_blank"}. +Read about Zarr support and the `filter` package [here](reading_zarr.html) ## netCDF-Java API Changes (5.4.x) diff --git a/docs/src/site/pages/netcdfJava/developer/FileTypes.md b/docs/src/site/pages/netcdfJava/developer/FileTypes.md index 20e5cef9dc..e9e9fc2528 100644 --- a/docs/src/site/pages/netcdfJava/developer/FileTypes.md +++ b/docs/src/site/pages/netcdfJava/developer/FileTypes.md @@ -50,4 +50,5 @@ To register your format/IOServiceProvider, or to send corrections and additions | UAMIV | CAMx UAM-IV formatted files | `cdm-radial` | | | UniversalRadarFormat | Universal Radar Format | `cdm-radial` | | | USPLN | US Precision Lightning Network | `cdm-misc` | | -| VIS5D | Vis5D grid file | `cdm-vs5d` | \ No newline at end of file +| VIS5D | Vis5D grid file | `cdm-vs5d` | | +| Zarr | Zarr dataset | `cdm-zarr` | | \ No newline at end of file diff --git a/docs/src/site/pages/netcdfJava_tutorial/runtime/runtimeloading.md b/docs/src/site/pages/netcdfJava_tutorial/runtime/runtimeloading.md index d10b2fbb2c..3475e6afc0 100644 --- a/docs/src/site/pages/netcdfJava_tutorial/runtime/runtimeloading.md +++ b/docs/src/site/pages/netcdfJava_tutorial/runtime/runtimeloading.md @@ -88,6 +88,19 @@ See [GribTables](grib_tables.html) for more information about lookup tables. The file must be a [BUFR table lookup file](bufr_tables.html). +### Register a `Filter` implementation: +Users-supplied filters must be provided using the [Service Provider](https://docs.oracle.com/javase/tutorial/ext/basics/spi.html){:target="_blank"} +mechanism and included in a JAR on the classpath, where it is dynamically loaded at runtime. +In your JAR, include a file named `META-INF/services/ucar.nc2.filter.FilterProvider` containing the name(s) of your implementations, eg: + +~~~ +ucar.nc2.filter.BloscFilter +ucar.nc2.filter.GZipFilter +~~~ + +Your `FilterProvider` classes must implement the `ucar.nc2.filter.FilterProvider` interface. +See [here](reading_zarr.html#implementing-a-filter) for details on implementing user-supplied filters. + ## Runtime Configuration Instead of calling the above routines in your code, you can pass the CDM library an XML configuration file. diff --git a/docs/src/test/java/examples/ZarrExamples.java b/docs/src/test/java/examples/ZarrExamples.java new file mode 100644 index 0000000000..daba26327f --- /dev/null +++ b/docs/src/test/java/examples/ZarrExamples.java @@ -0,0 +1,143 @@ +package examples; + +import ucar.ma2.Array; +import ucar.ma2.InvalidRangeException; +import ucar.ma2.Section; +import ucar.nc2.Group; +import ucar.nc2.NetcdfFile; +import ucar.nc2.NetcdfFiles; +import ucar.nc2.Variable; +import ucar.nc2.filter.Filter; +import ucar.nc2.filter.FilterProvider; +import ucar.nc2.filter.Filters; +import ucar.nc2.filter.Shuffle; +import ucar.nc2.iosp.AbstractIOServiceProvider; +import ucar.nc2.util.CancelTask; +import ucar.unidata.io.RandomAccessFile; +import ucar.unidata.util.test.TestDir; + +import java.io.IOException; +import java.util.Map; + +public class ZarrExamples { + + private static String pathToDirectoryStore = ""; + private static String pathToZipStore = ""; + private static String pathToObjectStore = ""; + + public static void readZarrStores() throws IOException { + // a local file path + NetcdfFile directoryStoreZarr = NetcdfFiles.open(pathToDirectoryStore); + // a local file path + '.zip' + NetcdfFile zipdirectoryStoreZarr = NetcdfFiles.open(pathToZipStore); + // an object store path, sarting with 'cdms3:' and ending with 'delimiter=' + the store delimiter + NetcdfFile objectStoreZarr = NetcdfFiles.open(pathToObjectStore); + } + + public static void implementFilter() { + byte[] dataOut = null; /* DOCS-IGNORE */ + /* INSERT public */class MyFilter extends Filter { + + @Override + public byte[] encode(byte[] dataIn) throws IOException { + // your encoding implementation here + return dataOut; + } + + @Override + public byte[] decode(byte[] dataIn) throws IOException { + // your decoding implementation here + return dataOut; + } + } + } + + public static void implementFilterProvider() { + /* INSERT public */class MyFilterProvider implements FilterProvider { + + @Override + public String getName() { + // returns a string identifier for your filter + return "myFilter"; // see notes on filter names and ids + } + + @Override + public int getId() { + // returns a numeric identifier for your filter + return 32768; // see notes on filter names and ids + } + + @Override + public Filter create(Map properties) { + return new MyFilter(properties); // return an instance of your filter + } + } + } + + public static void implementFilterProvider2() { + /* INSERT public */class DefaultFilterProvider implements FilterProvider { + + @Override + public String getName() { + return "defaultFilter"; + } + + @Override + public int getId() { + return -1; + } + + @Override + public boolean canProvide(String name) { + return true; + } + + @Override + public boolean canProvide(int id) { + return true; + } + + @Override + public Filter create(Map properties) { + return new DefaultFilter(properties); // return an instance of your filter + } + } + } + + /** + * Shell classes for code snippets + */ + static class MyFilter extends Filter{ + + public MyFilter(Map properties) { + super(); + } + + @Override + public byte[] encode(byte[] dataIn) throws IOException { + return new byte[0]; + } + + @Override + public byte[] decode(byte[] dataIn) throws IOException { + return new byte[0]; + } + } + + static class DefaultFilter extends Filter { + + public DefaultFilter(Map properties) { + super(); + } + + @Override + public byte[] encode(byte[] dataIn) throws IOException { + return new byte[0]; + } + + @Override + public byte[] decode(byte[] dataIn) throws IOException { + return new byte[0]; + } + } +} diff --git a/docs/src/test/java/examples/writingiosp/LightningExampleTutorial.java b/docs/src/test/java/examples/writingiosp/LightningExampleTutorial.java index e0412a8cad..102b2a92fe 100644 --- a/docs/src/test/java/examples/writingiosp/LightningExampleTutorial.java +++ b/docs/src/test/java/examples/writingiosp/LightningExampleTutorial.java @@ -59,7 +59,7 @@ public String getFileTypeDescription() { return "Data from lightning data test file"; } } - return new UspLightning(); + return new UspLightning(); /* DOCS-IGNORE */ } /** diff --git a/docs/src/test/java/tests/TestZarrExamples.java b/docs/src/test/java/tests/TestZarrExamples.java new file mode 100644 index 0000000000..407d7f1d03 --- /dev/null +++ b/docs/src/test/java/tests/TestZarrExamples.java @@ -0,0 +1,25 @@ +package tests; + +import examples.ZarrExamples; +import org.junit.Assert; +import org.junit.Test; + +import java.io.FileNotFoundException; + +public class TestZarrExamples { + + @Test + public void testReadZarrExample() { + Assert.assertThrows(FileNotFoundException.class, () -> { + ZarrExamples.readZarrStores(); + }); + } + + @Test + public void testImplementFilterAndProvider() { + // just verify the example code compiles with no deprecations + ZarrExamples.implementFilter(); + ZarrExamples.implementFilterProvider(); + ZarrExamples.implementFilterProvider2(); + } +} From ca1d9ef26968f22deade1f5b90e4325b8849890a Mon Sep 17 00:00:00 2001 From: haileyajohnson Date: Mon, 27 Dec 2021 15:49:45 -0800 Subject: [PATCH 2/2] spotless --- docs/src/test/java/examples/ZarrExamples.java | 5 +++-- docs/src/test/java/tests/TestZarrExamples.java | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/docs/src/test/java/examples/ZarrExamples.java b/docs/src/test/java/examples/ZarrExamples.java index daba26327f..78250bd529 100644 --- a/docs/src/test/java/examples/ZarrExamples.java +++ b/docs/src/test/java/examples/ZarrExamples.java @@ -35,7 +35,8 @@ public static void readZarrStores() throws IOException { } public static void implementFilter() { - byte[] dataOut = null; /* DOCS-IGNORE */ + byte[] dataOut = null; + /* DOCS-IGNORE */ /* INSERT public */class MyFilter extends Filter { @Override @@ -107,7 +108,7 @@ public Filter create(Map properties) { /** * Shell classes for code snippets */ - static class MyFilter extends Filter{ + static class MyFilter extends Filter { public MyFilter(Map properties) { super(); diff --git a/docs/src/test/java/tests/TestZarrExamples.java b/docs/src/test/java/tests/TestZarrExamples.java index 407d7f1d03..e95a74e69c 100644 --- a/docs/src/test/java/tests/TestZarrExamples.java +++ b/docs/src/test/java/tests/TestZarrExamples.java @@ -10,9 +10,9 @@ public class TestZarrExamples { @Test public void testReadZarrExample() { - Assert.assertThrows(FileNotFoundException.class, () -> { - ZarrExamples.readZarrStores(); - }); + Assert.assertThrows(FileNotFoundException.class, () -> { + ZarrExamples.readZarrStores(); + }); } @Test