Merge pull request #33 from davidbrochart/update_binder

Update binder environment
xtensor-stack · Jan 29, 2021 · 1db74eb · 1db74eb
2 parents 4774e98 + 9e3efb8
commit 1db74eb
Show file tree

Hide file tree

Showing 8 changed files with 75 additions and 43 deletions.
diff --git a/README.md b/README.md
@@ -2,7 +2,7 @@
 
 [![Azure Pipelines](https://dev.azure.com/xtensor-stack/xtensor-stack/_apis/build/status/xtensor-stack.xtensor-zarr?branchName=master)](https://dev.azure.com/xtensor-stack/xtensor-stack/_build/latest?definitionId=9&branchName=master)
 [![ReadTheDocs](https://readthedocs.org/projects/xtensor-zarr/badge/?version=latest)](http://xtensor-zarr.readthedocs.io/en/latest)
-[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/xtensor-stack/xtensor-zarr/master?filepath=examples%2Fzarr_v3.ipynb)
+[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/xtensor-stack/xtensor-zarr/master?urlpath=lab%2Ftree%2Fexamples)
 [![Join the Gitter Chat](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/QuantStack/Lobby?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
 Implementation of the Zarr version 3.0 core protocol based on xtensor

diff --git a/binder/environment.yml b/binder/environment.yml
@@ -3,13 +3,14 @@ channels:
   - conda-forge
 dependencies:
   - python=3.8
-  - notebook
-  - xeus-cling=0.5.1
-  - xwidgets=0.17.0
+  - jupyterlab=3
+  - xeus-cling=0.12.0
+  - xwidgets=0.25.0
+  - jupyterlab_widgets=1.0.0
   - ffmpeg=4.3.1  # openimageio is being added as a runtime requirement to xtensor-io
   - widgetsnbextension
-  - xtensor-io=0.10.1
-  - zarray
+  - xtensor-io=0.12.1
+  - zarray=0.0.3
   - bash_kernel
   - nlohmann_json
   - cpp-filesystem

diff --git a/examples/explore_hierarchy.ipynb b/examples/explore_hierarchy.ipynb
@@ -146,7 +146,7 @@
    "outputs": [],
    "source": [
     ">xcpp14\n",
-    "auto h1 = xt::get_zarr_hierarchy(\"test.zr3\");"
+    "auto h = xt::get_zarr_hierarchy(\"test.zr3\");"
    ]
   },
   {
@@ -170,8 +170,8 @@
    "outputs": [],
    "source": [
     ">xcpp14\n",
-    "std::string children1 = h1.get_children(\"/\").dump();\n",
-    "children1"
+    "std::string children = h.get_children(\"/\").dump();\n",
+    "children"
    ]
   },
   {
@@ -181,8 +181,8 @@
    "outputs": [],
    "source": [
     ">xcpp14\n",
-    "std::string children2 = h1.get_children(\"/marvin\").dump();\n",
-    "children2"
+    "std::string children = h.get_children(\"/marvin\").dump();\n",
+    "children"
    ]
   },
   {
@@ -192,8 +192,8 @@
    "outputs": [],
    "source": [
     ">xcpp14\n",
-    "std::string children3 = h1.get_children(\"/tricia\").dump();\n",
-    "children3"
+    "std::string children = h.get_children(\"/tricia\").dump();\n",
+    "children"
    ]
   },
   {
@@ -203,8 +203,8 @@
    "outputs": [],
    "source": [
     ">xcpp14\n",
-    "std::string attrs1 = h1[\"/tricia/mcmillan\"].get_group().attrs().dump();\n",
-    "attrs1"
+    "std::string attrs = h[\"/tricia/mcmillan\"].get_group().attrs().dump();\n",
+    "children"
    ]
   },
   {
@@ -221,7 +221,7 @@
    "outputs": [],
    "source": [
     ">xcpp14\n",
-    "std::string nodes = h1.get_nodes().dump();\n",
+    "std::string nodes = h.get_nodes().dump();\n",
     "nodes"
    ]
   },
@@ -239,8 +239,8 @@
    "outputs": [],
    "source": [
     ">xcpp14\n",
-    "nlohmann::json attrs2 = {{\"heart\", \"gold\"}, {\"improbability\", \"infinite\"}};\n",
-    "auto g1 = h1.create_group(\"/foo/bar\", attrs2);"
+    "nlohmann::json attrs = {{\"heart\", \"gold\"}, {\"improbability\", \"infinite\"}};\n",
+    "auto g = h.create_group(\"/foo/bar\", attrs);"
    ]
   },
   {

diff --git a/examples/zarr_v2.ipynb b/examples/zarr_v2.ipynb
@@ -176,7 +176,7 @@
    "source": [
     ">xcpp14\n",
     "xt::xzarr_register_compressor<xt::xzarr_file_system_store, xt::xio_blosc_config>();\n",
-    "auto h = xt::get_zarr_hierarchy(\"marie_curie.zarr\", \"2\");\n",
+    "auto h = xt::get_zarr_hierarchy(\"marie_curie.zarr\");\n",
     "auto z = h.get_array(\"\");"
    ]
   },

diff --git a/examples/zarr_v3.ipynb b/examples/zarr_v3.ipynb
@@ -150,8 +150,8 @@
    "outputs": [],
    "source": [
     ">xcpp14\n",
-    "auto h1 = xt::get_zarr_hierarchy(\"h_zarrita.zr3\");\n",
-    "xt::zarray z1 = h1.get_array(\"/arthur/dent\");"
+    "auto h = xt::get_zarr_hierarchy(\"h_zarrita.zr3\");\n",
+    "xt::zarray z = h.get_array(\"/arthur/dent\");"
    ]
   },
   {
@@ -168,8 +168,8 @@
    "outputs": [],
    "source": [
     ">xcpp14\n",
-    "auto a1 = z1.get_array<double>();\n",
-    "a1"
+    "auto a = z.get_array<double>();\n",
+    "a"
    ]
   },
   {
@@ -188,17 +188,14 @@
     ">xcpp14\n",
     "std::vector<size_t> shape = {4, 4};\n",
     "std::vector<size_t> chunk_shape = {2, 2};\n",
-    "nlohmann::json attrs = {{\"question\", \"life\"}, {\"answer\", 42}};\n",
-    "std::size_t pool_size = 1;\n",
-    "double fill_value = 6.6;\n",
-    "auto h2 = xt::create_zarr_hierarchy(\"h_xtensor.zr3\");\n",
+    "auto h = xt::create_zarr_hierarchy(\"h_xtensor.zr3\");\n",
     "xt::xzarr_create_array_options<> o;\n",
     "o.chunk_memory_layout = 'C';\n",
     "o.chunk_separator = '/';\n",
-    "o.attrs = attrs;\n",
-    "o.chunk_pool_size = pool_size;\n",
-    "o.fill_value = fill_value;\n",
-    "xt::zarray z2 = h2.create_array(\"/arthur/dent\", shape, chunk_shape, \"<f8\", o);"
+    "o.attrs = {{\"question\", \"life\"}, {\"answer\", 42}};\n",
+    "o.chunk_pool_size = 1;\n",
+    "o.fill_value = 6.6;\n",
+    "xt::zarray z = h.create_array(\"/arthur/dent\", shape, chunk_shape, \"<f8\", o);"
    ]
   },
   {
@@ -262,11 +259,11 @@
     ">xcpp14\n",
     "xt::xzarr_register_compressor<xt::xzarr_gcs_store, xt::xio_gzip_config>();\n",
     "gcs::Client client((gcs::ClientOptions(gcs::oauth2::CreateAnonymousCredentials())));\n",
-    "xt::xzarr_gcs_store s3(\"zarr-demo/v3/test.zr3\", client);\n",
-    "auto h3 = xt::get_zarr_hierarchy(s3);\n",
-    "xt::zarray z3 = h3.get_array(\"/arthur/dent\");\n",
-    "auto a3 = z3.get_array<int32_t>();\n",
-    "a3"
+    "xt::xzarr_gcs_store s(\"zarr-demo/v3/test.zr3\", client);\n",
+    "auto h = xt::get_zarr_hierarchy(s);\n",
+    "xt::zarray z = h.get_array(\"/arthur/dent\");\n",
+    "auto a = z.get_array<int32_t>();\n",
+    "a"
    ]
   }
  ],

diff --git a/include/xtensor-zarr/xzarr_array.hpp b/include/xtensor-zarr/xzarr_array.hpp
@@ -85,14 +85,20 @@ namespace xt
     {
         std::string s;
         nlohmann::json attrs;
+        std::vector<std::string> keys;
+        std::vector<std::string> prefixes;
         switch (zarr_version_major)
         {
             case 3:
                 s = store[std::string("meta/root") + path + ".array.json"];
                 break;
             case 2:
                 s = store[path + "/.zarray"];
-                attrs = std::string(store[path + "/.zattrs"]);
+                store.list_dir(path, keys, prefixes);
+                if (std::count(keys.begin(), keys.end(), ".zattrs"))
+                {
+                    attrs = std::string(store[path + "/.zattrs"]);
+                }
                 break;
             default:
                 break;

diff --git a/test/test_gcs.cpp b/test/test_gcs.cpp
@@ -8,6 +8,7 @@
 ****************************************************************************/
 
 #include "xtensor-io/xio_gzip.hpp"
+#include "xtensor-io/xio_blosc.hpp"
 #include "xtensor-zarr/xzarr_hierarchy.hpp"
 #include "xtensor-zarr/xzarr_gcs_store.hpp"
 #include "xtensor-zarr/xzarr_compressor.hpp"
@@ -54,4 +55,22 @@ namespace xt
             EXPECT_THROW(s2.erase(keys2[i]), std::runtime_error);
         }
     }
+
+    // TODO: enable test when zview is fixed
+    //TEST(gcs, view)
+    //{
+    //    //init_zsystem();
+    //    using dispatcher_type = zdispatcher_t<detail::xview_dummy_functor, 1, 1>;
+    //    dispatcher_type::init();
+    //    xzarr_register_compressor<xzarr_gcs_store, xio_blosc_config>();
+    //    gcs::Client client((gcs::ClientOptions(gcs::oauth2::CreateAnonymousCredentials())));
+    //    xzarr_gcs_store s("pangeo-data/gpm_imerg/late/chunk_time/precipitationCal", client);
+    //    std::string metadata = s.get(".zarray");
+    //    std::cerr << metadata << std::endl;
+    //    auto h = get_zarr_hierarchy(s, "2");
+    //    auto z = h.get_array("");
+    //    xstrided_slice_vector sv({1000, all(), all()});
+    //    zarray z2 = make_strided_view(z, sv);
+    //    auto a = z2.get_array<float>();
+    //}
 }
diff --git a/test/test_gdal.cpp b/test/test_gdal.cpp
@@ -16,18 +16,16 @@
 
 namespace xt
 {
-    TEST(gdal, write_read_array_gdal)
+    void write_read_array_gdal(const std::string& zarr_version)
     {
-        xzarr_register_compressor<xzarr_gdal_store, xio_gzip_config>();
-
         // write array
         std::vector<size_t> shape = {4, 4};
         std::vector<size_t> chunk_shape = {2, 2};
         nlohmann::json attrs = {{"question", "life"}, {"answer", 42}};
         std::size_t pool_size = 1;
         double fill_value = 6.6;
-        xzarr_gdal_store s1("/vsimem/test.zr3");
-        auto h1 = create_zarr_hierarchy(s1);
+        xzarr_gdal_store s1("/vsimem/test.zr" + zarr_version);
+        auto h1 = create_zarr_hierarchy(s1, zarr_version);
         xzarr_create_array_options<xio_gzip_config> o;
         o.chunk_memory_layout = 'C';
         o.chunk_separator = '/';
@@ -37,10 +35,21 @@ namespace xt
         zarray z1 = h1.create_array("/arthur/dent", shape, chunk_shape, "<f8", o);
 
         // read array
-        xzarr_gdal_store s2("/vsimem/test.zr3");
+        xzarr_gdal_store s2("/vsimem/test.zr" + zarr_version);
         auto h2 = get_zarr_hierarchy(s2);
         zarray z2 = h2.get_array("/arthur/dent");
         auto ref = ones<double>({4, 4}) * fill_value;
         EXPECT_EQ(ref, z2.get_array<double>());
     }
+
+    TEST(gdal, write_read_v2_array_gdal)
+    {
+        xzarr_register_compressor<xzarr_gdal_store, xio_gzip_config>();
+        write_read_array_gdal("2");
+    }
+
+    TEST(gdal, write_read_v3_array_gdal)
+    {
+        write_read_array_gdal("3");
+    }
 }