improve check tools

* check chunking of pixel_mask and data

* add option to check_h5_plugin to dynamically
  load other plugin .so file to compare speed
  and functionality.

README.md

@@ -31,6 +31,7 @@ Wikipedia: https://en.wikipedia.org/wiki/Alpe_di_Neggia
 
 /entry/instrument/detector/detectorSpecific/pixel_mask
     type: uint32
+    chunking must be disabled
     neggia will apply pixel_mask and set data to
           -1 for pixel_mask & 0b00001
           -2 for pixel_mask & 0b11110
@@ -43,6 +44,7 @@ Wikipedia: https://en.wikipedia.org/wiki/Alpe_di_Neggia
     'data_000001' to 'data_999999'
     for a single h5 file without links to external datasets
     '/entry/data/data' will be used to extract image data
+    all data must be chunked frame-wise
 ```
 
 You can check the compatibility requirements by running our test script

bin/CMakeLists.txt

@@ -7,3 +7,7 @@ add_executable(check_h5_plugin
   $<TARGET_OBJECTS:NEGGIA_USER>
   check_h5_plugin.cpp
   )
+
+target_link_libraries(check_h5_plugin
+  dl
+)

bin/check_h5_compatibility.py

@@ -149,6 +149,15 @@ def check_file(filename):
         )
 
     try:
+        pm_chunks = detectorSpecific["pixel_mask"].chunks
+        if pm_chunks is not None:
+            print_fail(
+                f"""'/entry/instrument/detector/detectorSpecific/pixel_mask' 2D data may not be chunked.
+       Please save your pixel mask with disabled chunking.
+       In python you can do this by adding argument 'chunks=None' to method create_dataset.
+"""
+            )
+        pm_shape = detectorSpecific["pixel_mask"].shape
         pm = detectorSpecific["pixel_mask"][()]
         if pm.dtype == np.uint32:
             print_ok(
@@ -185,6 +194,20 @@ def check_file(filename):
        neggia will ignore this entry."""
             )
         else:
+            data_chunks = master[path].chunks
+            data_shape = master[path].shape
+            if data_chunks is None:
+                print_fail(f"image data in '{path}' must be chunked")
+            if len(data_shape) != 3:
+                print_fail(f"expected 3-dimensional data for '{path}'")
+            if (data_shape[1], data_shape[2]) != (pm_shape[0], pm_shape[1]):
+                print_fail(
+                    f"""expected image data shape (any, {pm_shape[0]}, {pm_shape[1]}) but got {data_shape}
+       this shape is deduced from the shape of the pixel mask. (perhaps transposed?)"""
+                )
+            expected_chunks = (1, data_shape[1], data_shape[2])
+            if data_chunks != expected_chunks:
+                print_fail(f"expected chunks {expected_chunks} but got {data_chunks}")
             count_data_entries_correct_format += 1
     if count_data_entries_correct_format == 0:
         print_fail(f"no entries with correct format found in '/entry/data'")

bin/check_h5_plugin.cpp

@@ -1,29 +1,63 @@
 // SPDX-License-Identifier: MIT
 
 #include <dectris/neggia/plugin/H5ToXds.h>
+#include <dlfcn.h>
+#include <iomanip>
+#include <ios>
 #include <iostream>
 #include <memory>
 
+using dl_plugin_open = decltype(&plugin_open);
+using dl_plugin_get_header = decltype(&plugin_get_header);
+using dl_plugin_get_data = decltype(&plugin_get_data);
+using dl_plugin_close = decltype(&plugin_close);
+
 int main(int argc, char* argv[]) {
-    if (argc != 2) {
-        std::cerr << "Please provide the filename of the hdf5 file that you "
-                     "would like to test against neggia plugin\n";
-        return -1;
+    dl_plugin_open open_file;
+    dl_plugin_get_header get_header;
+    dl_plugin_get_data get_data;
+    dl_plugin_close close_file;
+    switch (argc) {
+        case 2: {
+            std::cerr << "using neggia methods directly\n";
+            open_file = plugin_open;
+            get_header = plugin_get_header;
+            get_data = plugin_get_data;
+            close_file = plugin_close;
+            break;
+        }
+        case 3: {
+            std::string so_filename = argv[2];
+            std::cerr << "dynamically linking methods from " << so_filename
+                      << "\n";
+            void* pluginHandle = dlopen(so_filename.c_str(), RTLD_NOW);
+            open_file = (dl_plugin_open)dlsym(pluginHandle, "plugin_open");
+            get_header = (dl_plugin_get_header)dlsym(pluginHandle,
+                                                     "plugin_get_header");
+            get_data =
+                    (dl_plugin_get_data)dlsym(pluginHandle, "plugin_get_data");
+            close_file = (dl_plugin_close)dlsym(pluginHandle, "plugin_close");
+            break;
+        }
+        default:
+            std::cerr
+                    << "Please provide the filename of the hdf5 file that you "
+                       "would like to test against neggia plugin\n";
+            return EXIT_FAILURE;
     }
+
     std::string filename = argv[1];
     int error_flag;
     int info_array[1024];
-    std::cerr << "\nOpening file " << filename << "\n";
-    plugin_open(filename.c_str(), info_array, &error_flag);
+    open_file(filename.c_str(), info_array, &error_flag);
     if (error_flag != 0) {
         std::cerr << "[FAIL] plugin_open returned error " << error_flag << "\n";
-        return -1;
+        return EXIT_FAILURE;
     }
     std::cerr << "[ OK ] plugin_open successful\n";
     int nx, ny, nbytes, nframes;
     float qx, qy;
-    plugin_get_header(&nx, &ny, &nbytes, &qx, &qy, &nframes, info_array,
-                      &error_flag);
+    get_header(&nx, &ny, &nbytes, &qx, &qy, &nframes, info_array, &error_flag);
     if (error_flag != 0) {
         std::cerr << "[FAIL] plugin_get_header returned error " << error_flag
                   << "\n";
@@ -33,35 +67,45 @@ int main(int argc, char* argv[]) {
         return -1;
     }
     std::cerr << "[ OK ] plugin_get_header successful\n";
-    std::cerr << "   nx      " << nx << "\n";
-    std::cerr << "   ny      " << ny << "\n";
-    std::cerr << "   nbytes  " << nbytes << "\n";
-    std::cerr << "   nframes " << nframes << "\n";
-    std::cerr << "   qx      " << qx << "\n";
-    std::cerr << "   qy      " << qy << "\n";
+    std::cerr << "    nx      " << nx << "\n";
+    std::cerr << "    ny      " << ny << "\n";
+    std::cerr << "    nbytes  " << nbytes << "\n";
+    std::cerr << "    nframes " << nframes << "\n";
+    std::cerr << "    qx      " << qx << "\n";
+    std::cerr << "    qy      " << qy << "\n";
+    if (nframes <= 0) {
+        std::cerr << "[FAIL] nframes must be positive\n";
+        return EXIT_FAILURE;
+    }
     if (qx <= 0.0 || qy <= 0.0) {
         std::cerr << "[WARN] qx/qy should be positive (pixel sizes) - either "
                      "missing or wrong\n";
     }
 
     auto dataArrayExtracted = std::unique_ptr<int[]>(new int[nx * ny]);
-
+    std::cerr << "  trying to extract all data frames. please wait...\n";
+    std::cerr << "    [" << std::setw(6) << std::right << "1"
+              << " / " << nframes << " ]\n";
     for (int frame = 1; frame <= nframes; ++frame) {
-        plugin_get_data(&frame, &nx, &ny, dataArrayExtracted.get(), info_array,
-                        &error_flag);
+        if ((frame % 100) == 0) {
+            std::cerr << "    [" << std::setw(6) << std::right << frame << " / "
+                      << nframes << " ]\n";
+        }
+        get_data(&frame, &nx, &ny, dataArrayExtracted.get(), info_array,
+                 &error_flag);
         if (error_flag != 0) {
             std::cerr << "[FAIL] plugin_get_data for frame " << frame
                       << " returned error " << error_flag << "\n";
-            return -1;
+            return EXIT_FAILURE;
         }
     }
 
-    plugin_close(&error_flag);
+    close_file(&error_flag);
     if (error_flag != 0) {
         std::cerr << "[FAIL] plugin_close returned error " << error_flag
                   << "\n";
-        return -1;
+        return EXIT_FAILURE;
     }
     std::cerr << "[ OK ] plugin_close successful\n";
-    return 0;
+    return EXIT_SUCCESS;
 }