H5: fix Huge Objects on Fractal Heap

cdm/src/main/java/ucar/nc2/iosp/hdf5/BTree2.java

@@ -84,6 +84,14 @@ public class BTree2 {
     }
   }
 
+  BTree2.Record1 getEntry1(int hugeObjectID) {
+    for (Entry2 entry : entryList) {
+      BTree2.Record1 record1 = (BTree2.Record1) entry.record;
+      if (record1.hugeObjectID == hugeObjectID) return record1;
+    }
+    return null;
+  }
+
   // these are part of the level 1A data structure, type = 0
   static class Entry2 {
     long childAddress, nrecords, totNrecords;

cdm/src/main/java/ucar/nc2/iosp/hdf5/FractalHeap.java

@@ -1,5 +1,6 @@
 package ucar.nc2.iosp.hdf5;
 
+import sun.reflect.generics.reflectiveObjects.NotImplementedException;
 import ucar.unidata.io.RandomAccessFile;
 import ucar.unidata.util.SpecialMathFunction;
 
@@ -76,7 +77,7 @@ public class FractalHeap {
   int maxSizeOfObjects;
   long nextHugeObjectId, freeSpace, managedSpace, allocatedManagedSpace, offsetDirectBlock,
           nManagedObjects, sizeHugeObjects, nHugeObjects, sizeTinyObjects, nTinyObjects;
-  long btreeAddress, freeSpaceTrackerAddress;
+  long btreeAddressHugeObjects, freeSpaceTrackerAddress;
 
   short maxHeapSize, startingNumRows, currentNumRows;
   long maxDirectBlockSize;
@@ -93,7 +94,7 @@ public class FractalHeap {
   byte[] ioFilterInfo;
 
   DoublingTable doublingTable;
-
+  BTree2 btreeHugeObjects;
 
 
   FractalHeap(H5header h5, String forWho, long address, MemTracker memTracker) throws IOException {
@@ -118,7 +119,7 @@ public class FractalHeap {
 
     maxSizeOfObjects = raf.readInt(); // greater than this are huge objects
     nextHugeObjectId = h5.readLength(); // next id to use for a huge object
-    btreeAddress = h5.readOffset(); // v2 btee to track huge objects
+    btreeAddressHugeObjects = h5.readOffset(); // v2 btee to track huge objects
     freeSpace = h5.readLength();  // total free space in managed direct blocks
     freeSpaceTrackerAddress = h5.readOffset();
     managedSpace = h5.readLength(); // total amount of managed space in the heap
@@ -150,7 +151,7 @@ public class FractalHeap {
     if (debugDetail || debugFractalHeap) {
       debugOut.println("FractalHeap for " + forWho + " version=" + version + " heapIdLen=" + heapIdLen + " ioFilterLen=" + ioFilterLen + " flags= " + flags);
       debugOut.println(" maxSizeOfObjects=" + maxSizeOfObjects + " nextHugeObjectId=" + nextHugeObjectId + " btreeAddress="
-              + btreeAddress + " managedSpace=" + managedSpace + " allocatedManagedSpace=" + allocatedManagedSpace + " freeSpace=" + freeSpace);
+              + btreeAddressHugeObjects + " managedSpace=" + managedSpace + " allocatedManagedSpace=" + allocatedManagedSpace + " freeSpace=" + freeSpace);
       debugOut.println(" nManagedObjects=" + nManagedObjects + " nHugeObjects= " + nHugeObjects + " nTinyObjects=" + nTinyObjects +
               " maxDirectBlockSize=" + maxDirectBlockSize + " maxHeapSize= 2^" + maxHeapSize);
       debugOut.println(" DoublingTable: tableWidth=" + tableWidth + " startingBlockSize=" + startingBlockSize);
@@ -193,7 +194,7 @@ public class FractalHeap {
   void showDetails(Formatter f) {
     f.format("FractalHeap version=" + version + " heapIdLen=" + heapIdLen + " ioFilterLen=" + ioFilterLen + " flags= " + flags + "%n");
     f.format(" maxSizeOfObjects=" + maxSizeOfObjects + " nextHugeObjectId=" + nextHugeObjectId + " btreeAddress="
-            + btreeAddress + " managedSpace=" + managedSpace + " allocatedManagedSpace=" + allocatedManagedSpace + " freeSpace=" + freeSpace + "%n");
+            + btreeAddressHugeObjects + " managedSpace=" + managedSpace + " allocatedManagedSpace=" + allocatedManagedSpace + " freeSpace=" + freeSpace + "%n");
     f.format(" nManagedObjects=" + nManagedObjects + " nHugeObjects= " + nHugeObjects + " nTinyObjects=" + nTinyObjects +
             " maxDirectBlockSize=" + maxDirectBlockSize + " maxHeapSize= 2^" + maxHeapSize + "%n");
     f.format(" rootBlockAddress=" + rootBlockAddress + " startingNumRows=" + startingNumRows + " currentNumRows=" + currentNumRows + "%n%n");
@@ -202,32 +203,86 @@ void showDetails(Formatter f) {
   }
 
 
-  DHeapId getHeapId(byte[] heapId) throws IOException {
+  DHeapId getFractalHeapId(byte[] heapId) throws IOException {
     return new DHeapId(heapId);
   }
 
   class DHeapId {
     int type;
-    int n, m;
+    int subtype;  // 1 = indirect no filter, 2 = indirect, filter 3 = direct, no filter, 4 = direct, filter
+    int n;        // the offset field size
+    int m;
     int offset; // This field is the offset of the object in the heap.
     int size;   // This field is the length of the object in the heap
 
     DHeapId(byte[] heapId) throws IOException {
       type = (heapId[0] & 0x30) >> 4;
-      n = maxHeapSize / 8;
-      m = h5.getNumBytesFromMax(maxDirectBlockSize - 1);
 
-      offset = h5.makeIntFromBytes(heapId, 1, n);
-      size = h5.makeIntFromBytes(heapId, 1 + n, m);
-      // System.out.println("Heap id =" + showBytes(heapId) + " type = " + type + " n= " + n + " m= " + m + " offset= " + offset + " size= " + size);
+      if (type == 0) {
+        n = maxHeapSize / 8;      // This field's size is the minimum number of bytes necessary to encode the Maximum Heap Size value
+        m = h5.getNumBytesFromMax(maxDirectBlockSize - 1);  // This field is the length of the object in the heap.
+        // It is determined by taking the minimum value of Maximum Direct Block Size and Maximum Size of Managed Objects in the Fractal Heap Header.
+        // Again, the minimum number of bytes needed to encode that value is used for the size of this field.
+
+        offset = h5.makeIntFromBytes(heapId, 1, n);
+        size = h5.makeIntFromBytes(heapId, 1 + n, m);
+      }
+
+      else if (type == 1) {
+        // how fun to guess the subtype
+        boolean hasBtree = (btreeAddressHugeObjects > 0);
+        boolean hasFilters = (ioFilterLen > 0);
+        if (hasBtree)
+          subtype = hasFilters ? 2 : 1;
+        else
+          subtype = hasFilters ? 4 : 3;
+
+        switch (subtype) {
+          case 1:
+            n = h5.getNumBytesFromMax(nManagedObjects);      // guess
+            offset = h5.makeIntFromBytes(heapId, 1, n);      // [16,1,0,0,0,0,0,0]
+            break;
+        }
+      } else if (type == 2) {
+        /* The sub-type for tiny heap IDs depends on whether the heap ID is large enough to store objects greater than 16 bytes or not.
+          If the heap ID length is 18 bytes or smaller, the "normal" tiny heap ID form is used. If the heap ID length is greater than 18 bytes in length,
+          the "extented" form is used. */
+        subtype = (heapId.length <= 18) ? 1 : 2; // 0 == normal, 1 = extended
+      }
+
+      else  {
+        throw new NotImplementedException(); // "DHeapId subtype = "+subtype);
+      }
+
+
     }
 
-    long getPos() {
-      return doublingTable.getPos(offset);
+    long getPos() throws IOException {
+      switch (type) {
+        case 0:
+          return doublingTable.getPos(offset);
+        case 1: {
+          switch (subtype) {
+            case 1:
+            case 2:
+              if (btreeHugeObjects == null) {
+                btreeHugeObjects = new BTree2(h5, "FractalHeap btreeHugeObjects", btreeAddressHugeObjects);
+                assert btreeHugeObjects.btreeType == subtype;
+              }
+              BTree2.Record1 record1 = btreeHugeObjects.getEntry1(offset);
+              return record1.hugeObjectAddress;
+            case 3:
+            case 4:
+              return offset;     // guess
+          }
+        }
+        default:
+          throw new RuntimeException("Unknown DHeapId type ="+type);
+      }
     }
 
     public String toString() {
-      return type + " " + n + " " + m + " " + offset + " " + size;
+      return type + "," + n + "," + m + "," + offset + "," + size;
     }
   }
 

cdm/src/main/java/ucar/nc2/iosp/hdf5/H5header.java

@@ -2413,11 +2413,11 @@ private void processAttributeInfoMessage(MessageAttributeInfo attInfo, List<Mess
         }
 
         // the heapId points to an Attribute Message in the fractal Heap
-        long pos = fractalHeap.getHeapId(heapId).getPos();
+        FractalHeap.DHeapId fractalHeapId = fractalHeap.getFractalHeapId(heapId);
+        long pos = fractalHeapId.getPos();
         if (pos > 0) {
-          raf.seek(pos);
           MessageAttribute attMessage = new MessageAttribute();
-          if (attMessage.read())
+          if (attMessage.read(pos))
             list.add(attMessage);
           if (debugBtree2) System.out.println("    attMessage=" + attMessage);
         }
@@ -2727,7 +2727,7 @@ int read(long filePos, int version, boolean creationOrderPresent, String objectN
 
       } else if (mtype == MessageType.Attribute) { // 12
         MessageAttribute data = new MessageAttribute();
-        data.read();
+        data.read(raf.getFilePointer());
         messData = data;
 
       } else if (mtype == MessageType.Comment) { // 13
@@ -3707,7 +3707,8 @@ public String getName() {
       return name;
     }
 
-    boolean read() throws IOException {
+    boolean read(long pos) throws IOException {
+      raf.seek(pos);
       if (debugPos) debugOut.println("   *MessageAttribute start pos= " + raf.getFilePointer());
       short nameSize, typeSize, spaceSize;
       byte flags = 0;
@@ -3732,9 +3733,10 @@ boolean read() throws IOException {
         nameSize = raf.readShort();
         typeSize = raf.readShort();
         spaceSize = raf.readShort();
-        log.error("bad version " + version + " at filePos " + raf.getFilePointer());
+        log.error("HDF5 MessageAttribute found bad version " + version + " at filePos " + raf.getFilePointer());
         // G:/work/galibert/IMOS_ANMN-NSW_AETVZ_20131127T230000Z_PH100_FV01_PH100-1311-Workhorse-ADCP-109.5_END-20140306T010000Z_C-20140521T053527Z.nc
-
+        // E:/work/antonio/2014_ch.nc
+        // return false;
       } else {
         log.error("bad version " + version + " at filePos " + raf.getFilePointer()); // buggery, may be HDF5 "more than 8 attributes" error
         return false;
@@ -3829,12 +3831,11 @@ void showFractalHeap(Formatter f) {
             }
 
             // the heapId points to an Attribute Message in the fractal Heap
-            FractalHeap.DHeapId dh = fractalHeap.getHeapId(heapId);
+            FractalHeap.DHeapId dh = fractalHeap.getFractalHeapId(heapId);
             f.format("   %2d %2d %2d %6d %4d %8d", dh.type, dh.n, dh.m, dh.offset, dh.size, dh.getPos());
             if (dh.getPos() > 0) {
-              raf.seek(dh.getPos());
               MessageAttribute attMessage = new MessageAttribute();
-              attMessage.read();
+              attMessage.read(dh.getPos());
               f.format(" %-30s", trunc(attMessage.getName(), 30));
             }
             f.format(" heapId=:");
@@ -3986,7 +3987,8 @@ private void readGroupNew(H5Group group, MessageGroupNew groupNewMessage, DataOb
         }
 
         // the heapId points to a Link message in the Fractal Heap
-        long pos = fractalHeap.getHeapId(heapId).getPos();
+        FractalHeap.DHeapId fractalHeapId = fractalHeap.getFractalHeapId(heapId);
+        long pos = fractalHeapId.getPos();
         if (pos < 0) continue;
         raf.seek(pos);
         MessageLink linkMessage = new MessageLink();

ui/src/main/java/ucar/nc2/ui/ToolsUI.java

@@ -1389,7 +1389,7 @@ else if (setUseRecordStructure)
       if ((null == message) && (ioe instanceof EOFException))
         message = "Premature End of File";
       JOptionPane.showMessageDialog(null, "NetcdfDataset.open cant open " + location + "\n" + message);
-      //if (!(ioe instanceof FileNotFoundException))
+      if (!(ioe instanceof FileNotFoundException))
         ioe.printStackTrace();
 
       ncfile = null;