Merge pull request #2332 from imunro/PQBIN2

Update PicoQuant reader to always buffer data.

components/formats-gpl/src/loci/formats/in/PQBinReader.java

@@ -34,20 +34,19 @@
 import loci.formats.FormatTools;
 import loci.formats.MetadataTools;
 import loci.formats.meta.MetadataStore;
-import ome.xml.model.primitives.PositiveFloat;
-
 import ome.units.quantity.Length;
-import ome.units.UNITS;
 
 /**
  * PQBinReader is the file format reader for PicoQuant .bin files.
  *
  * Please Note: This format holds FLIM data arranged so that each decay is stored contiguously. 
  * Therefore, as in other FLIM format readers e.g. SDTReader.java, on the first call to openBytes
- * the whole data cube ( x,y,t) (NB actually t not real-time T) is loaded from the file to  a buffer.
+ * the whole data cube ( x,y,t) (NB actually t not real-time T) is loaded from the file and buffered
  * On further calls to openBytes the appropriate 2D (x,y)plane (timebin) is returned from this buffer.
  * This is in the interest of significantly improved  performance when all the planes are requested one after another.
  * There will be a performance cost if a single plane is requested but this is highly unlikely for FLIM data.
+ * In order to limit the size of the buffer, beyond a certain size threshold only a subset of planes (a Block) are 
+ * retained in the buffer.
  */
 public class PQBinReader extends FormatReader {
 
@@ -67,9 +66,17 @@ public class PQBinReader extends FormatReader {
   protected byte[] dataStore = null;
 
   /**
-   * Whether to pre-load all lifetime bins for faster loading.
+   * Block of time bins currently stored for faster loading.
    */
-  protected boolean preLoad = true;
+  protected int currentBlock;
+
+  /**
+   * No of timeBins pre-loaded as a block
+  */
+  protected int blockLength ;  
+
+
+
 
   // -- Constructor --
   /**
@@ -122,73 +129,70 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h)
     int timeBin = no;
 
     int binSize = sizeX * sizeY  * bpp;  // size in Bytes of a single 2D timebin.
+
+    int blockSize = sizeX * sizeY * blockLength * bpp;
 
-    // if data is of a manageable size then pre-load for performance
-    if (preLoad)  {
-      if (dataStore == null)
-      {
-        // The whole plane (all timebins) is  copied into storage
-        // to allow different sub-plane sizes to be used for different timebins
-        dataStore = new byte[planeSize];
-        byte[] rowBuf = new byte[bpp * timeBins * sizeX];
-        in.seek(HEADER_SIZE);
-
-        for (int row = 0; row < sizeY; row++) {
-          in.read(rowBuf);
-
-          int input = 0;
-          for (int col = 0; col < sizeX; col++) {
-            // set output to first pixel of this row in 2D plane
-            // corresponding to zeroth timeBin
-            int output = (row * sizeX + col) * bpp;
-
-            for (int t = 0; t < timeBins; t++) {
-              for (int bb = 0; bb < bpp; bb++) {
-                dataStore[output + bb] = rowBuf[input + bb];
-              }
-              output += binSize;
-              input += bpp;
+    // pre-load data for performance
+    if (dataStore == null) {
+      dataStore = new byte[blockSize];
+      currentBlock = -1;
+    }
+
+    if (timeBin/blockLength != currentBlock)  {
+
+      currentBlock = timeBin/blockLength;
+
+      // A subset of whole timebins (a Block) is  copied into storage
+      // to allow different sub-plane sizes to be used for different timebins
+
+      byte[] rowBuf = new byte[bpp * timeBins * sizeX];
+      in.seek(HEADER_SIZE);
+
+      int endOfBlock =  (currentBlock + 1) * blockLength;
+      int storeLength;
+      if (endOfBlock > timeBins) {
+        storeLength = timeBins - (currentBlock * blockLength);
+      }
+      else {
+        storeLength = blockLength;
+      }
+
+      for (int row = 0; row < sizeY; row++) {
+        in.read(rowBuf);
+        for (int col = 0; col < sizeX; col++) {
+          // set output to first pixel of this row in 2D plane
+          // corresponding to zeroth timeBin
+          int output = ((row * sizeX) + col) * bpp;
+          int input = ((col * timeBins) + (currentBlock * blockLength)) * bpp;
+          // copy subset of decay into buffer.
+          for (int t = 0; t < storeLength; t++) {
+            for (int bb = 0; bb < bpp; bb++) {
+              dataStore[output + bb] = rowBuf[input + bb];
             }
+            output += binSize;
+            input += bpp;
           }
         }
       }
-        // chanStore loaded
-
-        // copy 2D plane  from chanStore  into buf
-      int iLineSize = sizeX * bpp;
-      int oLineSize = w * bpp;
-      // offset to correct timebin yth line and xth pixel
-      int input = (binSize * timeBin) + (y * iLineSize) + (x * bpp);
-      int output = 0;
+    }
+    // dataStore loaded
 
-      for (int row = 0; row < h; row++) {
-        System.arraycopy(dataStore, input, buf, output, oLineSize);
-        input += iLineSize;
-        output += oLineSize;
-      }
-    }    // endif preLoad
-    else  {   // load each plane individually for large data
-
-      byte[] rowBuf = new byte[bpp * timeBins * w];
-
-      in.seek(HEADER_SIZE +  (y * sizeX * bpp * timeBins));
+    // copy 2D plane  from dataStore  into buf
+    int iLineSize = sizeX * bpp;
+    int oLineSize = w * bpp;
+    // offset to correct timebin yth line and xth pixel
 
-      for (int row = 0; row < h; row++) {
-        in.skipBytes(x * bpp * timeBins);
-        in.read(rowBuf);
+    int binInStore = timeBin - (currentBlock * blockLength);
+
+    int input = (binSize * binInStore) + (y * iLineSize) + (x * bpp);
+    int output = 0;
 
-        for (int col = 0; col < w; col++) {
-          int output = (row * w + col) * bpp;
-          int input = (col * timeBins + timeBin) * bpp;
-          for (int bb = 0; bb < bpp; bb++) {
-            buf[output + bb] = rowBuf[input + bb];
-          }
-        }
+    for (int row = 0; row < h; row++) {
+      System.arraycopy(dataStore, input, buf, output, oLineSize);
+      input += iLineSize;
+      output += oLineSize;
+    }
 
-        in.skipBytes(bpp * timeBins * (sizeX - x - w));
-      }
-    }  // end else
-
     return buf;
   }
 
@@ -200,7 +204,8 @@ public void close(boolean fileOnly) throws IOException {
       // init preLoading
       dataStore = null;
       timeBins = 0;
-      preLoad = true;
+      currentBlock = -1;
+
     }
   }
 
@@ -220,8 +225,11 @@ protected void initFile(String id) throws FormatException, IOException {
     LOGGER.info("Reading header PQBin");
 
     // Header
-    m.sizeX = in.readInt();
-    m.sizeY = in.readInt();
+    int sizeX = in.readInt();
+    int sizeY = in.readInt();
+    m.sizeY = sizeY;
+    m.sizeX = sizeX;
+
     float pixResol = in.readFloat();    // Resolution of every Pixel in Image (in µm)
     m.sizeT = in.readInt();             // Number of DataPoints per Decay
 
@@ -249,14 +257,14 @@ protected void initFile(String id) throws FormatException, IOException {
     m.moduloT.end = m.moduloT.step * (m.sizeT - 1);
     m.moduloT.unit = "ps";
 
-    // disable pre-load mode for very large files
-    // threshold is set to smaller than the  size of the largest test file currently available
-    if ( m.sizeX * m.sizeY * m.sizeT  >  (900 * 200 * 200))  {
-      preLoad = false;
-    }
-    else  {
-      preLoad = true;
-    }
+    int sizeThreshold = 128 * 128 * 1024;  // Arbitararily chosen size limit for buffer
+    blockLength = 2048;   //default No of bins in buffer
+
+    while (blockLength * sizeX * sizeY > sizeThreshold)
+      blockLength = blockLength/2;
+
+    if (blockLength > timeBins)
+      blockLength = timeBins;
 
     MetadataStore store = makeFilterMetadata();
     MetadataTools.populatePixels(store, this);