From 15e32ab47bdc3a5766f5641e60d7bcd0ab7d74e4 Mon Sep 17 00:00:00 2001 From: Ian Munro Date: Wed, 16 Mar 2016 17:32:56 +0000 Subject: [PATCH 1/6] First roind if changes to buffer v.large .bin files. --- .../src/loci/formats/in/PQBinReader.java | 115 +++++++++--------- 1 file changed, 55 insertions(+), 60 deletions(-) diff --git a/components/formats-gpl/src/loci/formats/in/PQBinReader.java b/components/formats-gpl/src/loci/formats/in/PQBinReader.java index 85e134f9813..6bfde5e3174 100644 --- a/components/formats-gpl/src/loci/formats/in/PQBinReader.java +++ b/components/formats-gpl/src/loci/formats/in/PQBinReader.java @@ -67,9 +67,10 @@ public class PQBinReader extends FormatReader { protected byte[] dataStore = null; /** - * Whether to pre-load all lifetime bins for faster loading. + * Block of time bins currently stored for faster loading. */ - protected boolean preLoad = true; + int currentBlock; + // -- Constructor -- /** @@ -123,42 +124,64 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h) int binSize = sizeX * sizeY * bpp; // size in Bytes of a single 2D timebin. - // if data is of a manageable size then pre-load for performance - if (preLoad) { - if (dataStore == null) - { - // The whole plane (all timebins) is copied into storage - // to allow different sub-plane sizes to be used for different timebins - dataStore = new byte[planeSize]; - byte[] rowBuf = new byte[bpp * timeBins * sizeX]; - in.seek(HEADER_SIZE); - - for (int row = 0; row < sizeY; row++) { - in.read(rowBuf); + int blockLength = 512; // no of timeBins pre-loaded as a block + int blockSize = sizeX * sizeY * blockLength * bpp; + + // pre-load data for performance + + if (dataStore == null) { + dataStore = new byte[blockSize]; + currentBlock = -1; + } + + if (timeBin/blockLength != currentBlock) { + + currentBlock = timeBin/blockLength; + // A subset of timebins (a Block) is copied into storage + // to allow different sub-plane sizes to be used for different timebins + + byte[] rowBuf = new byte[bpp * timeBins * sizeX]; + in.seek(HEADER_SIZE); + + int endOfBlock = (currentBlock + 1) * blockLength; + int storeLength; + if (endOfBlock > timeBins) { + storeLength = timeBins - (currentBlock * blockLength); + } + else { + storeLength = blockLength; + } + + + for (int row = 0; row < sizeY; row++) { + in.read(rowBuf); - int input = 0; - for (int col = 0; col < sizeX; col++) { - // set output to first pixel of this row in 2D plane - // corresponding to zeroth timeBin - int output = (row * sizeX + col) * bpp; + int input = currentBlock * storeLength; + for (int col = 0; col < sizeX; col++) { + // set output to first pixel of this row in 2D plane + // corresponding to zeroth timeBin + int output = (row * sizeX + col) * bpp; - for (int t = 0; t < timeBins; t++) { - for (int bb = 0; bb < bpp; bb++) { - dataStore[output + bb] = rowBuf[input + bb]; - } - output += binSize; - input += bpp; + for (int t = 0; t < storeLength; t++) { + for (int bb = 0; bb < bpp; bb++) { + dataStore[output + bb] = rowBuf[input + bb]; } + output += binSize; + input += bpp; } } + } - // chanStore loaded + // dataStore loaded - // copy 2D plane from chanStore into buf + // copy 2D plane from dataStore into buf int iLineSize = sizeX * bpp; int oLineSize = w * bpp; // offset to correct timebin yth line and xth pixel - int input = (binSize * timeBin) + (y * iLineSize) + (x * bpp); + + int binInStore = timeBin - (currentBlock * blockLength); + + int input = (binSize * binInstore) + (y * iLineSize) + (x * bpp); int output = 0; for (int row = 0; row < h; row++) { @@ -166,29 +189,7 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h) input += iLineSize; output += oLineSize; } - } // endif preLoad - else { // load each plane individually for large data - - byte[] rowBuf = new byte[bpp * timeBins * w]; - - in.seek(HEADER_SIZE + (y * sizeX * bpp * timeBins)); - - for (int row = 0; row < h; row++) { - in.skipBytes(x * bpp * timeBins); - in.read(rowBuf); - - for (int col = 0; col < w; col++) { - int output = (row * w + col) * bpp; - int input = (col * timeBins + timeBin) * bpp; - for (int bb = 0; bb < bpp; bb++) { - buf[output + bb] = rowBuf[input + bb]; - } - } - - in.skipBytes(bpp * timeBins * (sizeX - x - w)); - } - } // end else - + } return buf; } @@ -200,7 +201,8 @@ public void close(boolean fileOnly) throws IOException { // init preLoading dataStore = null; timeBins = 0; - preLoad = true; + currentBlock = -1; + } } @@ -249,14 +251,7 @@ protected void initFile(String id) throws FormatException, IOException { m.moduloT.end = m.moduloT.step * (m.sizeT - 1); m.moduloT.unit = "ps"; - // disable pre-load mode for very large files - // threshold is set to smaller than the size of the largest test file currently available - if ( m.sizeX * m.sizeY * m.sizeT > (900 * 200 * 200)) { - preLoad = false; - } - else { - preLoad = true; - } + MetadataStore store = makeFilterMetadata(); MetadataTools.populatePixels(store, this); From 4620ad00497b163101d6af19d547c8b893714e54 Mon Sep 17 00:00:00 2001 From: Ian Munro Date: Wed, 16 Mar 2016 18:03:29 +0000 Subject: [PATCH 2/6] Second Round of changes. --- components/formats-gpl/src/loci/formats/in/PQBinReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/formats-gpl/src/loci/formats/in/PQBinReader.java b/components/formats-gpl/src/loci/formats/in/PQBinReader.java index 6bfde5e3174..10e271d680f 100644 --- a/components/formats-gpl/src/loci/formats/in/PQBinReader.java +++ b/components/formats-gpl/src/loci/formats/in/PQBinReader.java @@ -181,7 +181,7 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h) int binInStore = timeBin - (currentBlock * blockLength); - int input = (binSize * binInstore) + (y * iLineSize) + (x * bpp); + int input = (binSize * binInStore) + (y * iLineSize) + (x * bpp); int output = 0; for (int row = 0; row < h; row++) { From 97e3ea5694d4201f512f37d3a622c0408e12b4f6 Mon Sep 17 00:00:00 2001 From: Ian Munro Date: Thu, 7 Apr 2016 19:25:40 +0100 Subject: [PATCH 3/6] First working vesion using Blocks. --- .../src/loci/formats/in/PQBinReader.java | 68 ++++++++++++------- 1 file changed, 45 insertions(+), 23 deletions(-) diff --git a/components/formats-gpl/src/loci/formats/in/PQBinReader.java b/components/formats-gpl/src/loci/formats/in/PQBinReader.java index 10e271d680f..96d92a216f6 100644 --- a/components/formats-gpl/src/loci/formats/in/PQBinReader.java +++ b/components/formats-gpl/src/loci/formats/in/PQBinReader.java @@ -124,20 +124,29 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h) int binSize = sizeX * sizeY * bpp; // size in Bytes of a single 2D timebin. - int blockLength = 512; // no of timeBins pre-loaded as a block + int blockLength = 1024; // no of timeBins pre-loaded as a block + + + int blockSize = sizeX * sizeY * blockLength * bpp; + + // pre-load data for performance - if (dataStore == null) { + + LOGGER.info("About to set dataStore"); dataStore = new byte[blockSize]; currentBlock = -1; } + + if (timeBin/blockLength != currentBlock) { currentBlock = timeBin/blockLength; - // A subset of timebins (a Block) is copied into storage + + // A subset of whole timebins (a Block) is copied into storage // to allow different sub-plane sizes to be used for different timebins byte[] rowBuf = new byte[bpp * timeBins * sizeX]; @@ -152,16 +161,29 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h) storeLength = blockLength; } + LOGGER.info("About to preload!"); + LOGGER.info("currentBlock = " + Integer.toString(currentBlock)); + LOGGER.info("storeLength = " + Integer.toString(storeLength)); + LOGGER.info("sizeX = " + Integer.toString(sizeX)); + LOGGER.info("sizeY = " + Integer.toString(sizeY)); + LOGGER.info("timeBins = " + Integer.toString(timeBins)); for (int row = 0; row < sizeY; row++) { in.read(rowBuf); - - int input = currentBlock * storeLength; for (int col = 0; col < sizeX; col++) { // set output to first pixel of this row in 2D plane // corresponding to zeroth timeBin - int output = (row * sizeX + col) * bpp; - + int output = ((row * sizeX) + col) * bpp; + int input = ((col * timeBins) + (currentBlock * blockLength)) * bpp; + + //debug + if (row < 3 & col < 3) { + LOGGER.info("row = " + Integer.toString(row)); + LOGGER.info("col = " + Integer.toString(col)); + LOGGER.info("output = " + Integer.toString(output)); + LOGGER.info("input = " + Integer.toString(input)); + } + for (int t = 0; t < storeLength; t++) { for (int bb = 0; bb < bpp; bb++) { dataStore[output + bb] = rowBuf[input + bb]; @@ -170,26 +192,26 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h) input += bpp; } } - } - // dataStore loaded + } + // dataStore loaded - // copy 2D plane from dataStore into buf - int iLineSize = sizeX * bpp; - int oLineSize = w * bpp; - // offset to correct timebin yth line and xth pixel - - int binInStore = timeBin - (currentBlock * blockLength); - - int input = (binSize * binInStore) + (y * iLineSize) + (x * bpp); - int output = 0; + // copy 2D plane from dataStore into buf + int iLineSize = sizeX * bpp; + int oLineSize = w * bpp; + // offset to correct timebin yth line and xth pixel - for (int row = 0; row < h; row++) { - System.arraycopy(dataStore, input, buf, output, oLineSize); - input += iLineSize; - output += oLineSize; - } + int binInStore = timeBin - (currentBlock * blockLength); + + int input = (binSize * binInStore) + (y * iLineSize) + (x * bpp); + int output = 0; + + for (int row = 0; row < h; row++) { + System.arraycopy(dataStore, input, buf, output, oLineSize); + input += iLineSize; + output += oLineSize; } + return buf; } From b41ff92ab9ff8c4cdd7a0b3e16b2d9d3e31d92b0 Mon Sep 17 00:00:00 2001 From: Ian Munro Date: Thu, 7 Apr 2016 21:48:58 +0100 Subject: [PATCH 4/6] Removal of Debugging and assign buffer size automatically. --- .../src/loci/formats/in/PQBinReader.java | 54 ++++++++----------- 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/components/formats-gpl/src/loci/formats/in/PQBinReader.java b/components/formats-gpl/src/loci/formats/in/PQBinReader.java index 96d92a216f6..2b7380e41f8 100644 --- a/components/formats-gpl/src/loci/formats/in/PQBinReader.java +++ b/components/formats-gpl/src/loci/formats/in/PQBinReader.java @@ -34,20 +34,19 @@ import loci.formats.FormatTools; import loci.formats.MetadataTools; import loci.formats.meta.MetadataStore; -import ome.xml.model.primitives.PositiveFloat; - import ome.units.quantity.Length; -import ome.units.UNITS; /** * PQBinReader is the file format reader for PicoQuant .bin files. * * Please Note: This format holds FLIM data arranged so that each decay is stored contiguously. * Therefore, as in other FLIM format readers e.g. SDTReader.java, on the first call to openBytes - * the whole data cube ( x,y,t) (NB actually t not real-time T) is loaded from the file to a buffer. + * the whole data cube ( x,y,t) (NB actually t not real-time T) is loaded from the file and buffered * On further calls to openBytes the appropriate 2D (x,y)plane (timebin) is returned from this buffer. * This is in the interest of significantly improved performance when all the planes are requested one after another. * There will be a performance cost if a single plane is requested but this is highly unlikely for FLIM data. + * In order to limit the size of the buffer, beyond a certain size threshold only a subset of planes (a Block) are + * retained in the buffer. */ public class PQBinReader extends FormatReader { @@ -69,7 +68,14 @@ public class PQBinReader extends FormatReader { /** * Block of time bins currently stored for faster loading. */ - int currentBlock; + protected int currentBlock; + + /** + * No of timeBins pre-loaded as a block + */ + protected int blockLength ; + + // -- Constructor -- @@ -123,25 +129,15 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h) int timeBin = no; int binSize = sizeX * sizeY * bpp; // size in Bytes of a single 2D timebin. - - int blockLength = 1024; // no of timeBins pre-loaded as a block - - int blockSize = sizeX * sizeY * blockLength * bpp; - - // pre-load data for performance if (dataStore == null) { - - LOGGER.info("About to set dataStore"); dataStore = new byte[blockSize]; currentBlock = -1; } - - if (timeBin/blockLength != currentBlock) { currentBlock = timeBin/blockLength; @@ -161,13 +157,6 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h) storeLength = blockLength; } - LOGGER.info("About to preload!"); - LOGGER.info("currentBlock = " + Integer.toString(currentBlock)); - LOGGER.info("storeLength = " + Integer.toString(storeLength)); - LOGGER.info("sizeX = " + Integer.toString(sizeX)); - LOGGER.info("sizeY = " + Integer.toString(sizeY)); - LOGGER.info("timeBins = " + Integer.toString(timeBins)); - for (int row = 0; row < sizeY; row++) { in.read(rowBuf); for (int col = 0; col < sizeX; col++) { @@ -175,15 +164,7 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h) // corresponding to zeroth timeBin int output = ((row * sizeX) + col) * bpp; int input = ((col * timeBins) + (currentBlock * blockLength)) * bpp; - - //debug - if (row < 3 & col < 3) { - LOGGER.info("row = " + Integer.toString(row)); - LOGGER.info("col = " + Integer.toString(col)); - LOGGER.info("output = " + Integer.toString(output)); - LOGGER.info("input = " + Integer.toString(input)); - } - + for (int t = 0; t < storeLength; t++) { for (int bb = 0; bb < bpp; bb++) { dataStore[output + bb] = rowBuf[input + bb]; @@ -244,8 +225,11 @@ protected void initFile(String id) throws FormatException, IOException { LOGGER.info("Reading header PQBin"); // Header - m.sizeX = in.readInt(); - m.sizeY = in.readInt(); + int sizeX = in.readInt(); + int sizeY = in.readInt(); + m.sizeY = sizeY; + m.sizeX = sizeX; + float pixResol = in.readFloat(); // Resolution of every Pixel in Image (in µm) m.sizeT = in.readInt(); // Number of DataPoints per Decay @@ -273,7 +257,11 @@ protected void initFile(String id) throws FormatException, IOException { m.moduloT.end = m.moduloT.step * (m.sizeT - 1); m.moduloT.unit = "ps"; + int sizeThreshold = 128 * 128 * 1024; // Arbitararily chosen size limit for buffer + blockLength = 2048; //default No of bins in buffer + while (blockLength * sizeX * sizeY > sizeThreshold) + blockLength = blockLength/2; MetadataStore store = makeFilterMetadata(); MetadataTools.populatePixels(store, this); From add38c0dd9098df33f23b499db543f8ee359b565 Mon Sep 17 00:00:00 2001 From: Ian Munro Date: Thu, 7 Apr 2016 21:52:34 +0100 Subject: [PATCH 5/6] More comments. --- components/formats-gpl/src/loci/formats/in/PQBinReader.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/components/formats-gpl/src/loci/formats/in/PQBinReader.java b/components/formats-gpl/src/loci/formats/in/PQBinReader.java index 2b7380e41f8..81435fb79cf 100644 --- a/components/formats-gpl/src/loci/formats/in/PQBinReader.java +++ b/components/formats-gpl/src/loci/formats/in/PQBinReader.java @@ -164,7 +164,7 @@ public byte[] openBytes(int no, byte[] buf, int x, int y, int w, int h) // corresponding to zeroth timeBin int output = ((row * sizeX) + col) * bpp; int input = ((col * timeBins) + (currentBlock * blockLength)) * bpp; - + // copy subset of decay into buffer. for (int t = 0; t < storeLength; t++) { for (int bb = 0; bb < bpp; bb++) { dataStore[output + bb] = rowBuf[input + bb]; From b44feadf8d20e92ef1f38f5f2bc74ac9b7284868 Mon Sep 17 00:00:00 2001 From: Ian Munro Date: Mon, 11 Apr 2016 09:13:57 +0100 Subject: [PATCH 6/6] Ensure buffer is not over-allocated. --- components/formats-gpl/src/loci/formats/in/PQBinReader.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/components/formats-gpl/src/loci/formats/in/PQBinReader.java b/components/formats-gpl/src/loci/formats/in/PQBinReader.java index 81435fb79cf..4deed4ca19f 100644 --- a/components/formats-gpl/src/loci/formats/in/PQBinReader.java +++ b/components/formats-gpl/src/loci/formats/in/PQBinReader.java @@ -263,6 +263,9 @@ protected void initFile(String id) throws FormatException, IOException { while (blockLength * sizeX * sizeY > sizeThreshold) blockLength = blockLength/2; + if (blockLength > timeBins) + blockLength = timeBins; + MetadataStore store = makeFilterMetadata(); MetadataTools.populatePixels(store, this);