From ac5819b4ea421e242960f32baae9139d78f97c75 Mon Sep 17 00:00:00 2001 From: Mark Taylor Date: Thu, 3 Jul 2014 22:57:44 +0100 Subject: [PATCH] ttools: increase default xmatch blocksize from 10k to 50k rows Thomas Boch says it's OK. Also improve documentation of this value. --- topcat/src/docs/sun253.xml | 12 ++++++++++-- .../uk/ac/starlink/topcat/join/UploadMatchPanel.java | 2 +- .../ac/starlink/ttools/task/CdsUploadSkyMatch.java | 9 ++++++++- 3 files changed, 19 insertions(+), 4 deletions(-) diff --git a/topcat/src/docs/sun253.xml b/topcat/src/docs/sun253.xml index ace2526502..3b06255f85 100644 --- a/topcat/src/docs/sun253.xml +++ b/topcat/src/docs/sun253.xml @@ -16406,8 +16406,16 @@ about how the match will operate:

Block size

The number of rows uploaded to the X-Match service at a time. - This should not affect the result; it may affect performance, - though probably not very much. + This should not affect the result, though it may affect performance. +

+

Large blocksizes tend to be good (up to a point) for + reducing the total amount of time a large xmatch operation takes, + but they can make it harder to see the job progressing. + There is also the danger (for ALL-type find modes) + of exceeding the return size limit, which will result in + truncation of the returned result. + At time of writing, the upload limit is 100Mbyte (about 3Mrow), + and the maximum return size is 2Mrow.

diff --git a/topcat/src/main/uk/ac/starlink/topcat/join/UploadMatchPanel.java b/topcat/src/main/uk/ac/starlink/topcat/join/UploadMatchPanel.java index 437381387b..70ca139454 100644 --- a/topcat/src/main/uk/ac/starlink/topcat/join/UploadMatchPanel.java +++ b/topcat/src/main/uk/ac/starlink/topcat/join/UploadMatchPanel.java @@ -85,7 +85,7 @@ public class UploadMatchPanel extends JPanel { private static final long MAXREC = -1; private static final int[] BLOCK_SIZES = { 100, 1000, 5000, 10000, 20000, 50000, 100000, 200000, 500000, }; - private static final int DEFAULT_BLOCKSIZE = 10000; + private static final int DEFAULT_BLOCKSIZE = 50000; private static final ValueInfo SR_INFO = new DefaultValueInfo( "Radius", Double.class, "Search Radius" ); diff --git a/ttools/src/main/uk/ac/starlink/ttools/task/CdsUploadSkyMatch.java b/ttools/src/main/uk/ac/starlink/ttools/task/CdsUploadSkyMatch.java index 0f0d83cb7c..0cef9f6ab9 100644 --- a/ttools/src/main/uk/ac/starlink/ttools/task/CdsUploadSkyMatch.java +++ b/ttools/src/main/uk/ac/starlink/ttools/task/CdsUploadSkyMatch.java @@ -121,9 +121,16 @@ public CdsUploadSkyMatch() { "(about 3Mrow; this does not depend on the width of your table),", "and the maximum return size is 2Mrow.", "

", + "

Large blocksizes tend to be good (up to a point) for", + "reducing the total amount of time a large xmatch operation takes,", + "but they can make it harder to see the job progressing.", + "There is also the danger (for ALL-type find modes)", + "of exceeding the return size limit, which will result in", + "truncation of the returned result.", + "

", } ); chunkParam_.setMinimum( 1 ); - chunkParam_.setDefault( Integer.toString( 10 * 1000 ) ); + chunkParam_.setDefault( Integer.toString( 50 * 1000 ) ); findParam_ = new ChoiceParameter( "find", UserFindMode.class,