Skip to content

Commit

Permalink
ttools: fix upload matcher to cope with empty input tables
Browse files Browse the repository at this point in the history
The CDS upload service currently fails with an error, at least under
some circumstances (selection=all) if there are no rows in the input
table.  So add logic which is able, configurably, to fail the match
with a well-behaved IOException in the case of zero input rows rather
than attempt the match.

Once the CDS service copes with empty upload tables this should
perhaps be switched back, since an empty output table may be of
some use (at least you can get the columns).  Or maybe not, you
probably don't want to keep the result.
  • Loading branch information
mbtaylor committed Jul 3, 2014
1 parent 489b545 commit a2cd023
Show file tree
Hide file tree
Showing 4 changed files with 26 additions and 9 deletions.
Expand Up @@ -435,9 +435,11 @@ private MatchWorker createMatchWorker() {
JoinFixAction inFixAct = JoinFixAction.NO_ACTION;
JoinFixAction cdsFixAct = fixSelector_.getJoinFixAction();
boolean oneToOne = upMode.isOneToOne();
boolean uploadEmpty = CdsUploadMatcher.UPLOAD_EMPTY;
BlockUploader blocker =
new BlockUploader( umatcher, blocksize, maxrec, outName,
inFixAct, cdsFixAct, serviceMode, oneToOne );
inFixAct, cdsFixAct, serviceMode, oneToOne,
uploadEmpty );

/* Create and return the match worker. */
return new MatchWorker( blocker, upMode, tcModel, qsFact, storage );
Expand Down
22 changes: 15 additions & 7 deletions ttools/src/main/uk/ac/starlink/ttools/cone/BlockUploader.java
Expand Up @@ -31,6 +31,7 @@ public class BlockUploader {
private final JoinFixAction remoteFixAct_;
private final ServiceFindMode serviceMode_;
private final boolean oneToOne_;
private final boolean uploadEmpty_;

private static final Logger logger_ =
Logger.getLogger( "uk.ac.starlink.ttools.task" );
Expand All @@ -46,11 +47,15 @@ public class BlockUploader {
* @param remoteFixAct name deduplication policy for remote table
* @param serviceMode upload match mode
* @param oneToOne true iff output rows match 1:1 with input rows
* @param uploadEmpty determines behaviour if there are no input rows:
* true means attempt the match anyway,
* false means throw an IOException
*/
public BlockUploader( UploadMatcher umatcher, int blocksize, long maxrec,
String outName, JoinFixAction uploadFixAct,
JoinFixAction remoteFixAct,
ServiceFindMode serviceMode, boolean oneToOne ) {
ServiceFindMode serviceMode, boolean oneToOne,
boolean uploadEmpty ) {
umatcher_ = umatcher;
blocksize_ = blocksize;
maxrec_ = maxrec;
Expand All @@ -59,6 +64,7 @@ public BlockUploader( UploadMatcher umatcher, int blocksize, long maxrec,
remoteFixAct_ = remoteFixAct;
serviceMode_ = serviceMode;
oneToOne_ = oneToOne;
uploadEmpty_ = uploadEmpty;
if ( oneToOne_ && ! serviceMode.supportsOneToOne() ) {
throw new IllegalArgumentException( "Mode " + serviceMode
+ " doesn't support 1:1" );
Expand Down Expand Up @@ -114,9 +120,13 @@ public StarTable runMatch( StarTable inTable, QuerySequenceFactory qsFact,
while ( ! done && ( maxrec_ < 0 || totOut < maxrec_ ) ) {
PreviewBlockSequence blockSeq =
new PreviewBlockSequence( coneSeq, blocksize_ );
if ( blockSeq.hasNext() ) {
BlockSink blockSink =
new BlockSink( rawResultStore, iblock == 0 );
boolean isFirst = iblock == 0;
boolean hasNext = blockSeq.hasNext();
if ( isFirst && ! hasNext && ! uploadEmpty_ ) {
throw new IOException( "No candidate rows for upload match" );
}
if ( hasNext || isFirst ) {
BlockSink blockSink = new BlockSink( rawResultStore, isFirst );
long nRemain = maxrec_ >= 0 ? maxrec_ - totOut : -1;
boolean over =
umatcher_.streamRawResult( blockSeq, blockSink, rowMapper,
Expand All @@ -130,9 +140,7 @@ public StarTable runMatch( StarTable inTable, QuerySequenceFactory qsFact,
totOut += nOut;
iblock++;
}
else {
done = true;
}
done = ! hasNext;
};
int nblock = iblock;
coneSeq.close();
Expand Down
Expand Up @@ -51,8 +51,13 @@ public class CdsUploadMatcher implements UploadMatcher {
/** URL for the CDS Xmatch service. */
public static final String XMATCH_URL =
"http://cdsxmatch.u-strasbg.fr/xmatch/api/v1/sync";

/** Alias for Simbad flat view table. */
public static final String SIMBAD_NAME = "simbad";

/** Whether it is safe/recommended to upload empty tables to match. */
public static final boolean UPLOAD_EMPTY = false;

private static final String RA_NAME = "__UPLOAD_RA__";
private static final String DEC_NAME = "__UPLOAD_DEC__";
private static final String ID_NAME = "__UPLOAD_ID__";
Expand Down
Expand Up @@ -252,9 +252,11 @@ public TableProducer createProducer( Environment env )
final TableProducer inProd = createInputProducer( env );
final StoragePolicy storage =
LineTableEnvironment.getStoragePolicy( env );
boolean uploadEmpty = CdsUploadMatcher.UPLOAD_EMPTY;
final BlockUploader blocker =
new BlockUploader( umatcher, blocksize, maxrec, tableName,
inFixAct, cdsFixAct, serviceMode, oneToOne );
inFixAct, cdsFixAct, serviceMode, oneToOne,
uploadEmpty );

/* Create and return an object which will produce the result. */
return new TableProducer() {
Expand Down

0 comments on commit a2cd023

Please sign in to comment.