Skip to content

Commit

Permalink
smurf: Allow makemap to export compressed NOI model
Browse files Browse the repository at this point in the history
Since skyloop is to support noi.box_size, it will need to cache the NOI
model created by the first invocation of makemap. But this could be
another fill size time-series data set since previously the NOI model
could only be exprted as the Variance component of the RES model. Now, by
setting noi.export, the NOI model can be exported in compressed form
(i.e. only one value per noi.box_size time slices of raw data), without
lots of unneeded extras like quality and residuals, etc.

, but this
  • Loading branch information
David Berry committed Sep 27, 2013
1 parent a4e1407 commit dd0d74c
Show file tree
Hide file tree
Showing 11 changed files with 394 additions and 121 deletions.
39 changes: 28 additions & 11 deletions applications/smurf/defaults/smurf_makemap.def
Expand Up @@ -1550,6 +1550,23 @@ noi.dcsmooth = 50
# unsmoothed data.
noi.dcthresh = 25.0

#-----------------------------------------------------------------
# Name: noi.export
# Type: integer
# Default: 0
# Purpose: Controls the exporting of the NOI model.
# Description: If set to a non-zero value, export the values in the NOI
# model to an NDF. The name of the NDF is similar to that
# used by parameter "exportndf", but with a trailing
# suffix of "_noi" instead of "_res". Unlike "exportndf",
# the NOI values are stored in the "Data" array of the NDF,
# and only one value is stored for each box in the NOI model
# (see parameter "noi.box_size"). This makes the NDF much
# smaller than that produced by "exportndf". The box size
# (in samples) is stored in the NOI_BOXSIZE item in the
# SMURF extension of the NDF.
noi.export = 0

#-----------------------------------------------------------------
# Name: noi.fillgaps
# Type: integer
Expand All @@ -1561,18 +1578,18 @@ noi.fillgaps = 1

#-----------------------------------------------------------------
# Name: noi.import
# Type: string
# Default: <undef>
# Type: integer
# Default: 0
# Purpose: Controls the noise values used in the NOI model.
# Description: If set to "Data" or "Variance", import the the noise values
# to use in the NOI model from an NDF created by a previous
# run of makemap. The NDF is expected to have the same name as
# would be created by setting parameter "exportNDF" to "(noi)"
# in the makemap configuration. This means the file should end
# in "_res.sdf". The specific value supplied ("Data" or
# "Variance") indicates which NDF component contains the
# noise variances. Normally "Data" should be used.
noi.import = <undef>
# Description: If set to a non-zero value, import the noise values to use
# in the NOI model from an NDF created by a previous run of
# makemap, rather than calculating them from scratch. The
# NDF should have been created by setting parameter "noi.export"
# to 1 in the configuration for a previous run of makemap.
# (the file should end in "_noi.sdf"). Note, the parameter
# "noi.box_size" must be set to the same value it had in the
# previous run of makemap.
noi.import = 0

#-----------------------------------------------------------------
# Name: noi.spikebox
Expand Down
1 change: 1 addition & 0 deletions applications/smurf/libsmf/Makefile.am
Expand Up @@ -146,6 +146,7 @@ smf_expand_tilegroup.c \
smf_expmodel_dks.c \
smf_expmodel_two.c \
smf_expmodel_init.c \
smf_export_noi.c \
smf_ext2km.c \
smf_extracols.c \
smf_fft_2dazav.c \
Expand Down
9 changes: 6 additions & 3 deletions applications/smurf/libsmf/smf.h.source
Expand Up @@ -1014,6 +1014,8 @@ void smf_expmodel_two( const smfData *indata, smfData **outdata, int *status);

void smf_expmodel_init( const smfData *indata, smfData **outdata, int *status);

void smf_export_noi( smfData *noi, const char *name, int boxsize, int *status );

void smf_ext2km( int indf, const char *xname, AstKeyMap *keymap, int mode,
int *status );

Expand Down Expand Up @@ -1401,7 +1403,8 @@ void smf_import_array( smfData *refdata, const char *name, int bad,
int expand, double *dataptr, int *status );

int smf_import_noi( const char *name, smfDIMMHead *head,
AstKeyMap *keymap, double *dataptr, int *status );
AstKeyMap *keymap, double *dataptr, dim_t *noi_boxsize,
int *status );

void smf_inbeam_str ( smf_inbeam_t inbeam, char *inbeamstr, size_t lenstr, int * status );

Expand Down Expand Up @@ -1525,8 +1528,8 @@ void smf_model_create( ThrWorkForce *wf, const smfGroup *igroup,
dim_t nchunks, smf_modeltype mtype, int isTordered,
AstFrameSet *outfset, int moving,
int *lbnd_out, int *ubnd_out, fts2Port fts_port,
smfGroup **mgroup,
smfArray **mdata, AstKeyMap *keymap, int *status );
smfGroup **mgroup, smfArray **mdata,
AstKeyMap *keymap, dim_t *noi_boxsize, int *status );

void smf_model_createHdr( smfData *model, smf_modeltype type,
smfData *refdata, int *status );
Expand Down
10 changes: 7 additions & 3 deletions applications/smurf/libsmf/smf_calcmodel_noi.c
Expand Up @@ -211,6 +211,7 @@ void smf_calcmodel_noi( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
dim_t i; /* Loop counter */
dim_t ibolo; /* Bolometer index */
int ibox; /* Index of current noise box */
int import; /* Has an external NOI model been imported? */
dim_t itime; /* Time slice index */
dim_t idx=0; /* Index within subgroup */
JCMTState *instate=NULL; /* Pointer to input JCMTState */
Expand Down Expand Up @@ -274,6 +275,9 @@ void smf_calcmodel_noi( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
/* Did we already calculate the noise on each detector? */
astMapGet0I( kmap, "CALCFIRST", &calcfirst );

/* Did we import an external BOI model? */
astMapGet0I( kmap, "IMPORT", &import );

/* Initialize chisquared */
dat->chisquared[chunk] = 0;
nchisq = 0;
Expand Down Expand Up @@ -307,9 +311,7 @@ void smf_calcmodel_noi( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
the noise after the first iteration. We also skip this
calculation if we have already imported external noise values into
the NOI model. */

if( (flags & SMF__DIMM_FIRSTITER) && (!calcfirst) &&
(!astMapDefined( kmap, "IMPORT" )) ) {
if( (flags & SMF__DIMM_FIRSTITER) && (!calcfirst) && (!import) ) {

/* There are two forms for the NOI model: one constant noise value
for each bolometer, or "ntslice" noise values for each bolometer.
Expand All @@ -333,6 +335,7 @@ void smf_calcmodel_noi( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
}

var = astFree( var );
dat->noi_boxsize = ntslice;
}


Expand Down Expand Up @@ -473,6 +476,7 @@ void smf_calcmodel_noi( ThrWorkForce *wf, smfDIMMData *dat, int chunk,
}

var = astFree( var );
dat->noi_boxsize = boxsize;
}

/* Report an error if the number of samples for each bolometer in
Expand Down
2 changes: 1 addition & 1 deletion applications/smurf/libsmf/smf_clean_pca.c
Expand Up @@ -847,7 +847,7 @@ void smf_clean_pca( ThrWorkForce *wf, smfData *data, size_t t_first,

smf_model_create( wf, NULL, &data_array, NULL, NULL, NULL,NULL, NULL, 1,
SMF__GAI, data->isTordered, NULL, 0, NULL, NULL,
NO_FTS, &gain_group, &gain_array, keymap, status );
NO_FTS, &gain_group, &gain_array, keymap, NULL, status );

/* Compare bolometers to the template in order to flag outliers */
smf_find_gains( wf, 0, data, NULL, NULL, template, kmap, SMF__Q_GOOD,
Expand Down
4 changes: 2 additions & 2 deletions applications/smurf/libsmf/smf_clean_smfArray.c
Expand Up @@ -440,11 +440,11 @@ void smf_clean_smfArray( ThrWorkForce *wf, smfArray *array,
/* Create model containers for COM, GAI */
smf_model_create( wf, NULL, &array, NULL, NULL, NULL, NULL, NULL, 1, SMF__COM,
0, NULL, 0, NULL, NULL, NO_FTS, &comgroup, &comdata, keymap,
status );
NULL, status );

smf_model_create( wf, NULL, &array, NULL, NULL, NULL, NULL, NULL, 1, SMF__GAI,
0, NULL, 0, NULL, NULL, NO_FTS, &gaigroup, &gaidata, keymap,
status );
NULL, status );

/* Manually create quadata to share memory with the quality already
stored in array */
Expand Down
181 changes: 181 additions & 0 deletions applications/smurf/libsmf/smf_export_noi.c
@@ -0,0 +1,181 @@
/*
*+
* Name:
* smf_export_noi
* Purpose:
* Export the NOI model to an NDF.
* Language:
* Starlink ANSI C
* Type of Module:
* C function
* Invocation:
* void smf_export_noi( smfData *noi, const char *name, int boxsize,
* int *status )
* Arguments:
* noi = smfData * (Given)
* The NOI model.
* name = const char * (Given)
* The name of the NDF to create.
* boxsize = int (Given)
* The number of samples in one noise box, corresponding to
* NOI.BOX_SIZE.
* status = int* (Given and Returned)
* Pointer to global status.
* Description:
* This function exports the compressed NOI model to a new NDF. It can
* be imported again via function smf_import_noi.
* Authors:
* David S Berry (JAC, Hawaii)
* {enter_new_authors_here}
* History:
* 24-SEP-2013 (DSB):
* Original version.
* {enter_further_changes_here}
* Copyright:
* Copyright (C) 2013 Science & Technology Facilities Council.
* All Rights Reserved.
* Licence:
* This program is free software; you can redistribute it and/or
* modify it under the terms of the GNU General Public License as
* published by the Free Software Foundation; either version 3 of
* the License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be
* useful, but WITHOUT ANY WARRANTY; without even the implied
* warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
* PURPOSE. See the GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public
* License along with this program; if not, write to the Free
* Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston,
* MA 02110-1301, USA
* Bugs:
* {note_any_bugs_here}
*-
*/

/* Starlink includes */
#include "sae_par.h"
#include "mers.h"
#include "ndf.h"
#include "star/hds.h"
#include "ast.h"

/* SMURF includes */
#include "libsmf/smf.h"

void smf_export_noi( smfData *noi, const char *name, int boxsize, int *status ){

/* Local Variables */
HDSLoc *xloc = NULL;
dim_t ntslice;
dim_t nrows;
dim_t ncols;
dim_t nbolo;
double *ip;
double *dataptr;
double *dp;
double *pd;
int el;
int ibolo;
int indf;
int itime;
int lbnd[ 3 ];
int nz;
int place;
int ubnd[ 3 ];
size_t bstride;
int iz;

/* Check inherited status. */
if( *status != SAI__OK ) return;

/* Get the dimensions of the NOI model. */
smf_get_dims( noi, &nrows, &ncols, &nbolo, &ntslice, NULL, &bstride,
NULL, status );
if( ntslice == 1 ) boxsize = 0;

/* Determine the number of boxes to use. This is the length of 3rd axis.
of the new NDF. A boxsize of zero means "there is one box". */
nz = 1;
if( boxsize > 0 ) {
nz = ntslice/boxsize;
if( nz == 0 ) nz = 1;
} else if( ntslice > 1 ){
*status = SAI__ERROR;
errRepf("", "smf_export_noi: boxsize is zero but ntslice (%d) is "
"larger than 1 (programming error).", status, (int) ntslice );
}

/* Get a pointer to the NOI data values. */
dataptr = noi->pntr[ 0 ];

/* Create the NDF. */
ndfPlace( NULL, name, &place, status );
lbnd[ 0 ] = 1;
lbnd[ 1 ] = 1;
lbnd[ 2 ] = 1;
ubnd[ 0 ] = ncols;
ubnd[ 1 ] = nrows;
ubnd[ 2 ] = nz;
ndfNew( "_DOUBLE", 3, lbnd, ubnd, &place, &indf, status );

/* Map the Data array of the NDF and copy the NOI values into it. */
ndfMap( indf, "DATA", "_DOUBLE", "WRITE", (void **) &ip, &el, status );
if( *status == SAI__OK ) {

/* Initialise the time slice at the middle of the current box in the model. */
itime = ( nz == 1 ) ? 0 : boxsize/2;

/* Loop round all planes in the NDF. */
for( iz = 0; iz < nz; iz++ ) {

/* First deal with a time ordered model. The NOI model is filled with 1.0
values by smf_model_create, and can also be set to zero to indicate
missing values. Therefore convert both these values into VAL__BADD. */
if( bstride == 1 ){
pd = ip;
dp = dataptr + itime*nbolo;
for( ibolo = 0; ibolo < (int) nbolo; ibolo++,dp++ ) {
*(pd++) = (*dp == 0.0 || *dp == 1.0) ? VAL__BADD : *dp;
}

/* Now deal with a bolo ordered model. The NOI model is filled with 1.0
values by smf_model_create, and can also be set to zero to indicate
missing values. Therefore convert both these values into VAL__BADD. */
} else {
pd = ip;
dp = dataptr + itime;
for( ibolo = 0; ibolo < (int) nbolo; ibolo++ ) {
*(pd++) = (*dp == 0.0 || *dp == 1.0) ? VAL__BADD : *dp;
dp += ntslice;
}
}

/* Point to the start of the next plane in the NDF. */
ip += nbolo;
itime += boxsize;
}

/* Store the box size as an extension item in the NDF. */
ndfXnew( indf, SMURF__EXTNAME, SMURF__EXTTYPE, 0, NULL, &xloc, status );
ndfXpt0i( boxsize, indf, SMURF__EXTNAME, "NOI_BOXSIZE", status );
datAnnul( &xloc, status );
}

/* Annul the NDF identifier. */
ndfAnnul( &indf, status );

}

0 comments on commit dd0d74c

Please sign in to comment.