diff --git a/cdm/src/main/java/ucar/nc2/ft/fmrc/FmrcDataset.java b/cdm/src/main/java/ucar/nc2/ft/fmrc/FmrcDataset.java index 9affe17c28..86ac6eedfb 100644 --- a/cdm/src/main/java/ucar/nc2/ft/fmrc/FmrcDataset.java +++ b/cdm/src/main/java/ucar/nc2/ft/fmrc/FmrcDataset.java @@ -6,10 +6,38 @@ package ucar.nc2.ft.fmrc; import thredds.featurecollection.FeatureCollectionConfig; -import ucar.ma2.*; -import ucar.nc2.*; -import ucar.nc2.constants.*; -import ucar.nc2.dataset.*; +import ucar.ma2.Array; +import ucar.ma2.ArrayDouble; +import ucar.ma2.DataType; +import ucar.ma2.InvalidRangeException; +import ucar.ma2.MAMath; +import ucar.ma2.Range; +import ucar.ma2.Section; +import ucar.nc2.Attribute; +import ucar.nc2.Dimension; +import ucar.nc2.EnumTypedef; +import ucar.nc2.Group; +import ucar.nc2.NetcdfFile; +import ucar.nc2.ProxyReader; +import ucar.nc2.Structure; +import ucar.nc2.Variable; +import ucar.nc2.constants.AxisType; +import ucar.nc2.constants.CDM; +import ucar.nc2.constants.CF; +import ucar.nc2.constants.FeatureType; +import ucar.nc2.constants._Coordinate; +import ucar.nc2.dataset.CoordSysBuilder; +import ucar.nc2.dataset.CoordSysBuilderIF; +import ucar.nc2.dataset.CoordinateAxis; +import ucar.nc2.dataset.CoordinateSystem; +import ucar.nc2.dataset.CoordinateTransform; +import ucar.nc2.dataset.DatasetConstructor; +import ucar.nc2.dataset.DatasetUrl; +import ucar.nc2.dataset.NetcdfDataset; +import ucar.nc2.dataset.StructureDS; +import ucar.nc2.dataset.TransformType; +import ucar.nc2.dataset.VariableDS; +import ucar.nc2.dataset.VariableEnhanced; import ucar.nc2.dt.GridCoordSystem; import ucar.nc2.dt.GridDatatype; import ucar.nc2.dt.grid.GridDataset; @@ -20,7 +48,13 @@ import javax.annotation.concurrent.ThreadSafe; import java.io.IOException; -import java.util.*; +import java.util.ArrayList; +import java.util.Formatter; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.Set; /** * Helper class for Fmrc. @@ -33,553 +67,554 @@ *

* This replaces ucar.nc2.dt.fmrc.FmrcImpl * - * * @author caron * @since Jan 19, 2010 */ @ThreadSafe class FmrcDataset { - static private final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FmrcDataset.class); - static private final boolean debugEnhance = false, debugRead = false; + static private final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(FmrcDataset.class); + static private final boolean debugEnhance = false, debugRead = false; - private final FeatureCollectionConfig config; - // private final Element ncmlOuter, ncmlInner; + private final FeatureCollectionConfig config; + // private final Element ncmlOuter, ncmlInner; - //private List protoList; // the list of datasets in the proto that have proxy reader, so these need to exist. not implemented yet + //private List protoList; // the list of datasets in the proto that have proxy reader, so these need to exist. not implemented yet - // allow to build a new state while old state can still be safely used - private static class State { - NetcdfDataset proto; // once built, the proto doesnt change until setInventory is called with forceProto = true - FmrcInvLite lite; // lightweight version of the FmrcInv + // allow to build a new state while old state can still be safely used + private static class State { + NetcdfDataset proto; // once built, the proto doesnt change until setInventory is called with forceProto = true + FmrcInvLite lite; // lightweight version of the FmrcInv - private State(NetcdfDataset proto, FmrcInvLite lite) { - this.proto = proto; - this.lite = lite; + private State(NetcdfDataset proto, FmrcInvLite lite) { + this.proto = proto; + this.lite = lite; + } } - } - private State state; - private final Object lock= new Object(); - FmrcDataset(FeatureCollectionConfig config) { // }, Element ncmlInner, Element ncmlOuter) { - this.config = config; - //this.ncmlInner = ncmlInner; - //this.ncmlOuter = ncmlOuter; - } - - List getRunDates() { - State localState; - synchronized (lock) { - localState = state; - } - return localState.lite.getRunDates(); - } + private State state; + private final Object lock = new Object(); - List getForecastDates() { - State localState; - synchronized (lock) { - localState = state; + FmrcDataset(FeatureCollectionConfig config) { // }, Element ncmlInner, Element ncmlOuter) { + this.config = config; + //this.ncmlInner = ncmlInner; + //this.ncmlOuter = ncmlOuter; } - return localState.lite.getForecastDates(); - } - // for making offset datasets - double[] getForecastOffsets() { - State localState; - synchronized (lock) { - localState = state; + List getRunDates() { + State localState; + synchronized (lock) { + localState = state; + } + return localState.lite.getRunDates(); } - return localState.lite.getForecastOffsets(); - } - public CalendarDateRange getDateRangeForRun(CalendarDate run) { - State localState; - synchronized (lock) { - localState = state; - } - int runidx = localState.lite.findRunIndex(run); - if (runidx < 0) return null; - - double min = Double.MAX_VALUE; - double max = Double.MIN_VALUE; - for (FmrcInvLite.Gridset gs : localState.lite.gridSets) { - for (int i=0; i getForecastDates() { + State localState; + synchronized (lock) { + localState = state; + } + return localState.lite.getForecastDates(); } - return CalendarDateRange.of(FmrcInv.makeOffsetDate(localState.lite.base, min), FmrcInv.makeOffsetDate(localState.lite.base, max)); - } - public CalendarDateRange getDateRangeForOffset(double offset) { - State localState; - synchronized (lock) { - localState = state; + // for making offset datasets + double[] getForecastOffsets() { + State localState; + synchronized (lock) { + localState = state; + } + return localState.lite.getForecastOffsets(); } - List runs = localState.lite.getRunDates(); - int n = runs.size(); - return CalendarDateRange.of(FmrcInv.makeOffsetDate(runs.get(0), offset), FmrcInv.makeOffsetDate(runs.get(n-1), offset)); - } - /** - * Make the 2D dataset - * @param result use this empty NetcdfDataset, may be null (used by NcML) - * @return 2D dataset - * @throws IOException on read error - */ - GridDataset getNetcdfDataset2D(NetcdfDataset result) throws IOException { - State localState; - synchronized (lock) { - localState = state; + public CalendarDateRange getDateRangeForRun(CalendarDate run) { + State localState; + synchronized (lock) { + localState = state; + } + int runidx = localState.lite.findRunIndex(run); + if (runidx < 0) return null; + + double min = Double.MAX_VALUE; + double max = Double.MIN_VALUE; + for (FmrcInvLite.Gridset gs : localState.lite.gridSets) { + for (int i = 0; i < gs.noffsets; i++) { + double time = gs.getTimeCoord(runidx, i); + if (Double.isNaN(time)) continue; + min = Math.min(min, time); + max = Math.max(max, time); + } + } + return CalendarDateRange.of(FmrcInv.makeOffsetDate(localState.lite.base, min), FmrcInv.makeOffsetDate(localState.lite.base, max)); } - return buildDataset2D(result, localState.proto, localState.lite); - } - GridDataset getBest() throws IOException { - State localState; - synchronized (lock) { - localState = state; + public CalendarDateRange getDateRangeForOffset(double offset) { + State localState; + synchronized (lock) { + localState = state; + } + List runs = localState.lite.getRunDates(); + int n = runs.size(); + return CalendarDateRange.of(FmrcInv.makeOffsetDate(runs.get(0), offset), FmrcInv.makeOffsetDate(runs.get(n - 1), offset)); } - return buildDataset1D( localState.proto, localState.lite, localState.lite.makeBestDatasetInventory()); - } - GridDataset getBest(FeatureCollectionConfig.BestDataset bd) throws IOException { - State localState; - synchronized (lock) { - localState = state; - } - return buildDataset1D( localState.proto, localState.lite, localState.lite.makeBestDatasetInventory(bd)); - } - - GridDataset getRunTimeDataset(CalendarDate run) throws IOException { - State localState; - synchronized (lock) { - localState = state; + /** + * Make the 2D dataset + * + * @param result use this empty NetcdfDataset, may be null (used by NcML) + * @return 2D dataset + * @throws IOException on read error + */ + GridDataset getNetcdfDataset2D(NetcdfDataset result) throws IOException { + State localState; + synchronized (lock) { + localState = state; + } + return buildDataset2D(result, localState.proto, localState.lite); } - return buildDataset1D( localState.proto, localState.lite, localState.lite.makeRunTimeDatasetInventory( run)); - } - GridDataset getConstantForecastDataset(CalendarDate run) throws IOException { - State localState; - synchronized (lock) { - localState = state; + GridDataset getBest() throws IOException { + State localState; + synchronized (lock) { + localState = state; + } + return buildDataset1D(localState.proto, localState.lite, localState.lite.makeBestDatasetInventory()); } - return buildDataset1D( localState.proto, localState.lite, localState.lite.getConstantForecastDataset( run)); - } - GridDataset getConstantOffsetDataset(double offset) throws IOException { - State localState; - synchronized (lock) { - localState = state; + GridDataset getBest(FeatureCollectionConfig.BestDataset bd) throws IOException { + State localState; + synchronized (lock) { + localState = state; + } + return buildDataset1D(localState.proto, localState.lite, localState.lite.makeBestDatasetInventory(bd)); } - return buildDataset1D( localState.proto, localState.lite, localState.lite.getConstantOffsetDataset( offset)); - } - /** - * Set a new FmrcInv and optionally a proto dataset - * - * @param fmrcInv based on this inventory - * @param forceProto create a new proto, else use existing one - * @throws IOException on read error - */ - void setInventory(FmrcInv fmrcInv, boolean forceProto) throws IOException { - NetcdfDataset protoLocal = null; - - // make new proto if needed - if (state == null || forceProto) { - protoLocal = buildProto(fmrcInv, config.protoConfig); + GridDataset getRunTimeDataset(CalendarDate run) throws IOException { + State localState; + synchronized (lock) { + localState = state; + } + return buildDataset1D(localState.proto, localState.lite, localState.lite.makeRunTimeDatasetInventory(run)); } - // switch to FmrcInvLite to reduce memory usage - FmrcInvLite liteLocal = new FmrcInvLite(fmrcInv); - synchronized (lock) { - if (protoLocal == null && state != null) protoLocal = state.proto; - state = new State(protoLocal, liteLocal); + GridDataset getConstantForecastDataset(CalendarDate run) throws IOException { + State localState; + synchronized (lock) { + localState = state; + } + return buildDataset1D(localState.proto, localState.lite, localState.lite.getConstantForecastDataset(run)); } - } - - ///////////////////////////////////////////////////////////////////////////// - // the prototypical dataset - - private NetcdfDataset buildProto(FmrcInv fmrcInv, FeatureCollectionConfig.ProtoConfig protoConfig) throws IOException { - NetcdfDataset result = new NetcdfDataset(); // empty - // choose some run in the list - List list = fmrcInv.getFmrInv(); - if (list.size() == 0) { - logger.error("Fmrc collection is empty ="+fmrcInv.getName()); - throw new IllegalStateException("Fmrc collection is empty ="+fmrcInv.getName()); + GridDataset getConstantOffsetDataset(double offset) throws IOException { + State localState; + synchronized (lock) { + localState = state; + } + return buildDataset1D(localState.proto, localState.lite, localState.lite.getConstantOffsetDataset(offset)); } - int protoIdx = 0; - switch (protoConfig.choice) { - case First: - protoIdx = 0; - break; - case Random: - Random r = new Random(System.currentTimeMillis()); - protoIdx = r.nextInt(list.size() - 1); - break; - case Penultimate: - protoIdx = Math.max(list.size() - 2, 0); - break; - case Latest: - protoIdx = Math.max(list.size() - 1, 0); - break; - - case Run: - int runOffset = 0; - if (protoConfig.param != null) - runOffset = Integer.parseInt(protoConfig.param); - for (int i=0; i openFilesProto = new HashMap<>(); - - try { - // create the union of all objects in that run - // this covers the case where the variables are split across files - Set files = proto.getFiles(); - if (logger.isDebugEnabled()) - logger.debug("FmrcDataset: proto= " + proto.getName() + " " + proto.getRunDate() + " collection= " + fmrcInv.getName()); - for (GridDatasetInv file : files) { - NetcdfDataset ncfile = open(file.getLocation(), openFilesProto); - if (ncfile != null) - transferGroup(ncfile.getRootGroup(), result.getRootGroup(), result); - else - logger.warn("Failed to open "+file.getLocation()); - if (logger.isDebugEnabled()) logger.debug("FmrcDataset: proto dataset= " + file.getLocation()); - } - // some additional global attributes - Group root = result.getRootGroup(); - Attribute orgConv = root.findAttributeIgnoreCase(CDM.CONVENTIONS); - String convAtt = CoordSysBuilder.buildConventionAttribute("CF-1.4", (orgConv == null ? null : orgConv.getStringValue())); - root.addAttribute(new Attribute(CDM.CONVENTIONS, convAtt)); - root.addAttribute(new Attribute("cdm_data_type", FeatureType.GRID.toString())); - root.addAttribute(new Attribute(CF.FEATURE_TYPE, FeatureType.GRID.toString())); - root.addAttribute(new Attribute("location", "Proto "+fmrcInv.getName())); - - // remove some attributes that can cause trouble - root.remove(root.findAttribute(_Coordinate.ModelRunDate)); - - // protoList = new ArrayList(); - // these are the non-agg variables - store data or ProxyReader in proto - List copyList = new ArrayList<>(root.getVariables()); // use copy since we may be removing some variables - for (Variable v : copyList) { - // see if its a non-agg variable - FmrcInv.UberGrid grid = fmrcInv.findUberGrid(v.getFullName()); - if (grid == null) { // only non-agg vars need to be cached - Variable orgV = (Variable) v.getSPobject(); - if (orgV.getSize() > 10 * 1000 * 1000) { - logger.info("FMRCDataset build Proto cache >10M var= "+orgV.getNameAndDimensions()); - } else { - v.setCachedData(orgV.read()); // read from original - store in proto - } + // switch to FmrcInvLite to reduce memory usage + FmrcInvLite liteLocal = new FmrcInvLite(fmrcInv); + synchronized (lock) { + if (protoLocal == null && state != null) protoLocal = state.proto; + state = new State(protoLocal, liteLocal); } + } - v.setSPobject(null); // clear the reference to orgV for all of proto - } + ///////////////////////////////////////////////////////////////////////////// + // the prototypical dataset - result.finish(); - - // enhance the proto. becomes the template for coordSystems in the derived datasets - CoordSysBuilderIF builder = result.enhance(); - if (debugEnhance) System.out.printf("proto.enhance() parseInfo = %s%n", builder.getParseInfo()); - - // turn it into a GridDataset, so we can add standard metadata to result, not dependent on CoordSysBuilder - // also see ucar.nc2.dt.grid.NetcdfCFWriter - common code could be extracted - Formatter parseInfo = new Formatter(); - GridDataset gds = new ucar.nc2.dt.grid.GridDataset(result, parseInfo); // LOOK not sure coord axes will read ?? - if (debugEnhance) System.out.printf("proto GridDataset parseInfo = %s%n", parseInfo); - - // now make standard CF metadata for gridded data - for (GridDatatype grid : gds.getGrids()) { - Variable newV = result.findVariable(grid.getFullName()); - if (newV == null) { - logger.warn("FmrcDataset cant find "+grid.getFullName()+" in proto gds "); - continue; - } + private NetcdfDataset buildProto(FmrcInv fmrcInv, FeatureCollectionConfig.ProtoConfig protoConfig) throws IOException { + NetcdfDataset result = new NetcdfDataset(); // empty - // annotate Variable for CF - StringBuilder sbuff = new StringBuilder(); - GridCoordSystem gcs = grid.getCoordinateSystem(); - for (CoordinateAxis axis : gcs.getCoordinateAxes()) { - if ((axis.getAxisType() != AxisType.Time) && (axis.getAxisType() != AxisType.RunTime)) // these are added later - sbuff.append(axis.getFullName()).append(" "); + // choose some run in the list + List list = fmrcInv.getFmrInv(); + if (list.size() == 0) { + logger.error("Fmrc collection is empty =" + fmrcInv.getName()); + throw new IllegalStateException("Fmrc collection is empty =" + fmrcInv.getName()); } - newV.addAttribute(new Attribute(CF.COORDINATES, sbuff.toString())); // LOOK what about adding lat/lon variable - // looking for coordinate transform variables - for (CoordinateTransform ct : gcs.getCoordinateTransforms()) { - Variable ctv = result.findVariable(ct.getName()); - if ((ctv != null) && (ct.getTransformType() == TransformType.Projection)) - newV.addAttribute(new Attribute(CF.GRID_MAPPING, ctv.getFullName())); + int protoIdx = 0; + switch (protoConfig.choice) { + case First: + protoIdx = 0; + break; + case Random: + Random r = new Random(System.currentTimeMillis()); + protoIdx = r.nextInt(list.size() - 1); + break; + case Penultimate: + protoIdx = Math.max(list.size() - 2, 0); + break; + case Latest: + protoIdx = Math.max(list.size() - 1, 0); + break; + + case Run: + int runOffset = 0; + if (protoConfig.param != null) + runOffset = Integer.parseInt(protoConfig.param); + for (int i = 0; i < list.size(); i++) { + FmrInv fmr = list.get(i); + CalendarDate cd = fmr.getRunDate(); + int hour = cd.getHourOfDay(); + if (hour == runOffset) + protoIdx = i; // later ones override + } + break; } + FmrInv proto = list.get(protoIdx); // use this one - // LOOK is this needed ? - for (CoordinateAxis axis : gcs.getCoordinateAxes()) { - Variable coordV = result.findVariable(axis.getFullNameEscaped()); - if ((axis.getAxisType() == AxisType.Height) || (axis.getAxisType() == AxisType.Pressure) || (axis.getAxisType() == AxisType.GeoZ)) { - if (null != axis.getPositive()) - coordV.addAttribute(new Attribute(CF.POSITIVE, axis.getPositive())); - } - if (axis.getAxisType() == AxisType.Lat) { - coordV.addAttribute(new Attribute(CDM.UNITS, CDM.LAT_UNITS)); - coordV.addAttribute(new Attribute(CF.STANDARD_NAME, "latitude")); - } - if (axis.getAxisType() == AxisType.Lon) { - coordV.addAttribute(new Attribute(CDM.UNITS, CDM.LON_UNITS)); - coordV.addAttribute(new Attribute(CF.STANDARD_NAME, "longitude")); - } - if (axis.getAxisType() == AxisType.GeoX) { - coordV.addAttribute(new Attribute(CF.STANDARD_NAME, "projection_x_coordinate")); - } - if (axis.getAxisType() == AxisType.GeoY) { - coordV.addAttribute(new Attribute(CF.STANDARD_NAME, "projection_y_coordinate")); - } - if (axis.getAxisType() == AxisType.Time) { - Attribute att = axis.findAttribute("bounds"); // LOOK nasty : remove time bounds from proto - if ((att != null) && att.isString()) - result.removeVariable(null, att.getStringValue()); - } - } - } + Map openFilesProto = new HashMap<>(); - // more troublesome attributes, use pure CF - for (Variable v : result.getVariables()) { - Attribute att; - if (null != (att = v.findAttribute(_Coordinate.Axes))) - v.remove(att); - if (null != (att = v.findAttribute(_Coordinate.Systems))) - v.remove(att); - if (null != (att = v.findAttribute(_Coordinate.SystemFor))) - v.remove(att); - if (null != (att = v.findAttribute(_Coordinate.Transforms))) - v.remove(att); - } + try { + // create the union of all objects in that run + // this covers the case where the variables are split across files + Set files = proto.getFiles(); + if (logger.isDebugEnabled()) + logger.debug("FmrcDataset: proto= " + proto.getName() + " " + proto.getRunDate() + " collection= " + fmrcInv.getName()); + for (GridDatasetInv file : files) { + NetcdfDataset ncfile = open(file.getLocation(), openFilesProto); + if (ncfile != null) + transferGroup(ncfile.getRootGroup(), result.getRootGroup(), result); + else + logger.warn("Failed to open " + file.getLocation()); + if (logger.isDebugEnabled()) logger.debug("FmrcDataset: proto dataset= " + file.getLocation()); + } - // apply ncml if it exists - if (protoConfig.outerNcml != null) - NcMLReader.mergeNcMLdirect(result, protoConfig.outerNcml); + // some additional global attributes + Group root = result.getRootGroup(); + Attribute orgConv = root.findAttributeIgnoreCase(CDM.CONVENTIONS); + String convAtt = CoordSysBuilder.buildConventionAttribute("CF-1.4", (orgConv == null ? null : orgConv.getStringValue())); + root.addAttribute(new Attribute(CDM.CONVENTIONS, convAtt)); + root.addAttribute(new Attribute("cdm_data_type", FeatureType.GRID.toString())); + root.addAttribute(new Attribute(CF.FEATURE_TYPE, FeatureType.GRID.toString())); + root.addAttribute(new Attribute("location", "Proto " + fmrcInv.getName())); + + // remove some attributes that can cause trouble + root.remove(root.findAttribute(_Coordinate.ModelRunDate)); + + // protoList = new ArrayList(); + // these are the non-agg variables - store data or ProxyReader in proto + List copyList = new ArrayList<>(root.getVariables()); // use copy since we may be removing some variables + for (Variable v : copyList) { + // see if its a non-agg variable + FmrcInv.UberGrid grid = fmrcInv.findUberGrid(v.getFullName()); + if (grid == null) { // only non-agg vars need to be cached + Variable orgV = (Variable) v.getSPobject(); + if (orgV.getSize() > 10 * 1000 * 1000) { + logger.info("FMRCDataset build Proto cache >10M var= " + orgV.getNameAndDimensions()); + } + v.setCachedData(orgV.read()); // read from original - store in proto + } + + v.setSPobject(null); // clear the reference to orgV for all of proto + } - return result; + result.finish(); + + // enhance the proto. becomes the template for coordSystems in the derived datasets + CoordSysBuilderIF builder = result.enhance(); + if (debugEnhance) System.out.printf("proto.enhance() parseInfo = %s%n", builder.getParseInfo()); + + // turn it into a GridDataset, so we can add standard metadata to result, not dependent on CoordSysBuilder + // also see ucar.nc2.dt.grid.NetcdfCFWriter - common code could be extracted + Formatter parseInfo = new Formatter(); + GridDataset gds = new ucar.nc2.dt.grid.GridDataset(result, parseInfo); // LOOK not sure coord axes will read ?? + if (debugEnhance) System.out.printf("proto GridDataset parseInfo = %s%n", parseInfo); + + // now make standard CF metadata for gridded data + for (GridDatatype grid : gds.getGrids()) { + Variable newV = result.findVariable(grid.getFullName()); + if (newV == null) { + logger.warn("FmrcDataset cant find " + grid.getFullName() + " in proto gds "); + continue; + } + + // annotate Variable for CF + StringBuilder sbuff = new StringBuilder(); + GridCoordSystem gcs = grid.getCoordinateSystem(); + for (CoordinateAxis axis : gcs.getCoordinateAxes()) { + if ((axis.getAxisType() != AxisType.Time) && (axis.getAxisType() != AxisType.RunTime)) // these are added later + sbuff.append(axis.getFullName()).append(" "); + } + newV.addAttribute(new Attribute(CF.COORDINATES, sbuff.toString())); // LOOK what about adding lat/lon variable + + // looking for coordinate transform variables + for (CoordinateTransform ct : gcs.getCoordinateTransforms()) { + Variable ctv = result.findVariable(ct.getName()); + if ((ctv != null) && (ct.getTransformType() == TransformType.Projection)) + newV.addAttribute(new Attribute(CF.GRID_MAPPING, ctv.getFullName())); + } + + // LOOK is this needed ? + for (CoordinateAxis axis : gcs.getCoordinateAxes()) { + Variable coordV = result.findVariable(axis.getFullNameEscaped()); + if ((axis.getAxisType() == AxisType.Height) || (axis.getAxisType() == AxisType.Pressure) || (axis.getAxisType() == AxisType.GeoZ)) { + if (null != axis.getPositive()) + coordV.addAttribute(new Attribute(CF.POSITIVE, axis.getPositive())); + } + if (axis.getAxisType() == AxisType.Lat) { + coordV.addAttribute(new Attribute(CDM.UNITS, CDM.LAT_UNITS)); + coordV.addAttribute(new Attribute(CF.STANDARD_NAME, "latitude")); + } + if (axis.getAxisType() == AxisType.Lon) { + coordV.addAttribute(new Attribute(CDM.UNITS, CDM.LON_UNITS)); + coordV.addAttribute(new Attribute(CF.STANDARD_NAME, "longitude")); + } + if (axis.getAxisType() == AxisType.GeoX) { + coordV.addAttribute(new Attribute(CF.STANDARD_NAME, "projection_x_coordinate")); + } + if (axis.getAxisType() == AxisType.GeoY) { + coordV.addAttribute(new Attribute(CF.STANDARD_NAME, "projection_y_coordinate")); + } + if (axis.getAxisType() == AxisType.Time) { + Attribute att = axis.findAttribute("bounds"); // LOOK nasty : remove time bounds from proto + if ((att != null) && att.isString()) + result.removeVariable(null, att.getStringValue()); + } + } + } - } finally { - // data is read and cached, can close files now - closeAll(openFilesProto); - } - } + // more troublesome attributes, use pure CF + for (Variable v : result.getVariables()) { + Attribute att; + if (null != (att = v.findAttribute(_Coordinate.Axes))) + v.remove(att); + if (null != (att = v.findAttribute(_Coordinate.Systems))) + v.remove(att); + if (null != (att = v.findAttribute(_Coordinate.SystemFor))) + v.remove(att); + if (null != (att = v.findAttribute(_Coordinate.Transforms))) + v.remove(att); + } - // transfer the objects in src group to the target group, unless that name already exists - // Dimensions, Variables, Groups are not transferred, but an equivalent object is created with same metadata - // Attributes and EnumTypedef are transferred, these are immutable with no references to container + // apply ncml if it exists + if (protoConfig.outerNcml != null) + NcMLReader.mergeNcMLdirect(result, protoConfig.outerNcml); - private void transferGroup(Group srcGroup, Group targetGroup, NetcdfDataset target) throws IOException { - // group attributes - DatasetConstructor.transferGroupAttributes(srcGroup, targetGroup); + return result; - // dimensions - for (Dimension d : srcGroup.getDimensions()) { - if (null == targetGroup.findDimensionLocal(d.getShortName())) { - Dimension newd = new Dimension(d.getShortName(), d.getLength(), d.isShared(), d.isUnlimited(), d.isVariableLength()); - targetGroup.addDimension(newd); - } + } finally { + // data is read and cached, can close files now + closeAll(openFilesProto); + } } - // transfer variables - eliminate any references to component files - for (Variable v : srcGroup.getVariables()) { - Variable targetV = targetGroup.findVariable(v.getShortName()); + // transfer the objects in src group to the target group, unless that name already exists + // Dimensions, Variables, Groups are not transferred, but an equivalent object is created with same metadata + // Attributes and EnumTypedef are transferred, these are immutable with no references to container - if (null == targetV) { // add it - if (v instanceof Structure) { - targetV = new StructureDS(target, targetGroup, null, v.getShortName(), v.getDimensionsString(), v.getUnitsString(), v.getDescription()); - //LOOK - not adding the members here - what to do ?? + private void transferGroup(Group srcGroup, Group targetGroup, NetcdfDataset target) throws IOException { + // group attributes + DatasetConstructor.transferGroupAttributes(srcGroup, targetGroup); - } else { - targetV = new VariableDS(target, targetGroup, null, v.getShortName(), v.getDataType(), v.getDimensionsString(), v.getUnitsString(), v.getDescription()); + // dimensions + for (Dimension d : srcGroup.getDimensions()) { + if (null == targetGroup.findDimensionLocal(d.getShortName())) { + Dimension newd = new Dimension(d.getShortName(), d.getLength(), d.isShared(), d.isUnlimited(), d.isVariableLength()); + targetGroup.addDimension(newd); + } } - DatasetConstructor.transferVariableAttributes(v, targetV); - VariableDS vds = (VariableDS) v; - targetV.setSPobject(vds); //temporary, for non-agg variables when proto is made - if (vds.hasCachedDataRecurse()) { - if (vds.getSize() > 1000 * 1000) { - boolean wtf = vds.hasCachedDataRecurse(); - } - targetV.setCachedData(vds.read()); // + // transfer variables - eliminate any references to component files + for (Variable v : srcGroup.getVariables()) { + Variable targetV = targetGroup.findVariable(v.getShortName()); + + if (null == targetV) { // add it + if (v instanceof Structure) { + targetV = new StructureDS(target, targetGroup, null, v.getShortName(), v.getDimensionsString(), v.getUnitsString(), v.getDescription()); + //LOOK - not adding the members here - what to do ?? + + } else { + targetV = new VariableDS(target, targetGroup, null, v.getShortName(), v.getDataType(), v.getDimensionsString(), v.getUnitsString(), v.getDescription()); + } + + DatasetConstructor.transferVariableAttributes(v, targetV); + VariableDS vds = (VariableDS) v; + targetV.setSPobject(vds); //temporary, for non-agg variables when proto is made + if (vds.hasCachedDataRecurse()) { + if (vds.getSize() > 1000 * 1000) { + boolean wtf = vds.hasCachedDataRecurse(); + } + targetV.setCachedData(vds.read()); // + } + targetGroup.addVariable(targetV); + } } - targetGroup.addVariable(targetV); - } - } - // nested groups - check if target already has it - for (Group srcNested : srcGroup.getGroups()) { - Group nested = targetGroup.findGroup(srcNested.getShortName()); - if (null == nested) { - nested = new Group(target, targetGroup, srcNested.getShortName()); - targetGroup.addGroup(nested); - for (EnumTypedef et : srcNested.getEnumTypedefs()) { - targetGroup.addEnumeration(et); + // nested groups - check if target already has it + for (Group srcNested : srcGroup.getGroups()) { + Group nested = targetGroup.findGroup(srcNested.getShortName()); + if (null == nested) { + nested = new Group(target, targetGroup, srcNested.getShortName()); + targetGroup.addGroup(nested); + for (EnumTypedef et : srcNested.getEnumTypedefs()) { + targetGroup.addEnumeration(et); + } + } + transferGroup(srcNested, nested, target); } - } - transferGroup(srcNested, nested, target); } - } - ///////////////////////////////////////////////////////// - // constructing the dataset + ///////////////////////////////////////////////////////// + // constructing the dataset - // private static final String FTIME_PREFIX = ""; + // private static final String FTIME_PREFIX = ""; - private String getRunDimensionName() { - return "run"; - } + private String getRunDimensionName() { + return "run"; + } - private String makeCoordinateList(VariableDS aggVar, String timeCoordName, boolean is2D) { - String coords = ""; - Attribute att = aggVar.findAttribute(CF.COORDINATES); - if (att == null) - att = aggVar.findAttribute(_Coordinate.Axes); - if (att != null) - coords = att.getStringValue(); - - if (is2D) - return getRunDimensionName() + " " + timeCoordName + " " + coords; - else - return timeCoordName + "_" + getRunDimensionName() + " " + timeCoordName + " " + coords; - } + private String makeCoordinateList(VariableDS aggVar, String timeCoordName, boolean is2D) { + String coords = ""; + Attribute att = aggVar.findAttribute(CF.COORDINATES); + if (att == null) + att = aggVar.findAttribute(_Coordinate.Axes); + if (att != null) + coords = att.getStringValue(); - private void addAttributeInfo(NetcdfDataset result, String attName, String info) { - Attribute att = result.findGlobalAttribute(attName); - if (att == null) - result.addAttribute(null, new Attribute(attName, info)); - else { - String oldValue = att.getStringValue(); - result.addAttribute(null, new Attribute(attName, oldValue +" ;\n"+ info)); + if (is2D) + return getRunDimensionName() + " " + timeCoordName + " " + coords; + else + return timeCoordName + "_" + getRunDimensionName() + " " + timeCoordName + " " + coords; } - } - - /** - * Build the 2D time dataset, make it immutable so it can be shared across threads - * - * @param result place results in here, if null create a new one. must be threadsafe (immutable) - * @param proto current proto dataset - * @param lite current inventory - * @return resulting GridDataset - * @throws IOException on read error - */ - private GridDataset buildDataset2D(NetcdfDataset result, NetcdfDataset proto, FmrcInvLite lite) throws IOException { - if (lite == null) return null; - // make a copy, so that this object can coexist with previous incarnations - if (result == null) result = new NetcdfDataset(); - result.setLocation(lite.collectionName); - transferGroup(proto.getRootGroup(), result.getRootGroup(), result); - result.finish(); - //CoordSysBuilderIF builder = result.enhance(); - //if (debugEnhance) System.out.printf("buildDataset2D.enhance() parseInfo = %s%n", builder.getParseInfo()); - - addAttributeInfo(result, CDM.HISTORY, "FMRC 2D Dataset"); - - // create runtime aggregation dimension - double[] runOffset = lite.runOffset; - String runtimeDimName = getRunDimensionName(); - int nruns = runOffset.length; - Dimension runDim = new Dimension(runtimeDimName, nruns); - result.removeDimension(null, runtimeDimName); // remove previous declaration, if any - result.addDimension(null, runDim); - - // deal with promoteGlobalAttribute - // promoteGlobalAttributes((AggregationOuterDimension.DatasetOuterDimension) typicalDataset); - - ProxyReader2D proxyReader2D = new ProxyReader2D(); - - // extract a copy of the runtimes for thread safety - // List runTimes = new ArrayList(fmrcInv.getRunTimes()); - - // create runtime aggregation coordinate variable - DataType coordType = DataType.DOUBLE; // LOOK getCoordinateType(); - VariableDS runtimeCoordVar = new VariableDS(result, null, null, runtimeDimName, coordType, runtimeDimName, null, null); - runtimeCoordVar.addAttribute(new Attribute(CDM.LONG_NAME, "Run time for ForecastModelRunCollection")); - runtimeCoordVar.addAttribute(new ucar.nc2.Attribute("standard_name", "forecast_reference_time")); - runtimeCoordVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + lite.base)); - runtimeCoordVar.addAttribute(new ucar.nc2.Attribute(_Coordinate.AxisType, AxisType.RunTime.toString())); - result.removeVariable(null, runtimeCoordVar.getShortName()); - result.addVariable(null, runtimeCoordVar); - if (logger.isDebugEnabled()) logger.debug("FmrcDataset: added runtimeCoordVar " + runtimeCoordVar.getFullName()); - - // make the runtime coordinates - Array runCoordVals = ArrayDouble.factory(DataType.DOUBLE, new int[] {nruns}, runOffset); - runtimeCoordVar.setCachedData(runCoordVals); - - // make the time coordinate(s) as 2D - List nonAggVars = result.getVariables(); - for (FmrcInvLite.Gridset gridset : lite.gridSets) { - Group newGroup = result.getRootGroup(); // can it be different ?? - - //int noffsets = runSeq.getNTimeOffsets(); - Dimension timeDim = new Dimension(gridset.gridsetName, gridset.noffsets); - result.removeDimension(null, gridset.gridsetName); // remove previous declaration, if any - result.addDimension(null, timeDim); - - DataType dtype = DataType.DOUBLE; - String dims = getRunDimensionName() + " " + gridset.gridsetName; - VariableDS timeVar = new VariableDS(result, newGroup, null, gridset.gridsetName, dtype, dims, null, null); // LOOK could just make a CoordinateAxis1D - timeVar.addAttribute(new Attribute(CDM.LONG_NAME, "Forecast time for ForecastModelRunCollection")); - timeVar.addAttribute(new ucar.nc2.Attribute("standard_name", "time")); - timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + lite.base)); - timeVar.addAttribute(new ucar.nc2.Attribute(CDM.MISSING_VALUE, Double.NaN)); - timeVar.addAttribute(new ucar.nc2.Attribute(_Coordinate.AxisType, AxisType.Time.toString())); - - // remove the old one if any - newGroup.removeVariable(gridset.gridsetName); - newGroup.addVariable(timeVar); - - Array timeCoordVals = Array.factory(DataType.DOUBLE, timeVar.getShape(), gridset.timeOffset); - timeVar.setCachedData(timeCoordVals); - - if (gridset.timeBounds != null) { - String bname = timeVar.getShortName() + "_bounds"; - timeVar.addAttribute(new ucar.nc2.Attribute("bounds", bname)); - Dimension bd = ucar.nc2.dataset.DatasetConstructor.getBoundsDimension( result); - VariableDS boundsVar = new VariableDS(result, newGroup, null, bname, dtype, dims+" "+bd.getShortName(), null, null); - boundsVar.addAttribute(new Attribute(CDM.LONG_NAME, "bounds for "+ timeVar.getShortName())); - boundsVar.setCachedData(Array.factory( DataType.DOUBLE, new int[] {nruns, gridset.noffsets, 2}, gridset.timeBounds)); - newGroup.addVariable(boundsVar); - } - // promote all grid variables to agg variables - for (FmrcInvLite.Gridset.Grid ugrid : gridset.grids) { - VariableDS aggVar = (VariableDS) result.findVariable(ugrid.name); - if (aggVar == null) { // a ugrid is not in the proto - logger.error("buildDataset2D: cant find ugrid variable "+ugrid.name+" in collection "+lite.collectionName+debugMissingVar(proto, result)); - continue; // skip + private void addAttributeInfo(NetcdfDataset result, String attName, String info) { + Attribute att = result.findGlobalAttribute(attName); + if (att == null) + result.addAttribute(null, new Attribute(attName, info)); + else { + String oldValue = att.getStringValue(); + result.addAttribute(null, new Attribute(attName, oldValue + " ;\n" + info)); } + } - // create dimension list - List dimList = aggVar.getDimensions(); - dimList = dimList.subList(1, dimList.size()); // LOOK assumes time is outer dimension - dimList.add(0, timeDim); - dimList.add(0, runDim); - - aggVar.setDimensions(dimList); - aggVar.setProxyReader(proxyReader2D); - aggVar.setSPobject(ugrid); - nonAggVars.remove(aggVar); + /** + * Build the 2D time dataset, make it immutable so it can be shared across threads + * + * @param result place results in here, if null create a new one. must be threadsafe (immutable) + * @param proto current proto dataset + * @param lite current inventory + * @return resulting GridDataset + * @throws IOException on read error + */ + private GridDataset buildDataset2D(NetcdfDataset result, NetcdfDataset proto, FmrcInvLite lite) throws IOException { + if (lite == null) return null; + // make a copy, so that this object can coexist with previous incarnations + if (result == null) result = new NetcdfDataset(); + result.setLocation(lite.collectionName); + transferGroup(proto.getRootGroup(), result.getRootGroup(), result); + result.finish(); + //CoordSysBuilderIF builder = result.enhance(); + //if (debugEnhance) System.out.printf("buildDataset2D.enhance() parseInfo = %s%n", builder.getParseInfo()); + + addAttributeInfo(result, CDM.HISTORY, "FMRC 2D Dataset"); + + // create runtime aggregation dimension + double[] runOffset = lite.runOffset; + String runtimeDimName = getRunDimensionName(); + int nruns = runOffset.length; + Dimension runDim = new Dimension(runtimeDimName, nruns); + result.removeDimension(null, runtimeDimName); // remove previous declaration, if any + result.addDimension(null, runDim); + + // deal with promoteGlobalAttribute + // promoteGlobalAttributes((AggregationOuterDimension.DatasetOuterDimension) typicalDataset); + + ProxyReader2D proxyReader2D = new ProxyReader2D(); + + // extract a copy of the runtimes for thread safety + // List runTimes = new ArrayList(fmrcInv.getRunTimes()); + + // create runtime aggregation coordinate variable + DataType coordType = DataType.DOUBLE; // LOOK getCoordinateType(); + VariableDS runtimeCoordVar = new VariableDS(result, null, null, runtimeDimName, coordType, runtimeDimName, null, null); + runtimeCoordVar.addAttribute(new Attribute(CDM.LONG_NAME, "Run time for ForecastModelRunCollection")); + runtimeCoordVar.addAttribute(new ucar.nc2.Attribute("standard_name", "forecast_reference_time")); + runtimeCoordVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + lite.base)); + runtimeCoordVar.addAttribute(new ucar.nc2.Attribute(_Coordinate.AxisType, AxisType.RunTime.toString())); + result.removeVariable(null, runtimeCoordVar.getShortName()); + result.addVariable(null, runtimeCoordVar); + if (logger.isDebugEnabled()) + logger.debug("FmrcDataset: added runtimeCoordVar " + runtimeCoordVar.getFullName()); + + // make the runtime coordinates + Array runCoordVals = ArrayDouble.factory(DataType.DOUBLE, new int[]{nruns}, runOffset); + runtimeCoordVar.setCachedData(runCoordVals); + + // make the time coordinate(s) as 2D + List nonAggVars = result.getVariables(); + for (FmrcInvLite.Gridset gridset : lite.gridSets) { + Group newGroup = result.getRootGroup(); // can it be different ?? + + //int noffsets = runSeq.getNTimeOffsets(); + Dimension timeDim = new Dimension(gridset.gridsetName, gridset.noffsets); + result.removeDimension(null, gridset.gridsetName); // remove previous declaration, if any + result.addDimension(null, timeDim); + + DataType dtype = DataType.DOUBLE; + String dims = getRunDimensionName() + " " + gridset.gridsetName; + VariableDS timeVar = new VariableDS(result, newGroup, null, gridset.gridsetName, dtype, dims, null, null); // LOOK could just make a CoordinateAxis1D + timeVar.addAttribute(new Attribute(CDM.LONG_NAME, "Forecast time for ForecastModelRunCollection")); + timeVar.addAttribute(new ucar.nc2.Attribute("standard_name", "time")); + timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + lite.base)); + timeVar.addAttribute(new ucar.nc2.Attribute(CDM.MISSING_VALUE, Double.NaN)); + timeVar.addAttribute(new ucar.nc2.Attribute(_Coordinate.AxisType, AxisType.Time.toString())); + + // remove the old one if any + newGroup.removeVariable(gridset.gridsetName); + newGroup.addVariable(timeVar); + + Array timeCoordVals = Array.factory(DataType.DOUBLE, timeVar.getShape(), gridset.timeOffset); + timeVar.setCachedData(timeCoordVals); + + if (gridset.timeBounds != null) { + String bname = timeVar.getShortName() + "_bounds"; + timeVar.addAttribute(new ucar.nc2.Attribute("bounds", bname)); + Dimension bd = ucar.nc2.dataset.DatasetConstructor.getBoundsDimension(result); + VariableDS boundsVar = new VariableDS(result, newGroup, null, bname, dtype, dims + " " + bd.getShortName(), null, null); + boundsVar.addAttribute(new Attribute(CDM.LONG_NAME, "bounds for " + timeVar.getShortName())); + boundsVar.setCachedData(Array.factory(DataType.DOUBLE, new int[]{nruns, gridset.noffsets, 2}, gridset.timeBounds)); + newGroup.addVariable(boundsVar); + } - // we need to explicitly list the coordinate axes, because time coord is now 2D - String coords = makeCoordinateList(aggVar, gridset.gridsetName, true); - aggVar.removeAttribute(_Coordinate.Axes); - aggVar.addAttribute(new Attribute(CF.COORDINATES, coords)); + // promote all grid variables to agg variables + for (FmrcInvLite.Gridset.Grid ugrid : gridset.grids) { + VariableDS aggVar = (VariableDS) result.findVariable(ugrid.name); + if (aggVar == null) { // a ugrid is not in the proto + logger.error("buildDataset2D: cant find ugrid variable " + ugrid.name + " in collection " + lite.collectionName + debugMissingVar(proto, result)); + continue; // skip + } + + // create dimension list + List dimList = aggVar.getDimensions(); + dimList = dimList.subList(1, dimList.size()); // LOOK assumes time is outer dimension + dimList.add(0, timeDim); + dimList.add(0, runDim); + + aggVar.setDimensions(dimList); + aggVar.setProxyReader(proxyReader2D); + aggVar.setSPobject(ugrid); + nonAggVars.remove(aggVar); + + // we need to explicitly list the coordinate axes, because time coord is now 2D + String coords = makeCoordinateList(aggVar, gridset.gridsetName, true); + aggVar.removeAttribute(_Coordinate.Axes); + aggVar.addAttribute(new Attribute(CF.COORDINATES, coords)); /* transfer Coordinate Systems VariableDS protoV = (VariableDS) proto.findVariable(aggVar.getName()); @@ -587,36 +622,36 @@ private GridDataset buildDataset2D(NetcdfDataset result, NetcdfDataset proto, Fm CoordinateSystem cs = findReplacementCs(protoCs, gridset.gridsetName, result); aggVar.addCoordinateSystem(cs); } */ - } - } + } + } - result.finish(); // this puts the new dimensions into the global structures + result.finish(); // this puts the new dimensions into the global structures - //CoordSysBuilderIF builder = result.enhance(); - //if (debugEnhance) System.out.printf("parseInfo = %s%n", builder.getParseInfo()); + //CoordSysBuilderIF builder = result.enhance(); + //if (debugEnhance) System.out.printf("parseInfo = %s%n", builder.getParseInfo()); - // LOOK better not to do this when you only want the NetcdfDataset - Formatter parseInfo = new Formatter(); - GridDataset gds = new ucar.nc2.dt.grid.GridDataset(result, parseInfo); - if (debugEnhance) System.out.printf("GridDataset2D parseInfo = %s%n", parseInfo); - return gds; - } + // LOOK better not to do this when you only want the NetcdfDataset + Formatter parseInfo = new Formatter(); + GridDataset gds = new ucar.nc2.dt.grid.GridDataset(result, parseInfo); + if (debugEnhance) System.out.printf("GridDataset2D parseInfo = %s%n", parseInfo); + return gds; + } - private CoordinateSystem findReplacementCs(CoordinateSystem protoCs, String timeDim, NetcdfDataset result) { - CoordinateSystem replace = result.findCoordinateSystem(protoCs.getName()); - if (replace != null) return replace; + private CoordinateSystem findReplacementCs(CoordinateSystem protoCs, String timeDim, NetcdfDataset result) { + CoordinateSystem replace = result.findCoordinateSystem(protoCs.getName()); + if (replace != null) return replace; - List axes = new ArrayList<>(); - for (CoordinateAxis axis : protoCs.getCoordinateAxes()) { - CoordinateAxis ra = result.findCoordinateAxis(axis.getFullNameEscaped()); - axes.add(ra); - } + List axes = new ArrayList<>(); + for (CoordinateAxis axis : protoCs.getCoordinateAxes()) { + CoordinateAxis ra = result.findCoordinateAxis(axis.getFullNameEscaped()); + axes.add(ra); + } - // coord transforms are immutable and can be shared - CoordinateSystem cs = new CoordinateSystem(result, axes, protoCs.getCoordinateTransforms()); - result.addCoordinateSystem(cs); - return cs; - } + // coord transforms are immutable and can be shared + CoordinateSystem cs = new CoordinateSystem(result, axes, protoCs.getCoordinateTransforms()); + result.addCoordinateSystem(cs); + return cs; + } /* private ArrayDouble.D2 makeTimeCoordinateData2D(Date baseDate, FmrcInv.RunSeq runSeq, VariableDS timeVar) { // throws IOException { @@ -702,363 +737,361 @@ private TimeInstance findInventory(int runIdx, int timeIdx) { } */ - private class ProxyReader2D implements ProxyReader { - - @Override - public Array reallyRead(Variable mainv, CancelTask cancelTask) throws IOException { - try { - return reallyRead(mainv, mainv.getShapeAsSection(), cancelTask); - } catch (InvalidRangeException e) { - throw new IOException(e); - } - } + private class ProxyReader2D implements ProxyReader { - // here is where agg variables get read - - @Override - public Array reallyRead(Variable mainv, Section section, CancelTask cancelTask) throws IOException, InvalidRangeException { - FmrcInvLite.Gridset.Grid gridLite = (FmrcInvLite.Gridset.Grid) mainv.getSPobject(); - - // read the original type - if its been promoted to a new type, the conversion happens after this read - DataType dtype = (mainv instanceof VariableDS) ? ((VariableDS) mainv).getOriginalDataType() : mainv.getDataType(); - - Array allData = Array.factory(dtype, section.getShape()); - int destPos = 0; - - // assumes the first two dimensions are runtime and time: LOOK: ensemble ?? - List ranges = section.getRanges(); - Range runRange = ranges.get(0); - Range timeRange = ranges.get(1); - List innerSection = ranges.subList(2, ranges.size()); - - // keep track of open file - must be local variable for thread safety - HashMap openFilesRead = new HashMap<>(); - try { - - // iterate over the desired runs - for (int runIdx : runRange) { - //Date runDate = vstate.runTimes.get(runIdx); - - // iterate over the desired forecast times - for (int timeIdx : timeRange) { - Array result = null; - - // find the inventory for this grid, runtime, and hour - TimeInventory.Instance timeInv = gridLite.getInstance(runIdx, timeIdx); - if (timeInv != null) { - if (debugRead) System.out.printf("HIT %d %d ", runIdx, timeIdx); - result = read(timeInv, gridLite.name, innerSection, openFilesRead); // may return null - result = MAMath.convert(result, dtype); // just in case it need to be converted + @Override + public Array reallyRead(Variable mainv, CancelTask cancelTask) throws IOException { + try { + return reallyRead(mainv, mainv.getShapeAsSection(), cancelTask); + } catch (InvalidRangeException e) { + throw new IOException(e); } + } - // missing data - if (result == null) { - int[] shape = new Section(innerSection).getShape(); - result = ((VariableDS) mainv).getMissingDataArray(shape); // fill with missing values - if (debugRead) System.out.printf("MISS %d %d ", runIdx, timeIdx); + // here is where agg variables get read + + @Override + public Array reallyRead(Variable mainv, Section section, CancelTask cancelTask) throws IOException, InvalidRangeException { + FmrcInvLite.Gridset.Grid gridLite = (FmrcInvLite.Gridset.Grid) mainv.getSPobject(); + + // read the original type - if its been promoted to a new type, the conversion happens after this read + DataType dtype = (mainv instanceof VariableDS) ? ((VariableDS) mainv).getOriginalDataType() : mainv.getDataType(); + + Array allData = Array.factory(dtype, section.getShape()); + int destPos = 0; + + // assumes the first two dimensions are runtime and time: LOOK: ensemble ?? + List ranges = section.getRanges(); + Range runRange = ranges.get(0); + Range timeRange = ranges.get(1); + List innerSection = ranges.subList(2, ranges.size()); + + // keep track of open file - must be local variable for thread safety + HashMap openFilesRead = new HashMap<>(); + try { + + // iterate over the desired runs + for (int runIdx : runRange) { + //Date runDate = vstate.runTimes.get(runIdx); + + // iterate over the desired forecast times + for (int timeIdx : timeRange) { + Array result = null; + + // find the inventory for this grid, runtime, and hour + TimeInventory.Instance timeInv = gridLite.getInstance(runIdx, timeIdx); + if (timeInv != null) { + if (debugRead) System.out.printf("HIT %d %d ", runIdx, timeIdx); + result = read(timeInv, gridLite.name, innerSection, openFilesRead); // may return null + result = MAMath.convert(result, dtype); // just in case it need to be converted + } + + // missing data + if (result == null) { + int[] shape = new Section(innerSection).getShape(); + result = ((VariableDS) mainv).getMissingDataArray(shape); // fill with missing values + if (debugRead) System.out.printf("MISS %d %d ", runIdx, timeIdx); + } + + if (debugRead) + System.out.printf("%d %d reallyRead %s %d bytes start at %d total size is %d%n", + runIdx, timeIdx, mainv.getFullName(), result.getSize(), destPos, allData.getSize()); + + Array.arraycopy(result, 0, allData, destPos, (int) result.getSize()); + destPos += result.getSize(); + } + } + return allData; + + } finally { + // close any files used during this operation + closeAll(openFilesRead); } - - if (debugRead) - System.out.printf("%d %d reallyRead %s %d bytes start at %d total size is %d%n", - runIdx, timeIdx, mainv.getFullName(), result.getSize(), destPos, allData.getSize()); - - Array.arraycopy(result, 0, allData, destPos, (int) result.getSize()); - destPos += result.getSize(); - } } - return allData; - } finally { - // close any files used during this operation - closeAll(openFilesRead); - } } - } + ///////////////////////////////////////////////////////////////////////// + // 1D + + /** + * Build a dataset with a 1D time coordinate. + * + * @param proto current proto dataset + * @param lite current inventory + * @param timeInv use this to generate the time coordinates + * @return resulting GridDataset + * @throws IOException on read error + */ + private GridDataset buildDataset1D(NetcdfDataset proto, FmrcInvLite lite, TimeInventory timeInv) throws IOException { + if (timeInv == null) return null; + NetcdfDataset result = new NetcdfDataset(); // make a copy, so that this object can coexist with previous incarnations + result.setLocation(lite.collectionName); + transferGroup(proto.getRootGroup(), result.getRootGroup(), result); + result.finish(); + addAttributeInfo(result, CDM.HISTORY, "FMRC " + timeInv.getName() + " Dataset"); + + //DateFormatter dateFormatter = new DateFormatter(); + ProxyReader1D proxyReader1D = new ProxyReader1D(); + + // make the time coordinate(s) for each runSeq + List nonAggVars = result.getVariables(); + for (FmrcInvLite.Gridset gridset : lite.gridSets) { + Group group = result.getRootGroup(); // can it be different ?? + String timeDimName = gridset.gridsetName; + + // construct the dimension + int ntimes = timeInv.getTimeLength(gridset); + if (ntimes == 0) { // eg a constant offset dataset for variables that dont have that offset + // remove all variables that are in this gridset + for (FmrcInvLite.Gridset.Grid ugrid : gridset.grids) { + result.removeVariable(group, ugrid.name); + logger.warn("buildDataset1D " + timeInv.getName() + " remove " + ugrid.name); + } + continue; // skip the rest + } - ///////////////////////////////////////////////////////////////////////// - // 1D - - /** - * Build a dataset with a 1D time coordinate. - * - * @param proto current proto dataset - * @param lite current inventory - * @param timeInv use this to generate the time coordinates - * @return resulting GridDataset - * @throws IOException on read error - */ - private GridDataset buildDataset1D(NetcdfDataset proto, FmrcInvLite lite, TimeInventory timeInv) throws IOException { - if (timeInv == null) return null; - NetcdfDataset result = new NetcdfDataset(); // make a copy, so that this object can coexist with previous incarnations - result.setLocation(lite.collectionName); - transferGroup(proto.getRootGroup(), result.getRootGroup(), result); - result.finish(); - addAttributeInfo(result, CDM.HISTORY, "FMRC "+timeInv.getName()+" Dataset"); - - //DateFormatter dateFormatter = new DateFormatter(); - ProxyReader1D proxyReader1D = new ProxyReader1D(); - - // make the time coordinate(s) for each runSeq - List nonAggVars = result.getVariables(); - for (FmrcInvLite.Gridset gridset : lite.gridSets) { - Group group = result.getRootGroup(); // can it be different ?? - String timeDimName = gridset.gridsetName; - - // construct the dimension - int ntimes = timeInv.getTimeLength(gridset); - if (ntimes == 0) { // eg a constant offset dataset for variables that dont have that offset - // remove all variables that are in this gridset - for (FmrcInvLite.Gridset.Grid ugrid : gridset.grids) { - result.removeVariable(group, ugrid.name); - logger.warn("buildDataset1D "+timeInv.getName()+" remove "+ugrid.name); - } - continue; // skip the rest - } + Dimension timeDim = new Dimension(timeDimName, ntimes); + result.removeDimension(group, timeDimName); // remove previous declaration, if any + result.addDimension(group, timeDim); - Dimension timeDim = new Dimension(timeDimName, ntimes); - result.removeDimension(group, timeDimName); // remove previous declaration, if any - result.addDimension(group, timeDim); + // optional time coordinate + group.removeVariable(timeDimName); + FmrcInvLite.ValueB timeCoordValues = timeInv.getTimeCoords(gridset); + if (timeCoordValues != null) { + makeTimeCoordinate(result, group, timeDimName, lite.base, timeCoordValues); + } - // optional time coordinate - group.removeVariable(timeDimName); - FmrcInvLite.ValueB timeCoordValues = timeInv.getTimeCoords(gridset); - if (timeCoordValues != null) { - makeTimeCoordinate(result, group, timeDimName, lite.base, timeCoordValues); - } + // optional runtime coordinate + group.removeVariable(timeDimName + "_run"); + double[] runtimeCoordValues = timeInv.getRunTimeCoords(gridset); + if (runtimeCoordValues != null) { + makeRunTimeCoordinate(result, group, timeDimName, lite.base, runtimeCoordValues); + } - // optional runtime coordinate - group.removeVariable(timeDimName+"_run"); - double[] runtimeCoordValues = timeInv.getRunTimeCoords(gridset); - if (runtimeCoordValues != null) { - makeRunTimeCoordinate(result, group, timeDimName, lite.base, runtimeCoordValues); - } + // optional offset coordinate + group.removeVariable(timeDimName + "_offset"); + double[] offsetCoordValues = timeInv.getOffsetCoords(gridset); + if (offsetCoordValues != null) { + makeOffsetCoordinate(result, group, timeDimName, lite.base, offsetCoordValues); + } - // optional offset coordinate - group.removeVariable(timeDimName+"_offset"); - double[] offsetCoordValues = timeInv.getOffsetCoords(gridset); - if (offsetCoordValues != null) { - makeOffsetCoordinate(result, group, timeDimName, lite.base, offsetCoordValues); - } + // promote all grid variables to agg variables + for (FmrcInvLite.Gridset.Grid ugrid : gridset.grids) { + //BestInventory bestInv = makeBestInventory(bestTimeCoord, ugrid); - // promote all grid variables to agg variables - for (FmrcInvLite.Gridset.Grid ugrid : gridset.grids) { - //BestInventory bestInv = makeBestInventory(bestTimeCoord, ugrid); + VariableDS aggVar = (VariableDS) result.findVariable(ugrid.name); + if (aggVar == null) { // a ugrid is not in the proto + logger.error("buildDataset1D " + lite.collectionName + ": cant find ugrid variable " + ugrid.name + " in collection " + lite.collectionName + debugMissingVar(proto, result)); + continue; // skip + } - VariableDS aggVar = (VariableDS) result.findVariable(ugrid.name); - if (aggVar == null) { // a ugrid is not in the proto - logger.error("buildDataset1D "+lite.collectionName+": cant find ugrid variable "+ugrid.name+" in collection "+lite.collectionName+debugMissingVar(proto, result)); - continue; // skip - } + // create dimension list + List dimList = aggVar.getDimensions(); + dimList = dimList.subList(1, dimList.size()); // LOOK assumes time is outer dimension + dimList.add(0, timeDim); - // create dimension list - List dimList = aggVar.getDimensions(); - dimList = dimList.subList(1, dimList.size()); // LOOK assumes time is outer dimension - dimList.add(0, timeDim); + aggVar.setDimensions(dimList); + aggVar.setProxyReader(proxyReader1D); + aggVar.setSPobject(new Vstate1D(ugrid, timeInv)); + nonAggVars.remove(aggVar); - aggVar.setDimensions(dimList); - aggVar.setProxyReader(proxyReader1D); - aggVar.setSPobject( new Vstate1D(ugrid, timeInv)); - nonAggVars.remove(aggVar); + // we need to explicitly list the coordinate axes + String coords = makeCoordinateList(aggVar, timeDimName, false); + aggVar.removeAttribute(_Coordinate.Axes); + aggVar.addAttribute(new Attribute(CF.COORDINATES, coords)); // CF - // we need to explicitly list the coordinate axes - String coords = makeCoordinateList(aggVar, timeDimName, false); - aggVar.removeAttribute(_Coordinate.Axes); - aggVar.addAttribute(new Attribute(CF.COORDINATES, coords)); // CF + // if (logger.isDebugEnabled()) logger.debug("FmrcDataset: added grid " + aggVar.getName()); + } + } - // if (logger.isDebugEnabled()) logger.debug("FmrcDataset: added grid " + aggVar.getName()); - } - } + result.finish(); // this puts the new dimensions into the global structures - result.finish(); // this puts the new dimensions into the global structures + // these are the non-agg variables - get data or ProxyReader from proto + for (Variable v : nonAggVars) { + VariableDS protoV = (VariableDS) proto.findVariable(v.getFullNameEscaped()); + if (protoV.hasCachedDataRecurse()) { + v.setCachedData(protoV.read()); // read from original + } else { + v.setProxyReader(protoV.getProxyReader()); + } + } - // these are the non-agg variables - get data or ProxyReader from proto - for (Variable v : nonAggVars) { - VariableDS protoV = (VariableDS) proto.findVariable(v.getFullNameEscaped()); - if (protoV.hasCachedDataRecurse()) { - v.setCachedData(protoV.read()); // read from original - } else { - v.setProxyReader(protoV.getProxyReader()); - } - } + if (debugEnhance) { + CoordSysBuilderIF builder = result.enhance(); + System.out.printf("GridDataset1D parseInfo = %s%n", builder.getParseInfo()); + } - if (debugEnhance) { - CoordSysBuilderIF builder = result.enhance(); - System.out.printf("GridDataset1D parseInfo = %s%n", builder.getParseInfo()); + return new ucar.nc2.dt.grid.GridDataset(result); } - return new ucar.nc2.dt.grid.GridDataset(result); - } - - private String debugMissingVar(NetcdfFile proto, NetcdfFile result) { - Formatter f = new Formatter(); - f.format("%nresult dataset %s%n", result.getLocation()); - for (Variable v: result.getVariables()) - f.format(" %s%n", v.getNameAndDimensions()); - f.format("%n"); - f.format("proto dataset %s%n", proto.getLocation()); - for (Variable v: proto.getVariables()) - f.format(" %s%n", v.getNameAndDimensions()); - - return f.toString(); - } - - private VariableDS makeTimeCoordinate(NetcdfDataset result, Group group, String dimName, CalendarDate base, FmrcInvLite.ValueB valueb) { - DataType dtype = DataType.DOUBLE; - VariableDS timeVar = new VariableDS(result, group, null, dimName, dtype, dimName, null, null); // LOOK could just make a CoordinateAxis1D - timeVar.addAttribute(new Attribute(CDM.LONG_NAME, "Forecast time for ForecastModelRunCollection")); - timeVar.addAttribute(new ucar.nc2.Attribute("standard_name", "time")); - timeVar.addAttribute(new ucar.nc2.Attribute(CF.CALENDAR, base.getCalendar().name() )); - //timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + base)); - - //Ensure a valid udunit - timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + base.getTimeUnits())); - - timeVar.addAttribute(new ucar.nc2.Attribute(CDM.MISSING_VALUE, Double.NaN)); - timeVar.addAttribute(new ucar.nc2.Attribute(_Coordinate.AxisType, AxisType.Time.toString())); - - // construct the values - int ntimes = valueb.offset.length; - timeVar.setCachedData(Array.factory( DataType.DOUBLE, new int[] {ntimes}, valueb.offset)); - group.addVariable(timeVar); - - if (valueb.bounds != null) { - String bname = timeVar.getShortName() + "_bounds"; - timeVar.addAttribute(new ucar.nc2.Attribute("bounds", bname)); - Dimension bd = ucar.nc2.dataset.DatasetConstructor.getBoundsDimension( result); - VariableDS boundsVar = new VariableDS(result, group, null, bname, dtype, dimName+" " + bd.getShortName(), null, null); - boundsVar.addAttribute(new Attribute(CDM.LONG_NAME, "bounds for "+ timeVar.getShortName())); - boundsVar.setCachedData(Array.factory( DataType.DOUBLE, new int[] {ntimes, 2}, valueb.bounds)); - group.addVariable(boundsVar); + private String debugMissingVar(NetcdfFile proto, NetcdfFile result) { + Formatter f = new Formatter(); + f.format("%nresult dataset %s%n", result.getLocation()); + for (Variable v : result.getVariables()) + f.format(" %s%n", v.getNameAndDimensions()); + f.format("%n"); + f.format("proto dataset %s%n", proto.getLocation()); + for (Variable v : proto.getVariables()) + f.format(" %s%n", v.getNameAndDimensions()); + + return f.toString(); } - return timeVar; - } - - private VariableDS makeRunTimeCoordinate(NetcdfDataset result, Group group, String dimName, CalendarDate base, double[] values) { - DataType dtype = DataType.DOUBLE; - VariableDS timeVar = new VariableDS(result, group, null, dimName+"_run", dtype, dimName, null, null); // LOOK could just make a CoordinateAxis1D - timeVar.addAttribute(new Attribute(CDM.LONG_NAME, "run times for coordinate = " + dimName)); - timeVar.addAttribute(new ucar.nc2.Attribute("standard_name", "forecast_reference_time")); - timeVar.addAttribute(new ucar.nc2.Attribute(CF.CALENDAR, base.getCalendar().name() )); - //timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + base)); - timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + base.getTimeUnits() )); - - timeVar.addAttribute(new ucar.nc2.Attribute(CDM.MISSING_VALUE, Double.NaN)); - timeVar.addAttribute(new ucar.nc2.Attribute(_Coordinate.AxisType, AxisType.RunTime.toString())); // if theres already a time coord, dont put in coordSys - too complicated - - // construct the values - int ntimes = values.length; - ArrayDouble.D1 timeCoordVals = (ArrayDouble.D1) Array.factory( DataType.DOUBLE, new int[] {ntimes}, values); - timeVar.setCachedData(timeCoordVals); - group.addVariable(timeVar); - - return timeVar; - } - - private VariableDS makeOffsetCoordinate(NetcdfDataset result, Group group, String dimName, CalendarDate base, double[] values) { - DataType dtype = DataType.DOUBLE; - VariableDS timeVar = new VariableDS(result, group, null, dimName+"_offset", dtype, dimName, null, null); // LOOK could just make a CoordinateAxis1D - timeVar.addAttribute(new Attribute(CDM.LONG_NAME, "offset hour from start of run for coordinate = " + dimName)); - timeVar.addAttribute(new ucar.nc2.Attribute("standard_name", "forecast_period")); - timeVar.addAttribute(new ucar.nc2.Attribute(CF.CALENDAR, base.getCalendar().name() )); - timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + base)); - timeVar.addAttribute(new ucar.nc2.Attribute(CDM.MISSING_VALUE, Double.NaN)); - - // construct the values - int ntimes = values.length; - ArrayDouble.D1 timeCoordVals = (ArrayDouble.D1) Array.factory( DataType.DOUBLE, new int[] {ntimes}, values); - timeVar.setCachedData(timeCoordVals); - group.addVariable(timeVar); - - return timeVar; - } - - private static class Vstate1D { - FmrcInvLite.Gridset.Grid gridLite; - TimeInventory timeInv; + private VariableDS makeTimeCoordinate(NetcdfDataset result, Group group, String dimName, CalendarDate base, FmrcInvLite.ValueB valueb) { + DataType dtype = DataType.DOUBLE; + VariableDS timeVar = new VariableDS(result, group, null, dimName, dtype, dimName, null, null); // LOOK could just make a CoordinateAxis1D + timeVar.addAttribute(new Attribute(CDM.LONG_NAME, "Forecast time for ForecastModelRunCollection")); + timeVar.addAttribute(new ucar.nc2.Attribute("standard_name", "time")); + timeVar.addAttribute(new ucar.nc2.Attribute(CF.CALENDAR, base.getCalendar().name())); + //timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + base)); + + //Ensure a valid udunit + timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + base.getTimeUnits())); + + timeVar.addAttribute(new ucar.nc2.Attribute(CDM.MISSING_VALUE, Double.NaN)); + timeVar.addAttribute(new ucar.nc2.Attribute(_Coordinate.AxisType, AxisType.Time.toString())); + + // construct the values + int ntimes = valueb.offset.length; + timeVar.setCachedData(Array.factory(DataType.DOUBLE, new int[]{ntimes}, valueb.offset)); + group.addVariable(timeVar); + + if (valueb.bounds != null) { + String bname = timeVar.getShortName() + "_bounds"; + timeVar.addAttribute(new ucar.nc2.Attribute("bounds", bname)); + Dimension bd = ucar.nc2.dataset.DatasetConstructor.getBoundsDimension(result); + VariableDS boundsVar = new VariableDS(result, group, null, bname, dtype, dimName + " " + bd.getShortName(), null, null); + boundsVar.addAttribute(new Attribute(CDM.LONG_NAME, "bounds for " + timeVar.getShortName())); + boundsVar.setCachedData(Array.factory(DataType.DOUBLE, new int[]{ntimes, 2}, valueb.bounds)); + group.addVariable(boundsVar); + } - private Vstate1D(FmrcInvLite.Gridset.Grid gridLite, TimeInventory timeInv) { - this.gridLite = gridLite; - this.timeInv = timeInv; + return timeVar; } - } - - private class ProxyReader1D implements ProxyReader { - - @Override - public Array reallyRead(Variable mainv, CancelTask cancelTask) throws IOException { - try { - return reallyRead(mainv, mainv.getShapeAsSection(), cancelTask); - } catch (InvalidRangeException e) { - throw new IOException(e); - } + private VariableDS makeRunTimeCoordinate(NetcdfDataset result, Group group, String dimName, CalendarDate base, double[] values) { + DataType dtype = DataType.DOUBLE; + VariableDS timeVar = new VariableDS(result, group, null, dimName + "_run", dtype, dimName, null, null); // LOOK could just make a CoordinateAxis1D + timeVar.addAttribute(new Attribute(CDM.LONG_NAME, "run times for coordinate = " + dimName)); + timeVar.addAttribute(new ucar.nc2.Attribute("standard_name", "forecast_reference_time")); + timeVar.addAttribute(new ucar.nc2.Attribute(CF.CALENDAR, base.getCalendar().name())); + //timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + base)); + timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + base.getTimeUnits())); + + timeVar.addAttribute(new ucar.nc2.Attribute(CDM.MISSING_VALUE, Double.NaN)); + timeVar.addAttribute(new ucar.nc2.Attribute(_Coordinate.AxisType, AxisType.RunTime.toString())); // if theres already a time coord, dont put in coordSys - too complicated + + // construct the values + int ntimes = values.length; + ArrayDouble.D1 timeCoordVals = (ArrayDouble.D1) Array.factory(DataType.DOUBLE, new int[]{ntimes}, values); + timeVar.setCachedData(timeCoordVals); + group.addVariable(timeVar); + + return timeVar; } - // here is where 1D agg variables get read - @Override - public Array reallyRead(Variable mainv, Section section, CancelTask cancelTask) throws IOException, InvalidRangeException { - Vstate1D vstate = (Vstate1D) mainv.getSPobject(); - - // read the original type - if its been promoted to a new type, the conversion happens after this read - DataType dtype = (mainv instanceof VariableDS) ? ((VariableDS) mainv).getOriginalDataType() : mainv.getDataType(); - - Array allData = Array.factory(dtype, section.getShape()); - int destPos = 0; - - // assumes the first dimension is time: LOOK: what about ensemble ?? - List ranges = section.getRanges(); - Range timeRange = ranges.get(0); - List innerSection = ranges.subList(1, ranges.size()); - - // keep track of open files - must be local variable for thread safety - HashMap openFilesRead = new HashMap<>(); + private VariableDS makeOffsetCoordinate(NetcdfDataset result, Group group, String dimName, CalendarDate base, double[] values) { + DataType dtype = DataType.DOUBLE; + VariableDS timeVar = new VariableDS(result, group, null, dimName + "_offset", dtype, dimName, null, null); // LOOK could just make a CoordinateAxis1D + timeVar.addAttribute(new Attribute(CDM.LONG_NAME, "offset hour from start of run for coordinate = " + dimName)); + timeVar.addAttribute(new ucar.nc2.Attribute("standard_name", "forecast_period")); + timeVar.addAttribute(new ucar.nc2.Attribute(CF.CALENDAR, base.getCalendar().name())); + timeVar.addAttribute(new ucar.nc2.Attribute(CDM.UNITS, "hours since " + base)); + timeVar.addAttribute(new ucar.nc2.Attribute(CDM.MISSING_VALUE, Double.NaN)); + + // construct the values + int ntimes = values.length; + ArrayDouble.D1 timeCoordVals = (ArrayDouble.D1) Array.factory(DataType.DOUBLE, new int[]{ntimes}, values); + timeVar.setCachedData(timeCoordVals); + group.addVariable(timeVar); + + return timeVar; + } - try { + private static class Vstate1D { + FmrcInvLite.Gridset.Grid gridLite; + TimeInventory timeInv; - // iterate over the desired forecast times - for (int timeIdx : timeRange) { - Array result = null; + private Vstate1D(FmrcInvLite.Gridset.Grid gridLite, TimeInventory timeInv) { + this.gridLite = gridLite; + this.timeInv = timeInv; + } + } - // find the inventory for this grid, runtime, and hour - TimeInventory.Instance timeInv = vstate.timeInv.getInstance(vstate.gridLite, timeIdx); - if (timeInv == null) { - if (logger.isDebugEnabled()) - logger.debug("Missing Inventory timeInx="+timeIdx+ " for "+ mainv.getFullName()+" in "+state.lite.collectionName); - // vstate.timeInv.getInstance(vstate.gridLite, timeIdx); // allow debugger - } - - else if (timeInv.getDatasetLocation() != null) { - if (debugRead) System.out.printf("HIT %s%n", timeInv); - result = read(timeInv, mainv.getFullNameEscaped(), innerSection, openFilesRead); // may return null - result = MAMath.convert(result, dtype); // just in case it need to be converted - } + private class ProxyReader1D implements ProxyReader { - // may have missing data - if (result == null) { - int[] shape = new Section(innerSection).getShape(); - result = ((VariableDS) mainv).getMissingDataArray(shape); // fill with missing values - if (debugRead) System.out.printf("MISS %d ", timeIdx); - } + @Override + public Array reallyRead(Variable mainv, CancelTask cancelTask) throws IOException { + try { + return reallyRead(mainv, mainv.getShapeAsSection(), cancelTask); - if (debugRead) - System.out.printf("%d reallyRead %s %d bytes start at %d total size is %d%n", timeIdx, mainv.getFullName(), result.getSize(), destPos, allData.getSize()); + } catch (InvalidRangeException e) { + throw new IOException(e); + } + } - Array.arraycopy(result, 0, allData, destPos, (int) result.getSize()); - destPos += result.getSize(); + // here is where 1D agg variables get read + @Override + public Array reallyRead(Variable mainv, Section section, CancelTask cancelTask) throws IOException, InvalidRangeException { + Vstate1D vstate = (Vstate1D) mainv.getSPobject(); + + // read the original type - if its been promoted to a new type, the conversion happens after this read + DataType dtype = (mainv instanceof VariableDS) ? ((VariableDS) mainv).getOriginalDataType() : mainv.getDataType(); + + Array allData = Array.factory(dtype, section.getShape()); + int destPos = 0; + + // assumes the first dimension is time: LOOK: what about ensemble ?? + List ranges = section.getRanges(); + Range timeRange = ranges.get(0); + List innerSection = ranges.subList(1, ranges.size()); + + // keep track of open files - must be local variable for thread safety + HashMap openFilesRead = new HashMap<>(); + + try { + + // iterate over the desired forecast times + for (int timeIdx : timeRange) { + Array result = null; + + // find the inventory for this grid, runtime, and hour + TimeInventory.Instance timeInv = vstate.timeInv.getInstance(vstate.gridLite, timeIdx); + if (timeInv == null) { + if (logger.isDebugEnabled()) + logger.debug("Missing Inventory timeInx=" + timeIdx + " for " + mainv.getFullName() + " in " + state.lite.collectionName); + // vstate.timeInv.getInstance(vstate.gridLite, timeIdx); // allow debugger + } else if (timeInv.getDatasetLocation() != null) { + if (debugRead) System.out.printf("HIT %s%n", timeInv); + result = read(timeInv, mainv.getFullNameEscaped(), innerSection, openFilesRead); // may return null + result = MAMath.convert(result, dtype); // just in case it need to be converted + } + + // may have missing data + if (result == null) { + int[] shape = new Section(innerSection).getShape(); + result = ((VariableDS) mainv).getMissingDataArray(shape); // fill with missing values + if (debugRead) System.out.printf("MISS %d ", timeIdx); + } + + if (debugRead) + System.out.printf("%d reallyRead %s %d bytes start at %d total size is %d%n", timeIdx, mainv.getFullName(), result.getSize(), destPos, allData.getSize()); + + Array.arraycopy(result, 0, allData, destPos, (int) result.getSize()); + destPos += result.getSize(); + } + return allData; + + } finally { + // close any files used during this operation + closeAll(openFilesRead); + } } - return allData; - } finally { - // close any files used during this operation - closeAll(openFilesRead); - } } - } - - //////////////////////////////////////////////////////////////////////////////////////////////////// + //////////////////////////////////////////////////////////////////////////////////////////////////// /* private BestInventory makeBestInventory(TimeCoord.TimeResult bestTimeCoord, FmrcInv.UberGrid ugrid) { BestInventory bestInv = new BestInventory(bestTimeCoord, ugrid); @@ -1112,54 +1145,55 @@ private int findIndex(double offsetHour) { } */ - /////////////////////////////////////////////////////////////////////////////////////////////////// + /////////////////////////////////////////////////////////////////////////////////////////////////// - // the general case is to get only one time per read - probably not too inefficient, eg GRIB, except maybe for remote reads + // the general case is to get only one time per read - probably not too inefficient, eg GRIB, except maybe for remote reads - private Array read(TimeInventory.Instance timeInstance, String fullNameEsc, List innerSection, HashMap openFilesRead) throws IOException, InvalidRangeException { - NetcdfFile ncfile = open(timeInstance.getDatasetLocation(), openFilesRead); - if (ncfile == null) return null; // file might be deleted ?? + private Array read(TimeInventory.Instance timeInstance, String fullNameEsc, List innerSection, HashMap openFilesRead) throws IOException, InvalidRangeException { + NetcdfFile ncfile = open(timeInstance.getDatasetLocation(), openFilesRead); + if (ncfile == null) return null; // file might be deleted ?? - Variable v = ncfile.findVariable(fullNameEsc); - if (v == null) return null; // v could be missing, return missing data i think + Variable v = ncfile.findVariable(fullNameEsc); + if (v == null) return null; // v could be missing, return missing data i think - // assume time is first dimension LOOK: out of-order; ensemble; section different ?? - Range timeRange = new Range(timeInstance.getDatasetIndex(), timeInstance.getDatasetIndex()); - Section s = new Section(innerSection); - s.insertRange(0, timeRange); - return v.read(s); - } + // assume time is first dimension LOOK: out of-order; ensemble; section different ?? + Range timeRange = new Range(timeInstance.getDatasetIndex(), timeInstance.getDatasetIndex()); + Section s = new Section(innerSection); + s.insertRange(0, timeRange); + return v.read(s); + } - /** - * Open a file, keep track of open files - * @param location open this location - * @param openFiles keep track of open files - * @return file or null if not found - */ - private NetcdfDataset open(String location, Map openFiles) throws IOException { - NetcdfDataset ncd; + /** + * Open a file, keep track of open files + * + * @param location open this location + * @param openFiles keep track of open files + * @return file or null if not found + */ + private NetcdfDataset open(String location, Map openFiles) throws IOException { + NetcdfDataset ncd; - if (openFiles != null) { - ncd = openFiles.get(location); - if (ncd != null) return ncd; - } + if (openFiles != null) { + ncd = openFiles.get(location); + if (ncd != null) return ncd; + } - if (config.innerNcml == null) { - ncd = NetcdfDataset.acquireDataset(new DatasetUrl(null, location), true, null); // default enhance + if (config.innerNcml == null) { + ncd = NetcdfDataset.acquireDataset(new DatasetUrl(null, location), true, null); // default enhance - } else { - NetcdfFile nc = NetcdfDataset.acquireFile(new DatasetUrl(null, location), null); - ncd = NcMLReader.mergeNcML(nc, config.innerNcml); // create new dataset - ncd.enhance(); // now that the ncml is added, enhance "in place", ie modify the NetcdfDataset - } + } else { + NetcdfFile nc = NetcdfDataset.acquireFile(new DatasetUrl(null, location), null); + ncd = NcMLReader.mergeNcML(nc, config.innerNcml); // create new dataset + ncd.enhance(); // now that the ncml is added, enhance "in place", ie modify the NetcdfDataset + } - if (openFiles != null && ncd != null) { - openFiles.put(location, ncd); - } + if (openFiles != null && ncd != null) { + openFiles.put(location, ncd); + } - return ncd; - } + return ncd; + } /* from Aggregation.Dataset public NetcdfFile acquireFile(CancelTask cancelTask) throws IOException { @@ -1189,51 +1223,51 @@ public NetcdfFile acquireFile(CancelTask cancelTask) throws IOException { return ds; } */ - private void closeAll(Map openFiles) throws IOException { - for (NetcdfDataset ncfile : openFiles.values()) - ncfile.close(); - openFiles.clear(); - } + private void closeAll(Map openFiles) throws IOException { + for (NetcdfDataset ncfile : openFiles.values()) + ncfile.close(); + openFiles.clear(); + } - //////////////////////////////////////////////////////////////////////////////////// - // all normal (non agg, non cache) variables must use a proxy to acquire the file - // the openFiles map is null - should be ok since its a non-agg, so file gets opened (and closed) for each read. + //////////////////////////////////////////////////////////////////////////////////// + // all normal (non agg, non cache) variables must use a proxy to acquire the file + // the openFiles map is null - should be ok since its a non-agg, so file gets opened (and closed) for each read. - protected class DatasetProxyReader implements ProxyReader { - String location; + protected class DatasetProxyReader implements ProxyReader { + String location; - DatasetProxyReader(String location) { - this.location = location; - } + DatasetProxyReader(String location) { + this.location = location; + } - public Array reallyRead(Variable client, CancelTask cancelTask) throws IOException { - try (NetcdfFile ncfile= open(location, null)) { - if ((cancelTask != null) && cancelTask.isCancel()) return null; - Variable proxyV = findVariable(ncfile, client); - return proxyV.read(); - } - } + public Array reallyRead(Variable client, CancelTask cancelTask) throws IOException { + try (NetcdfFile ncfile = open(location, null)) { + if ((cancelTask != null) && cancelTask.isCancel()) return null; + Variable proxyV = findVariable(ncfile, client); + return proxyV.read(); + } + } - public Array reallyRead(Variable client, Section section, CancelTask cancelTask) throws IOException, InvalidRangeException { - try (NetcdfFile ncfile = open(location, null)) { - Variable proxyV = findVariable(ncfile, client); - if ((cancelTask != null) && cancelTask.isCancel()) return null; - return proxyV.read(section); - } + public Array reallyRead(Variable client, Section section, CancelTask cancelTask) throws IOException, InvalidRangeException { + try (NetcdfFile ncfile = open(location, null)) { + Variable proxyV = findVariable(ncfile, client); + if ((cancelTask != null) && cancelTask.isCancel()) return null; + return proxyV.read(section); + } + } } - } - protected Variable findVariable(NetcdfFile ncfile, Variable client) { - Variable v = ncfile.findVariable(client.getFullNameEscaped()); - if (v == null) { // might be renamed - VariableEnhanced ve = (VariableEnhanced) client; - v = ncfile.findVariable(ve.getOriginalName()); + protected Variable findVariable(NetcdfFile ncfile, Variable client) { + Variable v = ncfile.findVariable(client.getFullNameEscaped()); + if (v == null) { // might be renamed + VariableEnhanced ve = (VariableEnhanced) client; + v = ncfile.findVariable(ve.getOriginalName()); + } + return v; } - return v; - } - public void showDetails(Formatter out) { - out.format("==========================%nproto=%n%s%n", state.proto); - } + public void showDetails(Formatter out) { + out.format("==========================%nproto=%n%s%n", state.proto); + } }