diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java index 7eadbade5..c5cba7c20 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java @@ -1,19 +1,61 @@ package cwms.cda.api; import static com.codahale.metrics.MetricRegistry.name; -import static cwms.cda.api.Controllers.*; +import static cwms.cda.api.Controllers.BEGIN; +import static cwms.cda.api.Controllers.CREATE; +import static cwms.cda.api.Controllers.CREATE_AS_LRTS; +import static cwms.cda.api.Controllers.CURSOR; +import static cwms.cda.api.Controllers.DATUM; +import static cwms.cda.api.Controllers.DELETE; +import static cwms.cda.api.Controllers.END; +import static cwms.cda.api.Controllers.END_TIME_INCLUSIVE; +import static cwms.cda.api.Controllers.FORMAT; +import static cwms.cda.api.Controllers.GET_ALL; +import static cwms.cda.api.Controllers.GET_ONE; +import static cwms.cda.api.Controllers.INCLUDE_ENTRY_DATE; +import static cwms.cda.api.Controllers.MAX_VERSION; +import static cwms.cda.api.Controllers.NAME; +import static cwms.cda.api.Controllers.NOT_SUPPORTED_YET; +import static cwms.cda.api.Controllers.OFFICE; +import static cwms.cda.api.Controllers.OVERRIDE_PROTECTION; +import static cwms.cda.api.Controllers.PAGE; +import static cwms.cda.api.Controllers.PAGE_SIZE; +import static cwms.cda.api.Controllers.RESULTS; +import static cwms.cda.api.Controllers.SIZE; +import static cwms.cda.api.Controllers.START_TIME_INCLUSIVE; +import static cwms.cda.api.Controllers.STATUS_200; +import static cwms.cda.api.Controllers.STATUS_400; +import static cwms.cda.api.Controllers.STATUS_404; +import static cwms.cda.api.Controllers.STATUS_501; +import static cwms.cda.api.Controllers.STORE_RULE; +import static cwms.cda.api.Controllers.TIMESERIES; +import static cwms.cda.api.Controllers.TIMEZONE; +import static cwms.cda.api.Controllers.TIME_FORMAT_DESC; +import static cwms.cda.api.Controllers.UNIT; +import static cwms.cda.api.Controllers.UNITS; +import static cwms.cda.api.Controllers.UPDATE; +import static cwms.cda.api.Controllers.VERSION; +import static cwms.cda.api.Controllers.VERSION_DATE; +import static cwms.cda.api.Controllers.addDeprecatedContentTypeWarning; +import static cwms.cda.api.Controllers.queryParamAsClass; +import static cwms.cda.api.Controllers.queryParamAsZdt; +import static cwms.cda.api.Controllers.requiredParam; +import static cwms.cda.api.Controllers.requiredZdt; import com.codahale.metrics.Histogram; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.Timer; import cwms.cda.api.enums.UnitSystem; import cwms.cda.api.errors.CdaError; +import cwms.cda.api.errors.NotFoundException; import cwms.cda.data.dao.JooqDao; import cwms.cda.data.dao.StoreRule; import cwms.cda.data.dao.TimeSeriesDao; import cwms.cda.data.dao.TimeSeriesDaoImpl; import cwms.cda.data.dao.TimeSeriesDeleteOptions; import cwms.cda.data.dao.TimeSeriesRequestParameters; +import cwms.cda.data.dao.TimeSeriesVerticalDatumConverter; +import cwms.cda.data.dao.VerticalDatum; import cwms.cda.data.dto.TimeSeries; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; @@ -118,6 +160,7 @@ public TimeSeriesController(MetricRegistry metrics) { static { JavalinValidation.register(StoreRule.class, StoreRule::getStoreRule); + JavalinValidation.register(VerticalDatum.class, VerticalDatum::getVerticalDatum); } private Timer.Context markAndTime(String subject) { @@ -147,7 +190,16 @@ private Timer.Context markAndTime(String subject) { @OpenApiParam(name = STORE_RULE, type = StoreRule.class, description = STORE_RULE_DESC), @OpenApiParam(name = OVERRIDE_PROTECTION, type = Boolean.class, description = "A flag " + "to ignore the protected data quality when storing data. 'True' or 'False'" - + ", default is " + TimeSeriesDaoImpl.OVERRIDE_PROTECTION) + + ", default is " + TimeSeriesDaoImpl.OVERRIDE_PROTECTION), + @OpenApiParam(name = DATUM, type = VerticalDatum.class, description = "If the provided " + + "time-series includes an explicit vertical-datum-info attribute " + + "then it is assumed that the data is in the datum specified by the vertical-datum-info. " + + "If the input timeseries does not include vertical-datum-info and " + + "this parameter is not provided it is assumed that the data is in the as-stored " + + "datum and no conversion is necessary. " + + "If the input timeseries does not include vertical-datum-info and " + + "this parameter is provided it is assumed that the data is in the Datum named by the argument " + + "and should be converted to the as-stored datum before being saved.") }, method = HttpMethod.POST, path = "/timeseries", @@ -162,12 +214,18 @@ public void create(@NotNull Context ctx) { boolean overrideProtection = ctx.queryParamAsClass(OVERRIDE_PROTECTION, Boolean.class) .getOrDefault(TimeSeriesDaoImpl.OVERRIDE_PROTECTION); + VerticalDatum vd = ctx.queryParamAsClass(DATUM, VerticalDatum.class) + .getOrDefault(null); + try (final Timer.Context ignored = markAndTime(CREATE)) { DSLContext dsl = getDslContext(ctx); TimeSeriesDao dao = getTimeSeriesDao(dsl); TimeSeries timeSeries = deserializeTimeSeries(ctx); - dao.create(timeSeries, createAsLrts, storeRule, overrideProtection); + + vd = TimeSeriesVerticalDatumConverter.getVerticalDatum(timeSeries).orElse(vd); + + dao.create(timeSeries, createAsLrts, storeRule, overrideProtection, vd); ctx.status(HttpServletResponse.SC_OK); } catch (DataAccessException | IOException ex) { CdaError re = new CdaError("Internal Error"); @@ -435,11 +493,6 @@ public void getAll(@NotNull Context ctx) { if (version != null && version.equals("2")) { - if (datum != null) { - throw new IllegalArgumentException(String.format("Datum is not supported for:%s and %s", - Formats.JSONV2, Formats.XMLV2)); - } - String office = requiredParam(ctx, OFFICE); TimeSeriesRequestParameters requestParameters = new TimeSeriesRequestParameters.Builder() .withNames(names) @@ -453,6 +506,12 @@ public void getAll(@NotNull Context ctx) { .build(); TimeSeries ts = dao.getTimeseries(cursor, pageSize, requestParameters); + if(datum != null) { //this will be null for non-elevation ts + // user has requested a specific vertical datum + VerticalDatum vd = VerticalDatum.valueOf(datum); // the users request + ts = TimeSeriesVerticalDatumConverter.convertToVerticalDatum(ts, vd); + } + results = Formats.format(contentType, ts); ctx.status(HttpServletResponse.SC_OK); @@ -482,6 +541,11 @@ public void getAll(@NotNull Context ctx) { } addDeprecatedContentTypeWarning(ctx, contentType); requestResultSize.update(results.length()); + } catch (NotFoundException e) { + CdaError re = new CdaError("Not found."); + logger.log(Level.WARNING, re.toString(), e); + ctx.status(HttpServletResponse.SC_NOT_FOUND); + ctx.json(re); } catch (IllegalArgumentException ex) { CdaError re = new CdaError("Invalid arguments supplied"); logger.log(Level.SEVERE, re.toString(), ex); @@ -542,7 +606,16 @@ public void getOne(@NotNull Context ctx, @NotNull String id) { @OpenApiParam(name = CREATE_AS_LRTS, type = Boolean.class, description = ""), @OpenApiParam(name = STORE_RULE, type = StoreRule.class, description = STORE_RULE_DESC), @OpenApiParam(name = OVERRIDE_PROTECTION, type = Boolean.class, description = - "A flag to ignore the protected data quality when storing data. \"'true' or 'false'\"") + "A flag to ignore the protected data quality when storing data. \"'true' or 'false'\""), + @OpenApiParam(name = DATUM, type = VerticalDatum.class, description = "If the provided " + + "time-series includes an explicit vertical-datum-info attribute " + + "then it is assumed that the data is in the datum specified by the vertical-datum-info. " + + "If the input timeseries does not include vertical-datum-info and " + + "this parameter is not provided it is assumed that the data is in the as-stored " + + "datum and no conversion is necessary. " + + "If the input timeseries does not include vertical-datum-info and " + + "this parameter is provided it is assumed that the data is in the Datum named by the argument " + + "and should be converted to the as-stored datum before being saved.") }, method = HttpMethod.PATCH, path = "/timeseries/{timeseries}", @@ -563,7 +636,11 @@ public void update(@NotNull Context ctx, @NotNull String id) { boolean overrideProtection = ctx.queryParamAsClass(OVERRIDE_PROTECTION, Boolean.class) .getOrDefault(TimeSeriesDaoImpl.OVERRIDE_PROTECTION); - dao.store(timeSeries, createAsLrts, storeRule, overrideProtection); + VerticalDatum vd = ctx.queryParamAsClass(DATUM, VerticalDatum.class) + .getOrDefault(null); + vd = TimeSeriesVerticalDatumConverter.getVerticalDatum(timeSeries).orElse(vd); + + dao.store(timeSeries, createAsLrts, storeRule, overrideProtection, vd); ctx.status(HttpServletResponse.SC_OK); } catch (DataAccessException | IOException ex) { diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java index 264860a6c..424ebbc91 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java @@ -17,12 +17,12 @@ public interface TimeSeriesDao { void create(TimeSeries input); void create(TimeSeries input, - boolean createAsLrts, StoreRule replaceAll, boolean overrideProtection); + boolean createAsLrts, StoreRule replaceAll, boolean overrideProtection, VerticalDatum vd); void store(TimeSeries timeSeries, Timestamp versionDate); void store(TimeSeries timeSeries, boolean createAsLrts, - StoreRule replaceAll, boolean overrideProtection); + StoreRule replaceAll, boolean overrideProtection, VerticalDatum vd); void delete(String officeId, String tsId, TimeSeriesDeleteOptions options); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 097c1a83a..5c9d76eee 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -1,6 +1,15 @@ package cwms.cda.data.dao; +import cwms.cda.data.dao.rsql.FieldResolver; +import cwms.cda.data.dao.rsql.MapFieldResolver; +import cwms.cda.data.dao.rsql.RSQLConditionBuilder; +import cwms.cda.data.dto.filteredtimeseries.FilteredTimeSeries; +import cwms.cda.data.dto.catalog.TimeSeriesAlias; +import cwms.cda.helpers.DateUtils; + +import java.sql.Connection; + import static org.jooq.impl.DSL.asterisk; import static org.jooq.impl.DSL.countDistinct; import static org.jooq.impl.DSL.field; @@ -10,6 +19,9 @@ import static org.jooq.impl.DSL.partitionBy; import static org.jooq.impl.DSL.select; import static org.jooq.impl.DSL.selectDistinct; + +import org.jooq.ConnectionRunnable; +import usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID; import static org.jooq.impl.DSL.table; import static usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID2.AV_CWMS_TS_ID2; import static usace.cwms.db.jooq.codegen.tables.AV_TS_EXTENTS_UTC.AV_TS_EXTENTS_UTC; @@ -21,9 +33,6 @@ import com.google.common.cache.CacheStats; import cwms.cda.api.enums.UnitSystem; import cwms.cda.api.enums.VersionType; -import cwms.cda.data.dao.rsql.FieldResolver; -import cwms.cda.data.dao.rsql.MapFieldResolver; -import cwms.cda.data.dao.rsql.RSQLConditionBuilder; import cwms.cda.data.dto.Catalog; import cwms.cda.data.dto.CwmsDTOPaginated; import cwms.cda.data.dto.RecentValue; @@ -34,15 +43,10 @@ import cwms.cda.data.dto.TsvId; import cwms.cda.data.dto.VerticalDatumInfo; import cwms.cda.data.dto.catalog.CatalogEntry; -import cwms.cda.data.dto.catalog.TimeSeriesAlias; import cwms.cda.data.dto.catalog.TimeseriesCatalogEntry; -import cwms.cda.data.dto.filteredtimeseries.FilteredTimeSeries; -import cwms.cda.formatters.FormattingException; import cwms.cda.formatters.xml.XMLv1; -import cwms.cda.helpers.DateUtils; import java.math.BigDecimal; import java.math.BigInteger; -import java.sql.Connection; import java.sql.SQLException; import java.sql.Timestamp; import java.time.Duration; @@ -60,6 +64,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.logging.Level; @@ -95,7 +100,6 @@ import usace.cwms.db.jooq.codegen.packages.CWMS_LOC_PACKAGE; import usace.cwms.db.jooq.codegen.packages.CWMS_TS_PACKAGE; import usace.cwms.db.jooq.codegen.packages.CWMS_UTIL_PACKAGE; -import usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID; import usace.cwms.db.jooq.codegen.tables.AV_LOC; import usace.cwms.db.jooq.codegen.tables.AV_LOC_GRP_ASSGN; import usace.cwms.db.jooq.codegen.tables.AV_TSV; @@ -410,15 +414,6 @@ protected TimeSeries getRequestedTimeSeries(String page, int pageSize, @NotNull valid.field("interval", BigDecimal.class).as("interval"), valid.field("loc_part", String.class).as("loc_part"), valid.field("parm_part", String.class).as("parm_part"), - DSL.choose(valid.field("parm_part", String.class)) - .when( - "ELEV", - CWMS_LOC_PACKAGE.call_GET_VERTICAL_DATUM_INFO_F__2( - valid.field("loc_part", String.class), - valid.field("units", String.class), - valid.field("office_id", String.class))) - .otherwise("") - .as("VERTICAL_DATUM"), totalField, AV_CWMS_TS_ID2.INTERVAL_UTC_OFFSET, AV_CWMS_TS_ID2.TIME_ZONE_ID @@ -437,8 +432,15 @@ protected TimeSeries getRequestedTimeSeries(String page, int pageSize, @NotNull TimeSeries timeseries = metadataQuery.fetchOne(tsMetadata -> { - String vert = (String) tsMetadata.getValue("VERTICAL_DATUM"); - VerticalDatumInfo verticalDatumInfo = parseVerticalDatumInfo(vert); + String parmPart = tsMetadata.getValue("parm_part", String.class); + String locPart = tsMetadata.getValue("loc_part", String.class); + + // Fetch vertical datum info separately only when needed + VerticalDatumInfo verticalDatumInfo = null; + if (shouldFetchVerticalDatum(parmPart)) { + verticalDatumInfo = fetchVerticalDatumInfoSeparately( locPart, units, office); + } + VersionType finalDateVersionType = getVersionType(dsl, names, office, versionDate != null); return new TimeSeries(recordCursor, recordPageSize, tsMetadata.getValue("TOTAL", Integer.class), tsMetadata.getValue("NAME", String.class), @@ -540,6 +542,25 @@ protected TimeSeries getRequestedTimeSeries(String page, int pageSize, @NotNull return retVal; } + private boolean shouldFetchVerticalDatum(String parmPart) { + // Check if parameter requires vertical datum (e.g., "ELEV") + if (parmPart == null) { + return false; + } + String upperParm = parmPart.toUpperCase(); + return upperParm.equals("ELEV"); + } + + private VerticalDatumInfo fetchVerticalDatumInfoSeparately(String locPart, String units, String office) { + + return connectionResult(dsl, conn -> { + DSLContext dslContext = getDslContext(conn, office); + String result = CWMS_LOC_PACKAGE.call_GET_VERTICAL_DATUM_INFO_F__2(dslContext.configuration(), + locPart, units, office); + return parseVerticalDatumInfo(result); + }); + } + public void validateEntryDateSupport(boolean includeEntryDate) { if (includeEntryDate) { Record entryDateSupport = dsl.select(asterisk()).from(table("ALL_TYPES")) @@ -612,11 +633,7 @@ private static boolean isVersioned(DSLContext dsl, String tsId, String office) { public static VerticalDatumInfo parseVerticalDatumInfo(String body) { VerticalDatumInfo retVal = null; if (body != null && !body.isEmpty()) { - try { - retVal = new XMLv1().parseContent(body, VerticalDatumInfo.class); - } catch (FormattingException e) { - logger.log(Level.WARNING, e, () -> "Failed to parse:" + body); - } + retVal = new XMLv1().parseContent(body, VerticalDatumInfo.class); } return retVal; } @@ -1400,7 +1417,7 @@ public List findRecentsInRange(String office, String categoryId, St @Override public void create(TimeSeries input) { - create(input, false, StoreRule.REPLACE_ALL, TimeSeriesDaoImpl.OVERRIDE_PROTECTION); + create(input, false, StoreRule.REPLACE_ALL, TimeSeriesDaoImpl.OVERRIDE_PROTECTION, null); } /** @@ -1423,51 +1440,92 @@ public void create(TimeSeries input) { * * @param storeRule How to update the database if data exists. {@see cwms.cda.data.dao.StoreRule for more detail} * @param overrideProtection honor override protection + * @param vd The VerticalDatum in which specified elevations are interpreted. * */ @SuppressWarnings("unused") public void create(TimeSeries input, - boolean createAsLrts, StoreRule storeRule, boolean overrideProtection) { + boolean createAsLrts, StoreRule storeRule, boolean overrideProtection, VerticalDatum vd) { + + Timestamp versionDate; + if (input.getVersionDate() != null) { + versionDate = Timestamp.from(input.getVersionDate().toInstant()); + } else { + versionDate = null; + } + connection(dsl, connection -> { - int intervalForward = 0; - int intervalBackward = 0; - boolean activeFlag = true; - // the code does not need to be created before hand. - // do not add a call to create_ts_code - if (!input.getValues().isEmpty()) { - Timestamp versionDate = null; - if (input.getVersionDate() != null) { - versionDate = Timestamp.from(input.getVersionDate().toInstant()); + DSLContext dslContext = getDslContext(connection, input.getOfficeId()); + + withDefaultDatum(vd, dslContext, (conn)-> { + // the code does not need to be created before hand. + // do not add a call to create_ts_code + + if (!input.getValues().isEmpty()) { + store(dslContext, input.getOfficeId(), input.getName(), input.getUnits(), + versionDate, input.getValues(), createAsLrts, storeRule, + overrideProtection); } + }); + }); + } + + // + + /** + * The idea here is that this will check the current default datum, + * possible switch to the specified datum and + * then run the code and + * if the datum was previously switched + * then switch back to the initial datum. + * @param targetDatum The desired ver + * @param dslContext + * @param cr + */ + private void withDefaultDatum(@Nullable VerticalDatum targetDatum, DSLContext dslContext, ConnectionRunnable cr) { + String defaultVertDatum = CWMS_LOC_PACKAGE.call_GET_DEFAULT_VERTICAL_DATUM(dslContext.configuration()); + String targetName = (targetDatum != null) ? targetDatum.toString() : null; + boolean changeDefaultDatum = !Objects.equals(targetDatum, defaultVertDatum); + try { + if (changeDefaultDatum) { + CWMS_LOC_PACKAGE.call_SET_DEFAULT_VERTICAL_DATUM(dslContext.configuration(), targetName); + } - store(connection, input.getOfficeId(), input.getName(), input.getUnits(), - versionDate, input.getValues(), createAsLrts, storeRule, - overrideProtection); + connection(dslContext, cr); + }finally{ + if (changeDefaultDatum) { + // If we changed it we should restore. + CWMS_LOC_PACKAGE.call_SET_DEFAULT_VERTICAL_DATUM(dslContext.configuration(), defaultVertDatum); } - }); + } } @Override public void store(TimeSeries timeSeries, Timestamp versionDate) { - store(timeSeries, false, StoreRule.REPLACE_ALL, TimeSeriesDaoImpl.OVERRIDE_PROTECTION); + store(timeSeries, false, StoreRule.REPLACE_ALL, TimeSeriesDaoImpl.OVERRIDE_PROTECTION, null); } - public void store(TimeSeries input, boolean createAsLrts, StoreRule replaceAll, boolean overrideProtection) { - connection(dsl, connection -> { - Timestamp versionDate = null; - if (input.getVersionDate() != null) { - versionDate = Timestamp.from(input.getVersionDate().toInstant()); - } + public void store(TimeSeries input, boolean createAsLrts, StoreRule replaceAll, boolean overrideProtection, VerticalDatum vd) { + Timestamp versionDate; + if (input.getVersionDate() != null) { + versionDate = Timestamp.from(input.getVersionDate().toInstant()); + } else { + versionDate = null; + } - store(connection, input.getOfficeId(), input.getName(), input.getUnits(), - versionDate, input.getValues(), createAsLrts, replaceAll, overrideProtection); - }); + connection(dsl, connection -> storeWithDefaultDatum(input, createAsLrts, replaceAll, overrideProtection, vd, connection, versionDate)); + } + + private void storeWithDefaultDatum(TimeSeries input, boolean createAsLrts, StoreRule replaceAll, boolean overrideProtection, + VerticalDatum vd, Connection connection, Timestamp versionDate) throws Throwable { + DSLContext dslContext = getDslContext(connection, input.getOfficeId()); + withDefaultDatum(vd, dslContext, (conn)-> store(dslContext, input.getOfficeId(), input.getName(), input.getUnits(), + versionDate, input.getValues(), createAsLrts, replaceAll, overrideProtection)); } - private void store(Connection connection, String officeId, String tsId, String units, + private void store(DSLContext dslContext, String officeId, String tsId, String units, Timestamp versionDate, List values, boolean createAsLrts, - StoreRule storeRule, boolean overrideProtection) throws SQLException { - setOffice(connection,officeId); + StoreRule storeRule, boolean overrideProtection) { final ZTSV_ARRAY tsvArray = new ZTSV_ARRAY(); @@ -1481,9 +1539,10 @@ private void store(Connection connection, String officeId, String tsId, String u } } + if (versionDate != null) { try { - CWMS_TS_PACKAGE.call_SET_TSID_VERSIONED(getDslContext(connection, officeId).configuration(), + CWMS_TS_PACKAGE.call_SET_TSID_VERSIONED(dslContext.configuration(), tsId, "T", officeId); } catch (DataAccessException e) { if (e.getCause() instanceof SQLException) { @@ -1499,7 +1558,7 @@ private void store(Connection connection, String officeId, String tsId, String u } } } - CWMS_TS_PACKAGE.call_ZSTORE_TS(getDslContext(connection, officeId).configuration(), + CWMS_TS_PACKAGE.call_ZSTORE_TS(dslContext.configuration(), tsId, units, tsvArray, @@ -1510,21 +1569,6 @@ private void store(Connection connection, String officeId, String tsId, String u formatBool(createAsLrts)); } - public void update(TimeSeries input, boolean createAsLrts, StoreRule storeRule, - Timestamp versionDate, boolean overrideProtection) throws SQLException { - String name = input.getName(); - if (!timeseriesExists(name)) { - throw new SQLException("Cannot update a non-existant Timeseries. Create " + name + " " - + "first."); - } - connection(dsl, connection -> { - setOffice(connection,input.getOfficeId()); - store(connection, input.getOfficeId(), name, input.getUnits(), versionDate, - input.getValues(), createAsLrts, storeRule, overrideProtection); - }); - } - - protected BigDecimal retrieveTsCode(String tsId) { return dsl.select(AV_CWMS_TS_ID2.TS_CODE) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java new file mode 100644 index 000000000..c9d0c383b --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java @@ -0,0 +1,75 @@ +package cwms.cda.data.dao; + +import cwms.cda.data.dto.TimeSeries; +import cwms.cda.data.dto.VerticalDatumInfo; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +public final class TimeSeriesVerticalDatumConverter { + + + private TimeSeriesVerticalDatumConverter() { + throw new AssertionError("Utility class, don't instantiate"); + } + + public static TimeSeries convertToVerticalDatum(TimeSeries originalTimeSeries, VerticalDatum convertTo) { + VerticalDatum vd = getVerticalDatum(originalTimeSeries).orElse(convertTo); + if(Objects.equals(convertTo, vd)) { + return originalTimeSeries; //no conversion needed + } + TimeSeries retVal = originalTimeSeries; + VerticalDatumInfo vdi = originalTimeSeries.getVerticalDatumInfo(); + VerticalDatumInfo.Offset offset = vdi.getOffsetForDatum(convertTo); + if(offset != null) + { + List newValues = applyOffsetToValues(offset.getValue(), originalTimeSeries.getValues()); + VerticalDatumInfo newVerticalDatumInfo = vdi.convertedTo(offset); + retVal = new TimeSeries(originalTimeSeries.getPage(), + originalTimeSeries.getPageSize(), + originalTimeSeries.getTotal(), + originalTimeSeries.getName(), + originalTimeSeries.getOfficeId(), + originalTimeSeries.getBegin(), + originalTimeSeries.getEnd(), + originalTimeSeries.getUnits(), + originalTimeSeries.getInterval(), + newVerticalDatumInfo, + originalTimeSeries.getIntervalOffset(), + originalTimeSeries.getTimeZone(), + originalTimeSeries.getVersionDate(), + originalTimeSeries.getDateVersionType()) + .withValues(newValues); + } + return retVal; + } + + @NotNull + private static List applyOffsetToValues(Double offset, List originalValues) { + List newValues = new ArrayList<>(); + for (TimeSeries.Record record : originalValues) { + Double newValue = record.getValue() + offset; + TimeSeries.Record newRecord = new TimeSeries.Record(record.getDateTime(), newValue, record.getQualityCode()); + newValues.add(newRecord); + } + return newValues; + } + + public static Optional getVerticalDatum(TimeSeries timeSeries) { + return Optional.ofNullable(timeSeries) + .map(TimeSeries::getVerticalDatumInfo) + .map(VerticalDatumInfo::getNativeDatum) + .filter(s -> !s.isEmpty()) + .map(s -> { + if (s.equalsIgnoreCase(VerticalDatum.OTHER.toString())) { + throw new IllegalArgumentException("Vertical Datum of OTHER is not currently supported."); + } + return VerticalDatum.getVerticalDatum(s); + }); + } + +} diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/VerticalDatum.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/VerticalDatum.java new file mode 100644 index 000000000..39aaef9b4 --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/VerticalDatum.java @@ -0,0 +1,30 @@ +package cwms.cda.data.dao; + +public enum VerticalDatum { + NAVD88("NAVD88"), + NGVD29("NGVD29"), + NATIVE("NATIVE"), + OTHER("OTHER"); + + private final String rule; + + VerticalDatum(String rule) { + this.rule = rule; + } + + public static VerticalDatum getVerticalDatum(String input) { + VerticalDatum retval = null; + + if (input != null) { + input = input.replace("-", ""); + retval = VerticalDatum.valueOf(input.toUpperCase()); + } + return retval; + } + + @Override + public String toString() { + return rule; + } + +} diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index d1414527f..a3fcdb68f 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -244,6 +244,12 @@ public void addValue(Timestamp dateTime, Double value, int qualityCode, Timestam } } + public TimeSeries withValues(List values) { + this.values.clear(); + this.values.addAll(values); + return this; + } + public static List getColumnDescriptor() { List columns = new ArrayList<>(); for (Field f: Record.class.getDeclaredFields()) { diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/VerticalDatumInfo.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/VerticalDatumInfo.java index 88c811fd2..e4369a3cb 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/VerticalDatumInfo.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/VerticalDatumInfo.java @@ -1,11 +1,16 @@ package cwms.cda.data.dto; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonRootName; import com.fasterxml.jackson.databind.PropertyNamingStrategies; import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonNaming; import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import cwms.cda.data.dao.VerticalDatum; + +import java.util.ArrayList; +import java.util.List; @JsonRootName("vertical-datum-info") @JsonDeserialize(builder = VerticalDatumInfo.Builder.class) @@ -59,6 +64,54 @@ public String getLocalDatumName() { return localDatumName; } + @JsonIgnore + public VerticalDatumInfo.Offset getOffsetForDatum(VerticalDatum convertTo) { + VerticalDatumInfo.Offset retVal = null; + VerticalDatumInfo.Offset[] offsets = getOffsets(); + for (VerticalDatumInfo.Offset offset : offsets) { + if (offset.isForDatum(convertTo.toString())) { + retVal = offset; + break; + } + } + return retVal; + } + + @JsonIgnore + public VerticalDatumInfo convertedTo(VerticalDatumInfo.Offset convertToOffset) { + VerticalDatum convertTo = VerticalDatum.getVerticalDatum(convertToOffset.getToDatum()); + Double offsetValue = convertToOffset.getValue(); + return new VerticalDatumInfo.Builder() + .from(this) + .withElevation(getElevation() + offsetValue) + .withNativeDatum(convertToOffset.getToDatum()) + .withOffsets(buildConvertedOffsets(convertTo, convertToOffset)) + .build(); + } + + private VerticalDatumInfo.Offset[] buildConvertedOffsets(VerticalDatum convertTo, VerticalDatumInfo.Offset convertToOffset) { + List newOffsets = new ArrayList<>(); + + //add the reverse offset + Double conversionFactor = convertToOffset.getValue(); + double convertToOffsetToOriginal = -conversionFactor; + VerticalDatumInfo.Offset reverseOffset = new VerticalDatumInfo.Offset(convertToOffset.isEstimate(), getNativeDatum(), convertToOffsetToOriginal); + newOffsets.add(reverseOffset); + + //add the other offsets, adjusted + VerticalDatumInfo.Offset[] offsets = getOffsets(); + for (VerticalDatumInfo.Offset offset : offsets) { + String toDatum = offset.getToDatum(); + if (!offset.isForDatum(convertTo.toString())) { + Double newOffsetValue = convertToOffsetToOriginal + offset.getValue(); + boolean isEstimate = offset.isEstimate() || convertToOffset.isEstimate(); + VerticalDatumInfo.Offset newOffset = new VerticalDatumInfo.Offset(isEstimate, toDatum, newOffsetValue); + newOffsets.add(newOffset); + } + } + return newOffsets.toArray(new VerticalDatumInfo.Offset[]{}); + } + @JsonNaming(PropertyNamingStrategies.KebabCaseStrategy.class) public static class Offset { boolean estimate; @@ -88,6 +141,17 @@ public Offset(boolean isEstimate, String toDatum, Double value) { this.value = value; } + @JsonIgnore + public boolean isForDatum(String verticalDatum) { + if(verticalDatum == null && toDatum == null) { + return true; + } + if(verticalDatum == null || toDatum == null) { + return false; + } + return toDatum.replaceAll("-", "").equalsIgnoreCase(verticalDatum.replaceAll("-", "")); + } + @Override public boolean equals(Object o) { if (this == o) { @@ -182,6 +246,18 @@ public VerticalDatumInfo.Builder withLocalDatumName(String localDatumName) { return this; } + @JsonIgnore + public Builder from(VerticalDatumInfo vdi) { + this.office = vdi.getOffice(); + this.unit = vdi.getUnit(); + this.location = vdi.getLocation(); + this.nativeDatum = vdi.getNativeDatum(); + this.elevation = vdi.getElevation(); + this.offsets = vdi.getOffsets(); + this.localDatumName = vdi.getLocalDatumName(); + return this; + } + public VerticalDatumInfo build() { return new VerticalDatumInfo(this); } diff --git a/cwms-data-api/src/test/java/cwms/cda/api/LocationControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/LocationControllerTestIT.java index 630a26877..15d21f749 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/LocationControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/LocationControllerTestIT.java @@ -545,7 +545,6 @@ void test_create_update_null_elev_units() throws Exception { given() .log().ifValidationFails(LogDetail.ALL,true) .accept(Formats.JSON) - .header("Authorization", user.toHeaderValue()) .queryParam(OFFICE, user.getOperatingOffice()) .when() .redirects().follow(true) @@ -585,7 +584,6 @@ void test_create_update_null_elev_units() throws Exception { given() .log().ifValidationFails(LogDetail.ALL,true) .accept(Formats.JSON) - .header("Authorization", user.toHeaderValue()) .queryParam(OFFICE, user.getOperatingOffice()) .when() .redirects().follow(true) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/LockControllerIT.java b/cwms-data-api/src/test/java/cwms/cda/api/LockControllerIT.java index 5e70195ad..bdb8c811e 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/LockControllerIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/LockControllerIT.java @@ -116,7 +116,7 @@ final class LockControllerIT extends DataApiTestIT { .withName("TEST_LOCATION3") .withLocationKind("LOCK") .withDescription("Test Lock") - .withHorizontalDatum("NVGD29") + .withHorizontalDatum("NGVD29") .withTimeZoneName(ZoneId.of("UTC")) .withOfficeId("SPK") .withActive(true) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index f49b68cb9..961a5ae92 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -2,9 +2,9 @@ import static cwms.cda.api.Controllers.*; import static cwms.cda.data.dao.JooqDao.getDslContext; +import static helpers.FloatCloseTo.floatCloseTo; import static io.restassured.RestAssured.given; import static io.restassured.config.JsonConfig.jsonConfig; -import static io.restassured.internal.common.assertion.AssertParameter.notNull; import static org.hamcrest.Matchers.*; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -13,7 +13,10 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import cwms.cda.ApiServlet; -import cwms.cda.data.dto.Location; +import cwms.cda.data.dao.VerticalDatum; +import cwms.cda.data.dto.TimeSeries; +import cwms.cda.data.dto.VerticalDatumInfo; +import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; import cwms.cda.helpers.DatabaseHelpers.SCHEMA_VERSION; import cwms.cda.helpers.ZoneIdHelper; @@ -28,6 +31,9 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; import javax.servlet.http.HttpServletResponse; import io.restassured.response.ExtractableResponse; @@ -35,13 +41,13 @@ import io.restassured.response.ValidatableResponse; import mil.army.usace.hec.test.database.CwmsDatabaseContainer; import org.apache.commons.io.IOUtils; -import org.hamcrest.Matchers; import org.jooq.DSLContext; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import usace.cwms.db.jooq.codegen.packages.CWMS_LOC_PACKAGE; @Tag("integration") final class TimeseriesControllerTestIT extends DataApiTestIT { @@ -1215,7 +1221,7 @@ void test_v1_cant_version() throws Exception { } @Test - void test_v2_cant_datum() throws Exception { + void test_v2_can_datum() throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); @@ -1239,7 +1245,7 @@ void test_v2_cant_datum() throws Exception { .queryParam(NAME, ts.get(NAME).asText()) .queryParam(BEGIN, firstPoint) .queryParam(END, firstPoint) - .queryParam("datum", "NAVD88") + .queryParam(DATUM, VerticalDatum.NAVD88.toString()) .when() .redirects().follow(true) .redirects().max(3) @@ -1247,7 +1253,7 @@ void test_v2_cant_datum() throws Exception { .then() .log().ifValidationFails(LogDetail.ALL, true) .assertThat() - .statusCode(is(HttpServletResponse.SC_BAD_REQUEST)) + .statusCode(is(HttpServletResponse.SC_OK)) ; } @@ -1797,4 +1803,384 @@ enum GetAllTest this.expectedContentType = expectedContentType; } } + + @Test + void test_get_for_elev_has_datum() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/spk/elev_ts_create.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + + JsonNode ts = mapper.readTree(tsData); + String location = ts.get(NAME).asText().split("\\.")[0]; + String officeId = ts.get("office-id").asText(); + + createLocation(location, true, officeId); // This marks for delete at end of test. + updateLocation(location, true, officeId); + + TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; + + // inserting the time series + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization",user.toHeaderValue()) + .queryParam(OFFICE, officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + + // 1209654000000 as ms == Thursday, May 1, 2008 3:00:00 PM + + // get it back + String firstPoint = "2008-05-01T03:00:00.000Z"; + + // try once with auth + ValidatableResponse validatableResponse = given() + .log().ifValidationFails(LogDetail.ALL, true) + .header("Authorization",user.toHeaderValue()) + .accept(Formats.JSONV2) + .queryParam(OFFICE, officeId) + .queryParam(UNIT, "m") + .queryParam(NAME, ts.get(NAME).asText()) + .queryParam(BEGIN, firstPoint) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // verify that there is vertical-datum-info in the response. + validatableResponse.body("vertical-datum-info", notNullValue()) + .body("vertical-datum-info.location", equalTo(location)) + .body("vertical-datum-info.unit", equalTo("m")) + .body("vertical-datum-info.offsets.size()", equalTo(1)) + ; + + // Try again without auth + validatableResponse = given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .queryParam(OFFICE, officeId) + .queryParam(UNIT, "m") + .queryParam(NAME, ts.get(NAME).asText()) + .queryParam(BEGIN, firstPoint) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // verify that there is vertical-datum-info in the response. + validatableResponse.body("vertical-datum-info", notNullValue()) + .body("vertical-datum-info.location", equalTo(location)) + .body("vertical-datum-info.unit", equalTo("m")) + .body("vertical-datum-info.offsets.size()", equalTo(1)) + ; + + + validatableResponse = given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .queryParam(OFFICE, officeId) + .queryParam(UNIT, "m") + .queryParam(NAME, ts.get(NAME).asText()) + .queryParam(BEGIN, firstPoint) + .queryParam(DATUM, VerticalDatum.NAVD88.toString()) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // verify that there is vertical-datum-info in the response. + validatableResponse.body("vertical-datum-info", notNullValue()) + .body("vertical-datum-info.location", equalTo(location)) + .body("vertical-datum-info.unit", equalTo("m")) + .body("vertical-datum-info.offsets.size()", equalTo(1)) + ; + + ContentType contentType = Formats.parseHeader(Formats.JSONV2, TimeSeries.class); + TimeSeries timeSeries = Formats.parseContent(contentType, validatableResponse.extract().asString(), TimeSeries.class); + Double conversionFactor = Arrays.stream(timeSeries.getVerticalDatumInfo().getOffsets()).sequential() + .filter(o -> o.getToDatum().equalsIgnoreCase("NGVD-29")) + .findFirst() + .map(VerticalDatumInfo.Offset::getValue) + .orElseThrow(() -> new Exception("No conversion factor from NAVD88 to NGVD29 found")); + + ValidatableResponse validatableResponseConverted = given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .queryParam(OFFICE, officeId) + .queryParam(UNIT, "m") + .queryParam(NAME, ts.get(NAME).asText()) + .queryParam(BEGIN, firstPoint) + .queryParam(DATUM, "NGVD29") + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + // verify that there is vertical-datum-info in the response. + validatableResponseConverted.body("vertical-datum-info", notNullValue()) + .body("vertical-datum-info.location", equalTo(location)) + .body("vertical-datum-info.unit", equalTo("m")) + .body("vertical-datum-info.offsets.size()", equalTo(1)) + .body("values[0][1].toDouble()", closeTo( timeSeries.getValues().get(0).getValue() + conversionFactor, 0.001)) + ; + + + } + + private void updateLocation(String location, boolean active, String officeId) throws SQLException { + + String P_LOCATION_ID = location; + String P_LOCATION_TYPE = "SITE"; + Number P_ELEVATION = 11; + String P_ELEV_UNIT_ID = "m"; + + // Pretty sure this isn't supposed to have a dash. The create doesn't check. The default create just passes null. + // If it has a dash then the offsets don't work. + // select VERTICAL_DATUM, count(*) as COUNT + // from AT_PHYSICAL_LOCATION + // group by VERTICAL_DATUM + // order by COUNT desc + // has no entries with a dash in the name (unless we've run this test with a dash). + String P_VERTICAL_DATUM = VerticalDatum.NAVD88.toString(); + Number P_LATITUDE = 38.5757; // pretty sure that if these are 0,0 then its not inside the navd88 bounds and the offsets come back [] + Number P_LONGITUDE = -121.4789; + String P_HORIZONTAL_DATUM = "WGS84"; + String P_PUBLIC_NAME = "Integration Test Sac Dam"; + String P_LONG_NAME= null; + String P_DESCRIPTION = "for testing"; + String P_TIME_ZONE_ID = "UTC"; + String P_COUNTY_NAME = "Sacramento"; + String P_STATE_INITIAL = "CA"; + String P_ACTIVE = active ? "T" : "F"; + String P_DB_OFFICE_ID = officeId; + + CwmsDatabaseContainer db = CwmsDataApiSetupCallback.getDatabaseLink(); + db.connection(c -> { + DSLContext dslContext = getDslContext(c, officeId); + +// CWMS_LOC_PACKAGE.call_DELETE_LOCATION(dslContext.configuration(), P_LOCATION_ID, String.valueOf(DeleteRule.DELETE_LOC_CASCADE), P_DB_OFFICE_ID); +// CWMS_LOC_PACKAGE.call_CREATE_LOCATION(dslContext.configuration(), +// P_LOCATION_ID, P_LOCATION_TYPE, P_ELEVATION, P_ELEV_UNIT_ID, P_VERTICAL_DATUM, P_LATITUDE, P_LONGITUDE, +// P_HORIZONTAL_DATUM, P_PUBLIC_NAME, P_LONG_NAME, P_DESCRIPTION, P_TIME_ZONE_ID, P_COUNTY_NAME, P_STATE_INITIAL, +// P_ACTIVE, P_DB_OFFICE_ID); + + String P_IGNORENULLS = "F"; + CWMS_LOC_PACKAGE.call_UPDATE_LOCATION(dslContext.configuration(), + P_LOCATION_ID, P_LOCATION_TYPE, P_ELEVATION, P_ELEV_UNIT_ID, P_VERTICAL_DATUM, P_LATITUDE, P_LONGITUDE, + P_HORIZONTAL_DATUM, P_PUBLIC_NAME, P_LONG_NAME, P_DESCRIPTION, P_TIME_ZONE_ID, P_COUNTY_NAME, P_STATE_INITIAL, + P_ACTIVE, P_IGNORENULLS, P_DB_OFFICE_ID ); + + }); + + } + + + // vertical-datum parameter was recently added to the timeseries create call. + // The timeseries sent as the body to the create can optionally also include a vertical-datum-info element. This + // test is meant to verify 3 scenarios when the timeseries does not include vertical-datum-info: + // 1) no vertical-datum parameter is sent to the create call. + // 2) a NAVD88 vertical-datum parameter is sent to the create call. + // 3) a NGVD29 vertical-datum parameter is sent to the create call. + // + @Test + void test_create_without_vertical_datum_info() throws Exception { + if(getSchemaVersion() < SCHEMA_VERSION.LATEST_DEV.numeric()) + { + return; + } + ObjectMapper mapper = new ObjectMapper(); + + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/spk/elev_ts_create.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + + JsonNode ts = mapper.readTree(tsData); + String tsName = ts.get(NAME).asText(); + String location = tsName.split("\\.")[0]; + String officeId = ts.get("office-id").asText(); + + // Collect input times and values from the payload + List inputTimes = new ArrayList<>(); + List inputValues = new ArrayList<>(); + for (JsonNode row : ts.get("values")) { + inputTimes.add(row.get(0).asLong()); + inputValues.add(row.get(1).asDouble()); + } + long firstMillis = inputTimes.get(0); + long lastMillis = inputTimes.get(inputTimes.size() - 1); + String beginIso = java.time.Instant.ofEpochMilli(firstMillis).toString(); + // pad end by 1 hour to ensure inclusion + String endIso = java.time.Instant.ofEpochMilli(lastMillis + 3600_000L).toString(); + + createLocation(location, true, officeId); // This marks for delete at end of test. + updateLocation(location, true, officeId); + + TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; + + // Helper lambda to GET the series and return a ValidatableResponse + java.util.function.Supplier doGet = () -> + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .queryParam(OFFICE, officeId) + .queryParam(NAME, tsName) + .queryParam(UNIT, "m") + .queryParam(BEGIN, beginIso) + .queryParam(END, endIso) + .queryParam(TRIM, true) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .statusCode(is(HttpServletResponse.SC_OK)); + + // 1) No vertical-datum parameter provided + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization", user.toHeaderValue()) + .queryParam(OFFICE, officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // GET after scenario 1 and verify values equal input (NAVD88 native) + ValidatableResponse vr1 = doGet.get(); + + /* Response includes + "vertical-datum-info": { + "office": "SPK", + "unit": "m", + "location": "Sacramento Dam", + "native-datum": "NAVD-88", + "elevation": 11.0, + "offsets": [ + { + "estimate": true, + "to-datum": "NGVD-29", + "value": -0.7717 + } + ] + }*/ + + vr1.body("values.size()", equalTo(inputValues.size())); + for (int i = 0; i < inputValues.size(); i++) { + long expectedTime = inputTimes.get(i); + double expectedVal = inputValues.get(i); + vr1.body("values[" + i + "][0]", equalTo(expectedTime)) + .body("values[" + i + "][1]", floatCloseTo(expectedVal, 1e-6)); + } + + // 2) Provide NAVD88 vertical-datum parameter (matches location's datum) + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization", user.toHeaderValue()) + .queryParam(OFFICE, officeId) + .queryParam(DATUM, VerticalDatum.NAVD88.toString()) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // GET after scenario 2 and verify values equal input; also capture NGVD29 offset + ValidatableResponse vr2 = doGet.get(); + vr2.body("values.size()", equalTo(inputValues.size())); + for (int i = 0; i < inputValues.size(); i++) { + vr2.body("values[" + i + "][1]", floatCloseTo(inputValues.get(i), 1e-6)); + } + ExtractableResponse ex2 = vr2.extract(); + String body2 = ex2.asString(); + JsonNode resp2 = new ObjectMapper().readTree(body2); + JsonNode vdi = resp2.get("vertical-datum-info"); + Double offsetToNgvd29 = null; + if (vdi != null && vdi.has("offsets")) { + for (JsonNode off : vdi.get("offsets")) { + if ("NGVD-29".equalsIgnoreCase(off.get("to-datum").asText())) { + if (off.hasNonNull("value")) { + offsetToNgvd29 = off.get("value").asDouble(); + } + break; + } + } + } + assertNotNull(offsetToNgvd29, "Expected NGVD-29 offset to be present in vertical-datum-info"); + + // 3) Provide NGVD29 vertical-datum parameter (conversion should occur to as-stored datum) + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization", user.toHeaderValue()) + .queryParam(OFFICE, officeId) + .queryParam(DATUM, "NGVD29") + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // Compute expected NAVD88 = NGVD29 - offset(NAVD88->NGVD29) + java.util.List expectedNavd88 = new java.util.ArrayList<>(); + for (Double v : inputValues) { + expectedNavd88.add(v - offsetToNgvd29); + } + + // GET after scenario 3 and verify the conversion was applied + ValidatableResponse vr3 = doGet.get(); + vr3.body("values.size()", equalTo(expectedNavd88.size())); + for (int i = 0; i < expectedNavd88.size(); i++) { + vr3.body("values[" + i + "][1]", floatCloseTo(expectedNavd88.get(i), 1e-4)); + } + } + + + } diff --git a/cwms-data-api/src/test/java/cwms/cda/api/project/ProjectChildLocationHandlerIT.java b/cwms-data-api/src/test/java/cwms/cda/api/project/ProjectChildLocationHandlerIT.java index bc1bc1ab2..27a0e2789 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/project/ProjectChildLocationHandlerIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/project/ProjectChildLocationHandlerIT.java @@ -180,7 +180,7 @@ private Embankment buildTestEmbankment(Location location, CwmsId projId) { private Location buildTestLocation(String office, String name) { return new Location.Builder(name, "EMBANKMENT", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", office) + 50.0, 50.0, "NGVD29", office) .withElevation(10.0) .withElevationUnits("ft") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/LocationLevelsDaoTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/LocationLevelsDaoTest.java index 50009a0c7..f5810743f 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/LocationLevelsDaoTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/LocationLevelsDaoTest.java @@ -347,7 +347,7 @@ LocationLevel buildExampleLevel(String locationName) throws Exception } private Location buildTestLocation(String name) { - return new Location.Builder(name, "SITE", ZoneId.of("UTC"), 50.0, 50.0, "NVGD29", OFFICE_ID) + return new Location.Builder(name, "SITE", ZoneId.of("UTC"), 50.0, 50.0, "NGVD29", OFFICE_ID) .withElevation(10.0) .withCountyName("Sacramento") .withNation(Nation.US) diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/LocationsDaoTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/LocationsDaoTest.java index 637390ef0..f3881d93f 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/LocationsDaoTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/LocationsDaoTest.java @@ -101,7 +101,7 @@ private void cleanUpRoutine() throws Exception { private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "SITE", ZoneId.of("UTC"), 50.0, 50.0, - "NVGD29", "LRL") + "NGVD29", "LRL") .withElevation(10.0) .withCountyName("Sacramento") .withNation(Nation.US) diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverterTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverterTest.java new file mode 100644 index 000000000..2e3a8f619 --- /dev/null +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverterTest.java @@ -0,0 +1,124 @@ +package cwms.cda.data.dao; + +import cwms.cda.data.dto.TimeSeries; +import cwms.cda.data.dto.VerticalDatumInfo; +import cwms.cda.formatters.ContentType; +import cwms.cda.formatters.Formats; +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Test; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class TimeSeriesVerticalDatumConverterTest { + + @Test + void testConvertVerticalDatum() throws Exception + { + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/timeseries/ts_with_vertical.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + ContentType contentType = Formats.parseHeader(Formats.JSONV2, TimeSeries.class); + TimeSeries ts = Formats.parseContent(contentType, tsData, TimeSeries.class); + + Double conversionFactor = Arrays.stream(ts.getVerticalDatumInfo().getOffsets()).sequential() + .filter(o -> o.getToDatum().equalsIgnoreCase("NAVD-88")) + .findFirst() + .map(VerticalDatumInfo.Offset::getValue) + .orElseThrow(() -> new Exception("No conversion factor from NGVD29 to NAVD88 found")); + TimeSeries convertedTs = TimeSeriesVerticalDatumConverter.convertToVerticalDatum(ts, VerticalDatum.NAVD88); + assertFalse(convertedTs.getValues().isEmpty()); + assertEquals(convertedTs.getValues().size(), ts.getValues().size()); + for(int i=0; i< convertedTs.getValues().size(); i++) + { + assertEquals(ts.getValues().get(i).getValue() + conversionFactor, convertedTs.getValues().get(i).getValue(), 0.0001); + } + assertEquals(ts.getVerticalDatumInfo().getElevation() + conversionFactor, convertedTs.getVerticalDatumInfo().getElevation(), 0.0001); + assertEquals("NAVD-88", convertedTs.getVerticalDatumInfo().getNativeDatum()); + } + + @Test + void testConvertVerticalDatumRoundTrip() throws Exception + { + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/timeseries/ts_with_vertical.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + ContentType contentType = Formats.parseHeader(Formats.JSONV2, TimeSeries.class); + TimeSeries ts = Formats.parseContent(contentType, tsData, TimeSeries.class); + TimeSeries convertedTs = TimeSeriesVerticalDatumConverter.convertToVerticalDatum(ts, VerticalDatum.NAVD88); + TimeSeries convertedTsBack = TimeSeriesVerticalDatumConverter.convertToVerticalDatum(convertedTs, VerticalDatum.NGVD29); + + //verify round trip worked + assertFalse(convertedTsBack.getValues().isEmpty()); + assertEquals(convertedTsBack.getValues().size(), ts.getValues().size()); + for(int i=0; i< convertedTsBack.getValues().size(); i++) + { + assertEquals(convertedTsBack.getValues().get(i).getValue(), ts.getValues().get(i).getValue(), 0.0001); + } + assertEquals(ts.getVerticalDatumInfo().getElevation(), convertedTsBack.getVerticalDatumInfo().getElevation(), 0.0001); + assertEquals(ts.getVerticalDatumInfo().getNativeDatum(), convertedTsBack.getVerticalDatumInfo().getNativeDatum()); + } + + @Test + void testConvertVerticalDatumOffsetsUpdates() throws Exception + { + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/timeseries/ts_with_vertical.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + ContentType contentType = Formats.parseHeader(Formats.JSONV2, TimeSeries.class); + TimeSeries ts = Formats.parseContent(contentType, tsData, TimeSeries.class); + VerticalDatumInfo originalDatumInfo = ts.getVerticalDatumInfo(); + List madeupOffsets = new ArrayList<>(Arrays.asList(originalDatumInfo.getOffsets())); + //since we are tied to an enum to define what we support, using "NATIVE" as a made-up datum value for testing + madeupOffsets.add(new VerticalDatumInfo.Offset(true, VerticalDatum.NATIVE.toString(), 10.0)); + VerticalDatumInfo madeupVdi = new VerticalDatumInfo.Builder() + .from(ts.getVerticalDatumInfo()) + .withOffsets(madeupOffsets.toArray(new VerticalDatumInfo.Offset[]{})) + .build(); + ts = new TimeSeries(ts.getPage(), + ts.getPageSize(), + ts.getTotal(), + ts.getName(), + ts.getOfficeId(), + ts.getBegin(), + ts.getEnd(), + ts.getUnits(), + ts.getInterval(), + madeupVdi, + ts.getIntervalOffset(), + ts.getTimeZone(), + ts.getVersionDate(), + ts.getDateVersionType()) + .withValues(ts.getValues()); + TimeSeries convertedTs = TimeSeriesVerticalDatumConverter.convertToVerticalDatum(ts, VerticalDatum.NAVD88); + TimeSeries convertedTsToMadeUp = TimeSeriesVerticalDatumConverter.convertToVerticalDatum(convertedTs, VerticalDatum.NATIVE); + TimeSeries convertBackToOriginal = TimeSeriesVerticalDatumConverter.convertToVerticalDatum(convertedTsToMadeUp, VerticalDatum.NGVD29); + //verify we get back to original after multiple conversions between datums - this ensures that offsets are being updated properly + assertFalse(convertBackToOriginal.getValues().isEmpty()); + assertEquals(convertBackToOriginal.getValues().size(), ts.getValues().size()); + for(int i=0; i< convertBackToOriginal.getValues().size(); i++) + { + assertEquals(ts.getValues().get(i).getValue(), convertBackToOriginal.getValues().get(i).getValue(), 0.0001); + } + assertEquals(ts.getVerticalDatumInfo().getElevation(), convertBackToOriginal.getVerticalDatumInfo().getElevation(), 0.0001); + assertEquals(ts.getVerticalDatumInfo().getNativeDatum(), convertBackToOriginal.getVerticalDatumInfo().getNativeDatum()); + //verify all original offsets are present in round-trip conversion and values match + for(VerticalDatumInfo.Offset offset : convertBackToOriginal.getVerticalDatumInfo().getOffsets()) + { + VerticalDatum convertedBackToDatum = VerticalDatum.getVerticalDatum(offset.getToDatum()); + VerticalDatumInfo.Offset originalToDatum = ts.getVerticalDatumInfo().getOffsetForDatum(convertedBackToDatum); + assertNotNull(originalToDatum, "Round-trip conversion resulted in missing to-datum: " + convertedBackToDatum); + assertEquals(originalToDatum.getValue(), offset.getValue(), 0.0001); + } + } +} diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/EmbankmentDaoTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/EmbankmentDaoTest.java index adfe40e5d..f2834d6cc 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/EmbankmentDaoTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/EmbankmentDaoTest.java @@ -84,7 +84,7 @@ private Embankment buildTestEmbankment() { private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "EMBANKMENT", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withElevationUnits("ft") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LocationUtilTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LocationUtilTest.java index a9c354b38..5cc56d910 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LocationUtilTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LocationUtilTest.java @@ -84,7 +84,7 @@ private LookupType buildTestLookupType() { private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "EMBANKMENT", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withElevationUnits("m") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LockDaoIT.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LockDaoIT.java index 18983ef9f..c3700c77d 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LockDaoIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LockDaoIT.java @@ -433,7 +433,7 @@ private Lock buildTestLockSI() { private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "LOCK", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "SPK") + 50.0, 50.0, "NGVD29", "SPK") .withElevation(10.0) .withElevationUnits("ft") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LockDaoTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LockDaoTest.java index 29781061c..85be6721e 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LockDaoTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LockDaoTest.java @@ -151,7 +151,7 @@ private Lock buildTestLockWithNullLevels() { private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "LOCK", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withElevationUnits("ft") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/ProjectStructureIT.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/ProjectStructureIT.java index bcd51ea22..885a4bb37 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/ProjectStructureIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/ProjectStructureIT.java @@ -55,7 +55,7 @@ public static void setupProject() throws Exception { public static Location buildProjectLocation(String locationId) { return new Location.Builder(locationId, "PROJECT", ZoneId.of("UTC"), - 38.5613824, -121.7298432, "NVGD29", OFFICE_ID) + 38.5613824, -121.7298432, "NGVD29", OFFICE_ID) .withElevation(10.0) .withElevationUnits("m") .withLocationType("SITE") @@ -97,7 +97,7 @@ public static PROJECT_OBJ_T buildProject(Location location) { public static Location buildProjectStructureLocation(String locationId, String locationKind) { return new Location.Builder(locationId, locationKind, ZoneId.of("UTC"), - 38.5613824, -121.7298432, "NVGD29", OFFICE_ID) + 38.5613824, -121.7298432, "NGVD29", OFFICE_ID) .withPublicName("Integration Test " + locationId) .withLongName("Integration Test " + locationId + " " + locationKind) .withElevation(10.0) diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/TurbineDaoIT.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/TurbineDaoIT.java index acb8788c5..52104d990 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/TurbineDaoIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/TurbineDaoIT.java @@ -246,7 +246,7 @@ void testTurbineChangesRoundTrip() throws Exception { private static Location buildProjectLocation(String projectId) { return new Location.Builder(projectId, "PROJECT", ZoneId.of("UTC"), - 38.5613824, -121.7298432, "NVGD29", OFFICE) + 38.5613824, -121.7298432, "NGVD29", OFFICE) .withElevation(10.0) .withElevationUnits("m") .withLocationType("SITE") @@ -276,7 +276,7 @@ private static Turbine buildTestTurbine(Location location, String projectId) { private static Location buildTurbineLocation(String locationId) { return new Location.Builder(locationId, "TURBINE", ZoneId.of("UTC"), - 38.5613824, -121.7298432, "NVGD29", OFFICE) + 38.5613824, -121.7298432, "NGVD29", OFFICE) .withElevation(10.0) .withElevationUnits("m") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/LocationTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/LocationTest.java index a98ea4593..50468f369 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dto/LocationTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/LocationTest.java @@ -99,7 +99,7 @@ void testSerializationRoundTrip(String format) void canBuildNullLatLon(){ Location location = new Location.Builder("TEST_LOCATION2", "SITE", ZoneId.of("UTC"), null, null, // lat/lon are null in this test - "NVGD29", "LRL") + "NGVD29", "LRL") .withElevation(10.0) .withCountyName("Sacramento") .withNation(Nation.US) @@ -143,7 +143,7 @@ void test_alias_roundtrip() { Location location = new Location.Builder("TEST_LOCATION2", "SITE", ZoneId.of("UTC"), null, null, // lat/lon are null in this test - "NVGD29", "LRL") + "NGVD29", "LRL") .withElevation(10.0) .withCountyName("Sacramento") .withNation(Nation.US) @@ -186,7 +186,7 @@ void test_serialization_no_alias() void test_serialization_empty_alias() { Location location = new Location.Builder("TEST_LOCATION2", "SITE", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withCountyName("Sacramento") .withNation(Nation.US) @@ -211,7 +211,7 @@ void test_serialization_empty_alias() private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "SITE", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withCountyName("Sacramento") .withNation(Nation.US) @@ -228,7 +228,7 @@ private Location buildTestLocation() { private Location buildTestLocationNewLine() { return new Location.Builder("TEST_LOCATION2", "SITE", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withCountyName("Sacramento") .withNation(Nation.US) diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java index 8e1e13d50..51e866aef 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java @@ -4,6 +4,9 @@ import cwms.cda.formatters.Formats; import cwms.cda.formatters.json.JsonV2; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.sql.Timestamp; import java.time.Duration; import java.time.Instant; @@ -20,6 +23,8 @@ import cwms.cda.formatters.xml.XMLv2; import java.util.List; + +import org.apache.commons.io.IOUtils; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.Test; @@ -105,6 +110,33 @@ void testSerializerWithNulls() { assertNotNull(tsBody); } + @Test + void testDeserializeVerticalDatum() throws IOException { + InputStream stream; + + // verify that we can deserialize ts that don't have vertical datum info + stream = getClass().getClassLoader().getResourceAsStream( + "cwms/cda/api/timeseries/ts_no_vertical.json"); + assertNotNull(stream); + String input = IOUtils.toString(stream, StandardCharsets.UTF_8); + ObjectMapper om = buildObjectMapper(); + + TimeSeries ts = om.readValue(input, TimeSeries.class); + assertNotNull(ts); + assertNull(ts.getVerticalDatumInfo()); + + // verify that we can deserialize ts that do have vertical datum inf + stream = getClass().getClassLoader().getResourceAsStream( + "cwms/cda/api/timeseries/ts_with_vertical.json"); + assertNotNull(stream); + input = IOUtils.toString(stream, StandardCharsets.UTF_8); + + ts = om.readValue(input, TimeSeries.class); + assertNotNull(ts); + assertNotNull(ts.getVerticalDatumInfo()); + + } + @NotNull private TimeSeries buildTimeSeries() { return buildTimeSeries(null); diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/EmbankmentTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/EmbankmentTest.java index 44bd391c0..bf17e5cb4 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/EmbankmentTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/EmbankmentTest.java @@ -118,7 +118,7 @@ private Embankment buildTestEmbankment() { private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "EMBANKMENT", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withElevationUnits("m") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/LockTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/LockTest.java index 86216c639..5d657e58d 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/LockTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/LockTest.java @@ -126,7 +126,7 @@ private Lock buildTestLock() { private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "LOCK", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withElevationUnits("m") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/TurbineTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/TurbineTest.java index 8d44c5c0d..8b7b03e92 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/TurbineTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/TurbineTest.java @@ -92,7 +92,7 @@ private Turbine buildTestTurbine() { private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "TURBINE", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withElevationUnits("m") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/embankment.json b/cwms-data-api/src/test/resources/cwms/cda/api/embankment.json index 7cd636f6d..9a7f1d6c2 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/embankment.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/embankment.json @@ -18,7 +18,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": -95.992775, "published-latitude": 36.153980, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/lock.json b/cwms-data-api/src/test/resources/cwms/cda/api/lock.json index f83a706c5..43516f4c5 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/lock.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/lock.json @@ -18,7 +18,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": 38.5, "published-latitude": -121.7, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/project_location.json b/cwms-data-api/src/test/resources/cwms/cda/api/project_location.json index 21381afaa..85198f486 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/project_location.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/project_location.json @@ -13,7 +13,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": -95.992775, "published-latitude": 36.153980, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/project_location_lock.json b/cwms-data-api/src/test/resources/cwms/cda/api/project_location_lock.json index ecb69a63f..75c65c5db 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/project_location_lock.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/project_location_lock.json @@ -13,7 +13,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": -95.992775, "published-latitude": 36.153980, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/project_location_lock2.json b/cwms-data-api/src/test/resources/cwms/cda/api/project_location_lock2.json index 41813ab2c..a25f3f5b8 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/project_location_lock2.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/project_location_lock2.json @@ -13,7 +13,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": -95.992775, "published-latitude": 36.153980, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/project_location_turb.json b/cwms-data-api/src/test/resources/cwms/cda/api/project_location_turb.json index dda7840db..a69ba844d 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/project_location_turb.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/project_location_turb.json @@ -13,7 +13,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": -95.992775, "published-latitude": 36.153980, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/project_location_turb_changes.json b/cwms-data-api/src/test/resources/cwms/cda/api/project_location_turb_changes.json index 9538d2b74..43a0b8913 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/project_location_turb_changes.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/project_location_turb_changes.json @@ -13,7 +13,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": -95.992775, "published-latitude": 36.153980, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/spk/elev_ts_create.json b/cwms-data-api/src/test/resources/cwms/cda/api/spk/elev_ts_create.json new file mode 100644 index 000000000..128d3d919 --- /dev/null +++ b/cwms-data-api/src/test/resources/cwms/cda/api/spk/elev_ts_create.json @@ -0,0 +1,26 @@ +{ + "office-id": "SPK", + "name": "Sacramento Dam.Elev.Inst.1Hour.0.Raw", + "interval": "PT1H", + "interval-offset": 0, + "units": "m", + "time-zone": "UTC", + "values": [ + [ + 1209654000000, + 2.2, + 0 + ], + [ + 1209657600000, + 4.4, + 0 + ], + [ + 1209661200000, + 6.6, + 0 + ] + ] +} + diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/timeseries/ts_no_vertical.json b/cwms-data-api/src/test/resources/cwms/cda/api/timeseries/ts_no_vertical.json new file mode 100644 index 000000000..1c0f53333 --- /dev/null +++ b/cwms-data-api/src/test/resources/cwms/cda/api/timeseries/ts_no_vertical.json @@ -0,0 +1,23 @@ +{ + "begin" : "2021-06-21T14:00:00-07:00", + "end" : "2021-06-22T14:00:00-07:00", + "interval" : "PT0S", + "name" : "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST", + "office-id" : "LRL", + "total" : 0, + "value-columns" : [ { + "name" : "date-time", + "ordinal" : 1, + "datatype" : "java.sql.Timestamp" + }, { + "name" : "value", + "ordinal" : 2, + "datatype" : "java.lang.Double" + }, { + "name" : "quality-code", + "ordinal" : 3, + "datatype" : "int" + } ], + "values" : [ [ 1759252080000, 12.34567, 0 ], [ 1759252140000, 12.34567, 0 ], [ 1759252200000, null, 0 ] ], + "version-date" : "2025-07-22T14:00:00Z" +} \ No newline at end of file diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/timeseries/ts_with_vertical.json b/cwms-data-api/src/test/resources/cwms/cda/api/timeseries/ts_with_vertical.json new file mode 100644 index 000000000..3d6a2c431 --- /dev/null +++ b/cwms-data-api/src/test/resources/cwms/cda/api/timeseries/ts_with_vertical.json @@ -0,0 +1,36 @@ +{ + "begin" : "2021-06-21T14:00:00-07:00", + "end" : "2021-06-22T14:00:00-07:00", + "interval" : "PT0S", + "name" : "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST", + "office-id" : "LRL", + "total" : 0, + "value-columns" : [ { + "name" : "date-time", + "ordinal" : 1, + "datatype" : "java.sql.Timestamp" + }, { + "name" : "value", + "ordinal" : 2, + "datatype" : "java.lang.Double" + }, { + "name" : "quality-code", + "ordinal" : 3, + "datatype" : "int" + } ], + "values" : [ [ 1759253040000, 12.34567, 0 ], [ 1759253100000, 12.34567, 0 ] ], + "version-date" : "2025-07-22T14:00:00Z", + "vertical-datum-info" : { + "office" : "LRL", + "unit" : "m", + "location" : "Buckhorn", + "native-datum" : "NGVD-29", + "elevation" : 230.7, + "local-datum-name" : "Castle Rock", + "offsets" : [ { + "estimate" : true, + "to-datum" : "NAVD-88", + "value" : -0.1666 + } ] + } +} \ No newline at end of file diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/turbine.json b/cwms-data-api/src/test/resources/cwms/cda/api/turbine.json index f672cabb1..9f98ce9ba 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/turbine.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/turbine.json @@ -18,7 +18,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": -95.992775, "published-latitude": 36.153980, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/turbine_phys.json b/cwms-data-api/src/test/resources/cwms/cda/api/turbine_phys.json index 07a5a7d10..3a8a29d9f 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/turbine_phys.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/turbine_phys.json @@ -18,7 +18,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": -95.992775, "published-latitude": 36.153980, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/embankment.json b/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/embankment.json index 9f6e4c239..00f7c7ab6 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/embankment.json +++ b/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/embankment.json @@ -18,7 +18,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": 50.0, "published-latitude": 50.0, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/lock.json b/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/lock.json index de02eda93..c6f5ffcda 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/lock.json +++ b/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/lock.json @@ -18,7 +18,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": 50.0, "published-latitude": 50.0, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/turbine.json b/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/turbine.json index aaf3dd195..713472b1a 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/turbine.json +++ b/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/turbine.json @@ -18,7 +18,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": 50.0, "published-latitude": 50.0, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/data/timeseries.csv b/cwms-data-api/src/test/resources/cwms/cda/data/timeseries.csv index bbf674694..c7c218b6c 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/data/timeseries.csv +++ b/cwms-data-api/src/test/resources/cwms/cda/data/timeseries.csv @@ -1648,8 +1648,8 @@ SWG,TBLT2.Stage.Inst.15Minutes.0.Decodes-Raw,T,TBLT2,lake level+rain,0.0,ft,NGVD SWG,TBLT2.Stage.Inst.15Minutes.0.Decodes-Raw,T,TBLT2,lake level+rain,0.0,ft,null,30.7952778,-94.18,NAD27,B. A. Steinhagen Lake,8040000 - B,"8040000 - B. A. Steinhagen Lk at Town Bluff, TX",US/Central,Unknown County or County N/A,TX SWG,TBLT2.Stage.Inst.15Minutes.0.Decodes-Raw,T,TBLT2,lake level+rain,0.0,m,null,30.7952778,-94.18,NAD27,B. A. Steinhagen Lake,8040000 - B,"8040000 - B. A. Steinhagen Lk at Town Bluff, TX",US/Central,Unknown County or County N/A,TX SWG,TBLT2.Stage.Inst.15Minutes.0.Decodes-Raw,T,TBLT2,lake level+rain,0.0,m,NGVD29,30.7952778,-94.18,null,B. A. Steinhagen Lake,B. A. Steinhagen Lake,"8040000 - B. A. Steinhagen Lk at Town Bluff, TX",US/Central,Tyler,TX -SWG,TXKT2-CG1.Flow.Inst.1Hour.0.Decodes-Ratings,T,TXKT2-CG1,lake level,0.0,m,NVGD,33.3044444,-94.1605556,WGS130,Wright Patman Lk CG 1,null,null,US/Central,Unknown County or County N/A,TX -SWG,TXKT2-CG1.Flow.Inst.1Hour.0.Decodes-Ratings,T,TXKT2-CG1,lake level,0.0,ft,NVGD,33.3044444,-94.1605556,WGS130,Wright Patman Lk CG 1,null,null,US/Central,Unknown County or County N/A,TX +SWG,TXKT2-CG1.Flow.Inst.1Hour.0.Decodes-Ratings,T,TXKT2-CG1,lake level,0.0,m,NGVD,33.3044444,-94.1605556,WGS130,Wright Patman Lk CG 1,null,null,US/Central,Unknown County or County N/A,TX +SWG,TXKT2-CG1.Flow.Inst.1Hour.0.Decodes-Ratings,T,TXKT2-CG1,lake level,0.0,ft,NGVD,33.3044444,-94.1605556,WGS130,Wright Patman Lk CG 1,null,null,US/Central,Unknown County or County N/A,TX SWG,TXKT2-CG1.Flow.Inst.1Hour.0.Decodes-Ratings,T,TXKT2-CG1,lake level,0.0,ft,NGVD29,33.3044444,-94.1605556,WGS130,Wright Patman Lk CG 1,Wright Patman Lk CG 1,null,US/Central,Cass,TX SWG,TXKT2-CG1.Flow.Inst.1Hour.0.Decodes-Ratings,T,TXKT2-CG1,lake level,0.0,m,NGVD29,33.3044444,-94.1605556,WGS130,Wright Patman Lk CG 1,Wright Patman Lk CG 1,null,US/Central,Cass,TX SWL,Beaver_Dam.Flow-In.Ave.1Hour.1Hour.CCP-Comp,T,Beaver_Dam,Corps Reservoir,0.0,ft,NGVD29,36.421283,-93.847617,WGS84,Beaver Dam,"White at Beaver Dam, AR - Platform Collects Pool Elevation, ","White at Beaver Dam, AR - Platform Collects Pool Elevation, Tailwater elevation, Precip, and Turbine generation/release data (HP,HT,PC,QA,QD,QH,VA,VD,VY)",America/Chicago,Carroll,AR