From 445611754b3a17071e8a70486765e0076ae2b6ac Mon Sep 17 00:00:00 2001 From: rma-rripken <89810919+rma-rripken@users.noreply.github.com> Date: Tue, 30 Sep 2025 10:43:10 -0700 Subject: [PATCH 01/18] Add tests for time series with and without vertical datum information --- .../cda/api/TimeseriesControllerTestIT.java | 105 ++++++++++++++++++ .../cwms/cda/data/dto/TimeSeriesTest.java | 32 ++++++ .../cwms/cda/api/spk/elev_ts_create.json | 26 +++++ .../cda/api/timeseries/ts_no_vertical.json | 23 ++++ .../cda/api/timeseries/ts_with_vertical.json | 36 ++++++ 5 files changed, 222 insertions(+) create mode 100644 cwms-data-api/src/test/resources/cwms/cda/api/spk/elev_ts_create.json create mode 100644 cwms-data-api/src/test/resources/cwms/cda/api/timeseries/ts_no_vertical.json create mode 100644 cwms-data-api/src/test/resources/cwms/cda/api/timeseries/ts_with_vertical.json diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index f49b68cb9..2ec9f82c3 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -1797,4 +1797,109 @@ enum GetAllTest this.expectedContentType = expectedContentType; } } + + @Test + void test_get_for_elev_has_datum() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/spk/elev_ts_create.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + + JsonNode ts = mapper.readTree(tsData); + String location = ts.get(NAME).asText().split("\\.")[0]; + String officeId = ts.get("office-id").asText(); + + + // createLocation(location, true, officeId); // For now, I don't want it delete so I can debug. + manuallyCreateLocation(location, true, officeId); + + TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; + + // inserting the time series + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization",user.toHeaderValue()) + .queryParam(OFFICE, officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + + System.out.println("Data has been inserted for " + location); + + // 1209654000000 as ms == Thursday, May 1, 2008 3:00:00 PM + + // get it back + String firstPoint = "2008-05-01T03:00:00.000Z"; + + + ValidatableResponse validatableResponse = given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .header("Authorization", user.toHeaderValue()) + .queryParam(OFFICE, officeId) + .queryParam(UNIT, "m") + .queryParam(NAME, ts.get(NAME).asText()) + .queryParam(BEGIN, firstPoint) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + System.out.println(validatableResponse.extract().asString()); + // verify that there is vertical-datum-info in the response. + validatableResponse.body("vertical-datum-info", notNullValue()) + .body("vertical-datum-info.location", equalTo(location)) + .body("vertical-datum-info.unit", equalTo("m")) + .body("vertical-datum-info.offsets.size()", equalTo(1)) +// .body("vertical-datum-info.offsets[0].to-datum", equalTo("NAVD-88")) +// .body("vertical-datum-info.offsets[0].value", closeTo(-0.1666, 0.0001)) +// .body("vertical-datum-info.offsets[0].estimate", equalTo(true)) + ; + + + } + + private void manuallyCreateLocation(String location, boolean active, String officeId) throws SQLException { + double latitude = 0; + double longitude = 0; + String kind = "SITE"; + String timeZone = "UTC"; + String horizontalDatum = "WGS84"; + + + CwmsDatabaseContainer db = CwmsDataApiSetupCallback.getDatabaseLink(); + db.connection((c) -> { + try (PreparedStatement stmt = c.prepareStatement(createLocationQuery)) { + stmt.setString(1, location); + stmt.setString(2, active ? "T" : "F"); + stmt.setString(3, officeId); + stmt.setString(4, timeZone); + stmt.setDouble(5, latitude); + stmt.setDouble(6, longitude); + stmt.setString(7, horizontalDatum); + stmt.setString(8, kind); + stmt.execute(); + + } catch (SQLException ex) { + throw new RuntimeException("Unable to create location", ex); + } + }, "cwms_20"); + + } + } diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java index 8e1e13d50..51e866aef 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/TimeSeriesTest.java @@ -4,6 +4,9 @@ import cwms.cda.formatters.Formats; import cwms.cda.formatters.json.JsonV2; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; import java.sql.Timestamp; import java.time.Duration; import java.time.Instant; @@ -20,6 +23,8 @@ import cwms.cda.formatters.xml.XMLv2; import java.util.List; + +import org.apache.commons.io.IOUtils; import org.jetbrains.annotations.NotNull; import org.junit.jupiter.api.Test; @@ -105,6 +110,33 @@ void testSerializerWithNulls() { assertNotNull(tsBody); } + @Test + void testDeserializeVerticalDatum() throws IOException { + InputStream stream; + + // verify that we can deserialize ts that don't have vertical datum info + stream = getClass().getClassLoader().getResourceAsStream( + "cwms/cda/api/timeseries/ts_no_vertical.json"); + assertNotNull(stream); + String input = IOUtils.toString(stream, StandardCharsets.UTF_8); + ObjectMapper om = buildObjectMapper(); + + TimeSeries ts = om.readValue(input, TimeSeries.class); + assertNotNull(ts); + assertNull(ts.getVerticalDatumInfo()); + + // verify that we can deserialize ts that do have vertical datum inf + stream = getClass().getClassLoader().getResourceAsStream( + "cwms/cda/api/timeseries/ts_with_vertical.json"); + assertNotNull(stream); + input = IOUtils.toString(stream, StandardCharsets.UTF_8); + + ts = om.readValue(input, TimeSeries.class); + assertNotNull(ts); + assertNotNull(ts.getVerticalDatumInfo()); + + } + @NotNull private TimeSeries buildTimeSeries() { return buildTimeSeries(null); diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/spk/elev_ts_create.json b/cwms-data-api/src/test/resources/cwms/cda/api/spk/elev_ts_create.json new file mode 100644 index 000000000..128d3d919 --- /dev/null +++ b/cwms-data-api/src/test/resources/cwms/cda/api/spk/elev_ts_create.json @@ -0,0 +1,26 @@ +{ + "office-id": "SPK", + "name": "Sacramento Dam.Elev.Inst.1Hour.0.Raw", + "interval": "PT1H", + "interval-offset": 0, + "units": "m", + "time-zone": "UTC", + "values": [ + [ + 1209654000000, + 2.2, + 0 + ], + [ + 1209657600000, + 4.4, + 0 + ], + [ + 1209661200000, + 6.6, + 0 + ] + ] +} + diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/timeseries/ts_no_vertical.json b/cwms-data-api/src/test/resources/cwms/cda/api/timeseries/ts_no_vertical.json new file mode 100644 index 000000000..1c0f53333 --- /dev/null +++ b/cwms-data-api/src/test/resources/cwms/cda/api/timeseries/ts_no_vertical.json @@ -0,0 +1,23 @@ +{ + "begin" : "2021-06-21T14:00:00-07:00", + "end" : "2021-06-22T14:00:00-07:00", + "interval" : "PT0S", + "name" : "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST", + "office-id" : "LRL", + "total" : 0, + "value-columns" : [ { + "name" : "date-time", + "ordinal" : 1, + "datatype" : "java.sql.Timestamp" + }, { + "name" : "value", + "ordinal" : 2, + "datatype" : "java.lang.Double" + }, { + "name" : "quality-code", + "ordinal" : 3, + "datatype" : "int" + } ], + "values" : [ [ 1759252080000, 12.34567, 0 ], [ 1759252140000, 12.34567, 0 ], [ 1759252200000, null, 0 ] ], + "version-date" : "2025-07-22T14:00:00Z" +} \ No newline at end of file diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/timeseries/ts_with_vertical.json b/cwms-data-api/src/test/resources/cwms/cda/api/timeseries/ts_with_vertical.json new file mode 100644 index 000000000..3d6a2c431 --- /dev/null +++ b/cwms-data-api/src/test/resources/cwms/cda/api/timeseries/ts_with_vertical.json @@ -0,0 +1,36 @@ +{ + "begin" : "2021-06-21T14:00:00-07:00", + "end" : "2021-06-22T14:00:00-07:00", + "interval" : "PT0S", + "name" : "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST", + "office-id" : "LRL", + "total" : 0, + "value-columns" : [ { + "name" : "date-time", + "ordinal" : 1, + "datatype" : "java.sql.Timestamp" + }, { + "name" : "value", + "ordinal" : 2, + "datatype" : "java.lang.Double" + }, { + "name" : "quality-code", + "ordinal" : 3, + "datatype" : "int" + } ], + "values" : [ [ 1759253040000, 12.34567, 0 ], [ 1759253100000, 12.34567, 0 ] ], + "version-date" : "2025-07-22T14:00:00Z", + "vertical-datum-info" : { + "office" : "LRL", + "unit" : "m", + "location" : "Buckhorn", + "native-datum" : "NGVD-29", + "elevation" : 230.7, + "local-datum-name" : "Castle Rock", + "offsets" : [ { + "estimate" : true, + "to-datum" : "NAVD-88", + "value" : -0.1666 + } ] + } +} \ No newline at end of file From f6734b8fbd4d4038c2d8c321228f3af804170f26 Mon Sep 17 00:00:00 2001 From: rma-rripken <89810919+rma-rripken@users.noreply.github.com> Date: Wed, 1 Oct 2025 10:12:12 -0700 Subject: [PATCH 02/18] Remove authorization header from get requests. --- .../cda/api/TimeseriesControllerTestIT.java | 37 +++++++++---------- 1 file changed, 18 insertions(+), 19 deletions(-) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index 2ec9f82c3..4d0079061 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -1812,27 +1812,27 @@ void test_get_for_elev_has_datum() throws Exception { String officeId = ts.get("office-id").asText(); - // createLocation(location, true, officeId); // For now, I don't want it delete so I can debug. + // createLocation(location, true, officeId); // For now, I don't want it deleted so I can debug. manuallyCreateLocation(location, true, officeId); TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; // inserting the time series given() - .log().ifValidationFails(LogDetail.ALL, true) - .accept(Formats.JSONV2) - .contentType(Formats.JSONV2) - .body(tsData) - .header("Authorization",user.toHeaderValue()) - .queryParam(OFFICE, officeId) - .when() - .redirects().follow(true) - .redirects().max(3) - .post("/timeseries/") - .then() - .log().ifValidationFails(LogDetail.ALL,true) - .assertThat() - .statusCode(is(HttpServletResponse.SC_OK)); + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization",user.toHeaderValue()) + .queryParam(OFFICE, officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); System.out.println("Data has been inserted for " + location); @@ -1846,18 +1846,17 @@ void test_get_for_elev_has_datum() throws Exception { ValidatableResponse validatableResponse = given() .log().ifValidationFails(LogDetail.ALL, true) .accept(Formats.JSONV2) - .header("Authorization", user.toHeaderValue()) .queryParam(OFFICE, officeId) .queryParam(UNIT, "m") .queryParam(NAME, ts.get(NAME).asText()) .queryParam(BEGIN, firstPoint) - .when() + .when() .redirects().follow(true) .redirects().max(3) .get("/timeseries/") - .then() + .then() .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() + .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); System.out.println(validatableResponse.extract().asString()); From d647522c796d1b2fbf8ecdf6256ebd0b80492ded Mon Sep 17 00:00:00 2001 From: rma-rripken <89810919+rma-rripken@users.noreply.github.com> Date: Wed, 1 Oct 2025 10:13:25 -0700 Subject: [PATCH 03/18] Call full form of create_location so that I can test with vertical_datum set. --- .../cda/api/TimeseriesControllerTestIT.java | 45 ++++++++++--------- 1 file changed, 24 insertions(+), 21 deletions(-) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index 4d0079061..a1c3833f8 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -42,6 +42,7 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.EnumSource; +import usace.cwms.db.jooq.codegen.packages.CWMS_LOC_PACKAGE; @Tag("integration") final class TimeseriesControllerTestIT extends DataApiTestIT { @@ -1874,30 +1875,32 @@ void test_get_for_elev_has_datum() throws Exception { } private void manuallyCreateLocation(String location, boolean active, String officeId) throws SQLException { - double latitude = 0; - double longitude = 0; - String kind = "SITE"; - String timeZone = "UTC"; - String horizontalDatum = "WGS84"; + String P_LOCATION_ID = location; + String P_LOCATION_TYPE = "SITE"; + Number P_ELEVATION = 0; + String P_ELEV_UNIT_ID = "m"; + String P_VERTICAL_DATUM = "NAVD-88"; + Number P_LATITUDE = 0; + Number P_LONGITUDE = 0; + String P_HORIZONTAL_DATUM = "WGS84"; + String P_PUBLIC_NAME = null; + String P_LONG_NAME= null; + String P_DESCRIPTION = null; + String P_TIME_ZONE_ID = "UTC"; + String P_COUNTY_NAME = null; + String P_STATE_INITIAL = null; + String P_ACTIVE = active ? "T" : "F"; + String P_DB_OFFICE_ID = officeId; CwmsDatabaseContainer db = CwmsDataApiSetupCallback.getDatabaseLink(); - db.connection((c) -> { - try (PreparedStatement stmt = c.prepareStatement(createLocationQuery)) { - stmt.setString(1, location); - stmt.setString(2, active ? "T" : "F"); - stmt.setString(3, officeId); - stmt.setString(4, timeZone); - stmt.setDouble(5, latitude); - stmt.setDouble(6, longitude); - stmt.setString(7, horizontalDatum); - stmt.setString(8, kind); - stmt.execute(); - - } catch (SQLException ex) { - throw new RuntimeException("Unable to create location", ex); - } - }, "cwms_20"); + db.connection(c -> { + DSLContext dslContext = getDslContext(c, officeId); + CWMS_LOC_PACKAGE.call_CREATE_LOCATION(dslContext.configuration(), + P_LOCATION_ID, P_LOCATION_TYPE, P_ELEVATION, P_ELEV_UNIT_ID, P_VERTICAL_DATUM, P_LATITUDE, P_LONGITUDE, + P_HORIZONTAL_DATUM, P_PUBLIC_NAME, P_LONG_NAME, P_DESCRIPTION, P_TIME_ZONE_ID, P_COUNTY_NAME, P_STATE_INITIAL, + P_ACTIVE, P_DB_OFFICE_ID); + }); } From a4f029f31d86bccb2eb4092524edeb7935c69aab Mon Sep 17 00:00:00 2001 From: rma-rripken <89810919+rma-rripken@users.noreply.github.com> Date: Wed, 1 Oct 2025 14:23:55 -0700 Subject: [PATCH 04/18] Removing Authorization header from GET requests --- .../src/test/java/cwms/cda/api/LocationControllerTestIT.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/LocationControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/LocationControllerTestIT.java index 630a26877..15d21f749 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/LocationControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/LocationControllerTestIT.java @@ -545,7 +545,6 @@ void test_create_update_null_elev_units() throws Exception { given() .log().ifValidationFails(LogDetail.ALL,true) .accept(Formats.JSON) - .header("Authorization", user.toHeaderValue()) .queryParam(OFFICE, user.getOperatingOffice()) .when() .redirects().follow(true) @@ -585,7 +584,6 @@ void test_create_update_null_elev_units() throws Exception { given() .log().ifValidationFails(LogDetail.ALL,true) .accept(Formats.JSON) - .header("Authorization", user.toHeaderValue()) .queryParam(OFFICE, user.getOperatingOffice()) .when() .redirects().follow(true) From d1b2b526251a62b6b209667d7662bd22eb0eaee0 Mon Sep 17 00:00:00 2001 From: rma-rripken <89810919+rma-rripken@users.noreply.github.com> Date: Thu, 9 Oct 2025 16:56:24 -0700 Subject: [PATCH 05/18] The vertical datum should be NGVD. Somehow many of these example files used NVGD. --- .../src/test/java/cwms/cda/api/LockControllerIT.java | 2 +- .../cda/api/project/ProjectChildLocationHandlerIT.java | 2 +- .../java/cwms/cda/data/dao/LocationLevelsDaoTest.java | 2 +- .../test/java/cwms/cda/data/dao/LocationsDaoTest.java | 2 +- .../cda/data/dao/location/kind/EmbankmentDaoTest.java | 2 +- .../cda/data/dao/location/kind/LocationUtilTest.java | 2 +- .../cwms/cda/data/dao/location/kind/LockDaoIT.java | 2 +- .../cwms/cda/data/dao/location/kind/LockDaoTest.java | 2 +- .../cda/data/dao/location/kind/ProjectStructureIT.java | 4 ++-- .../cwms/cda/data/dao/location/kind/TurbineDaoIT.java | 4 ++-- .../src/test/java/cwms/cda/data/dto/LocationTest.java | 10 +++++----- .../cda/data/dto/location/kind/EmbankmentTest.java | 2 +- .../java/cwms/cda/data/dto/location/kind/LockTest.java | 2 +- .../cwms/cda/data/dto/location/kind/TurbineTest.java | 2 +- .../src/test/resources/cwms/cda/api/embankment.json | 2 +- .../src/test/resources/cwms/cda/api/lock.json | 2 +- .../test/resources/cwms/cda/api/project_location.json | 2 +- .../resources/cwms/cda/api/project_location_lock.json | 2 +- .../resources/cwms/cda/api/project_location_lock2.json | 2 +- .../resources/cwms/cda/api/project_location_turb.json | 2 +- .../cwms/cda/api/project_location_turb_changes.json | 2 +- .../src/test/resources/cwms/cda/api/turbine.json | 2 +- .../src/test/resources/cwms/cda/api/turbine_phys.json | 2 +- .../cwms/cda/data/dto/location/kind/embankment.json | 2 +- .../cwms/cda/data/dto/location/kind/lock.json | 2 +- .../cwms/cda/data/dto/location/kind/turbine.json | 2 +- .../src/test/resources/cwms/cda/data/timeseries.csv | 4 ++-- 27 files changed, 34 insertions(+), 34 deletions(-) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/LockControllerIT.java b/cwms-data-api/src/test/java/cwms/cda/api/LockControllerIT.java index 5e70195ad..bdb8c811e 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/LockControllerIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/LockControllerIT.java @@ -116,7 +116,7 @@ final class LockControllerIT extends DataApiTestIT { .withName("TEST_LOCATION3") .withLocationKind("LOCK") .withDescription("Test Lock") - .withHorizontalDatum("NVGD29") + .withHorizontalDatum("NGVD29") .withTimeZoneName(ZoneId.of("UTC")) .withOfficeId("SPK") .withActive(true) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/project/ProjectChildLocationHandlerIT.java b/cwms-data-api/src/test/java/cwms/cda/api/project/ProjectChildLocationHandlerIT.java index bc1bc1ab2..27a0e2789 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/project/ProjectChildLocationHandlerIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/project/ProjectChildLocationHandlerIT.java @@ -180,7 +180,7 @@ private Embankment buildTestEmbankment(Location location, CwmsId projId) { private Location buildTestLocation(String office, String name) { return new Location.Builder(name, "EMBANKMENT", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", office) + 50.0, 50.0, "NGVD29", office) .withElevation(10.0) .withElevationUnits("ft") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/LocationLevelsDaoTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/LocationLevelsDaoTest.java index 50009a0c7..f5810743f 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/LocationLevelsDaoTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/LocationLevelsDaoTest.java @@ -347,7 +347,7 @@ LocationLevel buildExampleLevel(String locationName) throws Exception } private Location buildTestLocation(String name) { - return new Location.Builder(name, "SITE", ZoneId.of("UTC"), 50.0, 50.0, "NVGD29", OFFICE_ID) + return new Location.Builder(name, "SITE", ZoneId.of("UTC"), 50.0, 50.0, "NGVD29", OFFICE_ID) .withElevation(10.0) .withCountyName("Sacramento") .withNation(Nation.US) diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/LocationsDaoTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/LocationsDaoTest.java index 637390ef0..f3881d93f 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/LocationsDaoTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/LocationsDaoTest.java @@ -101,7 +101,7 @@ private void cleanUpRoutine() throws Exception { private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "SITE", ZoneId.of("UTC"), 50.0, 50.0, - "NVGD29", "LRL") + "NGVD29", "LRL") .withElevation(10.0) .withCountyName("Sacramento") .withNation(Nation.US) diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/EmbankmentDaoTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/EmbankmentDaoTest.java index adfe40e5d..f2834d6cc 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/EmbankmentDaoTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/EmbankmentDaoTest.java @@ -84,7 +84,7 @@ private Embankment buildTestEmbankment() { private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "EMBANKMENT", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withElevationUnits("ft") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LocationUtilTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LocationUtilTest.java index a9c354b38..5cc56d910 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LocationUtilTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LocationUtilTest.java @@ -84,7 +84,7 @@ private LookupType buildTestLookupType() { private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "EMBANKMENT", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withElevationUnits("m") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LockDaoIT.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LockDaoIT.java index 18983ef9f..c3700c77d 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LockDaoIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LockDaoIT.java @@ -433,7 +433,7 @@ private Lock buildTestLockSI() { private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "LOCK", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "SPK") + 50.0, 50.0, "NGVD29", "SPK") .withElevation(10.0) .withElevationUnits("ft") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LockDaoTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LockDaoTest.java index 29781061c..85be6721e 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LockDaoTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/LockDaoTest.java @@ -151,7 +151,7 @@ private Lock buildTestLockWithNullLevels() { private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "LOCK", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withElevationUnits("ft") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/ProjectStructureIT.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/ProjectStructureIT.java index bcd51ea22..885a4bb37 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/ProjectStructureIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/ProjectStructureIT.java @@ -55,7 +55,7 @@ public static void setupProject() throws Exception { public static Location buildProjectLocation(String locationId) { return new Location.Builder(locationId, "PROJECT", ZoneId.of("UTC"), - 38.5613824, -121.7298432, "NVGD29", OFFICE_ID) + 38.5613824, -121.7298432, "NGVD29", OFFICE_ID) .withElevation(10.0) .withElevationUnits("m") .withLocationType("SITE") @@ -97,7 +97,7 @@ public static PROJECT_OBJ_T buildProject(Location location) { public static Location buildProjectStructureLocation(String locationId, String locationKind) { return new Location.Builder(locationId, locationKind, ZoneId.of("UTC"), - 38.5613824, -121.7298432, "NVGD29", OFFICE_ID) + 38.5613824, -121.7298432, "NGVD29", OFFICE_ID) .withPublicName("Integration Test " + locationId) .withLongName("Integration Test " + locationId + " " + locationKind) .withElevation(10.0) diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/TurbineDaoIT.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/TurbineDaoIT.java index acb8788c5..52104d990 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/TurbineDaoIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/location/kind/TurbineDaoIT.java @@ -246,7 +246,7 @@ void testTurbineChangesRoundTrip() throws Exception { private static Location buildProjectLocation(String projectId) { return new Location.Builder(projectId, "PROJECT", ZoneId.of("UTC"), - 38.5613824, -121.7298432, "NVGD29", OFFICE) + 38.5613824, -121.7298432, "NGVD29", OFFICE) .withElevation(10.0) .withElevationUnits("m") .withLocationType("SITE") @@ -276,7 +276,7 @@ private static Turbine buildTestTurbine(Location location, String projectId) { private static Location buildTurbineLocation(String locationId) { return new Location.Builder(locationId, "TURBINE", ZoneId.of("UTC"), - 38.5613824, -121.7298432, "NVGD29", OFFICE) + 38.5613824, -121.7298432, "NGVD29", OFFICE) .withElevation(10.0) .withElevationUnits("m") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/LocationTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/LocationTest.java index a98ea4593..50468f369 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dto/LocationTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/LocationTest.java @@ -99,7 +99,7 @@ void testSerializationRoundTrip(String format) void canBuildNullLatLon(){ Location location = new Location.Builder("TEST_LOCATION2", "SITE", ZoneId.of("UTC"), null, null, // lat/lon are null in this test - "NVGD29", "LRL") + "NGVD29", "LRL") .withElevation(10.0) .withCountyName("Sacramento") .withNation(Nation.US) @@ -143,7 +143,7 @@ void test_alias_roundtrip() { Location location = new Location.Builder("TEST_LOCATION2", "SITE", ZoneId.of("UTC"), null, null, // lat/lon are null in this test - "NVGD29", "LRL") + "NGVD29", "LRL") .withElevation(10.0) .withCountyName("Sacramento") .withNation(Nation.US) @@ -186,7 +186,7 @@ void test_serialization_no_alias() void test_serialization_empty_alias() { Location location = new Location.Builder("TEST_LOCATION2", "SITE", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withCountyName("Sacramento") .withNation(Nation.US) @@ -211,7 +211,7 @@ void test_serialization_empty_alias() private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "SITE", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withCountyName("Sacramento") .withNation(Nation.US) @@ -228,7 +228,7 @@ private Location buildTestLocation() { private Location buildTestLocationNewLine() { return new Location.Builder("TEST_LOCATION2", "SITE", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withCountyName("Sacramento") .withNation(Nation.US) diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/EmbankmentTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/EmbankmentTest.java index 44bd391c0..bf17e5cb4 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/EmbankmentTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/EmbankmentTest.java @@ -118,7 +118,7 @@ private Embankment buildTestEmbankment() { private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "EMBANKMENT", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withElevationUnits("m") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/LockTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/LockTest.java index 86216c639..5d657e58d 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/LockTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/LockTest.java @@ -126,7 +126,7 @@ private Lock buildTestLock() { private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "LOCK", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withElevationUnits("m") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/TurbineTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/TurbineTest.java index 8d44c5c0d..8b7b03e92 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/TurbineTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/location/kind/TurbineTest.java @@ -92,7 +92,7 @@ private Turbine buildTestTurbine() { private Location buildTestLocation() { return new Location.Builder("TEST_LOCATION2", "TURBINE", ZoneId.of("UTC"), - 50.0, 50.0, "NVGD29", "LRL") + 50.0, 50.0, "NGVD29", "LRL") .withElevation(10.0) .withElevationUnits("m") .withLocationType("SITE") diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/embankment.json b/cwms-data-api/src/test/resources/cwms/cda/api/embankment.json index 7cd636f6d..9a7f1d6c2 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/embankment.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/embankment.json @@ -18,7 +18,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": -95.992775, "published-latitude": 36.153980, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/lock.json b/cwms-data-api/src/test/resources/cwms/cda/api/lock.json index f83a706c5..43516f4c5 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/lock.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/lock.json @@ -18,7 +18,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": 38.5, "published-latitude": -121.7, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/project_location.json b/cwms-data-api/src/test/resources/cwms/cda/api/project_location.json index 21381afaa..85198f486 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/project_location.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/project_location.json @@ -13,7 +13,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": -95.992775, "published-latitude": 36.153980, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/project_location_lock.json b/cwms-data-api/src/test/resources/cwms/cda/api/project_location_lock.json index ecb69a63f..75c65c5db 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/project_location_lock.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/project_location_lock.json @@ -13,7 +13,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": -95.992775, "published-latitude": 36.153980, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/project_location_lock2.json b/cwms-data-api/src/test/resources/cwms/cda/api/project_location_lock2.json index 41813ab2c..a25f3f5b8 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/project_location_lock2.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/project_location_lock2.json @@ -13,7 +13,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": -95.992775, "published-latitude": 36.153980, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/project_location_turb.json b/cwms-data-api/src/test/resources/cwms/cda/api/project_location_turb.json index dda7840db..a69ba844d 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/project_location_turb.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/project_location_turb.json @@ -13,7 +13,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": -95.992775, "published-latitude": 36.153980, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/project_location_turb_changes.json b/cwms-data-api/src/test/resources/cwms/cda/api/project_location_turb_changes.json index 9538d2b74..43a0b8913 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/project_location_turb_changes.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/project_location_turb_changes.json @@ -13,7 +13,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": -95.992775, "published-latitude": 36.153980, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/turbine.json b/cwms-data-api/src/test/resources/cwms/cda/api/turbine.json index f672cabb1..9f98ce9ba 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/turbine.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/turbine.json @@ -18,7 +18,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": -95.992775, "published-latitude": 36.153980, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/api/turbine_phys.json b/cwms-data-api/src/test/resources/cwms/cda/api/turbine_phys.json index 07a5a7d10..3a8a29d9f 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/api/turbine_phys.json +++ b/cwms-data-api/src/test/resources/cwms/cda/api/turbine_phys.json @@ -18,7 +18,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": -95.992775, "published-latitude": 36.153980, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/embankment.json b/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/embankment.json index 9f6e4c239..00f7c7ab6 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/embankment.json +++ b/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/embankment.json @@ -18,7 +18,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": 50.0, "published-latitude": 50.0, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/lock.json b/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/lock.json index de02eda93..c6f5ffcda 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/lock.json +++ b/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/lock.json @@ -18,7 +18,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": 50.0, "published-latitude": 50.0, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/turbine.json b/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/turbine.json index aaf3dd195..713472b1a 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/turbine.json +++ b/cwms-data-api/src/test/resources/cwms/cda/data/dto/location/kind/turbine.json @@ -18,7 +18,7 @@ "nation": "US", "state-initial": "CA", "county-name": "Sacramento", - "horizontal-datum": "NVGD29", + "horizontal-datum": "NGVD29", "published-longitude": 50.0, "published-latitude": 50.0, "elevation": 10.0, diff --git a/cwms-data-api/src/test/resources/cwms/cda/data/timeseries.csv b/cwms-data-api/src/test/resources/cwms/cda/data/timeseries.csv index bbf674694..c7c218b6c 100644 --- a/cwms-data-api/src/test/resources/cwms/cda/data/timeseries.csv +++ b/cwms-data-api/src/test/resources/cwms/cda/data/timeseries.csv @@ -1648,8 +1648,8 @@ SWG,TBLT2.Stage.Inst.15Minutes.0.Decodes-Raw,T,TBLT2,lake level+rain,0.0,ft,NGVD SWG,TBLT2.Stage.Inst.15Minutes.0.Decodes-Raw,T,TBLT2,lake level+rain,0.0,ft,null,30.7952778,-94.18,NAD27,B. A. Steinhagen Lake,8040000 - B,"8040000 - B. A. Steinhagen Lk at Town Bluff, TX",US/Central,Unknown County or County N/A,TX SWG,TBLT2.Stage.Inst.15Minutes.0.Decodes-Raw,T,TBLT2,lake level+rain,0.0,m,null,30.7952778,-94.18,NAD27,B. A. Steinhagen Lake,8040000 - B,"8040000 - B. A. Steinhagen Lk at Town Bluff, TX",US/Central,Unknown County or County N/A,TX SWG,TBLT2.Stage.Inst.15Minutes.0.Decodes-Raw,T,TBLT2,lake level+rain,0.0,m,NGVD29,30.7952778,-94.18,null,B. A. Steinhagen Lake,B. A. Steinhagen Lake,"8040000 - B. A. Steinhagen Lk at Town Bluff, TX",US/Central,Tyler,TX -SWG,TXKT2-CG1.Flow.Inst.1Hour.0.Decodes-Ratings,T,TXKT2-CG1,lake level,0.0,m,NVGD,33.3044444,-94.1605556,WGS130,Wright Patman Lk CG 1,null,null,US/Central,Unknown County or County N/A,TX -SWG,TXKT2-CG1.Flow.Inst.1Hour.0.Decodes-Ratings,T,TXKT2-CG1,lake level,0.0,ft,NVGD,33.3044444,-94.1605556,WGS130,Wright Patman Lk CG 1,null,null,US/Central,Unknown County or County N/A,TX +SWG,TXKT2-CG1.Flow.Inst.1Hour.0.Decodes-Ratings,T,TXKT2-CG1,lake level,0.0,m,NGVD,33.3044444,-94.1605556,WGS130,Wright Patman Lk CG 1,null,null,US/Central,Unknown County or County N/A,TX +SWG,TXKT2-CG1.Flow.Inst.1Hour.0.Decodes-Ratings,T,TXKT2-CG1,lake level,0.0,ft,NGVD,33.3044444,-94.1605556,WGS130,Wright Patman Lk CG 1,null,null,US/Central,Unknown County or County N/A,TX SWG,TXKT2-CG1.Flow.Inst.1Hour.0.Decodes-Ratings,T,TXKT2-CG1,lake level,0.0,ft,NGVD29,33.3044444,-94.1605556,WGS130,Wright Patman Lk CG 1,Wright Patman Lk CG 1,null,US/Central,Cass,TX SWG,TXKT2-CG1.Flow.Inst.1Hour.0.Decodes-Ratings,T,TXKT2-CG1,lake level,0.0,m,NGVD29,33.3044444,-94.1605556,WGS130,Wright Patman Lk CG 1,Wright Patman Lk CG 1,null,US/Central,Cass,TX SWL,Beaver_Dam.Flow-In.Ave.1Hour.1Hour.CCP-Comp,T,Beaver_Dam,Corps Reservoir,0.0,ft,NGVD29,36.421283,-93.847617,WGS84,Beaver Dam,"White at Beaver Dam, AR - Platform Collects Pool Elevation, ","White at Beaver Dam, AR - Platform Collects Pool Elevation, Tailwater elevation, Precip, and Turbine generation/release data (HP,HT,PC,QA,QD,QH,VA,VD,VY)",America/Chicago,Carroll,AR From b00d230ebbe297ba8327cc694fea365116d7f467 Mon Sep 17 00:00:00 2001 From: rma-rripken <89810919+rma-rripken@users.noreply.github.com> Date: Fri, 10 Oct 2025 13:35:10 -0700 Subject: [PATCH 06/18] Work in-progress --- .../cwms/cda/api/TimeSeriesController.java | 104 ++++++- .../java/cwms/cda/data/dao/TimeSeriesDao.java | 4 +- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 173 +++++++++--- .../java/cwms/cda/data/dao/VerticalDatum.java | 29 ++ .../cda/api/TimeseriesControllerTestIT.java | 265 ++++++++++++++++-- 5 files changed, 492 insertions(+), 83 deletions(-) create mode 100644 cwms-data-api/src/main/java/cwms/cda/data/dao/VerticalDatum.java diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java index 7eadbade5..cc619a710 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java @@ -8,13 +8,10 @@ import com.codahale.metrics.Timer; import cwms.cda.api.enums.UnitSystem; import cwms.cda.api.errors.CdaError; -import cwms.cda.data.dao.JooqDao; -import cwms.cda.data.dao.StoreRule; -import cwms.cda.data.dao.TimeSeriesDao; -import cwms.cda.data.dao.TimeSeriesDaoImpl; -import cwms.cda.data.dao.TimeSeriesDeleteOptions; -import cwms.cda.data.dao.TimeSeriesRequestParameters; +import cwms.cda.api.errors.NotFoundException; +import cwms.cda.data.dao.*; import cwms.cda.data.dto.TimeSeries; +import cwms.cda.data.dto.VerticalDatumInfo; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; import cwms.cda.helpers.DateUtils; @@ -36,6 +33,7 @@ import java.sql.Timestamp; import java.time.ZoneId; import java.time.ZonedDateTime; +import java.util.Objects; import java.util.logging.Level; import java.util.logging.Logger; import javax.servlet.http.HttpServletResponse; @@ -43,6 +41,7 @@ import org.apache.http.client.utils.URIBuilder; import org.apache.http.client.utils.URLEncodedUtils; import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import org.jooq.DSLContext; import org.jooq.exception.DataAccessException; @@ -118,6 +117,7 @@ public TimeSeriesController(MetricRegistry metrics) { static { JavalinValidation.register(StoreRule.class, StoreRule::getStoreRule); + JavalinValidation.register(VerticalDatum.class, VerticalDatum::getVerticalDatum); } private Timer.Context markAndTime(String subject) { @@ -146,6 +146,17 @@ private Timer.Context markAndTime(String subject) { + "'True' or 'False', default is 'False'"), @OpenApiParam(name = STORE_RULE, type = StoreRule.class, description = STORE_RULE_DESC), @OpenApiParam(name = OVERRIDE_PROTECTION, type = Boolean.class, description = "A flag " + + "to ignore the protected data quality when storing data. 'True' or 'False'" + + ", default is " + TimeSeriesDaoImpl.OVERRIDE_PROTECTION), + @OpenApiParam(name = DATUM, type = VerticalDatum.class, description = "If the provided " + + "time-series includes an explicit vertical-datum-info attribute " + + "then it is assumed that the data is in the datum specified by the vertical-datum-info. " + + "If the input timeseries does not include vertical-datum-info and " + + "this parameter is not provided it is assumed that the data is in the as-stored " + + "datum and no conversion is necessary. " + + "If the input timeseries does not include vertical-datum-info and " + + "this parameter is provided it is assumed that the data is in the Datum named by the argument " + + "and should be converted to the as-stored datum before being saved.") + "to ignore the protected data quality when storing data. 'True' or 'False'" + ", default is " + TimeSeriesDaoImpl.OVERRIDE_PROTECTION) }, @@ -162,12 +173,18 @@ public void create(@NotNull Context ctx) { boolean overrideProtection = ctx.queryParamAsClass(OVERRIDE_PROTECTION, Boolean.class) .getOrDefault(TimeSeriesDaoImpl.OVERRIDE_PROTECTION); + VerticalDatum vd = ctx.queryParamAsClass(DATUM, VerticalDatum.class) + .getOrDefault(null); + try (final Timer.Context ignored = markAndTime(CREATE)) { DSLContext dsl = getDslContext(ctx); TimeSeriesDao dao = getTimeSeriesDao(dsl); TimeSeries timeSeries = deserializeTimeSeries(ctx); - dao.create(timeSeries, createAsLrts, storeRule, overrideProtection); + + vd = getVerticalDatum(timeSeries, vd); + + dao.create(timeSeries, createAsLrts, storeRule, overrideProtection, vd); ctx.status(HttpServletResponse.SC_OK); } catch (DataAccessException | IOException ex) { CdaError re = new CdaError("Internal Error"); @@ -176,6 +193,26 @@ public void create(@NotNull Context ctx) { } } + @Nullable + private static VerticalDatum getVerticalDatum(@Nullable TimeSeries timeSeries, @Nullable VerticalDatum vd) { + + if (timeSeries != null) { + VerticalDatumInfo vdi = timeSeries.getVerticalDatumInfo(); + if (vdi != null) { + String nativeDatum = vdi.getNativeDatum(); + if (nativeDatum != null && !nativeDatum.isEmpty()) { + if (nativeDatum.equalsIgnoreCase("OTHER")) { + throw new IllegalArgumentException("Vertical Datum of OTHER is not currently supported."); + } else { + vd = VerticalDatum.getVerticalDatum(nativeDatum); + } + } + } + } + + return vd; + } + protected DSLContext getDslContext(Context ctx) { return JooqDao.getDslContext(ctx); } @@ -356,6 +393,7 @@ public void delete(@NotNull Context ctx, @NotNull String timeseries) { + "whether to include the data entry date of each value in the response. Including the data entry " + "date will increase the size of the array containing each data value from three to four, " + "changing the format of the response. Default is false."), + @OpenApiParam(name = PAGE, description = "This end point can return large amounts " + "of data as a series of pages. This parameter is used to describes the " + "current location in the response stream. This is an opaque " @@ -435,11 +473,6 @@ public void getAll(@NotNull Context ctx) { if (version != null && version.equals("2")) { - if (datum != null) { - throw new IllegalArgumentException(String.format("Datum is not supported for:%s and %s", - Formats.JSONV2, Formats.XMLV2)); - } - String office = requiredParam(ctx, OFFICE); TimeSeriesRequestParameters requestParameters = new TimeSeriesRequestParameters.Builder() .withNames(names) @@ -453,6 +486,31 @@ public void getAll(@NotNull Context ctx) { .build(); TimeSeries ts = dao.getTimeseries(cursor, pageSize, requestParameters); + if(datum != null){ + // user has requested a specific vertical datum + VerticalDatum vd = VerticalDatum.valueOf(datum); // the users request + VerticalDatum tsVd = getVerticalDatum(ts, null); // what came back from sql + if( tsVd == null){ + // the ts came back without a vertical datum. + // We won't know why it came back without a vertical datum, + // maybe the location has lat/long 0,0 and is outside datum? + // Maybe the location doesn't have vertical datum set? + // maybe the ts isn't for Elev (this we could check) + // this doesn't seem like its our problem. We should probably just return the ts without vertical data info in it. + } else if( !Objects.equals(vd, tsVd)){ + // user requested a specific vertical datum but the ts came back with a different vertical datum. + // Here we need to convert + // verify that the vertical-datum-info contains the to-datum and from-datum + // throw exception if not found. + // find needed offset. + // convert the vertical-datum-info so that we switch the to-datum and from-datum to the requested vertical datum and fix the offset - this will also need to adjust other offset entries + // apply the offset to the elev in the vertical-datum-info + // update the ts with the new vertical-datum-info + // apply the offset to the ts values. + throw new UnsupportedOperationException("We need to do this conversion - not implemented yet."); + } + } + results = Formats.format(contentType, ts); ctx.status(HttpServletResponse.SC_OK); @@ -482,6 +540,11 @@ public void getAll(@NotNull Context ctx) { } addDeprecatedContentTypeWarning(ctx, contentType); requestResultSize.update(results.length()); + } catch (NotFoundException e) { + CdaError re = new CdaError("Not found."); + logger.log(Level.WARNING, re.toString(), e); + ctx.status(HttpServletResponse.SC_NOT_FOUND); + ctx.json(re); } catch (IllegalArgumentException ex) { CdaError re = new CdaError("Invalid arguments supplied"); logger.log(Level.SEVERE, re.toString(), ex); @@ -542,7 +605,16 @@ public void getOne(@NotNull Context ctx, @NotNull String id) { @OpenApiParam(name = CREATE_AS_LRTS, type = Boolean.class, description = ""), @OpenApiParam(name = STORE_RULE, type = StoreRule.class, description = STORE_RULE_DESC), @OpenApiParam(name = OVERRIDE_PROTECTION, type = Boolean.class, description = - "A flag to ignore the protected data quality when storing data. \"'true' or 'false'\"") + "A flag to ignore the protected data quality when storing data. \"'true' or 'false'\""), + @OpenApiParam(name = DATUM, type = VerticalDatum.class, description = "If the provided " + + "time-series includes an explicit vertical-datum-info attribute " + + "then it is assumed that the data is in the datum specified by the vertical-datum-info. " + + "If the input timeseries does not include vertical-datum-info and " + + "this parameter is not provided it is assumed that the data is in the as-stored " + + "datum and no conversion is necessary. " + + "If the input timeseries does not include vertical-datum-info and " + + "this parameter is provided it is assumed that the data is in the Datum named by the argument " + + "and should be converted to the as-stored datum before being saved.") }, method = HttpMethod.PATCH, path = "/timeseries/{timeseries}", @@ -563,7 +635,11 @@ public void update(@NotNull Context ctx, @NotNull String id) { boolean overrideProtection = ctx.queryParamAsClass(OVERRIDE_PROTECTION, Boolean.class) .getOrDefault(TimeSeriesDaoImpl.OVERRIDE_PROTECTION); - dao.store(timeSeries, createAsLrts, storeRule, overrideProtection); + VerticalDatum vd = ctx.queryParamAsClass(DATUM, VerticalDatum.class) + .getOrDefault(null); + vd = getVerticalDatum(timeSeries, vd); + + dao.store(timeSeries, createAsLrts, storeRule, overrideProtection, vd); ctx.status(HttpServletResponse.SC_OK); } catch (DataAccessException | IOException ex) { diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java index 264860a6c..424ebbc91 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java @@ -17,12 +17,12 @@ public interface TimeSeriesDao { void create(TimeSeries input); void create(TimeSeries input, - boolean createAsLrts, StoreRule replaceAll, boolean overrideProtection); + boolean createAsLrts, StoreRule replaceAll, boolean overrideProtection, VerticalDatum vd); void store(TimeSeries timeSeries, Timestamp versionDate); void store(TimeSeries timeSeries, boolean createAsLrts, - StoreRule replaceAll, boolean overrideProtection); + StoreRule replaceAll, boolean overrideProtection, VerticalDatum vd); void delete(String officeId, String tsId, TimeSeriesDeleteOptions options); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 097c1a83a..9ecdd29a2 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -1,6 +1,15 @@ package cwms.cda.data.dao; +import cwms.cda.data.dao.rsql.FieldResolver; +import cwms.cda.data.dao.rsql.MapFieldResolver; +import cwms.cda.data.dao.rsql.RSQLConditionBuilder; +import cwms.cda.data.dto.filteredtimeseries.FilteredTimeSeries; +import cwms.cda.data.dto.catalog.TimeSeriesAlias; +import cwms.cda.helpers.DateUtils; + +import java.util.*; + import static org.jooq.impl.DSL.asterisk; import static org.jooq.impl.DSL.countDistinct; import static org.jooq.impl.DSL.field; @@ -10,6 +19,7 @@ import static org.jooq.impl.DSL.partitionBy; import static org.jooq.impl.DSL.select; import static org.jooq.impl.DSL.selectDistinct; +import usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID; import static org.jooq.impl.DSL.table; import static usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID2.AV_CWMS_TS_ID2; import static usace.cwms.db.jooq.codegen.tables.AV_TS_EXTENTS_UTC.AV_TS_EXTENTS_UTC; @@ -410,15 +420,6 @@ protected TimeSeries getRequestedTimeSeries(String page, int pageSize, @NotNull valid.field("interval", BigDecimal.class).as("interval"), valid.field("loc_part", String.class).as("loc_part"), valid.field("parm_part", String.class).as("parm_part"), - DSL.choose(valid.field("parm_part", String.class)) - .when( - "ELEV", - CWMS_LOC_PACKAGE.call_GET_VERTICAL_DATUM_INFO_F__2( - valid.field("loc_part", String.class), - valid.field("units", String.class), - valid.field("office_id", String.class))) - .otherwise("") - .as("VERTICAL_DATUM"), totalField, AV_CWMS_TS_ID2.INTERVAL_UTC_OFFSET, AV_CWMS_TS_ID2.TIME_ZONE_ID @@ -437,8 +438,15 @@ protected TimeSeries getRequestedTimeSeries(String page, int pageSize, @NotNull TimeSeries timeseries = metadataQuery.fetchOne(tsMetadata -> { - String vert = (String) tsMetadata.getValue("VERTICAL_DATUM"); - VerticalDatumInfo verticalDatumInfo = parseVerticalDatumInfo(vert); + String parmPart = tsMetadata.getValue("parm_part", String.class); + String locPart = tsMetadata.getValue("loc_part", String.class); + + // Fetch vertical datum info separately only when needed + VerticalDatumInfo verticalDatumInfo = null; + if (parmPart != null && shouldFetchVerticalDatum(parmPart)) { + verticalDatumInfo = fetchVerticalDatumInfoSeparately( locPart, units, office); + } + VersionType finalDateVersionType = getVersionType(dsl, names, office, versionDate != null); return new TimeSeries(recordCursor, recordPageSize, tsMetadata.getValue("TOTAL", Integer.class), tsMetadata.getValue("NAME", String.class), @@ -540,6 +548,25 @@ protected TimeSeries getRequestedTimeSeries(String page, int pageSize, @NotNull return retVal; } + private boolean shouldFetchVerticalDatum(String parmPart) { + // Check if parameter requires vertical datum (e.g., "ELEV") + if (parmPart == null) { + return false; + } + String upperParm = parmPart.toUpperCase(); + return upperParm.equals("ELEV"); + } + + private VerticalDatumInfo fetchVerticalDatumInfoSeparately(String locPart, String units, String office) { + + return connectionResult(dsl, conn -> { + DSLContext dslContext = getDslContext(conn, office); + String result = CWMS_LOC_PACKAGE.call_GET_VERTICAL_DATUM_INFO_F__2(dslContext.configuration(), + locPart, units, office); + return parseVerticalDatumInfo(result); + }); + } + public void validateEntryDateSupport(boolean includeEntryDate) { if (includeEntryDate) { Record entryDateSupport = dsl.select(asterisk()).from(table("ALL_TYPES")) @@ -612,11 +639,7 @@ private static boolean isVersioned(DSLContext dsl, String tsId, String office) { public static VerticalDatumInfo parseVerticalDatumInfo(String body) { VerticalDatumInfo retVal = null; if (body != null && !body.isEmpty()) { - try { - retVal = new XMLv1().parseContent(body, VerticalDatumInfo.class); - } catch (FormattingException e) { - logger.log(Level.WARNING, e, () -> "Failed to parse:" + body); - } + retVal = new XMLv1().parseContent(body, VerticalDatumInfo.class); } return retVal; } @@ -1400,7 +1423,7 @@ public List findRecentsInRange(String office, String categoryId, St @Override public void create(TimeSeries input) { - create(input, false, StoreRule.REPLACE_ALL, TimeSeriesDaoImpl.OVERRIDE_PROTECTION); + create(input, false, StoreRule.REPLACE_ALL, TimeSeriesDaoImpl.OVERRIDE_PROTECTION, null); } /** @@ -1423,28 +1446,68 @@ public void create(TimeSeries input) { * * @param storeRule How to update the database if data exists. {@see cwms.cda.data.dao.StoreRule for more detail} * @param overrideProtection honor override protection + * @param vd The VerticalDatum in which specified elevations are interpreted. * */ @SuppressWarnings("unused") public void create(TimeSeries input, - boolean createAsLrts, StoreRule storeRule, boolean overrideProtection) { + boolean createAsLrts, StoreRule storeRule, boolean overrideProtection, VerticalDatum vd) { + + int intervalForward = 0; + int intervalBackward = 0; + boolean activeFlag = true; + Timestamp versionDate; + if (input.getVersionDate() != null) { + versionDate = Timestamp.from(input.getVersionDate().toInstant()); + } else { + versionDate = null; + } + connection(dsl, connection -> { - int intervalForward = 0; - int intervalBackward = 0; - boolean activeFlag = true; - // the code does not need to be created before hand. - // do not add a call to create_ts_code - if (!input.getValues().isEmpty()) { - Timestamp versionDate = null; - if (input.getVersionDate() != null) { - versionDate = Timestamp.from(input.getVersionDate().toInstant()); + DSLContext dslContext = getDslContext(connection, input.getOfficeId()); + + withDefaultDatum(vd, dslContext, (conn)-> { + // the code does not need to be created before hand. + // do not add a call to create_ts_code + + if (!input.getValues().isEmpty()) { + store(dslContext, input.getOfficeId(), input.getName(), input.getUnits(), + versionDate, input.getValues(), createAsLrts, storeRule, + overrideProtection); } + }); + }); + } + + // - store(connection, input.getOfficeId(), input.getName(), input.getUnits(), - versionDate, input.getValues(), createAsLrts, storeRule, - overrideProtection); + /** + * The idea here is that this will check the current default datum, + * possible switch to the specified datum and + * then run the code and + * if the datum was previously switched + * then switch back to the initial datum. + * @param targetDatum The desired ver + * @param dslContext + * @param cr + * @throws Throwable + */ + private void withDefaultDatum(@Nullable VerticalDatum targetDatum, DSLContext dslContext, ConnectionRunnable cr) throws Throwable { + final String defaultVertDatum = CWMS_LOC_PACKAGE.call_GET_DEFAULT_VERTICAL_DATUM(dslContext.configuration()); + String targetName = (targetDatum != null) ? targetDatum.toString() : null; + boolean changeDefaultDatum = !Objects.equals(targetDatum, defaultVertDatum); + try { + if (changeDefaultDatum) { + CWMS_LOC_PACKAGE.call_SET_DEFAULT_VERTICAL_DATUM(dslContext.configuration(), targetName); } - }); + + connection(dslContext, cr); + }finally{ + if (changeDefaultDatum) { + // If we changed it we should restore. + CWMS_LOC_PACKAGE.call_SET_DEFAULT_VERTICAL_DATUM(dslContext.configuration(), defaultVertDatum); + } + } } @Override @@ -1452,22 +1515,42 @@ public void store(TimeSeries timeSeries, Timestamp versionDate) { store(timeSeries, false, StoreRule.REPLACE_ALL, TimeSeriesDaoImpl.OVERRIDE_PROTECTION); } - public void store(TimeSeries input, boolean createAsLrts, StoreRule replaceAll, boolean overrideProtection) { + public void store(TimeSeries input, boolean createAsLrts, StoreRule replaceAll, boolean overrideProtection, VerticalDatum vd) { + Timestamp versionDate; + if (input.getVersionDate() != null) { + versionDate = Timestamp.from(input.getVersionDate().toInstant()); + } else { + versionDate = null; + } + connection(dsl, connection -> { - Timestamp versionDate = null; - if (input.getVersionDate() != null) { - versionDate = Timestamp.from(input.getVersionDate().toInstant()); - } + DSLContext dslContext = getDslContext(connection, input.getOfficeId()); + withDefaultDatum(vd, dslContext, (conn)-> { - store(connection, input.getOfficeId(), input.getName(), input.getUnits(), - versionDate, input.getValues(), createAsLrts, replaceAll, overrideProtection); + store(dslContext, input.getOfficeId(), input.getName(), input.getUnits(), + versionDate, input.getValues(), createAsLrts, replaceAll, overrideProtection); + }); }); } - private void store(Connection connection, String officeId, String tsId, String units, + + /** + * + * @param dslContext A DSLContext that is already in the correct office + * @param officeId + * @param tsId + * @param units + * @param versionDate + * @param values + * @param createAsLrts + * @param storeRule + * @param overrideProtection + * @throws SQLException + */ + private void store(DSLContext dslContext, String officeId, String tsId, String units, Timestamp versionDate, List values, boolean createAsLrts, StoreRule storeRule, boolean overrideProtection) throws SQLException { - setOffice(connection,officeId); + //setOffice(connection,officeId); final ZTSV_ARRAY tsvArray = new ZTSV_ARRAY(); @@ -1481,9 +1564,10 @@ private void store(Connection connection, String officeId, String tsId, String u } } + if (versionDate != null) { try { - CWMS_TS_PACKAGE.call_SET_TSID_VERSIONED(getDslContext(connection, officeId).configuration(), + CWMS_TS_PACKAGE.call_SET_TSID_VERSIONED(dslContext.configuration(), tsId, "T", officeId); } catch (DataAccessException e) { if (e.getCause() instanceof SQLException) { @@ -1499,7 +1583,7 @@ private void store(Connection connection, String officeId, String tsId, String u } } } - CWMS_TS_PACKAGE.call_ZSTORE_TS(getDslContext(connection, officeId).configuration(), + CWMS_TS_PACKAGE.call_ZSTORE_TS(dslContext.configuration(), tsId, units, tsvArray, @@ -1518,8 +1602,9 @@ public void update(TimeSeries input, boolean createAsLrts, StoreRule storeRule, + "first."); } connection(dsl, connection -> { - setOffice(connection,input.getOfficeId()); - store(connection, input.getOfficeId(), name, input.getUnits(), versionDate, + //setOffice(connection,input.getOfficeId()); + DSLContext dslContext = getDslContext(connection, input.getOfficeId()); + store(dslContext, input.getOfficeId(), name, input.getUnits(), versionDate, input.getValues(), createAsLrts, storeRule, overrideProtection); }); } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/VerticalDatum.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/VerticalDatum.java new file mode 100644 index 000000000..01217de2b --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/VerticalDatum.java @@ -0,0 +1,29 @@ +package cwms.cda.data.dao; + +public enum VerticalDatum { + NAVD88("NAVD88"), + NGVD29("NGVD29"), + NATIVE("NATIVE"); + + private final String rule; + + VerticalDatum(String rule) { + this.rule = rule; + } + + public static VerticalDatum getVerticalDatum(String input) { + VerticalDatum retval = null; + + if (input != null) { + input = input.replace("-", ""); + retval = VerticalDatum.valueOf(input.toUpperCase()); + } + return retval; + } + + @Override + public String toString() { + return rule; + } + +} diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index a1c3833f8..163c975fb 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -2,9 +2,9 @@ import static cwms.cda.api.Controllers.*; import static cwms.cda.data.dao.JooqDao.getDslContext; +import static helpers.FloatCloseTo.floatCloseTo; import static io.restassured.RestAssured.given; import static io.restassured.config.JsonConfig.jsonConfig; -import static io.restassured.internal.common.assertion.AssertParameter.notNull; import static org.hamcrest.Matchers.*; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -13,7 +13,7 @@ import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import cwms.cda.ApiServlet; -import cwms.cda.data.dto.Location; +import cwms.cda.data.dao.VerticalDatum; import cwms.cda.formatters.Formats; import cwms.cda.helpers.DatabaseHelpers.SCHEMA_VERSION; import cwms.cda.helpers.ZoneIdHelper; @@ -35,7 +35,6 @@ import io.restassured.response.ValidatableResponse; import mil.army.usace.hec.test.database.CwmsDatabaseContainer; import org.apache.commons.io.IOUtils; -import org.hamcrest.Matchers; import org.jooq.DSLContext; import org.junit.jupiter.api.Disabled; import org.junit.jupiter.api.Tag; @@ -1812,9 +1811,8 @@ void test_get_for_elev_has_datum() throws Exception { String location = ts.get(NAME).asText().split("\\.")[0]; String officeId = ts.get("office-id").asText(); - - // createLocation(location, true, officeId); // For now, I don't want it deleted so I can debug. - manuallyCreateLocation(location, true, officeId); + createLocation(location, true, officeId); // This marks for delete at end of test. + updateLocation(location, true, officeId); TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; @@ -1843,9 +1841,10 @@ void test_get_for_elev_has_datum() throws Exception { // get it back String firstPoint = "2008-05-01T03:00:00.000Z"; - + // try once with auth ValidatableResponse validatableResponse = given() .log().ifValidationFails(LogDetail.ALL, true) + .header("Authorization",user.toHeaderValue()) .accept(Formats.JSONV2) .queryParam(OFFICE, officeId) .queryParam(UNIT, "m") @@ -1866,42 +1865,262 @@ void test_get_for_elev_has_datum() throws Exception { .body("vertical-datum-info.location", equalTo(location)) .body("vertical-datum-info.unit", equalTo("m")) .body("vertical-datum-info.offsets.size()", equalTo(1)) -// .body("vertical-datum-info.offsets[0].to-datum", equalTo("NAVD-88")) -// .body("vertical-datum-info.offsets[0].value", closeTo(-0.1666, 0.0001)) -// .body("vertical-datum-info.offsets[0].estimate", equalTo(true)) ; + // Try again without auth + validatableResponse = given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .queryParam(OFFICE, officeId) + .queryParam(UNIT, "m") + .queryParam(NAME, ts.get(NAME).asText()) + .queryParam(BEGIN, firstPoint) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + System.out.println(validatableResponse.extract().asString()); + // verify that there is vertical-datum-info in the response. + validatableResponse.body("vertical-datum-info", notNullValue()) + .body("vertical-datum-info.location", equalTo(location)) + .body("vertical-datum-info.unit", equalTo("m")) + .body("vertical-datum-info.offsets.size()", equalTo(1)) + ; + } - private void manuallyCreateLocation(String location, boolean active, String officeId) throws SQLException { + private void updateLocation(String location, boolean active, String officeId) throws SQLException { String P_LOCATION_ID = location; String P_LOCATION_TYPE = "SITE"; - Number P_ELEVATION = 0; + Number P_ELEVATION = 11; String P_ELEV_UNIT_ID = "m"; - String P_VERTICAL_DATUM = "NAVD-88"; - Number P_LATITUDE = 0; - Number P_LONGITUDE = 0; + + // Pretty sure this isn't supposed to have a dash. The create doesn't check. The default create just passes null. + // If it has a dash then the offsets don't work. + // select VERTICAL_DATUM, count(*) as COUNT + // from AT_PHYSICAL_LOCATION + // group by VERTICAL_DATUM + // order by COUNT desc + // has no entries with a dash in the name (unless we've run this test with a dash). + String P_VERTICAL_DATUM = VerticalDatum.NAVD88.toString(); + Number P_LATITUDE = 38.5757; // pretty sure that if these are 0,0 then its not inside the navd88 bounds and the offsets come back [] + Number P_LONGITUDE = -121.4789; String P_HORIZONTAL_DATUM = "WGS84"; - String P_PUBLIC_NAME = null; + String P_PUBLIC_NAME = "Integration Test Sac Dam"; String P_LONG_NAME= null; - String P_DESCRIPTION = null; + String P_DESCRIPTION = "for testing"; String P_TIME_ZONE_ID = "UTC"; - String P_COUNTY_NAME = null; - String P_STATE_INITIAL = null; + String P_COUNTY_NAME = "Sacramento"; + String P_STATE_INITIAL = "CA"; String P_ACTIVE = active ? "T" : "F"; String P_DB_OFFICE_ID = officeId; CwmsDatabaseContainer db = CwmsDataApiSetupCallback.getDatabaseLink(); db.connection(c -> { DSLContext dslContext = getDslContext(c, officeId); - CWMS_LOC_PACKAGE.call_CREATE_LOCATION(dslContext.configuration(), - P_LOCATION_ID, P_LOCATION_TYPE, P_ELEVATION, P_ELEV_UNIT_ID, P_VERTICAL_DATUM, P_LATITUDE, P_LONGITUDE, - P_HORIZONTAL_DATUM, P_PUBLIC_NAME, P_LONG_NAME, P_DESCRIPTION, P_TIME_ZONE_ID, P_COUNTY_NAME, P_STATE_INITIAL, - P_ACTIVE, P_DB_OFFICE_ID); + +// CWMS_LOC_PACKAGE.call_DELETE_LOCATION(dslContext.configuration(), P_LOCATION_ID, String.valueOf(DeleteRule.DELETE_LOC_CASCADE), P_DB_OFFICE_ID); +// CWMS_LOC_PACKAGE.call_CREATE_LOCATION(dslContext.configuration(), +// P_LOCATION_ID, P_LOCATION_TYPE, P_ELEVATION, P_ELEV_UNIT_ID, P_VERTICAL_DATUM, P_LATITUDE, P_LONGITUDE, +// P_HORIZONTAL_DATUM, P_PUBLIC_NAME, P_LONG_NAME, P_DESCRIPTION, P_TIME_ZONE_ID, P_COUNTY_NAME, P_STATE_INITIAL, +// P_ACTIVE, P_DB_OFFICE_ID); + + String P_IGNORENULLS = "F"; + CWMS_LOC_PACKAGE.call_UPDATE_LOCATION(dslContext.configuration(), + P_LOCATION_ID, P_LOCATION_TYPE, P_ELEVATION, P_ELEV_UNIT_ID, P_VERTICAL_DATUM, P_LATITUDE, P_LONGITUDE, + P_HORIZONTAL_DATUM, P_PUBLIC_NAME, P_LONG_NAME, P_DESCRIPTION, P_TIME_ZONE_ID, P_COUNTY_NAME, P_STATE_INITIAL, + P_ACTIVE, P_IGNORENULLS, P_DB_OFFICE_ID ); + }); } + + // vertical-datum parameter was recently added to the timeseries create call. + // The timeseries sent as the body to the create can optionally also include a vertical-datum-info element. This + // test is meant to verify 3 scenarios when the timeseries does not include vertical-datum-info: + // 1) no vertical-datum parameter is sent to the create call. + // 2) a NAVD88 vertical-datum parameter is sent to the create call. + // 3) a NGVD29 vertical-datum parameter is sent to the create call. + // + @Test + void test_create_with_vertical_datum_parameter_but_no_vertical_datum_info() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/spk/elev_ts_create.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + + JsonNode ts = mapper.readTree(tsData); + String tsName = ts.get(NAME).asText(); + String location = tsName.split("\\.")[0]; + String officeId = ts.get("office-id").asText(); + + // Collect input times and values from the payload + java.util.List inputTimes = new java.util.ArrayList<>(); + java.util.List inputValues = new java.util.ArrayList<>(); + for (JsonNode row : ts.get("values")) { + inputTimes.add(row.get(0).asLong()); + inputValues.add(row.get(1).asDouble()); + } + long firstMillis = inputTimes.get(0); + long lastMillis = inputTimes.get(inputTimes.size() - 1); + String beginIso = java.time.Instant.ofEpochMilli(firstMillis).toString(); + // pad end by 1 hour to ensure inclusion + String endIso = java.time.Instant.ofEpochMilli(lastMillis + 3600_000L).toString(); + + createLocation(location, true, officeId); // This marks for delete at end of test. + updateLocation(location, true, officeId); + + TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; + + // Helper lambda to GET the series and return a ValidatableResponse + java.util.function.Supplier doGet = () -> + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .queryParam(OFFICE, officeId) + .queryParam(NAME, tsName) + .queryParam(UNIT, "m") + .queryParam(BEGIN, beginIso) + .queryParam(END, endIso) + .queryParam(TRIM, true) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .statusCode(is(HttpServletResponse.SC_OK)); + + // 1) No vertical-datum parameter provided + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization", user.toHeaderValue()) + .queryParam(OFFICE, officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // GET after scenario 1 and verify values equal input (NAVD88 native) + ValidatableResponse vr1 = doGet.get(); + System.out.println("1:" + vr1.extract().asString()); + + /* Response includes + "vertical-datum-info": { + "office": "SPK", + "unit": "m", + "location": "Sacramento Dam", + "native-datum": "NAVD-88", + "elevation": 11.0, + "offsets": [ + { + "estimate": true, + "to-datum": "NGVD-29", + "value": -0.7717 + } + ] + }*/ + + vr1.body("values.size()", equalTo(inputValues.size())); + for (int i = 0; i < inputValues.size(); i++) { + long expectedTime = inputTimes.get(i); + double expectedVal = inputValues.get(i); + vr1.body("values[" + i + "][0]", equalTo(expectedTime)) + .body("values[" + i + "][1]", floatCloseTo(expectedVal, 1e-6)); + } + + // 2) Provide NAVD88 vertical-datum parameter (matches location's datum) + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization", user.toHeaderValue()) + .queryParam(OFFICE, officeId) + .queryParam(VERTICAL_DATUM, "NAVD88") + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // GET after scenario 2 and verify values equal input; also capture NGVD29 offset + ValidatableResponse vr2 = doGet.get(); + System.out.println("2:" + vr2.extract().asString()); + vr2.body("values.size()", equalTo(inputValues.size())); + for (int i = 0; i < inputValues.size(); i++) { + vr2.body("values[" + i + "][1]", floatCloseTo(inputValues.get(i), 1e-6)); + } + ExtractableResponse ex2 = vr2.extract(); + String body2 = ex2.asString(); + JsonNode resp2 = new ObjectMapper().readTree(body2); + JsonNode vdi = resp2.get("vertical-datum-info"); + Double offsetToNgvd29 = null; + if (vdi != null && vdi.has("offsets")) { + for (JsonNode off : vdi.get("offsets")) { + if ("NGVD-29".equalsIgnoreCase(off.get("to-datum").asText())) { + if (off.hasNonNull("value")) { + offsetToNgvd29 = off.get("value").asDouble(); + } + break; + } + } + } + assertNotNull(offsetToNgvd29, "Expected NGVD-29 offset to be present in vertical-datum-info"); + + // 3) Provide NGVD29 vertical-datum parameter (conversion should occur to as-stored datum) + given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization", user.toHeaderValue()) + .queryParam(OFFICE, officeId) + .queryParam(VERTICAL_DATUM, "NGVD29") + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // Compute expected NAVD88 = NGVD29 - offset(NAVD88->NGVD29) + java.util.List expectedNavd88 = new java.util.ArrayList<>(); + for (Double v : inputValues) { + expectedNavd88.add(v - offsetToNgvd29); + } + + // GET after scenario 3 and verify conversion was applied + ValidatableResponse vr3 = doGet.get(); + System.out.println("3:" + vr3.extract().asString()); + vr3.body("values.size()", equalTo(expectedNavd88.size())); + for (int i = 0; i < expectedNavd88.size(); i++) { + vr3.body("values[" + i + "][1]", floatCloseTo(expectedNavd88.get(i), 1e-6)); + } + } + + + } From 3cb5837e8ed33d111728634ab091c6e34d1406c0 Mon Sep 17 00:00:00 2001 From: rma-rripken <89810919+rma-rripken@users.noreply.github.com> Date: Fri, 31 Oct 2025 11:21:31 -0700 Subject: [PATCH 07/18] Updating ts vertical datum test. --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 3 +-- .../cwms/cda/api/TimeseriesControllerTestIT.java | 16 +++++++++------- 2 files changed, 10 insertions(+), 9 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 9ecdd29a2..5668f4cb2 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -52,7 +52,6 @@ import cwms.cda.helpers.DateUtils; import java.math.BigDecimal; import java.math.BigInteger; -import java.sql.Connection; import java.sql.SQLException; import java.sql.Timestamp; import java.time.Duration; @@ -1512,7 +1511,7 @@ private void withDefaultDatum(@Nullable VerticalDatum targetDatum, DSLContext ds @Override public void store(TimeSeries timeSeries, Timestamp versionDate) { - store(timeSeries, false, StoreRule.REPLACE_ALL, TimeSeriesDaoImpl.OVERRIDE_PROTECTION); + store(timeSeries, false, StoreRule.REPLACE_ALL, TimeSeriesDaoImpl.OVERRIDE_PROTECTION, null); } public void store(TimeSeries input, boolean createAsLrts, StoreRule replaceAll, boolean overrideProtection, VerticalDatum vd) { diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index 163c975fb..10c5e8854 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -28,6 +28,8 @@ import java.sql.PreparedStatement; import java.sql.SQLException; import java.time.ZonedDateTime; +import java.util.ArrayList; +import java.util.List; import javax.servlet.http.HttpServletResponse; import io.restassured.response.ExtractableResponse; @@ -1951,7 +1953,7 @@ private void updateLocation(String location, boolean active, String officeId) th // 3) a NGVD29 vertical-datum parameter is sent to the create call. // @Test - void test_create_with_vertical_datum_parameter_but_no_vertical_datum_info() throws Exception { + void test_create_without_vertical_datum_info() throws Exception { ObjectMapper mapper = new ObjectMapper(); InputStream resource = this.getClass().getResourceAsStream( @@ -1965,8 +1967,8 @@ void test_create_with_vertical_datum_parameter_but_no_vertical_datum_info() thro String officeId = ts.get("office-id").asText(); // Collect input times and values from the payload - java.util.List inputTimes = new java.util.ArrayList<>(); - java.util.List inputValues = new java.util.ArrayList<>(); + List inputTimes = new ArrayList<>(); + List inputValues = new ArrayList<>(); for (JsonNode row : ts.get("values")) { inputTimes.add(row.get(0).asLong()); inputValues.add(row.get(1).asDouble()); @@ -2054,7 +2056,7 @@ void test_create_with_vertical_datum_parameter_but_no_vertical_datum_info() thro .body(tsData) .header("Authorization", user.toHeaderValue()) .queryParam(OFFICE, officeId) - .queryParam(VERTICAL_DATUM, "NAVD88") + .queryParam(DATUM, "NAVD88") .when() .redirects().follow(true) .redirects().max(3) @@ -2096,7 +2098,7 @@ void test_create_with_vertical_datum_parameter_but_no_vertical_datum_info() thro .body(tsData) .header("Authorization", user.toHeaderValue()) .queryParam(OFFICE, officeId) - .queryParam(VERTICAL_DATUM, "NGVD29") + .queryParam(DATUM, "NGVD29") .when() .redirects().follow(true) .redirects().max(3) @@ -2112,12 +2114,12 @@ void test_create_with_vertical_datum_parameter_but_no_vertical_datum_info() thro expectedNavd88.add(v - offsetToNgvd29); } - // GET after scenario 3 and verify conversion was applied + // GET after scenario 3 and verify the conversion was applied ValidatableResponse vr3 = doGet.get(); System.out.println("3:" + vr3.extract().asString()); vr3.body("values.size()", equalTo(expectedNavd88.size())); for (int i = 0; i < expectedNavd88.size(); i++) { - vr3.body("values[" + i + "][1]", floatCloseTo(expectedNavd88.get(i), 1e-6)); + vr3.body("values[" + i + "][1]", floatCloseTo(expectedNavd88.get(i), 1e-4)); } } From 5943a9cfe5d2fde190987d36b7a99bde8a29cf6d Mon Sep 17 00:00:00 2001 From: Bryson Spilman Date: Thu, 13 Nov 2025 15:53:44 -0800 Subject: [PATCH 08/18] CDA-45 - adds conversion of datum. Adds unit test and integration test. --- .../cwms/cda/api/TimeSeriesController.java | 21 +------ .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 16 ----- .../dao/TimeSeriesVerticalDatumConverter.java | 63 +++++++++++++++++++ .../java/cwms/cda/data/dto/TimeSeries.java | 11 ++++ .../cda/api/TimeseriesControllerTestIT.java | 62 ++++++++++++++++++ .../TimeSeriesVerticalDatumConverterTest.java | 43 +++++++++++++ 6 files changed, 180 insertions(+), 36 deletions(-) create mode 100644 cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java create mode 100644 cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverterTest.java diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java index cc619a710..2d5ab8115 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java @@ -489,26 +489,7 @@ public void getAll(@NotNull Context ctx) { if(datum != null){ // user has requested a specific vertical datum VerticalDatum vd = VerticalDatum.valueOf(datum); // the users request - VerticalDatum tsVd = getVerticalDatum(ts, null); // what came back from sql - if( tsVd == null){ - // the ts came back without a vertical datum. - // We won't know why it came back without a vertical datum, - // maybe the location has lat/long 0,0 and is outside datum? - // Maybe the location doesn't have vertical datum set? - // maybe the ts isn't for Elev (this we could check) - // this doesn't seem like its our problem. We should probably just return the ts without vertical data info in it. - } else if( !Objects.equals(vd, tsVd)){ - // user requested a specific vertical datum but the ts came back with a different vertical datum. - // Here we need to convert - // verify that the vertical-datum-info contains the to-datum and from-datum - // throw exception if not found. - // find needed offset. - // convert the vertical-datum-info so that we switch the to-datum and from-datum to the requested vertical datum and fix the offset - this will also need to adjust other offset entries - // apply the offset to the elev in the vertical-datum-info - // update the ts with the new vertical-datum-info - // apply the offset to the ts values. - throw new UnsupportedOperationException("We need to do this conversion - not implemented yet."); - } + ts = TimeSeriesVerticalDatumConverter.convertToVerticalDatum(ts, vd); } results = Formats.format(contentType, ts); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 5668f4cb2..98a4b0138 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -1593,22 +1593,6 @@ private void store(DSLContext dslContext, String officeId, String tsId, String u formatBool(createAsLrts)); } - public void update(TimeSeries input, boolean createAsLrts, StoreRule storeRule, - Timestamp versionDate, boolean overrideProtection) throws SQLException { - String name = input.getName(); - if (!timeseriesExists(name)) { - throw new SQLException("Cannot update a non-existant Timeseries. Create " + name + " " - + "first."); - } - connection(dsl, connection -> { - //setOffice(connection,input.getOfficeId()); - DSLContext dslContext = getDslContext(connection, input.getOfficeId()); - store(dslContext, input.getOfficeId(), name, input.getUnits(), versionDate, - input.getValues(), createAsLrts, storeRule, overrideProtection); - }); - } - - protected BigDecimal retrieveTsCode(String tsId) { return dsl.select(AV_CWMS_TS_ID2.TS_CODE) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java new file mode 100644 index 000000000..1e8104ea8 --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java @@ -0,0 +1,63 @@ +package cwms.cda.data.dao; + +import cwms.cda.data.dto.TimeSeries; +import cwms.cda.data.dto.VerticalDatumInfo; +import org.jetbrains.annotations.Nullable; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; + +public final class TimeSeriesVerticalDatumConverter { + + + private TimeSeriesVerticalDatumConverter() { + throw new AssertionError("Utility class, don't instantiate"); + } + + public static TimeSeries convertToVerticalDatum(TimeSeries timeSeries, VerticalDatum convertTo) { + if(timeSeries.getVerticalDatumInfo() == null || Objects.equals(convertTo, getVerticalDatum(timeSeries))) { + return timeSeries; //no conversion needed + } + TimeSeries retVal = new TimeSeries(timeSeries); + VerticalDatumInfo vdi = timeSeries.getVerticalDatumInfo(); + VerticalDatumInfo.Offset[] offsets = vdi.getOffsets(); + for (VerticalDatumInfo.Offset offset : offsets) { + String toDatum = offset.getToDatum(); + if (toDatum.replaceAll("-", "").equalsIgnoreCase(convertTo.name())) { + Double conversionFactor = offset.getValue(); + List values = retVal.getValues(); + List newValues = new ArrayList<>(); + for (TimeSeries.Record record : values) { + Double newValue = record.getValue() + conversionFactor; + TimeSeries.Record newRecord = new TimeSeries.Record(record.getDateTime(), newValue, record.getQualityCode()); + newValues.add(newRecord); + } + values.clear(); + values.addAll(newValues); + } + } + return retVal; + } + + private static VerticalDatum getVerticalDatum(@Nullable TimeSeries timeSeries) { + + VerticalDatum retVal = null; + if (timeSeries != null) { + VerticalDatumInfo vdi = timeSeries.getVerticalDatumInfo(); + if (vdi != null) { + String nativeDatum = vdi.getNativeDatum(); + if (nativeDatum != null && !nativeDatum.isEmpty()) { + if (nativeDatum.equalsIgnoreCase("OTHER")) { + throw new IllegalArgumentException("Vertical Datum of OTHER is not currently supported."); + } else { + retVal = VerticalDatum.getVerticalDatum(nativeDatum); + } + } + } + } + + return retVal; + } + +} diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index d1414527f..b1011ae83 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -134,6 +134,17 @@ public TimeSeries(String page, int pageSize, Integer total, String name, String values = new ArrayList<>(); } + public TimeSeries(TimeSeries timeSeries) + { + this(timeSeries.getPage(), timeSeries.getPageSize(), timeSeries.getTotal(), + timeSeries.getName(), timeSeries.getOfficeId(), timeSeries.getBegin(), + timeSeries.getEnd(), timeSeries.getUnits(), timeSeries.getInterval(), + timeSeries.getVerticalDatumInfo(), timeSeries.getIntervalOffset(), + timeSeries.getTimeZone(), timeSeries.getVersionDate(), + timeSeries.getDateVersionType()); + this.values = new ArrayList<>(timeSeries.getValues()); + } + public String getName() { return name; } diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index 10c5e8854..da7781b68 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -14,6 +14,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import cwms.cda.ApiServlet; import cwms.cda.data.dao.VerticalDatum; +import cwms.cda.data.dto.TimeSeries; +import cwms.cda.data.dto.VerticalDatumInfo; +import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; import cwms.cda.helpers.DatabaseHelpers.SCHEMA_VERSION; import cwms.cda.helpers.ZoneIdHelper; @@ -29,6 +32,7 @@ import java.sql.SQLException; import java.time.ZonedDateTime; import java.util.ArrayList; +import java.util.Arrays; import java.util.List; import javax.servlet.http.HttpServletResponse; @@ -1895,6 +1899,64 @@ void test_get_for_elev_has_datum() throws Exception { ; + validatableResponse = given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .queryParam(OFFICE, officeId) + .queryParam(UNIT, "m") + .queryParam(NAME, ts.get(NAME).asText()) + .queryParam(BEGIN, firstPoint) + .queryParam(DATUM, "NAVD88") + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + System.out.println(validatableResponse.extract().asString()); + // verify that there is vertical-datum-info in the response. + validatableResponse.body("vertical-datum-info", notNullValue()) + .body("vertical-datum-info.location", equalTo(location)) + .body("vertical-datum-info.unit", equalTo("m")) + .body("vertical-datum-info.offsets.size()", equalTo(1)) + ; + + ContentType contentType = Formats.parseHeader(Formats.JSONV2, TimeSeries.class); + TimeSeries timeSeries = Formats.parseContent(contentType, validatableResponse.extract().asString(), TimeSeries.class); + Double conversionFactor = Arrays.stream(timeSeries.getVerticalDatumInfo().getOffsets()).sequential() + .filter(o -> o.getToDatum().equalsIgnoreCase("NGVD-29")) + .findFirst() + .map(VerticalDatumInfo.Offset::getValue) + .orElseThrow(() -> new Exception("No conversion factor from NAVD88 to NGVD29 found")); + + ValidatableResponse validatableResponseConverted = given() + .log().ifValidationFails(LogDetail.ALL, true) + .accept(Formats.JSONV2) + .queryParam(OFFICE, officeId) + .queryParam(UNIT, "m") + .queryParam(NAME, ts.get(NAME).asText()) + .queryParam(BEGIN, firstPoint) + .queryParam(DATUM, "NGVD29") + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL, true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + // verify that there is vertical-datum-info in the response. + validatableResponseConverted.body("vertical-datum-info", notNullValue()) + .body("vertical-datum-info.location", equalTo(location)) + .body("vertical-datum-info.unit", equalTo("m")) + .body("vertical-datum-info.offsets.size()", equalTo(1)) + .body("values[0][1].toDouble()", closeTo( timeSeries.getValues().get(0).getValue() + conversionFactor, 0.001)) + ; + + } private void updateLocation(String location, boolean active, String officeId) throws SQLException { diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverterTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverterTest.java new file mode 100644 index 000000000..392bf2aa5 --- /dev/null +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverterTest.java @@ -0,0 +1,43 @@ +package cwms.cda.data.dao; + +import cwms.cda.data.dto.TimeSeries; +import cwms.cda.data.dto.VerticalDatumInfo; +import cwms.cda.formatters.ContentType; +import cwms.cda.formatters.Formats; +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Test; + +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.util.Arrays; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +public class TimeSeriesVerticalDatumConverterTest { + + @Test + void testConvertVerticalDatum() throws Exception + { + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/timeseries/ts_with_vertical.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + ContentType contentType = Formats.parseHeader(Formats.JSONV2, TimeSeries.class); + TimeSeries ts = Formats.parseContent(contentType, tsData, TimeSeries.class); + + Double conversionFactor = Arrays.stream(ts.getVerticalDatumInfo().getOffsets()).sequential() + .filter(o -> o.getToDatum().equalsIgnoreCase("NAVD-88")) + .findFirst() + .map(VerticalDatumInfo.Offset::getValue) + .orElseThrow(() -> new Exception("No conversion factor from NGVD29 to NAVD88 found")); + TimeSeries convertedTs = TimeSeriesVerticalDatumConverter.convertToVerticalDatum(ts, VerticalDatum.NAVD88); + assertFalse(convertedTs.getValues().isEmpty()); + assertEquals(convertedTs.getValues().size(), ts.getValues().size()); + for(int i=0; i< convertedTs.getValues().size(); i++) + { + assertEquals(convertedTs.getValues().get(i).getValue(), ts.getValues().get(i).getValue() + conversionFactor, 0.0001); + } + } +} From 751543c95c6e5db48f68f2ed879680cd25436f98 Mon Sep 17 00:00:00 2001 From: Bryson Spilman Date: Fri, 14 Nov 2025 12:46:20 -0800 Subject: [PATCH 09/18] CDA-45 - Updated offsets upon conversion and updates vertical datum info when converted. Adds rountrip unit tests. --- .../cwms/cda/api/TimeSeriesController.java | 2 +- .../dao/TimeSeriesVerticalDatumConverter.java | 95 ++++++++++++++++--- .../java/cwms/cda/data/dto/TimeSeries.java | 17 ++-- .../cwms/cda/data/dto/VerticalDatumInfo.java | 13 +++ .../TimeSeriesVerticalDatumConverterTest.java | 83 +++++++++++++++- 5 files changed, 182 insertions(+), 28 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java index 2d5ab8115..4b447e37b 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java @@ -486,7 +486,7 @@ public void getAll(@NotNull Context ctx) { .build(); TimeSeries ts = dao.getTimeseries(cursor, pageSize, requestParameters); - if(datum != null){ + if(datum != null) { //this will be null for non-elevation ts // user has requested a specific vertical datum VerticalDatum vd = VerticalDatum.valueOf(datum); // the users request ts = TimeSeriesVerticalDatumConverter.convertToVerticalDatum(ts, vd); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java index 1e8104ea8..743d0a03a 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java @@ -2,6 +2,7 @@ import cwms.cda.data.dto.TimeSeries; import cwms.cda.data.dto.VerticalDatumInfo; +import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; import java.util.ArrayList; @@ -15,31 +16,95 @@ private TimeSeriesVerticalDatumConverter() { throw new AssertionError("Utility class, don't instantiate"); } - public static TimeSeries convertToVerticalDatum(TimeSeries timeSeries, VerticalDatum convertTo) { - if(timeSeries.getVerticalDatumInfo() == null || Objects.equals(convertTo, getVerticalDatum(timeSeries))) { - return timeSeries; //no conversion needed + public static TimeSeries convertToVerticalDatum(TimeSeries originalTimeSeries, VerticalDatum convertTo) { + if(originalTimeSeries.getVerticalDatumInfo() == null || Objects.equals(convertTo, getVerticalDatum(originalTimeSeries))) { + return originalTimeSeries; //no conversion needed } - TimeSeries retVal = new TimeSeries(timeSeries); - VerticalDatumInfo vdi = timeSeries.getVerticalDatumInfo(); + TimeSeries retVal = originalTimeSeries; + VerticalDatumInfo vdi = originalTimeSeries.getVerticalDatumInfo(); + VerticalDatumInfo.Offset offset = getOffsetForDatum(vdi, convertTo); + if(offset != null) + { + List newValues = convertElevationValues(offset, originalTimeSeries); + VerticalDatumInfo newVerticalDatumInfo = convertVerticalDatumInfo(vdi, convertTo, offset); + retVal = new TimeSeries(originalTimeSeries.getPage(), + originalTimeSeries.getPageSize(), + originalTimeSeries.getTotal(), + originalTimeSeries.getName(), + originalTimeSeries.getOfficeId(), + originalTimeSeries.getBegin(), + originalTimeSeries.getEnd(), + originalTimeSeries.getUnits(), + originalTimeSeries.getInterval(), + newVerticalDatumInfo, + originalTimeSeries.getIntervalOffset(), + originalTimeSeries.getTimeZone(), + originalTimeSeries.getVersionDate(), + originalTimeSeries.getDateVersionType()) + .withValues(newValues); + } + return retVal; + } + + @NotNull + private static List convertElevationValues(VerticalDatumInfo.Offset offset, TimeSeries retVal) { + Double conversionFactor = offset.getValue(); + List originalValues = retVal.getValues(); + List newValues = new ArrayList<>(); + for (TimeSeries.Record record : originalValues) { + Double newValue = record.getValue() + conversionFactor; + TimeSeries.Record newRecord = new TimeSeries.Record(record.getDateTime(), newValue, record.getQualityCode()); + newValues.add(newRecord); + } + return newValues; + } + + static VerticalDatumInfo.Offset getOffsetForDatum(VerticalDatumInfo vdi, VerticalDatum convertTo) { + VerticalDatumInfo.Offset retVal = null; VerticalDatumInfo.Offset[] offsets = vdi.getOffsets(); for (VerticalDatumInfo.Offset offset : offsets) { String toDatum = offset.getToDatum(); if (toDatum.replaceAll("-", "").equalsIgnoreCase(convertTo.name())) { - Double conversionFactor = offset.getValue(); - List values = retVal.getValues(); - List newValues = new ArrayList<>(); - for (TimeSeries.Record record : values) { - Double newValue = record.getValue() + conversionFactor; - TimeSeries.Record newRecord = new TimeSeries.Record(record.getDateTime(), newValue, record.getQualityCode()); - newValues.add(newRecord); - } - values.clear(); - values.addAll(newValues); + retVal = offset; + break; } } return retVal; } + private static VerticalDatumInfo convertVerticalDatumInfo(VerticalDatumInfo vdi, VerticalDatum convertTo, VerticalDatumInfo.Offset convertToOffset) { + Double conversionFactor = convertToOffset.getValue(); + return new VerticalDatumInfo.Builder() + .from(vdi) + .withElevation(vdi.getElevation() + conversionFactor) + .withNativeDatum(convertToOffset.getToDatum()) + .withOffsets(buildConvertedOffsets(vdi, convertTo, convertToOffset)) + .build(); + } + + private static VerticalDatumInfo.Offset[] buildConvertedOffsets(VerticalDatumInfo vdi, VerticalDatum convertTo, VerticalDatumInfo.Offset convertToOffset) { + List newOffsets = new ArrayList<>(); + + //add the reverse offset + Double conversionFactor = convertToOffset.getValue(); + double convertToOffsetToOriginal = -conversionFactor; + VerticalDatumInfo.Offset reverseOffset = new VerticalDatumInfo.Offset(convertToOffset.isEstimate(), vdi.getNativeDatum(), convertToOffsetToOriginal); + newOffsets.add(reverseOffset); + + //add the other offsets, adjusted + VerticalDatumInfo.Offset[] offsets = vdi.getOffsets(); + for (VerticalDatumInfo.Offset offset : offsets) { + String toDatum = offset.getToDatum(); + if (!toDatum.replaceAll("-", "").equalsIgnoreCase(convertTo.name())) { + Double newOffsetValue = convertToOffsetToOriginal + offset.getValue(); + boolean isEstimate = offset.isEstimate() || convertToOffset.isEstimate(); + VerticalDatumInfo.Offset newOffset = new VerticalDatumInfo.Offset(isEstimate, toDatum, newOffsetValue); + newOffsets.add(newOffset); + } + } + return newOffsets.toArray(new VerticalDatumInfo.Offset[]{}); + } + private static VerticalDatum getVerticalDatum(@Nullable TimeSeries timeSeries) { VerticalDatum retVal = null; diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index b1011ae83..a3fcdb68f 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -134,17 +134,6 @@ public TimeSeries(String page, int pageSize, Integer total, String name, String values = new ArrayList<>(); } - public TimeSeries(TimeSeries timeSeries) - { - this(timeSeries.getPage(), timeSeries.getPageSize(), timeSeries.getTotal(), - timeSeries.getName(), timeSeries.getOfficeId(), timeSeries.getBegin(), - timeSeries.getEnd(), timeSeries.getUnits(), timeSeries.getInterval(), - timeSeries.getVerticalDatumInfo(), timeSeries.getIntervalOffset(), - timeSeries.getTimeZone(), timeSeries.getVersionDate(), - timeSeries.getDateVersionType()); - this.values = new ArrayList<>(timeSeries.getValues()); - } - public String getName() { return name; } @@ -255,6 +244,12 @@ public void addValue(Timestamp dateTime, Double value, int qualityCode, Timestam } } + public TimeSeries withValues(List values) { + this.values.clear(); + this.values.addAll(values); + return this; + } + public static List getColumnDescriptor() { List columns = new ArrayList<>(); for (Field f: Record.class.getDeclaredFields()) { diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/VerticalDatumInfo.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/VerticalDatumInfo.java index 88c811fd2..0495d243a 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/VerticalDatumInfo.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/VerticalDatumInfo.java @@ -1,5 +1,6 @@ package cwms.cda.data.dto; +import com.fasterxml.jackson.annotation.JsonIgnore; import com.fasterxml.jackson.annotation.JsonInclude; import com.fasterxml.jackson.annotation.JsonRootName; import com.fasterxml.jackson.databind.PropertyNamingStrategies; @@ -182,6 +183,18 @@ public VerticalDatumInfo.Builder withLocalDatumName(String localDatumName) { return this; } + @JsonIgnore + public Builder from(VerticalDatumInfo vdi) { + this.office = vdi.getOffice(); + this.unit = vdi.getUnit(); + this.location = vdi.getLocation(); + this.nativeDatum = vdi.getNativeDatum(); + this.elevation = vdi.getElevation(); + this.offsets = vdi.getOffsets(); + this.localDatumName = vdi.getLocalDatumName(); + return this; + } + public VerticalDatumInfo build() { return new VerticalDatumInfo(this); } diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverterTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverterTest.java index 392bf2aa5..e27052c21 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverterTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverterTest.java @@ -9,7 +9,9 @@ import java.io.InputStream; import java.nio.charset.StandardCharsets; +import java.util.ArrayList; import java.util.Arrays; +import java.util.List; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertFalse; @@ -37,7 +39,86 @@ void testConvertVerticalDatum() throws Exception assertEquals(convertedTs.getValues().size(), ts.getValues().size()); for(int i=0; i< convertedTs.getValues().size(); i++) { - assertEquals(convertedTs.getValues().get(i).getValue(), ts.getValues().get(i).getValue() + conversionFactor, 0.0001); + assertEquals(ts.getValues().get(i).getValue() + conversionFactor, convertedTs.getValues().get(i).getValue(), 0.0001); + } + assertEquals(ts.getVerticalDatumInfo().getElevation() + conversionFactor, convertedTs.getVerticalDatumInfo().getElevation(), 0.0001); + assertEquals("NAVD-88", convertedTs.getVerticalDatumInfo().getNativeDatum()); + } + + @Test + void testConvertVerticalDatumRoundTrip() throws Exception + { + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/timeseries/ts_with_vertical.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + ContentType contentType = Formats.parseHeader(Formats.JSONV2, TimeSeries.class); + TimeSeries ts = Formats.parseContent(contentType, tsData, TimeSeries.class); + TimeSeries convertedTs = TimeSeriesVerticalDatumConverter.convertToVerticalDatum(ts, VerticalDatum.NAVD88); + TimeSeries convertedTsBack = TimeSeriesVerticalDatumConverter.convertToVerticalDatum(convertedTs, VerticalDatum.NGVD29); + + //verify round trip worked + assertFalse(convertedTsBack.getValues().isEmpty()); + assertEquals(convertedTsBack.getValues().size(), ts.getValues().size()); + for(int i=0; i< convertedTsBack.getValues().size(); i++) + { + assertEquals(convertedTsBack.getValues().get(i).getValue(), ts.getValues().get(i).getValue(), 0.0001); + } + assertEquals(ts.getVerticalDatumInfo().getElevation(), convertedTsBack.getVerticalDatumInfo().getElevation(), 0.0001); + assertEquals(ts.getVerticalDatumInfo().getNativeDatum(), convertedTsBack.getVerticalDatumInfo().getNativeDatum()); + } + + @Test + void testConvertVerticalDatumOffsetsUpdates() throws Exception + { + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/timeseries/ts_with_vertical.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + ContentType contentType = Formats.parseHeader(Formats.JSONV2, TimeSeries.class); + TimeSeries ts = Formats.parseContent(contentType, tsData, TimeSeries.class); + VerticalDatumInfo originalDatumInfo = ts.getVerticalDatumInfo(); + List madeupOffsets = new ArrayList<>(Arrays.asList(originalDatumInfo.getOffsets())); + //since we are tied to an enum to define what we support, using "NATIVE" as a made-up datum value for testing + madeupOffsets.add(new VerticalDatumInfo.Offset(true, VerticalDatum.NATIVE.toString(), 10.0)); + VerticalDatumInfo madeupVdi = new VerticalDatumInfo.Builder() + .from(ts.getVerticalDatumInfo()) + .withOffsets(madeupOffsets.toArray(new VerticalDatumInfo.Offset[]{})) + .build(); + ts = new TimeSeries(ts.getPage(), + ts.getPageSize(), + ts.getTotal(), + ts.getName(), + ts.getOfficeId(), + ts.getBegin(), + ts.getEnd(), + ts.getUnits(), + ts.getInterval(), + madeupVdi, + ts.getIntervalOffset(), + ts.getTimeZone(), + ts.getVersionDate(), + ts.getDateVersionType()) + .withValues(ts.getValues()); + TimeSeries convertedTs = TimeSeriesVerticalDatumConverter.convertToVerticalDatum(ts, VerticalDatum.NAVD88); + TimeSeries convertedTsToMadeUp = TimeSeriesVerticalDatumConverter.convertToVerticalDatum(convertedTs, VerticalDatum.NATIVE); + TimeSeries convertBackToOriginal = TimeSeriesVerticalDatumConverter.convertToVerticalDatum(convertedTsToMadeUp, VerticalDatum.NGVD29); + //verify we get back to original after multiple conversions between datums - this ensures that offsets are being updated properly + assertFalse(convertBackToOriginal.getValues().isEmpty()); + assertEquals(convertBackToOriginal.getValues().size(), ts.getValues().size()); + for(int i=0; i< convertBackToOriginal.getValues().size(); i++) + { + assertEquals(ts.getValues().get(i).getValue(), convertBackToOriginal.getValues().get(i).getValue(), 0.0001); + } + assertEquals(ts.getVerticalDatumInfo().getElevation(), convertBackToOriginal.getVerticalDatumInfo().getElevation(), 0.0001); + assertEquals(ts.getVerticalDatumInfo().getNativeDatum(), convertBackToOriginal.getVerticalDatumInfo().getNativeDatum()); + //verify all original offsets are present in round-trip conversion and values match + for(VerticalDatumInfo.Offset offset : convertBackToOriginal.getVerticalDatumInfo().getOffsets()) + { + VerticalDatum convertedBackToDatum = VerticalDatum.getVerticalDatum(offset.getToDatum()); + VerticalDatumInfo.Offset originalToDatum = TimeSeriesVerticalDatumConverter.getOffsetForDatum(ts.getVerticalDatumInfo(), convertedBackToDatum); + assertNotNull(originalToDatum, "Round-trip conversion resulted in missing to-datum: " + convertedBackToDatum); + assertEquals(originalToDatum.getValue(), offset.getValue(), 0.0001); } } } From 45c50a2b70263e8c6e2c07be97c01f757996344f Mon Sep 17 00:00:00 2001 From: Bryson Spilman Date: Fri, 14 Nov 2025 13:03:27 -0800 Subject: [PATCH 10/18] CDA-45 - refactor of timeseries controller after rebase --- .../main/java/cwms/cda/api/TimeSeriesController.java | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java index 4b447e37b..5d8b85f45 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java @@ -9,7 +9,14 @@ import cwms.cda.api.enums.UnitSystem; import cwms.cda.api.errors.CdaError; import cwms.cda.api.errors.NotFoundException; -import cwms.cda.data.dao.*; +import cwms.cda.data.dao.JooqDao; +import cwms.cda.data.dao.StoreRule; +import cwms.cda.data.dao.TimeSeriesDao; +import cwms.cda.data.dao.TimeSeriesDaoImpl; +import cwms.cda.data.dao.TimeSeriesDeleteOptions; +import cwms.cda.data.dao.TimeSeriesRequestParameters; +import cwms.cda.data.dao.TimeSeriesVerticalDatumConverter; +import cwms.cda.data.dao.VerticalDatum; import cwms.cda.data.dto.TimeSeries; import cwms.cda.data.dto.VerticalDatumInfo; import cwms.cda.formatters.ContentType; @@ -33,7 +40,6 @@ import java.sql.Timestamp; import java.time.ZoneId; import java.time.ZonedDateTime; -import java.util.Objects; import java.util.logging.Level; import java.util.logging.Logger; import javax.servlet.http.HttpServletResponse; @@ -393,7 +399,6 @@ public void delete(@NotNull Context ctx, @NotNull String timeseries) { + "whether to include the data entry date of each value in the response. Including the data entry " + "date will increase the size of the array containing each data value from three to four, " + "changing the format of the response. Default is false."), - @OpenApiParam(name = PAGE, description = "This end point can return large amounts " + "of data as a series of pages. This parameter is used to describes the " + "current location in the response stream. This is an opaque " From 08b0c5c4ab2c236c89b4c88c0157728f69c33b6a Mon Sep 17 00:00:00 2001 From: Bryson Spilman Date: Fri, 14 Nov 2025 13:24:05 -0800 Subject: [PATCH 11/18] CDA-45 - Added back ConnectionRunnable that got removed during rebase --- .../main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 98a4b0138..d69939680 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -19,6 +19,8 @@ import static org.jooq.impl.DSL.partitionBy; import static org.jooq.impl.DSL.select; import static org.jooq.impl.DSL.selectDistinct; + +import org.jooq.ConnectionRunnable; import usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID; import static org.jooq.impl.DSL.table; import static usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID2.AV_CWMS_TS_ID2; @@ -1524,11 +1526,8 @@ public void store(TimeSeries input, boolean createAsLrts, StoreRule replaceAll, connection(dsl, connection -> { DSLContext dslContext = getDslContext(connection, input.getOfficeId()); - withDefaultDatum(vd, dslContext, (conn)-> { - - store(dslContext, input.getOfficeId(), input.getName(), input.getUnits(), - versionDate, input.getValues(), createAsLrts, replaceAll, overrideProtection); - }); + withDefaultDatum(vd, dslContext, (conn)-> store(dslContext, input.getOfficeId(), input.getName(), input.getUnits(), + versionDate, input.getValues(), createAsLrts, replaceAll, overrideProtection)); }); } From 6b81f8eda5f7e863660036c28493c4ccbfb930ab Mon Sep 17 00:00:00 2001 From: Bryson Spilman Date: Mon, 17 Nov 2025 11:34:51 -0800 Subject: [PATCH 12/18] CDA-45 - Improving method naming and refactoring code to be more clear. General code cleanup --- .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 15 +++++++++------ .../dao/TimeSeriesVerticalDatumConverter.java | 17 +++++++---------- .../cwms/cda/data/dto/VerticalDatumInfo.java | 11 +++++++++++ 3 files changed, 27 insertions(+), 16 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index d69939680..484c631ac 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -8,6 +8,7 @@ import cwms.cda.data.dto.catalog.TimeSeriesAlias; import cwms.cda.helpers.DateUtils; +import java.sql.Connection; import java.util.*; import static org.jooq.impl.DSL.asterisk; @@ -1524,11 +1525,14 @@ public void store(TimeSeries input, boolean createAsLrts, StoreRule replaceAll, versionDate = null; } - connection(dsl, connection -> { - DSLContext dslContext = getDslContext(connection, input.getOfficeId()); - withDefaultDatum(vd, dslContext, (conn)-> store(dslContext, input.getOfficeId(), input.getName(), input.getUnits(), - versionDate, input.getValues(), createAsLrts, replaceAll, overrideProtection)); - }); + connection(dsl, connection -> storeWithDefaultDatum(input, createAsLrts, replaceAll, overrideProtection, vd, connection, versionDate)); + } + + private void storeWithDefaultDatum(TimeSeries input, boolean createAsLrts, StoreRule replaceAll, boolean overrideProtection, + VerticalDatum vd, Connection connection, Timestamp versionDate) throws Throwable { + DSLContext dslContext = getDslContext(connection, input.getOfficeId()); + withDefaultDatum(vd, dslContext, (conn)-> store(dslContext, input.getOfficeId(), input.getName(), input.getUnits(), + versionDate, input.getValues(), createAsLrts, replaceAll, overrideProtection)); } @@ -1548,7 +1552,6 @@ public void store(TimeSeries input, boolean createAsLrts, StoreRule replaceAll, private void store(DSLContext dslContext, String officeId, String tsId, String units, Timestamp versionDate, List values, boolean createAsLrts, StoreRule storeRule, boolean overrideProtection) throws SQLException { - //setOffice(connection,officeId); final ZTSV_ARRAY tsvArray = new ZTSV_ARRAY(); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java index 743d0a03a..3f9b24c0a 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java @@ -25,7 +25,7 @@ public static TimeSeries convertToVerticalDatum(TimeSeries originalTimeSeries, V VerticalDatumInfo.Offset offset = getOffsetForDatum(vdi, convertTo); if(offset != null) { - List newValues = convertElevationValues(offset, originalTimeSeries); + List newValues = applyOffsetToValues(offset.getValue(), originalTimeSeries.getValues()); VerticalDatumInfo newVerticalDatumInfo = convertVerticalDatumInfo(vdi, convertTo, offset); retVal = new TimeSeries(originalTimeSeries.getPage(), originalTimeSeries.getPageSize(), @@ -47,12 +47,10 @@ public static TimeSeries convertToVerticalDatum(TimeSeries originalTimeSeries, V } @NotNull - private static List convertElevationValues(VerticalDatumInfo.Offset offset, TimeSeries retVal) { - Double conversionFactor = offset.getValue(); - List originalValues = retVal.getValues(); + private static List applyOffsetToValues(Double offset, List originalValues) { List newValues = new ArrayList<>(); for (TimeSeries.Record record : originalValues) { - Double newValue = record.getValue() + conversionFactor; + Double newValue = record.getValue() + offset; TimeSeries.Record newRecord = new TimeSeries.Record(record.getDateTime(), newValue, record.getQualityCode()); newValues.add(newRecord); } @@ -63,8 +61,7 @@ static VerticalDatumInfo.Offset getOffsetForDatum(VerticalDatumInfo vdi, Vertica VerticalDatumInfo.Offset retVal = null; VerticalDatumInfo.Offset[] offsets = vdi.getOffsets(); for (VerticalDatumInfo.Offset offset : offsets) { - String toDatum = offset.getToDatum(); - if (toDatum.replaceAll("-", "").equalsIgnoreCase(convertTo.name())) { + if (offset.isForDatum(convertTo.toString())) { retVal = offset; break; } @@ -73,10 +70,10 @@ static VerticalDatumInfo.Offset getOffsetForDatum(VerticalDatumInfo vdi, Vertica } private static VerticalDatumInfo convertVerticalDatumInfo(VerticalDatumInfo vdi, VerticalDatum convertTo, VerticalDatumInfo.Offset convertToOffset) { - Double conversionFactor = convertToOffset.getValue(); + Double offsetValue = convertToOffset.getValue(); return new VerticalDatumInfo.Builder() .from(vdi) - .withElevation(vdi.getElevation() + conversionFactor) + .withElevation(vdi.getElevation() + offsetValue) .withNativeDatum(convertToOffset.getToDatum()) .withOffsets(buildConvertedOffsets(vdi, convertTo, convertToOffset)) .build(); @@ -95,7 +92,7 @@ private static VerticalDatumInfo.Offset[] buildConvertedOffsets(VerticalDatumInf VerticalDatumInfo.Offset[] offsets = vdi.getOffsets(); for (VerticalDatumInfo.Offset offset : offsets) { String toDatum = offset.getToDatum(); - if (!toDatum.replaceAll("-", "").equalsIgnoreCase(convertTo.name())) { + if (!offset.isForDatum(convertTo.toString())) { Double newOffsetValue = convertToOffsetToOriginal + offset.getValue(); boolean isEstimate = offset.isEstimate() || convertToOffset.isEstimate(); VerticalDatumInfo.Offset newOffset = new VerticalDatumInfo.Offset(isEstimate, toDatum, newOffsetValue); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/VerticalDatumInfo.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/VerticalDatumInfo.java index 0495d243a..87885d43d 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/VerticalDatumInfo.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/VerticalDatumInfo.java @@ -89,6 +89,17 @@ public Offset(boolean isEstimate, String toDatum, Double value) { this.value = value; } + @JsonIgnore + public boolean isForDatum(String verticalDatum) { + if(verticalDatum == null && toDatum == null) { + return true; + } + if(verticalDatum == null || toDatum == null) { + return false; + } + return toDatum.replaceAll("-", "").equalsIgnoreCase(verticalDatum.replaceAll("-", "")); + } + @Override public boolean equals(Object o) { if (this == o) { From f6fdb9b639c56fbb9d14e0ca897e6e10526e91a4 Mon Sep 17 00:00:00 2001 From: Bryson Spilman Date: Mon, 17 Nov 2025 13:46:29 -0800 Subject: [PATCH 13/18] CDA-45 - moved some non-ts-specific offset conversions and getter into VerticalDatumInfo/Offset classes --- .../dao/TimeSeriesVerticalDatumConverter.java | 49 +---------------- .../java/cwms/cda/data/dao/VerticalDatum.java | 2 + .../cwms/cda/data/dto/VerticalDatumInfo.java | 52 +++++++++++++++++++ .../TimeSeriesVerticalDatumConverterTest.java | 2 +- 4 files changed, 57 insertions(+), 48 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java index 3f9b24c0a..1f6d78efb 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java @@ -22,11 +22,11 @@ public static TimeSeries convertToVerticalDatum(TimeSeries originalTimeSeries, V } TimeSeries retVal = originalTimeSeries; VerticalDatumInfo vdi = originalTimeSeries.getVerticalDatumInfo(); - VerticalDatumInfo.Offset offset = getOffsetForDatum(vdi, convertTo); + VerticalDatumInfo.Offset offset = vdi.getOffsetForDatum(convertTo); if(offset != null) { List newValues = applyOffsetToValues(offset.getValue(), originalTimeSeries.getValues()); - VerticalDatumInfo newVerticalDatumInfo = convertVerticalDatumInfo(vdi, convertTo, offset); + VerticalDatumInfo newVerticalDatumInfo = vdi.convertedTo(offset); retVal = new TimeSeries(originalTimeSeries.getPage(), originalTimeSeries.getPageSize(), originalTimeSeries.getTotal(), @@ -57,51 +57,6 @@ private static List applyOffsetToValues(Double offset, List newOffsets = new ArrayList<>(); - - //add the reverse offset - Double conversionFactor = convertToOffset.getValue(); - double convertToOffsetToOriginal = -conversionFactor; - VerticalDatumInfo.Offset reverseOffset = new VerticalDatumInfo.Offset(convertToOffset.isEstimate(), vdi.getNativeDatum(), convertToOffsetToOriginal); - newOffsets.add(reverseOffset); - - //add the other offsets, adjusted - VerticalDatumInfo.Offset[] offsets = vdi.getOffsets(); - for (VerticalDatumInfo.Offset offset : offsets) { - String toDatum = offset.getToDatum(); - if (!offset.isForDatum(convertTo.toString())) { - Double newOffsetValue = convertToOffsetToOriginal + offset.getValue(); - boolean isEstimate = offset.isEstimate() || convertToOffset.isEstimate(); - VerticalDatumInfo.Offset newOffset = new VerticalDatumInfo.Offset(isEstimate, toDatum, newOffsetValue); - newOffsets.add(newOffset); - } - } - return newOffsets.toArray(new VerticalDatumInfo.Offset[]{}); - } - private static VerticalDatum getVerticalDatum(@Nullable TimeSeries timeSeries) { VerticalDatum retVal = null; diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/VerticalDatum.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/VerticalDatum.java index 01217de2b..6574e9aa4 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/VerticalDatum.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/VerticalDatum.java @@ -1,5 +1,7 @@ package cwms.cda.data.dao; +import cwms.cda.data.dto.VerticalDatumInfo; + public enum VerticalDatum { NAVD88("NAVD88"), NGVD29("NGVD29"), diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/VerticalDatumInfo.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/VerticalDatumInfo.java index 87885d43d..e4369a3cb 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/VerticalDatumInfo.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/VerticalDatumInfo.java @@ -7,6 +7,10 @@ import com.fasterxml.jackson.databind.annotation.JsonDeserialize; import com.fasterxml.jackson.databind.annotation.JsonNaming; import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import cwms.cda.data.dao.VerticalDatum; + +import java.util.ArrayList; +import java.util.List; @JsonRootName("vertical-datum-info") @JsonDeserialize(builder = VerticalDatumInfo.Builder.class) @@ -60,6 +64,54 @@ public String getLocalDatumName() { return localDatumName; } + @JsonIgnore + public VerticalDatumInfo.Offset getOffsetForDatum(VerticalDatum convertTo) { + VerticalDatumInfo.Offset retVal = null; + VerticalDatumInfo.Offset[] offsets = getOffsets(); + for (VerticalDatumInfo.Offset offset : offsets) { + if (offset.isForDatum(convertTo.toString())) { + retVal = offset; + break; + } + } + return retVal; + } + + @JsonIgnore + public VerticalDatumInfo convertedTo(VerticalDatumInfo.Offset convertToOffset) { + VerticalDatum convertTo = VerticalDatum.getVerticalDatum(convertToOffset.getToDatum()); + Double offsetValue = convertToOffset.getValue(); + return new VerticalDatumInfo.Builder() + .from(this) + .withElevation(getElevation() + offsetValue) + .withNativeDatum(convertToOffset.getToDatum()) + .withOffsets(buildConvertedOffsets(convertTo, convertToOffset)) + .build(); + } + + private VerticalDatumInfo.Offset[] buildConvertedOffsets(VerticalDatum convertTo, VerticalDatumInfo.Offset convertToOffset) { + List newOffsets = new ArrayList<>(); + + //add the reverse offset + Double conversionFactor = convertToOffset.getValue(); + double convertToOffsetToOriginal = -conversionFactor; + VerticalDatumInfo.Offset reverseOffset = new VerticalDatumInfo.Offset(convertToOffset.isEstimate(), getNativeDatum(), convertToOffsetToOriginal); + newOffsets.add(reverseOffset); + + //add the other offsets, adjusted + VerticalDatumInfo.Offset[] offsets = getOffsets(); + for (VerticalDatumInfo.Offset offset : offsets) { + String toDatum = offset.getToDatum(); + if (!offset.isForDatum(convertTo.toString())) { + Double newOffsetValue = convertToOffsetToOriginal + offset.getValue(); + boolean isEstimate = offset.isEstimate() || convertToOffset.isEstimate(); + VerticalDatumInfo.Offset newOffset = new VerticalDatumInfo.Offset(isEstimate, toDatum, newOffsetValue); + newOffsets.add(newOffset); + } + } + return newOffsets.toArray(new VerticalDatumInfo.Offset[]{}); + } + @JsonNaming(PropertyNamingStrategies.KebabCaseStrategy.class) public static class Offset { boolean estimate; diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverterTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverterTest.java index e27052c21..2e3a8f619 100644 --- a/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverterTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverterTest.java @@ -116,7 +116,7 @@ void testConvertVerticalDatumOffsetsUpdates() throws Exception for(VerticalDatumInfo.Offset offset : convertBackToOriginal.getVerticalDatumInfo().getOffsets()) { VerticalDatum convertedBackToDatum = VerticalDatum.getVerticalDatum(offset.getToDatum()); - VerticalDatumInfo.Offset originalToDatum = TimeSeriesVerticalDatumConverter.getOffsetForDatum(ts.getVerticalDatumInfo(), convertedBackToDatum); + VerticalDatumInfo.Offset originalToDatum = ts.getVerticalDatumInfo().getOffsetForDatum(convertedBackToDatum); assertNotNull(originalToDatum, "Round-trip conversion resulted in missing to-datum: " + convertedBackToDatum); assertEquals(originalToDatum.getValue(), offset.getValue(), 0.0001); } From d48b9635dc2ce97a16fcdaeb2e9f709de4f10f19 Mon Sep 17 00:00:00 2001 From: Bryson Spilman Date: Mon, 1 Dec 2025 12:14:54 -0800 Subject: [PATCH 14/18] CDA-45 - Cleaning up code. --- .../cwms/cda/api/TimeSeriesController.java | 26 ++------------ .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 36 +++---------------- .../dao/TimeSeriesVerticalDatumConverter.java | 29 +++++++-------- .../java/cwms/cda/data/dao/VerticalDatum.java | 5 ++- .../cda/api/TimeseriesControllerTestIT.java | 30 +++++++--------- 5 files changed, 34 insertions(+), 92 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java index 5d8b85f45..128d5fdfb 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java @@ -18,7 +18,6 @@ import cwms.cda.data.dao.TimeSeriesVerticalDatumConverter; import cwms.cda.data.dao.VerticalDatum; import cwms.cda.data.dto.TimeSeries; -import cwms.cda.data.dto.VerticalDatumInfo; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; import cwms.cda.helpers.DateUtils; @@ -47,7 +46,6 @@ import org.apache.http.client.utils.URIBuilder; import org.apache.http.client.utils.URLEncodedUtils; import org.jetbrains.annotations.NotNull; -import org.jetbrains.annotations.Nullable; import org.jooq.DSLContext; import org.jooq.exception.DataAccessException; @@ -188,7 +186,7 @@ public void create(@NotNull Context ctx) { TimeSeriesDao dao = getTimeSeriesDao(dsl); TimeSeries timeSeries = deserializeTimeSeries(ctx); - vd = getVerticalDatum(timeSeries, vd); + vd = TimeSeriesVerticalDatumConverter.getVerticalDatum(timeSeries).orElse(vd); dao.create(timeSeries, createAsLrts, storeRule, overrideProtection, vd); ctx.status(HttpServletResponse.SC_OK); @@ -199,26 +197,6 @@ public void create(@NotNull Context ctx) { } } - @Nullable - private static VerticalDatum getVerticalDatum(@Nullable TimeSeries timeSeries, @Nullable VerticalDatum vd) { - - if (timeSeries != null) { - VerticalDatumInfo vdi = timeSeries.getVerticalDatumInfo(); - if (vdi != null) { - String nativeDatum = vdi.getNativeDatum(); - if (nativeDatum != null && !nativeDatum.isEmpty()) { - if (nativeDatum.equalsIgnoreCase("OTHER")) { - throw new IllegalArgumentException("Vertical Datum of OTHER is not currently supported."); - } else { - vd = VerticalDatum.getVerticalDatum(nativeDatum); - } - } - } - } - - return vd; - } - protected DSLContext getDslContext(Context ctx) { return JooqDao.getDslContext(ctx); } @@ -623,7 +601,7 @@ public void update(@NotNull Context ctx, @NotNull String id) { VerticalDatum vd = ctx.queryParamAsClass(DATUM, VerticalDatum.class) .getOrDefault(null); - vd = getVerticalDatum(timeSeries, vd); + vd = TimeSeriesVerticalDatumConverter.getVerticalDatum(timeSeries).orElse(vd); dao.store(timeSeries, createAsLrts, storeRule, overrideProtection, vd); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index 484c631ac..5c9d76eee 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -9,7 +9,6 @@ import cwms.cda.helpers.DateUtils; import java.sql.Connection; -import java.util.*; import static org.jooq.impl.DSL.asterisk; import static org.jooq.impl.DSL.countDistinct; @@ -34,9 +33,6 @@ import com.google.common.cache.CacheStats; import cwms.cda.api.enums.UnitSystem; import cwms.cda.api.enums.VersionType; -import cwms.cda.data.dao.rsql.FieldResolver; -import cwms.cda.data.dao.rsql.MapFieldResolver; -import cwms.cda.data.dao.rsql.RSQLConditionBuilder; import cwms.cda.data.dto.Catalog; import cwms.cda.data.dto.CwmsDTOPaginated; import cwms.cda.data.dto.RecentValue; @@ -47,12 +43,8 @@ import cwms.cda.data.dto.TsvId; import cwms.cda.data.dto.VerticalDatumInfo; import cwms.cda.data.dto.catalog.CatalogEntry; -import cwms.cda.data.dto.catalog.TimeSeriesAlias; import cwms.cda.data.dto.catalog.TimeseriesCatalogEntry; -import cwms.cda.data.dto.filteredtimeseries.FilteredTimeSeries; -import cwms.cda.formatters.FormattingException; import cwms.cda.formatters.xml.XMLv1; -import cwms.cda.helpers.DateUtils; import java.math.BigDecimal; import java.math.BigInteger; import java.sql.SQLException; @@ -72,6 +64,7 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Objects; import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.logging.Level; @@ -107,7 +100,6 @@ import usace.cwms.db.jooq.codegen.packages.CWMS_LOC_PACKAGE; import usace.cwms.db.jooq.codegen.packages.CWMS_TS_PACKAGE; import usace.cwms.db.jooq.codegen.packages.CWMS_UTIL_PACKAGE; -import usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID; import usace.cwms.db.jooq.codegen.tables.AV_LOC; import usace.cwms.db.jooq.codegen.tables.AV_LOC_GRP_ASSGN; import usace.cwms.db.jooq.codegen.tables.AV_TSV; @@ -445,7 +437,7 @@ protected TimeSeries getRequestedTimeSeries(String page, int pageSize, @NotNull // Fetch vertical datum info separately only when needed VerticalDatumInfo verticalDatumInfo = null; - if (parmPart != null && shouldFetchVerticalDatum(parmPart)) { + if (shouldFetchVerticalDatum(parmPart)) { verticalDatumInfo = fetchVerticalDatumInfoSeparately( locPart, units, office); } @@ -1455,9 +1447,6 @@ public void create(TimeSeries input) { public void create(TimeSeries input, boolean createAsLrts, StoreRule storeRule, boolean overrideProtection, VerticalDatum vd) { - int intervalForward = 0; - int intervalBackward = 0; - boolean activeFlag = true; Timestamp versionDate; if (input.getVersionDate() != null) { versionDate = Timestamp.from(input.getVersionDate().toInstant()); @@ -1492,10 +1481,9 @@ public void create(TimeSeries input, * @param targetDatum The desired ver * @param dslContext * @param cr - * @throws Throwable */ - private void withDefaultDatum(@Nullable VerticalDatum targetDatum, DSLContext dslContext, ConnectionRunnable cr) throws Throwable { - final String defaultVertDatum = CWMS_LOC_PACKAGE.call_GET_DEFAULT_VERTICAL_DATUM(dslContext.configuration()); + private void withDefaultDatum(@Nullable VerticalDatum targetDatum, DSLContext dslContext, ConnectionRunnable cr) { + String defaultVertDatum = CWMS_LOC_PACKAGE.call_GET_DEFAULT_VERTICAL_DATUM(dslContext.configuration()); String targetName = (targetDatum != null) ? targetDatum.toString() : null; boolean changeDefaultDatum = !Objects.equals(targetDatum, defaultVertDatum); try { @@ -1535,23 +1523,9 @@ private void storeWithDefaultDatum(TimeSeries input, boolean createAsLrts, Store versionDate, input.getValues(), createAsLrts, replaceAll, overrideProtection)); } - - /** - * - * @param dslContext A DSLContext that is already in the correct office - * @param officeId - * @param tsId - * @param units - * @param versionDate - * @param values - * @param createAsLrts - * @param storeRule - * @param overrideProtection - * @throws SQLException - */ private void store(DSLContext dslContext, String officeId, String tsId, String units, Timestamp versionDate, List values, boolean createAsLrts, - StoreRule storeRule, boolean overrideProtection) throws SQLException { + StoreRule storeRule, boolean overrideProtection) { final ZTSV_ARRAY tsvArray = new ZTSV_ARRAY(); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java index 1f6d78efb..c9d0c383b 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesVerticalDatumConverter.java @@ -8,6 +8,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Objects; +import java.util.Optional; public final class TimeSeriesVerticalDatumConverter { @@ -17,7 +18,8 @@ private TimeSeriesVerticalDatumConverter() { } public static TimeSeries convertToVerticalDatum(TimeSeries originalTimeSeries, VerticalDatum convertTo) { - if(originalTimeSeries.getVerticalDatumInfo() == null || Objects.equals(convertTo, getVerticalDatum(originalTimeSeries))) { + VerticalDatum vd = getVerticalDatum(originalTimeSeries).orElse(convertTo); + if(Objects.equals(convertTo, vd)) { return originalTimeSeries; //no conversion needed } TimeSeries retVal = originalTimeSeries; @@ -57,24 +59,17 @@ private static List applyOffsetToValues(Double offset, List getVerticalDatum(TimeSeries timeSeries) { + return Optional.ofNullable(timeSeries) + .map(TimeSeries::getVerticalDatumInfo) + .map(VerticalDatumInfo::getNativeDatum) + .filter(s -> !s.isEmpty()) + .map(s -> { + if (s.equalsIgnoreCase(VerticalDatum.OTHER.toString())) { throw new IllegalArgumentException("Vertical Datum of OTHER is not currently supported."); - } else { - retVal = VerticalDatum.getVerticalDatum(nativeDatum); } - } - } - } - - return retVal; + return VerticalDatum.getVerticalDatum(s); + }); } } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/VerticalDatum.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/VerticalDatum.java index 6574e9aa4..39aaef9b4 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/VerticalDatum.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/VerticalDatum.java @@ -1,11 +1,10 @@ package cwms.cda.data.dao; -import cwms.cda.data.dto.VerticalDatumInfo; - public enum VerticalDatum { NAVD88("NAVD88"), NGVD29("NGVD29"), - NATIVE("NATIVE"); + NATIVE("NATIVE"), + OTHER("OTHER"); private final String rule; diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index da7781b68..eada055a4 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -1840,8 +1840,6 @@ void test_get_for_elev_has_datum() throws Exception { .statusCode(is(HttpServletResponse.SC_OK)); - System.out.println("Data has been inserted for " + location); - // 1209654000000 as ms == Thursday, May 1, 2008 3:00:00 PM // get it back @@ -1865,7 +1863,6 @@ void test_get_for_elev_has_datum() throws Exception { .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - System.out.println(validatableResponse.extract().asString()); // verify that there is vertical-datum-info in the response. validatableResponse.body("vertical-datum-info", notNullValue()) .body("vertical-datum-info.location", equalTo(location)) @@ -1881,16 +1878,15 @@ void test_get_for_elev_has_datum() throws Exception { .queryParam(UNIT, "m") .queryParam(NAME, ts.get(NAME).asText()) .queryParam(BEGIN, firstPoint) - .when() + .when() .redirects().follow(true) .redirects().max(3) .get("/timeseries/") - .then() + .then() .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() + .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - System.out.println(validatableResponse.extract().asString()); // verify that there is vertical-datum-info in the response. validatableResponse.body("vertical-datum-info", notNullValue()) .body("vertical-datum-info.location", equalTo(location)) @@ -1907,16 +1903,15 @@ void test_get_for_elev_has_datum() throws Exception { .queryParam(NAME, ts.get(NAME).asText()) .queryParam(BEGIN, firstPoint) .queryParam(DATUM, "NAVD88") - .when() + .when() .redirects().follow(true) .redirects().max(3) .get("/timeseries/") - .then() + .then() .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() + .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); - System.out.println(validatableResponse.extract().asString()); // verify that there is vertical-datum-info in the response. validatableResponse.body("vertical-datum-info", notNullValue()) .body("vertical-datum-info.location", equalTo(location)) @@ -1940,13 +1935,13 @@ void test_get_for_elev_has_datum() throws Exception { .queryParam(NAME, ts.get(NAME).asText()) .queryParam(BEGIN, firstPoint) .queryParam(DATUM, "NGVD29") - .when() + .when() .redirects().follow(true) .redirects().max(3) .get("/timeseries/") - .then() + .then() .log().ifValidationFails(LogDetail.ALL, true) - .assertThat() + .assertThat() .statusCode(is(HttpServletResponse.SC_OK)); // verify that there is vertical-datum-info in the response. validatableResponseConverted.body("vertical-datum-info", notNullValue()) @@ -2016,6 +2011,10 @@ private void updateLocation(String location, boolean active, String officeId) th // @Test void test_create_without_vertical_datum_info() throws Exception { + if(getSchemaVersion() < SCHEMA_VERSION.LATEST_DEV.numeric()) + { + return; + } ObjectMapper mapper = new ObjectMapper(); InputStream resource = this.getClass().getResourceAsStream( @@ -2084,7 +2083,6 @@ void test_create_without_vertical_datum_info() throws Exception { // GET after scenario 1 and verify values equal input (NAVD88 native) ValidatableResponse vr1 = doGet.get(); - System.out.println("1:" + vr1.extract().asString()); /* Response includes "vertical-datum-info": { @@ -2130,7 +2128,6 @@ void test_create_without_vertical_datum_info() throws Exception { // GET after scenario 2 and verify values equal input; also capture NGVD29 offset ValidatableResponse vr2 = doGet.get(); - System.out.println("2:" + vr2.extract().asString()); vr2.body("values.size()", equalTo(inputValues.size())); for (int i = 0; i < inputValues.size(); i++) { vr2.body("values[" + i + "][1]", floatCloseTo(inputValues.get(i), 1e-6)); @@ -2178,7 +2175,6 @@ void test_create_without_vertical_datum_info() throws Exception { // GET after scenario 3 and verify the conversion was applied ValidatableResponse vr3 = doGet.get(); - System.out.println("3:" + vr3.extract().asString()); vr3.body("values.size()", equalTo(expectedNavd88.size())); for (int i = 0; i < expectedNavd88.size(); i++) { vr3.body("values[" + i + "][1]", floatCloseTo(expectedNavd88.get(i), 1e-4)); From 33259515b4d8d53b151e3de2831485036dc75154 Mon Sep 17 00:00:00 2001 From: Bryson Spilman Date: Mon, 1 Dec 2025 15:19:28 -0800 Subject: [PATCH 15/18] CDA-45 - fixed timeseries controller after rebase --- .../cwms/cda/api/TimeSeriesController.java | 47 +++++++++++++++++-- 1 file changed, 42 insertions(+), 5 deletions(-) diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java index 128d5fdfb..c5cba7c20 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java @@ -1,7 +1,46 @@ package cwms.cda.api; import static com.codahale.metrics.MetricRegistry.name; -import static cwms.cda.api.Controllers.*; +import static cwms.cda.api.Controllers.BEGIN; +import static cwms.cda.api.Controllers.CREATE; +import static cwms.cda.api.Controllers.CREATE_AS_LRTS; +import static cwms.cda.api.Controllers.CURSOR; +import static cwms.cda.api.Controllers.DATUM; +import static cwms.cda.api.Controllers.DELETE; +import static cwms.cda.api.Controllers.END; +import static cwms.cda.api.Controllers.END_TIME_INCLUSIVE; +import static cwms.cda.api.Controllers.FORMAT; +import static cwms.cda.api.Controllers.GET_ALL; +import static cwms.cda.api.Controllers.GET_ONE; +import static cwms.cda.api.Controllers.INCLUDE_ENTRY_DATE; +import static cwms.cda.api.Controllers.MAX_VERSION; +import static cwms.cda.api.Controllers.NAME; +import static cwms.cda.api.Controllers.NOT_SUPPORTED_YET; +import static cwms.cda.api.Controllers.OFFICE; +import static cwms.cda.api.Controllers.OVERRIDE_PROTECTION; +import static cwms.cda.api.Controllers.PAGE; +import static cwms.cda.api.Controllers.PAGE_SIZE; +import static cwms.cda.api.Controllers.RESULTS; +import static cwms.cda.api.Controllers.SIZE; +import static cwms.cda.api.Controllers.START_TIME_INCLUSIVE; +import static cwms.cda.api.Controllers.STATUS_200; +import static cwms.cda.api.Controllers.STATUS_400; +import static cwms.cda.api.Controllers.STATUS_404; +import static cwms.cda.api.Controllers.STATUS_501; +import static cwms.cda.api.Controllers.STORE_RULE; +import static cwms.cda.api.Controllers.TIMESERIES; +import static cwms.cda.api.Controllers.TIMEZONE; +import static cwms.cda.api.Controllers.TIME_FORMAT_DESC; +import static cwms.cda.api.Controllers.UNIT; +import static cwms.cda.api.Controllers.UNITS; +import static cwms.cda.api.Controllers.UPDATE; +import static cwms.cda.api.Controllers.VERSION; +import static cwms.cda.api.Controllers.VERSION_DATE; +import static cwms.cda.api.Controllers.addDeprecatedContentTypeWarning; +import static cwms.cda.api.Controllers.queryParamAsClass; +import static cwms.cda.api.Controllers.queryParamAsZdt; +import static cwms.cda.api.Controllers.requiredParam; +import static cwms.cda.api.Controllers.requiredZdt; import com.codahale.metrics.Histogram; import com.codahale.metrics.MetricRegistry; @@ -150,8 +189,8 @@ private Timer.Context markAndTime(String subject) { + "'True' or 'False', default is 'False'"), @OpenApiParam(name = STORE_RULE, type = StoreRule.class, description = STORE_RULE_DESC), @OpenApiParam(name = OVERRIDE_PROTECTION, type = Boolean.class, description = "A flag " - + "to ignore the protected data quality when storing data. 'True' or 'False'" + - ", default is " + TimeSeriesDaoImpl.OVERRIDE_PROTECTION), + + "to ignore the protected data quality when storing data. 'True' or 'False'" + + ", default is " + TimeSeriesDaoImpl.OVERRIDE_PROTECTION), @OpenApiParam(name = DATUM, type = VerticalDatum.class, description = "If the provided " + "time-series includes an explicit vertical-datum-info attribute " + "then it is assumed that the data is in the datum specified by the vertical-datum-info. " @@ -161,8 +200,6 @@ private Timer.Context markAndTime(String subject) { + "If the input timeseries does not include vertical-datum-info and " + "this parameter is provided it is assumed that the data is in the Datum named by the argument " + "and should be converted to the as-stored datum before being saved.") - + "to ignore the protected data quality when storing data. 'True' or 'False'" - + ", default is " + TimeSeriesDaoImpl.OVERRIDE_PROTECTION) }, method = HttpMethod.POST, path = "/timeseries", From 47ea71d11295febae0a57fc7c06c55e1eeabb8cf Mon Sep 17 00:00:00 2001 From: Bryson Spilman Date: Mon, 1 Dec 2025 15:26:01 -0800 Subject: [PATCH 16/18] CDA-45 - Added authorization heder to location IT to match develop --- .../src/test/java/cwms/cda/api/LocationControllerTestIT.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/LocationControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/LocationControllerTestIT.java index 15d21f749..630a26877 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/LocationControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/LocationControllerTestIT.java @@ -545,6 +545,7 @@ void test_create_update_null_elev_units() throws Exception { given() .log().ifValidationFails(LogDetail.ALL,true) .accept(Formats.JSON) + .header("Authorization", user.toHeaderValue()) .queryParam(OFFICE, user.getOperatingOffice()) .when() .redirects().follow(true) @@ -584,6 +585,7 @@ void test_create_update_null_elev_units() throws Exception { given() .log().ifValidationFails(LogDetail.ALL,true) .accept(Formats.JSON) + .header("Authorization", user.toHeaderValue()) .queryParam(OFFICE, user.getOperatingOffice()) .when() .redirects().follow(true) From c4ee10e43b00d6e5d25b4f5fadf43c998e335c7d Mon Sep 17 00:00:00 2001 From: Bryson Spilman Date: Tue, 2 Dec 2025 08:57:50 -0800 Subject: [PATCH 17/18] CDA-45 - Updates test that failed with datum query param as we now support it --- .../test/java/cwms/cda/api/TimeseriesControllerTestIT.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index eada055a4..9afacb5fa 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -1221,7 +1221,7 @@ void test_v1_cant_version() throws Exception { } @Test - void test_v2_cant_datum() throws Exception { + void test_v2_can_datum() throws Exception { InputStream resource = this.getClass().getResourceAsStream( "/cwms/cda/api/lrl/1day_offset.json"); assertNotNull(resource); @@ -1253,7 +1253,7 @@ void test_v2_cant_datum() throws Exception { .then() .log().ifValidationFails(LogDetail.ALL, true) .assertThat() - .statusCode(is(HttpServletResponse.SC_BAD_REQUEST)) + .statusCode(is(HttpServletResponse.SC_OK)) ; } From 7bf8f21aed0fc73a5e59d0fe46220003bf4ac7b1 Mon Sep 17 00:00:00 2001 From: Bryson Spilman Date: Tue, 2 Dec 2025 10:12:43 -0800 Subject: [PATCH 18/18] CDA-45 - Removed unnecessary authorization header in GET test. removed some hardcoded values in timeseries controller IT --- .../test/java/cwms/cda/api/LocationControllerTestIT.java | 2 -- .../test/java/cwms/cda/api/TimeseriesControllerTestIT.java | 6 +++--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/LocationControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/LocationControllerTestIT.java index 630a26877..15d21f749 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/LocationControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/LocationControllerTestIT.java @@ -545,7 +545,6 @@ void test_create_update_null_elev_units() throws Exception { given() .log().ifValidationFails(LogDetail.ALL,true) .accept(Formats.JSON) - .header("Authorization", user.toHeaderValue()) .queryParam(OFFICE, user.getOperatingOffice()) .when() .redirects().follow(true) @@ -585,7 +584,6 @@ void test_create_update_null_elev_units() throws Exception { given() .log().ifValidationFails(LogDetail.ALL,true) .accept(Formats.JSON) - .header("Authorization", user.toHeaderValue()) .queryParam(OFFICE, user.getOperatingOffice()) .when() .redirects().follow(true) diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java index 9afacb5fa..961a5ae92 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeseriesControllerTestIT.java @@ -1245,7 +1245,7 @@ void test_v2_can_datum() throws Exception { .queryParam(NAME, ts.get(NAME).asText()) .queryParam(BEGIN, firstPoint) .queryParam(END, firstPoint) - .queryParam("datum", "NAVD88") + .queryParam(DATUM, VerticalDatum.NAVD88.toString()) .when() .redirects().follow(true) .redirects().max(3) @@ -1902,7 +1902,7 @@ void test_get_for_elev_has_datum() throws Exception { .queryParam(UNIT, "m") .queryParam(NAME, ts.get(NAME).asText()) .queryParam(BEGIN, firstPoint) - .queryParam(DATUM, "NAVD88") + .queryParam(DATUM, VerticalDatum.NAVD88.toString()) .when() .redirects().follow(true) .redirects().max(3) @@ -2116,7 +2116,7 @@ void test_create_without_vertical_datum_info() throws Exception { .body(tsData) .header("Authorization", user.toHeaderValue()) .queryParam(OFFICE, officeId) - .queryParam(DATUM, "NAVD88") + .queryParam(DATUM, VerticalDatum.NAVD88.toString()) .when() .redirects().follow(true) .redirects().max(3)