diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesRecentController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesRecentController.java index da762699d..1aa56839d 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesRecentController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesRecentController.java @@ -179,7 +179,7 @@ public void handle(@NotNull Context ctx) { // just group provided latestValues = dao.findRecentsInRange(office, categoryId, groupId, pastLimit, futureLimit, unitSystem); } else { - latestValues = dao.findMostRecentsInRange(tsIds, pastLimit, futureLimit, unitSystem); + latestValues = dao.findMostRecentsInRange(office, tsIds, pastLimit, futureLimit, unitSystem); } String formatHeader = ctx.header(Header.ACCEPT); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java index 29be65bb2..4422f8ca5 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java @@ -35,7 +35,7 @@ String getTimeseries(String format, String names, String office, String unit, St List findRecentsInRange(String office, String categoryId, String groupId, Timestamp pastLimit, Timestamp futureLimit, UnitSystem unitSystem); - List findMostRecentsInRange(List tsIds, Timestamp pastLimit, + List findMostRecentsInRange(String office, List tsIds, Timestamp pastLimit, Timestamp futureLimit, UnitSystem unitSystem); } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index dce0f3344..f8e04be90 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -1,15 +1,10 @@ package cwms.cda.data.dao; import cwms.cda.helpers.DateUtils; -import static org.jooq.impl.DSL.asterisk; -import static org.jooq.impl.DSL.countDistinct; -import static org.jooq.impl.DSL.field; -import static org.jooq.impl.DSL.max; -import static org.jooq.impl.DSL.name; -import static org.jooq.impl.DSL.partitionBy; -import static org.jooq.impl.DSL.select; -import static org.jooq.impl.DSL.selectDistinct; -import static org.jooq.impl.DSL.table; + +import static org.jooq.impl.DSL.*; +import static org.jooq.impl.DSL.inline; +import static org.jooq.impl.DSL.unquotedName; import static usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID2.AV_CWMS_TS_ID2; import static usace.cwms.db.jooq.codegen.tables.AV_TS_EXTENTS_UTC.AV_TS_EXTENTS_UTC; @@ -38,10 +33,7 @@ import java.sql.Connection; import java.sql.SQLException; import java.sql.Timestamp; -import java.time.Duration; -import java.time.Instant; -import java.time.ZoneId; -import java.time.ZonedDateTime; +import java.time.*; import java.time.format.DateTimeFormatter; import java.util.ArrayList; import java.util.Arrays; @@ -57,25 +49,7 @@ import java.util.stream.Collectors; import org.jetbrains.annotations.NotNull; import org.jetbrains.annotations.Nullable; -import org.jooq.CommonTableExpression; -import org.jooq.Condition; -import org.jooq.DSLContext; -import org.jooq.Field; -import org.jooq.Record; -import org.jooq.Record1; -import org.jooq.Record3; -import org.jooq.Record4; -import org.jooq.Record7; -import org.jooq.Result; -import org.jooq.SQL; -import org.jooq.SelectConditionStep; -import org.jooq.SelectHavingStep; -import org.jooq.SelectJoinStep; -import org.jooq.SelectSeekStep2; -import org.jooq.Table; -import org.jooq.TableField; -import org.jooq.TableLike; -import org.jooq.TableOnConditionStep; +import org.jooq.*; import org.jooq.conf.ParamType; import org.jooq.exception.DataAccessException; import org.jooq.impl.DSL; @@ -84,18 +58,26 @@ import usace.cwms.db.jooq.codegen.packages.CWMS_LOC_PACKAGE; import usace.cwms.db.jooq.codegen.packages.CWMS_TS_PACKAGE; import usace.cwms.db.jooq.codegen.packages.CWMS_UTIL_PACKAGE; -import usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID; -import usace.cwms.db.jooq.codegen.tables.AV_LOC; -import usace.cwms.db.jooq.codegen.tables.AV_LOC_GRP_ASSGN; -import usace.cwms.db.jooq.codegen.tables.AV_TSV; -import usace.cwms.db.jooq.codegen.tables.AV_TSV_DQU; -import usace.cwms.db.jooq.codegen.tables.AV_TS_GRP_ASSGN; +import usace.cwms.db.jooq.codegen.tables.*; import usace.cwms.db.jooq.codegen.udt.records.ZTSV_ARRAY; import usace.cwms.db.jooq.codegen.udt.records.ZTSV_TYPE; public class TimeSeriesDaoImpl extends JooqDao implements TimeSeriesDao { private static final Logger logger = Logger.getLogger(TimeSeriesDaoImpl.class.getName()); + /** + * String constants for accessing alias table columns in FindMostRecentsRange + */ + private static final String DATE_TIME = "DATE_TIME"; + private static final String VERSION_DATE = "VERSION_DATE"; + private static final String DATA_ENTRY_DATE = "DATA_ENTRY_DATE"; + private static final String QUALITY_CODE = "QUALITY_CODE"; + private static final String START_DATE = "START_DATE"; + private static final String END_DATE = "END_DATE"; + private static final String TS_CODE = "TS_CODE"; + private static final String VALUE = "VALUE"; + private static final String CWMS_TS_ID = "CWMS_TS_ID"; + public static final boolean OVERRIDE_PROTECTION = true; public static final int TS_ID_MISSING_CODE = 20001; public static final String MAX_DATE_TIME = "max_date_time"; @@ -264,9 +246,9 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String ); // Give the TVQ (time, value, quality) columns names - Field dateTimeCol = field("DATE_TIME", Timestamp.class).as("DATE_TIME"); - Field valueCol = field("VALUE", Double.class).as("VALUE"); - Field qualityCol = field("QUALITY_CODE", Integer.class).as("QUALITY_CODE"); + Field dateTimeCol = field(DATE_TIME, Timestamp.class).as(DATE_TIME); + Field valueCol = field(VALUE, Double.class).as(VALUE); + Field qualityCol = field(QUALITY_CODE, Integer.class).as(QUALITY_CODE); Long beginTimeMilli = beginTime.toInstant().toEpochMilli(); Long endTimeMilli = endTime.toInstant().toEpochMilli(); @@ -379,7 +361,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String Field qualityNormCol = CWMS_TS_PACKAGE.call_NORMALIZE_QUALITY( DSL.nvl(qualityCol, DSL.inline(5))).as("QUALITY_NORM"); - Field dataEntryDate = field("DATA_ENTRY_DATE", Timestamp.class).as("data_entry_date"); + Field dataEntryDate = field(DATA_ENTRY_DATE, Timestamp.class).as(DATA_ENTRY_DATE); TimeSeries retVal = null; if (pageSize != 0) { @@ -915,82 +897,123 @@ private Tsv buildTsvFromViewRow(usace.cwms.db.jooq.codegen.tables.records.AV_TSV @Override - public List findMostRecentsInRange(List tsIds, Timestamp pastdate, + public List findMostRecentsInRange(String office, List tsIds, Timestamp pastdate, Timestamp futuredate, UnitSystem unitSystem) { List retval = Collections.emptyList(); if (tsIds != null && !tsIds.isEmpty()) { - String tsFieldName = "TSVIEW_CWMS_TS_ID"; - Field tsField = AV_CWMS_TS_ID2.CWMS_TS_ID.as(tsFieldName); + String tsFieldName = CWMS_TS_ID; + + // create baseIds alias + CommonTableExpression baseIds = name("base_ids").as( + selectDistinct(AV_CWMS_TS_ID2.TS_CODE, AV_CWMS_TS_ID2.CWMS_TS_ID) + .from(AV_CWMS_TS_ID2) + .where(AV_CWMS_TS_ID2.CWMS_TS_ID.in(tsIds))); + + // convert timestamp to date + java.sql.Date startDate = new java.sql.Date(pastdate.getTime()); + java.sql.Date endDate = new java.sql.Date(futuredate.getTime()); - Field maxDateField = max(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME) - .over(partitionBy(AV_TSV_DQU.AV_TSV_DQU.TS_CODE)) - .as(MAX_DATE_TIME); + // helper subquery for SELECT ts_code FROM base_ids + Select> tsCodeSubquery = select(baseIds.field(AV_CWMS_TS_ID2.TS_CODE)) + .from(baseIds); + + // references to appropriate year tables + Table AT_TSV_2023_TABLE = table(name("AT_TSV_2023")); + Table AT_TSV_2024_TABLE = table(name("AT_TSV_2024")); + + // create avTsvLimited alias + CommonTableExpression avTsvLimited = name("av_tsv_limited").as( + select(asterisk()) + .from(AT_TSV_2023_TABLE) + .where(field(name(AT_TSV_2023_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate)) + .and(field(name(AT_TSV_2023_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery)) + .unionAll( + select(asterisk()) + .from(AT_TSV_2024_TABLE) + .where(field(name(AT_TSV_2024_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate)) + .and(field(name(AT_TSV_2024_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery)) + ) + ); - Field defUnitsField = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( - CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(AV_TSV_DQU.AV_TSV_DQU.TS_CODE), - DSL.val(unitSystem, String.class)) + // Create table and field references for AT_TS_EXTENTS + Table AT_TS_EXTENTS_TABLE = table(name("AT_TS_EXTENTS")); + Field AT_TS_EXTENTS_TS_CODE = field(name("AT_TS_EXTENTS", TS_CODE), BigDecimal.class); + Field AT_TS_EXTENTS_VERSION_TIME = field(name("AT_TS_EXTENTS", "VERSION_TIME"), java.sql.Date.class); + Field AT_TS_EXTENTS_EARLIEST_ENTRY_TIME = field(name("AT_TS_EXTENTS", "EARLIEST_ENTRY_TIME"), Timestamp.class); + Field AT_TS_EXTENTS_LATEST_ENTRY_TIME = field(name("AT_TS_EXTENTS", "LATEST_ENTRY_TIME"), Timestamp.class); + + // Extract repeated TsCode and DateTime + Field avTsvLimitedTsCode = field(name(avTsvLimited.getName(), TS_CODE), BigDecimal.class); + Field avTsvLimitedDateTime = field(name(avTsvLimited.getName(), DATE_TIME), java.sql.Date.class); + + // create max_values alias + CommonTableExpression maxValues = name("max_values").as( + select( + field(name(baseIds.getName(), CWMS_TS_ID), String.class), + avTsvLimitedTsCode, + avTsvLimitedDateTime, + field(name(avTsvLimited.getName(), VALUE), BigDecimal.class), + field(name(avTsvLimited.getName(), VERSION_DATE), java.sql.Date.class), + field(name(avTsvLimited.getName(), DATA_ENTRY_DATE), java.sql.Date.class), + field(name(avTsvLimited.getName(), QUALITY_CODE), Integer.class), + AT_TS_EXTENTS_EARLIEST_ENTRY_TIME.as(START_DATE), + AT_TS_EXTENTS_LATEST_ENTRY_TIME.as(END_DATE), + max(avTsvLimitedDateTime) + .over(partitionBy(avTsvLimitedTsCode)) + .as(MAX_DATE_TIME) + ) + .from(avTsvLimited) + .join(baseIds).on(field(name(baseIds.getName(), TS_CODE), BigDecimal.class).equal(avTsvLimitedTsCode)) + .join(AT_TS_EXTENTS_TABLE).on( + AT_TS_EXTENTS_TS_CODE.equal(avTsvLimitedTsCode) + .and(AT_TS_EXTENTS_VERSION_TIME + .equal(field(name(avTsvLimited.getName(), VERSION_DATE), java.sql.Date.class))) + ) + ); + + Field getDefaultUnits = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( + CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name(maxValues.getName(), TS_CODE), BigDecimal.class)), + DSL.val(unitSystem, String.class)) .as(DEFAULT_UNITS); - SelectConditionStep innerSelect = dsl.select( - AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID, - AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID, - AV_TSV_DQU.AV_TSV_DQU.TS_CODE, - AV_TSV_DQU.AV_TSV_DQU.UNIT_ID, - AV_TSV_DQU.AV_TSV_DQU.DATE_TIME, - AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE, - AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE, - AV_TSV_DQU.AV_TSV_DQU.VALUE, - AV_TSV_DQU.AV_TSV_DQU.QUALITY_CODE, - AV_TSV_DQU.AV_TSV_DQU.START_DATE, - AV_TSV_DQU.AV_TSV_DQU.END_DATE, - defUnitsField, - maxDateField, - tsField + Field getSIUnits = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( + CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name(maxValues.getName(), TS_CODE), BigDecimal.class)), + DSL.val("SI", String.class)); + + Field getENUnits = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS( + CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name(maxValues.getName(), TS_CODE), BigDecimal.class)), + DSL.val("EN", String.class)); + Field value = field(name(maxValues.getName(), VALUE), Double.class); + + Field convertUnits = CWMS_UTIL_PACKAGE.call_CONVERT_UNITS( + value, + getSIUnits, + getENUnits + ); + + // Final query + ResultQuery query = dsl.with(baseIds) + .with(avTsvLimited) + .with(maxValues) + .select( + field(name(maxValues.getName(), CWMS_TS_ID), String.class), + field(name(maxValues.getName(), DATE_TIME), java.sql.Date.class), + field(name(maxValues.getName(), VERSION_DATE), java.sql.Date.class), + field(name(maxValues.getName(), DATA_ENTRY_DATE), java.sql.Date.class), + field(name(maxValues.getName(), QUALITY_CODE), Integer.class), + field(name(maxValues.getName(), START_DATE), java.sql.Date.class), + field(name(maxValues.getName(), END_DATE), java.sql.Date.class), + getDefaultUnits.as(DEFAULT_UNITS), + field(name(maxValues.getName(), DATE_TIME), java.sql.Date.class).as(MAX_DATE_TIME), + convertUnits.as("value_at_max_date") ) - .from(AV_TSV_DQU.AV_TSV_DQU.join(AV_CWMS_TS_ID2) - .on(AV_TSV_DQU.AV_TSV_DQU.TS_CODE.eq( - AV_CWMS_TS_ID2.TS_CODE.cast(Long.class)))) - .where( - AV_CWMS_TS_ID2.CWMS_TS_ID.in(tsIds) - .and(AV_TSV_DQU.AV_TSV_DQU.VALUE.isNotNull()) - .and(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME.lt(futuredate)) - .and(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME.gt(pastdate)) - .and(AV_TSV_DQU.AV_TSV_DQU.START_DATE.le(futuredate)) - .and(AV_TSV_DQU.AV_TSV_DQU.END_DATE.gt(pastdate))); - - // We want to use some of the fields from the innerSelect statement in our WHERE clause - // Its cleaner if we call them out individually. - Field dateTimeField = innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME); - Field unitField = innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.UNIT_ID); - - // We want to return fields from the innerSelect. - // Note: Although they are both fields, jOOQ treats - // innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE) - // differently than - // AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE - // Using the innerSelect field makes DATA_ENTRY_DATE correctly map to Timestamp - // and the generated sql refers to columns from the alias_??? table. - Field[] queryFields = new Field[]{ - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.TS_CODE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VALUE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.QUALITY_CODE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.START_DATE), - innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.END_DATE), - unitField, - dateTimeField, - innerSelect.field(tsField) - }; - - SelectConditionStep query = dsl.select(queryFields) - .from(innerSelect) - .where(dateTimeField.eq(maxDateField).and(unitField.eq(defUnitsField))); + .from(maxValues) + .where(field(name(maxValues.getName(), DATE_TIME), java.sql.Date.class) + .eq(field(name(maxValues.getName(), MAX_DATE_TIME), java.sql.Date.class))); logger.fine(() -> query.getSQL(ParamType.INLINED)); + // TODO: Update query.fetch() and function return since we no longer return an instance of RecentValue retval = query.fetch(r -> buildRecentValue(AV_TSV_DQU.AV_TSV_DQU, r, tsFieldName)); } return retval;