Skip to content

Fix Timeseries/recent endpoint speed #1183

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
wants to merge 9 commits into
base: develop
Choose a base branch
from
Draft

Fix Timeseries/recent endpoint speed #1183

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
9 commits
Select commit Hold shift + click to select a range
bbc0e22
Add office-id param to FindMostRecentsInRange, use as another filter
rohaan-rma Jul 8, 2025
eca1f9a
Make whereCondition field separate since office may be null
rohaan-rma Jul 8, 2025
5b24dc6
Merge branch 'develop' of https://github.com/USACE/cwms-data-api into…
rohaan-rma Jul 9, 2025
ddad365
Work in Progress on JOOQ for new query, added in lots of boilerplate,…
rohaan-rma Jul 9, 2025
e7cf33f
Added most of the new query logic using JOOQ
rohaan-rma Jul 10, 2025
ad996c7
Add JOOQ query except final query fetch
rohaan-rma Jul 10, 2025
20cd7e2
Merge branch 'develop' into timeseries-endpoint-speed
rohaan-rma Jul 10, 2025
885368a
Fix date imports
rohaan-rma Jul 10, 2025
c650cba
Fix field accessing issue
rohaan-rma Jul 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesRecentController.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ public void handle(@NotNull Context ctx) {
// just group provided
latestValues = dao.findRecentsInRange(office, categoryId, groupId, pastLimit, futureLimit, unitSystem);
} else {
latestValues = dao.findMostRecentsInRange(tsIds, pastLimit, futureLimit, unitSystem);
latestValues = dao.findMostRecentsInRange(office, tsIds, pastLimit, futureLimit, unitSystem);
}

String formatHeader = ctx.header(Header.ACCEPT);
Expand Down
2 changes: 1 addition & 1 deletion cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@ String getTimeseries(String format, String names, String office, String unit, St
List<RecentValue> findRecentsInRange(String office, String categoryId, String groupId,
Timestamp pastLimit, Timestamp futureLimit, UnitSystem unitSystem);

List<RecentValue> findMostRecentsInRange(List<String> tsIds, Timestamp pastLimit,
List<RecentValue> findMostRecentsInRange(String office, List<String> tsIds, Timestamp pastLimit,
Timestamp futureLimit, UnitSystem unitSystem);

}
237 changes: 130 additions & 107 deletions cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,15 +1,10 @@
package cwms.cda.data.dao;

import cwms.cda.helpers.DateUtils;
import static org.jooq.impl.DSL.asterisk;
import static org.jooq.impl.DSL.countDistinct;
import static org.jooq.impl.DSL.field;
import static org.jooq.impl.DSL.max;
import static org.jooq.impl.DSL.name;
import static org.jooq.impl.DSL.partitionBy;
import static org.jooq.impl.DSL.select;
import static org.jooq.impl.DSL.selectDistinct;
import static org.jooq.impl.DSL.table;

import static org.jooq.impl.DSL.*;
import static org.jooq.impl.DSL.inline;
import static org.jooq.impl.DSL.unquotedName;
import static usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID2.AV_CWMS_TS_ID2;
import static usace.cwms.db.jooq.codegen.tables.AV_TS_EXTENTS_UTC.AV_TS_EXTENTS_UTC;

Expand Down Expand Up @@ -38,10 +33,7 @@
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.*;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.Arrays;
Expand All @@ -57,25 +49,7 @@
import java.util.stream.Collectors;
import org.jetbrains.annotations.NotNull;
import org.jetbrains.annotations.Nullable;
import org.jooq.CommonTableExpression;
import org.jooq.Condition;
import org.jooq.DSLContext;
import org.jooq.Field;
import org.jooq.Record;
import org.jooq.Record1;
import org.jooq.Record3;
import org.jooq.Record4;
import org.jooq.Record7;
import org.jooq.Result;
import org.jooq.SQL;
import org.jooq.SelectConditionStep;
import org.jooq.SelectHavingStep;
import org.jooq.SelectJoinStep;
import org.jooq.SelectSeekStep2;
import org.jooq.Table;
import org.jooq.TableField;
import org.jooq.TableLike;
import org.jooq.TableOnConditionStep;
import org.jooq.*;
import org.jooq.conf.ParamType;
import org.jooq.exception.DataAccessException;
import org.jooq.impl.DSL;
Expand All @@ -84,18 +58,26 @@
import usace.cwms.db.jooq.codegen.packages.CWMS_LOC_PACKAGE;
import usace.cwms.db.jooq.codegen.packages.CWMS_TS_PACKAGE;
import usace.cwms.db.jooq.codegen.packages.CWMS_UTIL_PACKAGE;
import usace.cwms.db.jooq.codegen.tables.AV_CWMS_TS_ID;
import usace.cwms.db.jooq.codegen.tables.AV_LOC;
import usace.cwms.db.jooq.codegen.tables.AV_LOC_GRP_ASSGN;
import usace.cwms.db.jooq.codegen.tables.AV_TSV;
import usace.cwms.db.jooq.codegen.tables.AV_TSV_DQU;
import usace.cwms.db.jooq.codegen.tables.AV_TS_GRP_ASSGN;
import usace.cwms.db.jooq.codegen.tables.*;
import usace.cwms.db.jooq.codegen.udt.records.ZTSV_ARRAY;
import usace.cwms.db.jooq.codegen.udt.records.ZTSV_TYPE;

public class TimeSeriesDaoImpl extends JooqDao<TimeSeries> implements TimeSeriesDao {
private static final Logger logger = Logger.getLogger(TimeSeriesDaoImpl.class.getName());

/**
* String constants for accessing alias table columns in FindMostRecentsRange
*/
private static final String DATE_TIME = "DATE_TIME";
private static final String VERSION_DATE = "VERSION_DATE";
private static final String DATA_ENTRY_DATE = "DATA_ENTRY_DATE";
private static final String QUALITY_CODE = "QUALITY_CODE";
private static final String START_DATE = "START_DATE";
private static final String END_DATE = "END_DATE";
private static final String TS_CODE = "TS_CODE";
private static final String VALUE = "VALUE";
private static final String CWMS_TS_ID = "CWMS_TS_ID";

public static final boolean OVERRIDE_PROTECTION = true;
public static final int TS_ID_MISSING_CODE = 20001;
public static final String MAX_DATE_TIME = "max_date_time";
Expand Down Expand Up @@ -264,9 +246,9 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String
);

// Give the TVQ (time, value, quality) columns names
Field<Timestamp> dateTimeCol = field("DATE_TIME", Timestamp.class).as("DATE_TIME");
Field<Double> valueCol = field("VALUE", Double.class).as("VALUE");
Field<Integer> qualityCol = field("QUALITY_CODE", Integer.class).as("QUALITY_CODE");
Field<Timestamp> dateTimeCol = field(DATE_TIME, Timestamp.class).as(DATE_TIME);
Field<Double> valueCol = field(VALUE, Double.class).as(VALUE);
Field<Integer> qualityCol = field(QUALITY_CODE, Integer.class).as(QUALITY_CODE);

Long beginTimeMilli = beginTime.toInstant().toEpochMilli();
Long endTimeMilli = endTime.toInstant().toEpochMilli();
Expand Down Expand Up @@ -379,7 +361,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String

Field<BigDecimal> qualityNormCol = CWMS_TS_PACKAGE.call_NORMALIZE_QUALITY(
DSL.nvl(qualityCol, DSL.inline(5))).as("QUALITY_NORM");
Field<Timestamp> dataEntryDate = field("DATA_ENTRY_DATE", Timestamp.class).as("data_entry_date");
Field<Timestamp> dataEntryDate = field(DATA_ENTRY_DATE, Timestamp.class).as(DATA_ENTRY_DATE);

TimeSeries retVal = null;
if (pageSize != 0) {
Expand Down Expand Up @@ -915,82 +897,123 @@ private Tsv buildTsvFromViewRow(usace.cwms.db.jooq.codegen.tables.records.AV_TSV


@Override
public List<RecentValue> findMostRecentsInRange(List<String> tsIds, Timestamp pastdate,
public List<RecentValue> findMostRecentsInRange(String office, List<String> tsIds, Timestamp pastdate,
Timestamp futuredate, UnitSystem unitSystem) {
List<RecentValue> retval = Collections.emptyList();

if (tsIds != null && !tsIds.isEmpty()) {
String tsFieldName = "TSVIEW_CWMS_TS_ID";
Field<String> tsField = AV_CWMS_TS_ID2.CWMS_TS_ID.as(tsFieldName);
String tsFieldName = CWMS_TS_ID;

// create baseIds alias
CommonTableExpression<?> baseIds = name("base_ids").as(
selectDistinct(AV_CWMS_TS_ID2.TS_CODE, AV_CWMS_TS_ID2.CWMS_TS_ID)
.from(AV_CWMS_TS_ID2)
.where(AV_CWMS_TS_ID2.CWMS_TS_ID.in(tsIds)));

// convert timestamp to date
java.sql.Date startDate = new java.sql.Date(pastdate.getTime());
java.sql.Date endDate = new java.sql.Date(futuredate.getTime());

Field<Timestamp> maxDateField = max(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME)
.over(partitionBy(AV_TSV_DQU.AV_TSV_DQU.TS_CODE))
.as(MAX_DATE_TIME);
// helper subquery for SELECT ts_code FROM base_ids
Select<Record1<BigDecimal>> tsCodeSubquery = select(baseIds.field(AV_CWMS_TS_ID2.TS_CODE))
.from(baseIds);

// references to appropriate year tables
Table<?> AT_TSV_2023_TABLE = table(name("AT_TSV_2023"));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

These should be AT_TSV_PREV_YEAR and AT_TSV_CURRENT_YEAR and calculated at usage. Otherwise we'll have to remember to update them constantly.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yup you're correct. I just had those as place holders for now.

Table<?> AT_TSV_2024_TABLE = table(name("AT_TSV_2024"));

// create avTsvLimited alias
CommonTableExpression<?> avTsvLimited = name("av_tsv_limited").as(
select(asterisk())
.from(AT_TSV_2023_TABLE)
.where(field(name(AT_TSV_2023_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate))
.and(field(name(AT_TSV_2023_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery))
.unionAll(
select(asterisk())
.from(AT_TSV_2024_TABLE)
.where(field(name(AT_TSV_2024_TABLE.getName(), DATE_TIME), java.sql.Date.class).between(startDate, endDate))
.and(field(name(AT_TSV_2024_TABLE.getName(), TS_CODE), BigDecimal.class).in(tsCodeSubquery))
)
);

Field<String> defUnitsField = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS(
CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(AV_TSV_DQU.AV_TSV_DQU.TS_CODE),
DSL.val(unitSystem, String.class))
// Create table and field references for AT_TS_EXTENTS
Table<?> AT_TS_EXTENTS_TABLE = table(name("AT_TS_EXTENTS"));
Field<BigDecimal> AT_TS_EXTENTS_TS_CODE = field(name("AT_TS_EXTENTS", TS_CODE), BigDecimal.class);
Field<java.sql.Date> AT_TS_EXTENTS_VERSION_TIME = field(name("AT_TS_EXTENTS", "VERSION_TIME"), java.sql.Date.class);
Field<Timestamp> AT_TS_EXTENTS_EARLIEST_ENTRY_TIME = field(name("AT_TS_EXTENTS", "EARLIEST_ENTRY_TIME"), Timestamp.class);
Field<Timestamp> AT_TS_EXTENTS_LATEST_ENTRY_TIME = field(name("AT_TS_EXTENTS", "LATEST_ENTRY_TIME"), Timestamp.class);

// Extract repeated TsCode and DateTime
Field<BigDecimal> avTsvLimitedTsCode = field(name(avTsvLimited.getName(), TS_CODE), BigDecimal.class);
Field<java.sql.Date> avTsvLimitedDateTime = field(name(avTsvLimited.getName(), DATE_TIME), java.sql.Date.class);

// create max_values alias
CommonTableExpression<?> maxValues = name("max_values").as(
select(
field(name(baseIds.getName(), CWMS_TS_ID), String.class),
avTsvLimitedTsCode,
avTsvLimitedDateTime,
field(name(avTsvLimited.getName(), VALUE), BigDecimal.class),
field(name(avTsvLimited.getName(), VERSION_DATE), java.sql.Date.class),
field(name(avTsvLimited.getName(), DATA_ENTRY_DATE), java.sql.Date.class),
field(name(avTsvLimited.getName(), QUALITY_CODE), Integer.class),
AT_TS_EXTENTS_EARLIEST_ENTRY_TIME.as(START_DATE),
AT_TS_EXTENTS_LATEST_ENTRY_TIME.as(END_DATE),
max(avTsvLimitedDateTime)
.over(partitionBy(avTsvLimitedTsCode))
.as(MAX_DATE_TIME)
)
.from(avTsvLimited)
.join(baseIds).on(field(name(baseIds.getName(), TS_CODE), BigDecimal.class).equal(avTsvLimitedTsCode))
.join(AT_TS_EXTENTS_TABLE).on(
AT_TS_EXTENTS_TS_CODE.equal(avTsvLimitedTsCode)
.and(AT_TS_EXTENTS_VERSION_TIME
.equal(field(name(avTsvLimited.getName(), VERSION_DATE), java.sql.Date.class)))
)
);

Field<String> getDefaultUnits = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS(
CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name(maxValues.getName(), TS_CODE), BigDecimal.class)),
DSL.val(unitSystem, String.class))
.as(DEFAULT_UNITS);

SelectConditionStep<? extends Record> innerSelect = dsl.select(
AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID,
AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID,
AV_TSV_DQU.AV_TSV_DQU.TS_CODE,
AV_TSV_DQU.AV_TSV_DQU.UNIT_ID,
AV_TSV_DQU.AV_TSV_DQU.DATE_TIME,
AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE,
AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE,
AV_TSV_DQU.AV_TSV_DQU.VALUE,
AV_TSV_DQU.AV_TSV_DQU.QUALITY_CODE,
AV_TSV_DQU.AV_TSV_DQU.START_DATE,
AV_TSV_DQU.AV_TSV_DQU.END_DATE,
defUnitsField,
maxDateField,
tsField
Field<String> getSIUnits = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS(
CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name(maxValues.getName(), TS_CODE), BigDecimal.class)),
DSL.val("SI", String.class));

Field<String> getENUnits = CWMS_UTIL_PACKAGE.call_GET_DEFAULT_UNITS(
CWMS_TS_PACKAGE.call_GET_BASE_PARAMETER_ID(field(name(maxValues.getName(), TS_CODE), BigDecimal.class)),
DSL.val("EN", String.class));
Field<Double> value = field(name(maxValues.getName(), VALUE), Double.class);

Field<Double> convertUnits = CWMS_UTIL_PACKAGE.call_CONVERT_UNITS(
value,
getSIUnits,
getENUnits
);

// Final query
ResultQuery<?> query = dsl.with(baseIds)
.with(avTsvLimited)
.with(maxValues)
.select(
field(name(maxValues.getName(), CWMS_TS_ID), String.class),
field(name(maxValues.getName(), DATE_TIME), java.sql.Date.class),
field(name(maxValues.getName(), VERSION_DATE), java.sql.Date.class),
field(name(maxValues.getName(), DATA_ENTRY_DATE), java.sql.Date.class),
field(name(maxValues.getName(), QUALITY_CODE), Integer.class),
field(name(maxValues.getName(), START_DATE), java.sql.Date.class),
field(name(maxValues.getName(), END_DATE), java.sql.Date.class),
getDefaultUnits.as(DEFAULT_UNITS),
field(name(maxValues.getName(), DATE_TIME), java.sql.Date.class).as(MAX_DATE_TIME),
convertUnits.as("value_at_max_date")
)
.from(AV_TSV_DQU.AV_TSV_DQU.join(AV_CWMS_TS_ID2)
.on(AV_TSV_DQU.AV_TSV_DQU.TS_CODE.eq(
AV_CWMS_TS_ID2.TS_CODE.cast(Long.class))))
.where(
AV_CWMS_TS_ID2.CWMS_TS_ID.in(tsIds)
.and(AV_TSV_DQU.AV_TSV_DQU.VALUE.isNotNull())
.and(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME.lt(futuredate))
.and(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME.gt(pastdate))
.and(AV_TSV_DQU.AV_TSV_DQU.START_DATE.le(futuredate))
.and(AV_TSV_DQU.AV_TSV_DQU.END_DATE.gt(pastdate)));

// We want to use some of the fields from the innerSelect statement in our WHERE clause
// Its cleaner if we call them out individually.
Field<Timestamp> dateTimeField = innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.DATE_TIME);
Field<String> unitField = innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.UNIT_ID);

// We want to return fields from the innerSelect.
// Note: Although they are both fields, jOOQ treats
// innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE)
// differently than
// AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE
// Using the innerSelect field makes DATA_ENTRY_DATE correctly map to Timestamp
// and the generated sql refers to columns from the alias_??? table.
Field[] queryFields = new Field[]{
innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.CWMS_TS_ID),
innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.OFFICE_ID),
innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.TS_CODE),
innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE),
innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE),
innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.VALUE),
innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.QUALITY_CODE),
innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.START_DATE),
innerSelect.field(AV_TSV_DQU.AV_TSV_DQU.END_DATE),
unitField,
dateTimeField,
innerSelect.field(tsField)
};

SelectConditionStep<? extends Record> query = dsl.select(queryFields)
.from(innerSelect)
.where(dateTimeField.eq(maxDateField).and(unitField.eq(defUnitsField)));
.from(maxValues)
.where(field(name(maxValues.getName(), DATE_TIME), java.sql.Date.class)
.eq(field(name(maxValues.getName(), MAX_DATE_TIME), java.sql.Date.class)));

logger.fine(() -> query.getSQL(ParamType.INLINED));
// TODO: Update query.fetch() and function return since we no longer return an instance of RecentValue
retval = query.fetch(r -> buildRecentValue(AV_TSV_DQU.AV_TSV_DQU, r, tsFieldName));
}
return retval;
Expand Down
Loading