From be857f26881bfaeb7e106cb3d21c9ecf670d46fb Mon Sep 17 00:00:00 2001 From: ryan Date: Wed, 18 Jun 2025 14:29:41 -0700 Subject: [PATCH] Added filtered time series retrieval API. Introduced `TimeSeriesRequestParameters` and 'FilteredTimeSeriesParameters' to clean up TimeSeriesDaoImpl. Added RSQL query capability --- cwms-data-api/build.gradle | 2 + .../src/main/java/cwms/cda/ApiServlet.java | 23 +- .../main/java/cwms/cda/api/Controllers.java | 10 +- .../cda/api/LevelsAsTimeSeriesController.java | 12 +- .../cwms/cda/api/MeasurementController.java | 27 +- .../cwms/cda/api/TimeSeriesController.java | 122 +++--- .../cda/api/TimeSeriesFilteredController.java | 288 +++++++++++++ .../cwms/cda/api/rating/RatingController.java | 11 +- .../AccountingCatalogController.java | 6 +- .../dao/FilteredTimeSeriesParameters.java | 104 +++++ .../java/cwms/cda/data/dao/TimeSeriesDao.java | 22 + .../cwms/cda/data/dao/TimeSeriesDaoImpl.java | 200 +++++++-- .../data/dao/TimeSeriesRequestParameters.java | 128 ++++++ .../cwms/cda/data/dao/rsql/FieldResolver.java | 8 + .../cda/data/dao/rsql/JooqFieldResolver.java | 73 ++++ .../cda/data/dao/rsql/MapFieldResolver.java | 31 ++ .../data/dao/rsql/RSQLConditionBuilder.java | 54 +++ .../dao/rsql/RSQLJooqConditionVisitor.java | 131 ++++++ .../java/cwms/cda/data/dto/TimeSeries.java | 9 + .../FilteredTimeSeries.java | 61 +++ .../main/java/cwms/cda/helpers/DateUtils.java | 45 ++- cwms-data-api/src/main/webapp/rsql.html | 118 ++++++ cwms-data-api/src/main/webapp/times.html | 122 ++++++ .../cda/api/TimeSeriesControllerTest.java | 104 +---- .../TimeSeriesFilteredControllerTestIT.java | 382 ++++++++++++++++++ .../data/dao/rsql/MapFieldResolverTest.java | 74 ++++ .../dao/rsql/RSQLConditionBuilderTest.java | 187 +++++++++ .../rsql/RSQLJooqConditionVisitorTest.java | 325 +++++++++++++++ .../cda/data/dao/rsql/RSQLParserTest.java | 52 +++ .../FilteredTimeSeriesTest.java | 164 ++++++++ .../java/cwms/cda/helpers/DateUtilsTest.java | 35 +- .../filtered_timeseries_test.json | 33 ++ 32 files changed, 2688 insertions(+), 275 deletions(-) create mode 100644 cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesFilteredController.java create mode 100644 cwms-data-api/src/main/java/cwms/cda/data/dao/FilteredTimeSeriesParameters.java create mode 100644 cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesRequestParameters.java create mode 100644 cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/FieldResolver.java create mode 100644 cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/JooqFieldResolver.java create mode 100644 cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/MapFieldResolver.java create mode 100644 cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/RSQLConditionBuilder.java create mode 100644 cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/RSQLJooqConditionVisitor.java create mode 100644 cwms-data-api/src/main/java/cwms/cda/data/dto/filteredtimeseries/FilteredTimeSeries.java create mode 100644 cwms-data-api/src/main/webapp/rsql.html create mode 100644 cwms-data-api/src/main/webapp/times.html create mode 100644 cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesFilteredControllerTestIT.java create mode 100644 cwms-data-api/src/test/java/cwms/cda/data/dao/rsql/MapFieldResolverTest.java create mode 100644 cwms-data-api/src/test/java/cwms/cda/data/dao/rsql/RSQLConditionBuilderTest.java create mode 100644 cwms-data-api/src/test/java/cwms/cda/data/dao/rsql/RSQLJooqConditionVisitorTest.java create mode 100644 cwms-data-api/src/test/java/cwms/cda/data/dao/rsql/RSQLParserTest.java create mode 100644 cwms-data-api/src/test/java/cwms/cda/data/dto/filteredtimeseries/FilteredTimeSeriesTest.java create mode 100644 cwms-data-api/src/test/resources/cwms/cda/data/dto/filteredtimeseries/filtered_timeseries_test.json diff --git a/cwms-data-api/build.gradle b/cwms-data-api/build.gradle index 290ac598a..be5dbc5ed 100644 --- a/cwms-data-api/build.gradle +++ b/cwms-data-api/build.gradle @@ -110,6 +110,8 @@ dependencies { exclude group: "jakarta.xml.bind", module: "*" } + implementation 'cz.jirutka.rsql:rsql-parser:2.1.0' + compileOnly(libs.javaee.web.api) compileOnly(libs.cwms.tomcat.auth) diff --git a/cwms-data-api/src/main/java/cwms/cda/ApiServlet.java b/cwms-data-api/src/main/java/cwms/cda/ApiServlet.java index 2b66f1b86..6f0f7aacd 100644 --- a/cwms-data-api/src/main/java/cwms/cda/ApiServlet.java +++ b/cwms-data-api/src/main/java/cwms/cda/ApiServlet.java @@ -93,6 +93,7 @@ import cwms.cda.api.TimeSeriesGroupController; import cwms.cda.api.TimeSeriesIdentifierDescriptorController; import cwms.cda.api.TimeSeriesRecentController; +import cwms.cda.api.TimeSeriesFilteredController; import cwms.cda.api.TimeZoneController; import cwms.cda.api.TurbineChangesDeleteController; import cwms.cda.api.TurbineChangesGetController; @@ -321,7 +322,7 @@ public void init(ServletConfig config) throws ServletException { metrics = (MetricRegistry)config.getServletContext() .getAttribute(MetricsServlet.METRICS_REGISTRY); totalRequests = metrics.meter("cwms.dataapi.total_requests"); - + super.init(config); } @@ -552,6 +553,10 @@ protected void configureRoutes() { get(recentPath, new TimeSeriesRecentController(metrics)); addCacheControl(recentPath, 5, TimeUnit.MINUTES); + String filteredPath = "/timeseries/filtered"; + get(filteredPath, new TimeSeriesFilteredController(metrics)); + addCacheControl(filteredPath, 5, TimeUnit.MINUTES); + cdaCrudCache(format("/standard-text-id/{%s}", Controllers.STANDARD_TEXT_ID), new StandardTextController(metrics), requiredRoles,1, TimeUnit.DAYS); @@ -568,9 +573,9 @@ protected void configureRoutes() { addCacheControl(textBinaryValuePath, 1, TimeUnit.DAYS); String timeSeriesProfilePath = "/timeseries/profile/"; - get(format(timeSeriesProfilePath + "{%s}/{%s}", Controllers.LOCATION_ID, Controllers.PARAMETER_ID), + get(format( "%s{%s}/{%s}", timeSeriesProfilePath, Controllers.LOCATION_ID, Controllers.PARAMETER_ID), new TimeSeriesProfileController(metrics)); - delete(format(timeSeriesProfilePath + "/{%s}/{%s}", Controllers.LOCATION_ID, + delete(format( "%s/{%s}/{%s}", timeSeriesProfilePath, Controllers.LOCATION_ID, Controllers.PARAMETER_ID), new TimeSeriesProfileDeleteController(metrics), requiredRoles); get(format(timeSeriesProfilePath, Controllers.LOCATION_ID, Controllers.PARAMETER_ID), @@ -578,20 +583,20 @@ protected void configureRoutes() { post(timeSeriesProfilePath, new TimeSeriesProfileCreateController(metrics), requiredRoles); String timeSeriesProfileParserPath = "/timeseries/profile-parser/"; - get(format(timeSeriesProfileParserPath + "{%s}/{%s}/", Controllers.LOCATION_ID, + get(format( "%s{%s}/{%s}/", timeSeriesProfileParserPath, Controllers.LOCATION_ID, Controllers.PARAMETER_ID), new TimeSeriesProfileParserController(metrics)); post(timeSeriesProfileParserPath, new TimeSeriesProfileParserCreateController(metrics), requiredRoles); - delete(format(timeSeriesProfileParserPath + "{%s}/{%s}/", Controllers.LOCATION_ID, + delete(format( "%s{%s}/{%s}/", timeSeriesProfileParserPath, Controllers.LOCATION_ID, Controllers.PARAMETER_ID), new TimeSeriesProfileParserDeleteController(metrics), requiredRoles); get(timeSeriesProfileParserPath, new TimeSeriesProfileParserCatalogController(metrics)); String timeSeriesProfileInstancePath = "/timeseries/profile-instance/"; - get(format(timeSeriesProfileInstancePath + "{%s}/{%s}/{%s}/", Controllers.LOCATION_ID, + get(format("%s{%s}/{%s}/{%s}/", timeSeriesProfileInstancePath, Controllers.LOCATION_ID, Controllers.PARAMETER_ID, Controllers.VERSION), new TimeSeriesProfileInstanceController(metrics)); post(timeSeriesProfileInstancePath, new TimeSeriesProfileInstanceCreateController(metrics), requiredRoles); - delete(format(timeSeriesProfileInstancePath + "{%s}/{%s}/{%s}/", Controllers.LOCATION_ID, + delete(format("%s{%s}/{%s}/{%s}/", timeSeriesProfileInstancePath, Controllers.LOCATION_ID, Controllers.PARAMETER_ID, Controllers.VERSION), new TimeSeriesProfileInstanceDeleteController(metrics), requiredRoles); get(timeSeriesProfileInstancePath, new TimeSeriesProfileInstanceCatalogController(metrics)); @@ -629,7 +634,7 @@ protected void configureRoutes() { String measTimeExtents = measurements + "time-extents"; get(measTimeExtents,new MeasurementTimeExtentsGetController(metrics)); addCacheControl(measTimeExtents, 5, TimeUnit.MINUTES); - cdaCrudCache(format(measurements + "{%s}", LOCATION_ID), + cdaCrudCache(format( "%s{%s}", measurements, LOCATION_ID), new cwms.cda.api.MeasurementController(metrics), requiredRoles,5, TimeUnit.MINUTES); cdaCrudCache("/blobs/{blob-id}", new BlobController(metrics), requiredRoles,5, TimeUnit.MINUTES); @@ -939,7 +944,7 @@ private void getOpenApiOptions(JavalinConfig config) { String provider = CdaAccessManager.class.getSimpleName(); - + Components components = new Components(); final ArrayList secReqs = new ArrayList<>(); authenticator.getActiveProviders().forEach(identityProvider -> { diff --git a/cwms-data-api/src/main/java/cwms/cda/api/Controllers.java b/cwms-data-api/src/main/java/cwms/cda/api/Controllers.java index b8be777f1..fb4d5253d 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/Controllers.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/Controllers.java @@ -120,7 +120,6 @@ public final class Controllers { public static final String INTERVAL = "interval"; public static final String CATEGORY_ID = "category-id"; public static final String CATEGORY_ID_MASK = "category-id-mask"; - public static final String EXAMPLE_DATE = "2021-06-10T13:00:00-07:00"; public static final String VERSION_DATE = "version-date"; public static final String CREATE_AS_LRTS = "create-as-lrts"; @@ -160,7 +159,12 @@ public final class Controllers { public static final String REPLACE_ASSIGNED_LOCS = "replace-assigned-locs"; public static final String REPLACE_ASSIGNED_TS = "replace-assigned-ts"; public static final String TS_IDS = "ts-ids"; + + public static final String EXAMPLE_DATE = "2021-06-10T13:00:00-07:00"; public static final String DATE_FORMAT = "YYYY-MM-dd'T'hh:mm:ss[Z'['VV']']"; + public static final String TIME_FORMAT_DESC = "The format for this field is ISO 8601 extended" + + ", with optional offset and timezone, i.e., '" + DATE_FORMAT + "', e.g., '" + EXAMPLE_DATE + "'." ; + public static final String INCLUDE_ASSIGNED = "include-assigned"; public static final String ANY_MASK = "*"; public static final String OFFICE_MASK = "office-mask"; @@ -226,6 +230,10 @@ public final class Controllers { private static final String DEPRECATED_HEADER = "CWMS-DATA-Format-Deprecated"; private static final String DEPRECATED_TAB = "2024-11-01 TAB is not used often."; private static final String DEPRECATED_CSV = "2024-11-01 CSV is not used often."; + public static final String MIN_VALUE = "min-value"; + public static final String MAX_VALUE = "max-value"; + public static final String FILTER_NULLS = "filter-nulls"; + public static final String QUERY = "query"; static { diff --git a/cwms-data-api/src/main/java/cwms/cda/api/LevelsAsTimeSeriesController.java b/cwms-data-api/src/main/java/cwms/cda/api/LevelsAsTimeSeriesController.java index f4d277f7a..9927c069c 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/LevelsAsTimeSeriesController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/LevelsAsTimeSeriesController.java @@ -47,7 +47,6 @@ import java.time.ZoneId; import java.time.ZonedDateTime; -import static com.codahale.metrics.MetricRegistry.name; import static cwms.cda.api.Controllers.*; import static cwms.cda.data.dao.JooqDao.getDslContext; @@ -77,16 +76,13 @@ private Timer.Context markAndTime(String subject) { @OpenApiParam(name = BEGIN, description = "Specifies the " + "start of the time window for data to be included in the response. " + "If this field is not specified, any required time window begins 24" - + " hours prior to the specified or default end time. The format for " - + "this field is ISO 8601 extended, with optional offset and " - + "timezone, i.e., '" - + DATE_FORMAT + "', e.g., '" + EXAMPLE_DATE + "'."), + + " hours prior to the specified or default end time. " + + TIME_FORMAT_DESC), @OpenApiParam(name = END, description = "Specifies the " + "end of the time window for data to be included in the response. If" + " this field is not specified, any required time window ends at the" - + " current time. The format for this field is ISO 8601 extended, " - + "with optional timezone, i.e., '" - + DATE_FORMAT + "', e.g., '" + EXAMPLE_DATE + "'."), + + " current time. " + + TIME_FORMAT_DESC), @OpenApiParam(name = TIMEZONE, description = "Specifies " + "the time zone of the values of the begin and end fields (unless " + "otherwise specified), as well as the time zone of any times in the" diff --git a/cwms-data-api/src/main/java/cwms/cda/api/MeasurementController.java b/cwms-data-api/src/main/java/cwms/cda/api/MeasurementController.java index 8f2607176..90a72bb21 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/MeasurementController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/MeasurementController.java @@ -30,9 +30,7 @@ import static cwms.cda.api.Controllers.AGENCY; import static cwms.cda.api.Controllers.BEGIN; import static cwms.cda.api.Controllers.CREATE; -import static cwms.cda.api.Controllers.DATE_FORMAT; import static cwms.cda.api.Controllers.DELETE; -import static cwms.cda.api.Controllers.EXAMPLE_DATE; import static cwms.cda.api.Controllers.FAIL_IF_EXISTS; import static cwms.cda.api.Controllers.GET_ALL; import static cwms.cda.api.Controllers.GET_ONE; @@ -50,6 +48,7 @@ import static cwms.cda.api.Controllers.OFFICE_MASK; import static cwms.cda.api.Controllers.QUALITY; import static cwms.cda.api.Controllers.TIMEZONE; +import static cwms.cda.api.Controllers.TIME_FORMAT_DESC; import static cwms.cda.api.Controllers.UNIT_SYSTEM; import static cwms.cda.api.Controllers.queryParamAsDouble; import static cwms.cda.api.Controllers.queryParamAsInstant; @@ -100,14 +99,10 @@ private Timer.Context markAndTime(String subject) { @OpenApiParam(name = ID_MASK, description = "Location id mask for filtering measurements. Use null to retrieve measurements for all locations."), @OpenApiParam(name = MIN_NUMBER, description = "Minimum measurement number-id for filtering measurements."), @OpenApiParam(name = MAX_NUMBER, description = "Maximum measurement number-id for filtering measurements."), - @OpenApiParam(name = BEGIN, description = "The start of the time " - + "window to delete. The format for this field is ISO 8601 extended, with " - + "optional offset and timezone, i.e., '" + DATE_FORMAT + "', e.g., '" - + EXAMPLE_DATE + "'. A null value is treated as an unbounded start."), - @OpenApiParam(name = END, description = "The end of the time " - + "window to delete.The format for this field is ISO 8601 extended, with " - + "optional offset and timezone, i.e., '" + DATE_FORMAT + "', e.g., '" - + EXAMPLE_DATE + "'.A null value is treated as an unbounded end."), + @OpenApiParam(name = BEGIN, description = "The start of the time window to delete. " + + TIME_FORMAT_DESC + " A null value is treated as an unbounded start."), + @OpenApiParam(name = END, description = "The end of the time window to delete." + + TIME_FORMAT_DESC + " A null value is treated as an unbounded end."), @OpenApiParam(name = TIMEZONE, description = "This field specifies a default timezone " + "to be used if the format of the " + BEGIN + "and " + END + " parameters do not include offset or time zone information. " @@ -226,14 +221,10 @@ public void update(@NotNull Context ctx, @NotNull String locationId) { }, queryParams = { @OpenApiParam(name = OFFICE, required = true, description = "Specifies the office of the measurements to delete"), - @OpenApiParam(name = BEGIN, required = true, description = "The start of the time " - + "window to delete. The format for this field is ISO 8601 extended, with " - + "optional offset and timezone, i.e., '" + DATE_FORMAT + "', e.g., '" - + EXAMPLE_DATE + "'."), - @OpenApiParam(name = END, required = true, description = "The end of the time " - + "window to delete.The format for this field is ISO 8601 extended, with " - + "optional offset and timezone, i.e., '" + DATE_FORMAT + "', e.g., '" - + EXAMPLE_DATE + "'."), + @OpenApiParam(name = BEGIN, required = true, description = "The start of the time window to delete. " + + TIME_FORMAT_DESC), + @OpenApiParam(name = END, required = true, description = "The end of the time window to delete." + + TIME_FORMAT_DESC), @OpenApiParam(name = TIMEZONE, description = "This field specifies a default timezone " + "to be used if the format of the " + BEGIN + "and " + END + " parameters do not include offset or time zone information. " diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java index 32daf9aab..66092b4f5 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesController.java @@ -14,6 +14,7 @@ import cwms.cda.data.dao.TimeSeriesDao; import cwms.cda.data.dao.TimeSeriesDaoImpl; import cwms.cda.data.dao.TimeSeriesDeleteOptions; +import cwms.cda.data.dao.TimeSeriesRequestParameters; import cwms.cda.data.dto.TimeSeries; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; @@ -31,17 +32,17 @@ import io.javalin.plugin.openapi.annotations.OpenApiResponse; import java.io.IOException; import java.io.StringWriter; -import java.io.UnsupportedEncodingException; -import java.net.URLEncoder; +import java.net.URISyntaxException; import java.nio.charset.StandardCharsets; import java.sql.Timestamp; import java.time.ZoneId; import java.time.ZonedDateTime; -import java.time.format.DateTimeFormatter; import java.util.logging.Level; import java.util.logging.Logger; import javax.servlet.http.HttpServletResponse; import org.apache.commons.io.IOUtils; +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.client.utils.URLEncodedUtils; import org.jetbrains.annotations.NotNull; import org.jooq.DSLContext; import org.jooq.exception.DataAccessException; @@ -193,13 +194,9 @@ protected TimeSeriesDao getTimeSeriesDao(DSLContext dsl) { @OpenApiParam(name = OFFICE, required = true, description = "Specifies the office of " + "the timeseries to be deleted."), @OpenApiParam(name = BEGIN, required = true, description = "The start of the time " - + "window to delete. The format for this field is ISO 8601 extended, with " - + "optional offset and timezone, i.e., '" + DATE_FORMAT + "', e.g., '" - + EXAMPLE_DATE + "'."), + + "window to delete. " + TIME_FORMAT_DESC), @OpenApiParam(name = END, required = true, description = "The end of the time " - + "window to delete.The format for this field is ISO 8601 extended, with " - + "optional offset and timezone, i.e., '" + DATE_FORMAT + "', e.g., '" - + EXAMPLE_DATE + "'."), + + "window to delete. " + TIME_FORMAT_DESC), @OpenApiParam(name = TIMEZONE, description = "This field specifies a default timezone " + "to be used if the format of the " + BEGIN + ", " + END + ", or " + VERSION_DATE + " parameters do not include offset or time zone information. " @@ -279,7 +276,7 @@ public void delete(@NotNull Context ctx, @NotNull String timeseries) { @OpenApi( queryParams = { @OpenApiParam(name = NAME, required = true, description = "Specifies the " - + "name(s) of the time series whose data is to be included in the " + + "name of the time series whose data is to be included in the " + "response. A case insensitive comparison is used to match names."), @OpenApiParam(name = OFFICE, description = "Specifies the" + " owning office of the time series(s) whose data is to be included " @@ -310,9 +307,8 @@ public void delete(@NotNull Context ctx, @NotNull String timeseries) { + "\n* `Other` Any units returned in the response to the units URI " + "request that is appropriate for the requested parameters."), @OpenApiParam(name = VERSION_DATE, description = "Specifies the version date of a " - + "time series trace to be selected. The format for this field is ISO 8601 " - + "extended, i.e., 'format', e.g., '2021-06-10T13:00:00-0700' .If field is " - + "empty, query will return a max aggregate for the timeseries. " + + "time series trace to be selected. " + TIME_FORMAT_DESC + + "If field is empty, query will return a max aggregate for the timeseries. " + "Only supported for:" + Formats.JSONV2 + " and " + Formats.XMLV2), @OpenApiParam(name = DATUM, description = "Specifies the " + "elevation datum of the response. This field affects only elevation" @@ -325,16 +321,13 @@ public void delete(@NotNull Context ctx, @NotNull String timeseries) { @OpenApiParam(name = BEGIN, description = "Specifies the " + "start of the time window for data to be included in the response. " + "If this field is not specified, any required time window begins 24" - + " hours prior to the specified or default end time. The format for " - + "this field is ISO 8601 extended, with optional offset and " - + "timezone, i.e., '" - + DATE_FORMAT + "', e.g., '" + EXAMPLE_DATE + "'."), + + " hours prior to the specified or default end time. " + + TIME_FORMAT_DESC), @OpenApiParam(name = END, description = "Specifies the " + "end of the time window for data to be included in the response. If" + " this field is not specified, any required time window ends at the" - + " current time. The format for this field is ISO 8601 extended, " - + "with optional timezone, i.e., '" - + DATE_FORMAT + "', e.g., '" + EXAMPLE_DATE + "'."), + + " current time. " + + TIME_FORMAT_DESC), @OpenApiParam(name = TIMEZONE, description = "Specifies " + "the time zone of the values of the begin and end fields (unless " + "otherwise specified). " @@ -446,26 +439,24 @@ public void getAll(@NotNull Context ctx) { } String office = requiredParam(ctx, OFFICE); - TimeSeries ts = dao.getTimeseries(cursor, pageSize, names, office, units, - beginZdt, endZdt, versionDate, trim.getOrDefault(true), includeEntryDate); + TimeSeriesRequestParameters requestParameters = new TimeSeriesRequestParameters.Builder() + .withNames(names) + .withOffice(office) + .withUnits(units) + .withBeginTime(beginZdt) + .withEndTime(endZdt) + .withVersionDate(versionDate) + .withShouldTrim(trim.getOrDefault(true)) + .withIncludeEntryDate(includeEntryDate) + .build(); + TimeSeries ts = dao.getTimeseries(cursor, pageSize, requestParameters); results = Formats.format(contentType, ts); ctx.status(HttpServletResponse.SC_OK); - // Send back the link to the next page in the response header - StringBuilder linkValue = new StringBuilder(600); - linkValue.append(String.format("<%s>; rel=self; type=\"%s\"", - buildRequestUrl(ctx, ts, ts.getPage()), contentType)); + addLinkHeader(ctx, ts, contentType); - if (ts.getNextPage() != null) { - linkValue.append(","); - linkValue.append(String.format("<%s>; rel=next; type=\"%s\"", - buildRequestUrl(ctx, ts, ts.getNextPage()), - contentType)); - } - - ctx.header("Link", linkValue.toString()); ctx.result(results).contentType(contentType.toString()); } else { if (versionDate != null) { @@ -502,6 +493,28 @@ public void getAll(@NotNull Context ctx) { } } + private void addLinkHeader(@NotNull Context ctx, TimeSeries ts, ContentType contentType) { + try { + // Send back the link to the next page in the response header + StringBuilder linkValue = new StringBuilder(600); + String pageUrl = buildRequestUrl(ctx, ts, ts.getPage()); + linkValue.append(String.format("<%s>; rel=self; type=\"%s\"", + pageUrl, contentType)); + + if (ts.getNextPage() != null) { + linkValue.append(","); + String nextPageUrl = buildRequestUrl(ctx, ts, ts.getNextPage()); + linkValue.append(String.format("<%s>; rel=next; type=\"%s\"", + nextPageUrl, + contentType)); + } + + ctx.header("Link", linkValue.toString()); + } catch (URISyntaxException ex) { + logger.log(Level.WARNING, null, ex); + } + } + @OpenApi(ignore = true) @Override public void getOne(@NotNull Context ctx, @NotNull String id) { @@ -581,35 +594,20 @@ private TimeSeries deserializeTimeSeries(Context ctx) throws IOException { * @param ts the TimeSeries object that was used to generate the result * @return a URL that references the same query, but with a different "page" parameter */ - private String buildRequestUrl(Context ctx, TimeSeries ts, String cursor) { - StringBuffer result = ctx.req.getRequestURL(); - try { - result.append(String.format("?name=%s", URLEncoder.encode(ts.getName(), - StandardCharsets.UTF_8.toString()))); - result.append(String.format("&office=%s", URLEncoder.encode(ts.getOfficeId(), - StandardCharsets.UTF_8.toString()))); - result.append(String.format("&unit=%s", URLEncoder.encode(ts.getUnits(), - StandardCharsets.UTF_8.toString()))); - result.append(String.format("&begin=%s", - URLEncoder.encode(ts.getBegin().format(DateTimeFormatter.ISO_ZONED_DATE_TIME), - StandardCharsets.UTF_8.toString()))); - result.append(String.format("&end=%s", - URLEncoder.encode(ts.getEnd().format(DateTimeFormatter.ISO_ZONED_DATE_TIME), - StandardCharsets.UTF_8.toString()))); - - String format = ctx.queryParam(FORMAT); - if (format != null && !format.isEmpty()) { - result.append(String.format("&format=%s", format)); - } + public String buildRequestUrl(Context ctx, TimeSeries ts, String cursor) throws URISyntaxException { + URIBuilder builder = new URIBuilder(ctx.req.getRequestURL().toString()); // requestURL stops just before '?' - if (cursor != null && !cursor.isEmpty()) { - result.append(String.format("&page=%s", URLEncoder.encode(cursor, - StandardCharsets.UTF_8.toString()))); - } - } catch (UnsupportedEncodingException ex) { - // We shouldn't get here - logger.log(Level.WARNING, null, ex); + // Instead of adding specific parameters and risk forgetting to add one to this method + // Lets add all the previous parameters and then (cont.) + builder.setParameters(URLEncodedUtils.parse(ctx.req.getQueryString(), StandardCharsets.UTF_8)); + + // (cont.) override or add the page parameter with the new cursor value + if (cursor != null && !cursor.isEmpty()) { + builder.setParameter("page", cursor); } - return result.toString(); + + return builder.build().toString(); } + + } diff --git a/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesFilteredController.java b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesFilteredController.java new file mode 100644 index 000000000..697ad411b --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/api/TimeSeriesFilteredController.java @@ -0,0 +1,288 @@ +package cwms.cda.api; + +import static com.codahale.metrics.MetricRegistry.name; +import static cwms.cda.api.Controllers.*; + + +import com.codahale.metrics.Histogram; +import com.codahale.metrics.MetricRegistry; +import com.codahale.metrics.Timer; +import cwms.cda.api.enums.UnitSystem; +import cwms.cda.api.errors.CdaError; +import cwms.cda.api.errors.NotFoundException; +import cwms.cda.data.dao.FilteredTimeSeriesParameters; +import cwms.cda.data.dao.JooqDao; +import cwms.cda.data.dao.TimeSeriesDao; +import cwms.cda.data.dao.TimeSeriesDaoImpl; +import cwms.cda.data.dao.TimeSeriesRequestParameters; +import cwms.cda.data.dto.TimeSeries; + +import cwms.cda.data.dto.filteredtimeseries.FilteredTimeSeries; +import cwms.cda.formatters.ContentType; +import cwms.cda.formatters.Formats; +import cwms.cda.helpers.DateUtils; +import io.javalin.core.util.Header; +import io.javalin.core.validation.Validator; +import io.javalin.http.Context; +import io.javalin.http.Handler; +import io.javalin.plugin.openapi.annotations.HttpMethod; +import io.javalin.plugin.openapi.annotations.OpenApi; +import io.javalin.plugin.openapi.annotations.OpenApiContent; +import io.javalin.plugin.openapi.annotations.OpenApiParam; +import io.javalin.plugin.openapi.annotations.OpenApiResponse; +import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.util.logging.Level; +import java.util.logging.Logger; + +import javax.servlet.http.HttpServletResponse; + +import org.apache.http.client.utils.URIBuilder; +import org.apache.http.client.utils.URLEncodedUtils; +import org.jetbrains.annotations.NotNull; +import org.jooq.DSLContext; + +public class TimeSeriesFilteredController implements Handler { + private static final Logger logger = Logger.getLogger(TimeSeriesFilteredController.class.getName()); + public static final String TAG = "TimeSeries"; + + private static final int DEFAULT_PAGE_SIZE = 500; + + private final MetricRegistry metrics; + private final Histogram requestResultSize; + + public TimeSeriesFilteredController(MetricRegistry metrics) { + this.metrics = metrics; + String className = this.getClass().getName(); + requestResultSize = this.metrics.histogram((name(className, RESULTS, SIZE))); + } + + private Timer.Context markAndTime(String subject) { + return Controllers.markAndTime(metrics, getClass().getName(), subject); + } + + private DSLContext getDslContext(Context ctx) { + return JooqDao.getDslContext(ctx); + } + + private TimeSeriesDao getTimeSeriesDao(DSLContext dsl) { + return new TimeSeriesDaoImpl(dsl, metrics); + } + + @OpenApi( + queryParams = { + @OpenApiParam(name = NAME, required = true, description = "Specifies the " + + "name of the time series whose data is to be included in the " + + "response. A case insensitive comparison is used to match names."), + @OpenApiParam(name = OFFICE, description = "Specifies the" + + " owning office of the time series(s) whose data is to be included " + + "in the response. " + + "Required for:" + Formats.JSONV2 + " and " + Formats.XMLV2 + ". " + + "For other formats, if this field is not specified, matching location " + + "level information from all offices shall be returned."), + @OpenApiParam(name = UNIT, description = "Specifies the " + + "unit or unit system of the response. Valid values for the unit " + + "field are: " + + "\n* `EN` (default) Specifies English unit system. " + + "Location level values will be in the default English units for " + + "their parameters." + + "\n* `SI` Specifies the SI unit system. " + + "Location level values will be in the default SI units for their " + + "parameters." + + "\n* `Other` Any unit returned in the response to the units URI " + + "request that is appropriate for the requested parameters."), + @OpenApiParam(name = VERSION_DATE, description = "Specifies the version date of a " + + "time series trace to be selected. " + + TIME_FORMAT_DESC + + " If field is empty, query will return a max aggregate for the timeseries. " + + "Only supported for:" + Formats.JSONV2 + " and " + Formats.XMLV2), + @OpenApiParam(name = BEGIN, description = "Specifies the " + + "start of the time window for data to be included in the response. " + + "If this field is not specified, any required time window begins 24" + + " hours prior to the specified or default end time. " + + TIME_FORMAT_DESC), + @OpenApiParam(name = END, description = "Specifies the " + + "end of the time window for data to be included in the response. If" + + " this field is not specified, any required time window ends at the" + + " current time. " + + TIME_FORMAT_DESC), + @OpenApiParam(name = TIMEZONE, description = "Specifies " + + "the time zone of the values of the begin and end fields (unless " + + "otherwise specified). " + + "For " + Formats.JSONV2 + " and " + Formats.XMLV2 + + " the results are returned in UTC. For other formats this parameter " + + "affects the time zone of times in the " + + "response. If this field is not specified, the default time zone " + + "of UTC shall be used.\r\nIgnored if begin was specified with " + + "offset and timezone."), + @OpenApiParam(name = Controllers.TRIM, type = Boolean.class, description = "Specifies " + + "whether to trim missing values from the beginning and end of the " + + "retrieved values. " + + "Only supported for:" + Formats.JSONV2 + " and " + Formats.XMLV2 + ". " + + "Default is true."), + @OpenApiParam(name = INCLUDE_ENTRY_DATE, type = Boolean.class, description = "Specifies " + + "whether to include the data entry date of each value in the response. Including the data entry " + + "date will increase the size of the array containing each data value from three to four, " + + "changing the format of the response. Default is false."), + @OpenApiParam(name = MIN_VALUE, type = Double.class, description = "Specifies " + + "the minimum value to include in the results. Values below this threshold will be excluded."), + @OpenApiParam(name = MAX_VALUE, type = Double.class, description = "Specifies " + + "the maximum value to include in the results. Values above this threshold will be excluded."), + @OpenApiParam(name = FILTER_NULLS, type = Boolean.class, description = "Specifies " + + "whether to exclude null values from the results. Default is false."), + @OpenApiParam(name = QUERY, description = "Specifies " + + "an RSQL-like query string to filter the results. " + + "Expressions may reference \"value, datetime, quality, data_entry_date\""), + @OpenApiParam(name = PAGE, description = "This end point can return large amounts " + + "of data as a series of pages. This parameter is used to describes the " + + "current location in the response stream. This is an opaque " + + "value, and can be obtained from the 'next-page' value in the response."), + @OpenApiParam(name = PAGE_SIZE, + type = Integer.class, + description = "How many entries per page returned. " + + "Default " + DEFAULT_PAGE_SIZE + ".") + }, + responses = { + @OpenApiResponse(status = STATUS_200, + description = "A list of elements of the data set you've selected.", + content = { + @OpenApiContent(from = TimeSeries.class, type = Formats.JSONV2), + @OpenApiContent(from = TimeSeries.class, type = Formats.XMLV2), + @OpenApiContent(from = TimeSeries.class, type = Formats.XML), + @OpenApiContent(from = TimeSeries.class, type = Formats.JSON), + @OpenApiContent(from = TimeSeries.class, type = ""),}), + @OpenApiResponse(status = STATUS_400, description = "Invalid parameter combination"), + @OpenApiResponse(status = STATUS_404, description = "The provided combination of " + + "parameters did not find a timeseries."), + @OpenApiResponse(status = STATUS_501, description = "Requested format is not " + + "implemented") + }, + method = HttpMethod.GET, + path = "/timeseries", + tags = TAG + ) + @Override + public void handle(@NotNull Context ctx) { + + try (final Timer.Context ignored = markAndTime(GET_ALL)) { + DSLContext dsl = getDslContext(ctx); + + TimeSeriesDao dao = getTimeSeriesDao(dsl); + String format = ""; + String names = requiredParam(ctx, NAME); + + String unit = ctx.queryParamAsClass(UNIT, String.class) + .getOrDefault(UnitSystem.EN.getValue()); + + String begin = ctx.queryParam(BEGIN); + String end = ctx.queryParam(END); + String timezone = ctx.queryParamAsClass(TIMEZONE, String.class) + .getOrDefault("UTC"); + Validator trim = ctx.queryParamAsClass(Controllers.TRIM, Boolean.class); + + ZonedDateTime versionDate = queryParamAsZdt(ctx, VERSION_DATE); + + boolean includeEntryDate = ctx.queryParamAsClass(INCLUDE_ENTRY_DATE, Boolean.class) + .getOrDefault(false); + + // The following parameters are only used for jsonv2 and xmlv2 + String cursor = queryParamAsClass(ctx, new String[]{PAGE, CURSOR}, + String.class, "", metrics, name(TimeSeriesController.class.getName(), + GET_ALL)); + + int pageSize = queryParamAsClass(ctx, new String[]{PAGE_SIZE}, + Integer.class, DEFAULT_PAGE_SIZE, metrics, + name(TimeSeriesController.class.getName(), GET_ALL)); + + String acceptHeader = ctx.header(Header.ACCEPT); + ContentType contentType = Formats.parseHeaderAndQueryParm(acceptHeader, format, TimeSeries.class); + + ZoneId tz = ZoneId.of(timezone, ZoneId.SHORT_IDS); + begin = begin != null ? begin : "PT-24H"; + + ZonedDateTime beginZdt = DateUtils.parseUserDate(begin, timezone); + ZonedDateTime endZdt = end != null + ? DateUtils.parseUserDate(end, timezone) + : ZonedDateTime.now(tz); + + String office = requiredParam(ctx, OFFICE); + + FilteredTimeSeriesParameters ftsParams = FilteredTimeSeriesParameters.Builder.from(ctx) + .build(); + + TimeSeriesRequestParameters tsParams = new TimeSeriesRequestParameters.Builder() + .withNames(names) + .withOffice(office) + .withUnits(unit) + .withBeginTime(beginZdt) + .withEndTime(endZdt) + .withVersionDate(versionDate) + .withShouldTrim(trim.getOrDefault(true)) + .withIncludeEntryDate(includeEntryDate) + .build(); + + + FilteredTimeSeries fts = dao.getTimeseries(cursor, pageSize, tsParams, ftsParams); + + String results = Formats.format(contentType, fts); + + ctx.status(HttpServletResponse.SC_OK); + + addLinkHeader(ctx, fts, contentType); + + ctx.result(results).contentType(contentType.toString()); + + addDeprecatedContentTypeWarning(ctx, contentType); + requestResultSize.update(results.length()); + } catch (NotFoundException e) { + CdaError re = new CdaError("Not found."); + logger.log(Level.WARNING, re.toString(), e); + ctx.status(HttpServletResponse.SC_NOT_FOUND); + ctx.json(re); + } catch (IllegalArgumentException ex) { + CdaError re = new CdaError("Invalid arguments supplied"); + logger.log(Level.SEVERE, re.toString(), ex); + ctx.status(HttpServletResponse.SC_BAD_REQUEST); + ctx.json(re); + } + } + + private void addLinkHeader(@NotNull Context ctx, FilteredTimeSeries fts, ContentType contentType) { + // Send back the link to the next page in the response header + try { + // Send back the link to the next page in the response header + StringBuilder linkValue = new StringBuilder(600); + String pageUrl = buildRequestUrl(ctx, fts.getPage()); + linkValue.append(String.format("<%s>; rel=self; type=\"%s\"", + pageUrl, contentType)); + + if (fts.getNextPage() != null) { + linkValue.append(","); + String nextPageUrl = buildRequestUrl(ctx, fts.getNextPage()); + linkValue.append(String.format("<%s>; rel=next; type=\"%s\"", + nextPageUrl, + contentType)); + } + + ctx.header("Link", linkValue.toString()); + } catch (URISyntaxException ex) { + logger.log(Level.WARNING, null, ex); + } + } + + public String buildRequestUrl(Context ctx, String cursor) throws URISyntaxException { + URIBuilder builder = new URIBuilder(ctx.req.getRequestURL().toString()); // requestURL stops just before ? + builder.setParameters(URLEncodedUtils.parse(ctx.req.getQueryString(), StandardCharsets.UTF_8)); + + // override or add the paging cursor + if (cursor != null && !cursor.isEmpty()) { + builder.setParameter("page", cursor); + } + + return builder.build().toString(); + } + + +} diff --git a/cwms-data-api/src/main/java/cwms/cda/api/rating/RatingController.java b/cwms-data-api/src/main/java/cwms/cda/api/rating/RatingController.java index 92c568b1f..137ba9879 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/rating/RatingController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/rating/RatingController.java @@ -28,11 +28,9 @@ import static cwms.cda.api.Controllers.AT; import static cwms.cda.api.Controllers.BEGIN; import static cwms.cda.api.Controllers.CREATE; -import static cwms.cda.api.Controllers.DATE_FORMAT; import static cwms.cda.api.Controllers.DATUM; import static cwms.cda.api.Controllers.DELETE; import static cwms.cda.api.Controllers.END; -import static cwms.cda.api.Controllers.EXAMPLE_DATE; import static cwms.cda.api.Controllers.FORMAT; import static cwms.cda.api.Controllers.GET_ALL; import static cwms.cda.api.Controllers.GET_ONE; @@ -48,6 +46,7 @@ import static cwms.cda.api.Controllers.STATUS_501; import static cwms.cda.api.Controllers.STORE_TEMPLATE; import static cwms.cda.api.Controllers.TIMEZONE; +import static cwms.cda.api.Controllers.TIME_FORMAT_DESC; import static cwms.cda.api.Controllers.UNIT; import static cwms.cda.api.Controllers.UPDATE; import static cwms.cda.api.Controllers.VERSION_DATE; @@ -212,13 +211,9 @@ private String removeTemplate(String xml) { @OpenApiParam(name = OFFICE, required = true, description = "Specifies the office of " + "the ratings to be deleted."), @OpenApiParam(name = BEGIN, required = true, description = "The start of the time " - + "window to delete. The format for this field is ISO 8601 extended, " - + "with optional offset and timezone, i.e., '" - + DATE_FORMAT + "', e.g., '" + EXAMPLE_DATE + "'."), + + "window to delete. " + TIME_FORMAT_DESC), @OpenApiParam(name = END, required = true, description = "The end of the time window" - + " to delete. The format for this field is ISO 8601 extended, with optional " - + "offset and timezone, i.e., '" - + DATE_FORMAT + "', e.g., '" + EXAMPLE_DATE + "'."), + + " to delete. " + TIME_FORMAT_DESC), @OpenApiParam(name = TIMEZONE, description = "This field specifies a default " + "timezone to be used if the format of the " + BEGIN + ", " + END + ", or " + VERSION_DATE + " parameters do not include " diff --git a/cwms-data-api/src/main/java/cwms/cda/api/watersupply/AccountingCatalogController.java b/cwms-data-api/src/main/java/cwms/cda/api/watersupply/AccountingCatalogController.java index 88b8549ba..53a7b69d1 100644 --- a/cwms-data-api/src/main/java/cwms/cda/api/watersupply/AccountingCatalogController.java +++ b/cwms-data-api/src/main/java/cwms/cda/api/watersupply/AccountingCatalogController.java @@ -39,6 +39,7 @@ import static cwms.cda.api.Controllers.STATUS_404; import static cwms.cda.api.Controllers.STATUS_501; import static cwms.cda.api.Controllers.TIMEZONE; +import static cwms.cda.api.Controllers.TIME_FORMAT_DESC; import static cwms.cda.api.Controllers.UNIT; import static cwms.cda.api.Controllers.WATER_USER; import static cwms.cda.api.Controllers.requiredInstant; @@ -96,10 +97,9 @@ protected WaterSupplyAccountingDao getWaterSupplyAccountingDao(DSLContext dsl) { @OpenApi( queryParams = { @OpenApiParam(name = START, description = "The start time of the time window for " - + "pump accounting entries to retrieve. The format for this field is ISO 8601 extended, " - + "with optional offset and timezone", required = true), + + "pump accounting entries to retrieve. " + TIME_FORMAT_DESC, required = true), @OpenApiParam(name = END, description = "The end time of the time window for pump " - + "accounting entries to retrieve.", required = true), + + "accounting entries to retrieve." + TIME_FORMAT_DESC, required = true), @OpenApiParam(name = TIMEZONE, description = "This field specifies a default timezone " + "to be used if the format of the " + END + " or " + BEGIN + " parameters do not include offset or time zone information. " diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/FilteredTimeSeriesParameters.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/FilteredTimeSeriesParameters.java new file mode 100644 index 000000000..2830b7aac --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/FilteredTimeSeriesParameters.java @@ -0,0 +1,104 @@ +package cwms.cda.data.dao; + +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.annotation.JsonDeserialize; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; +import io.javalin.http.Context; +import org.jetbrains.annotations.NotNull; + +import static cwms.cda.api.Controllers.FILTER_NULLS; +import static cwms.cda.api.Controllers.MAX_VALUE; +import static cwms.cda.api.Controllers.MIN_VALUE; +import static cwms.cda.api.Controllers.QUERY; + +@JsonDeserialize(builder = FilteredTimeSeriesParameters.Builder.class) +@JsonNaming(PropertyNamingStrategies.KebabCaseStrategy.class) +public class FilteredTimeSeriesParameters { + private final Double minValue; + private final Double maxValue; + private final boolean filterNulls; + private final String query; + + private FilteredTimeSeriesParameters(Builder builder) { + this.minValue = builder.minValue; + this.maxValue = builder.maxValue; + this.filterNulls = builder.filterNulls; + this.query = builder.query; + } + + public Double getMinValue() { + return minValue; + } + + public Double getMaxValue() { + return maxValue; + } + + public boolean isFilterNulls() { + return filterNulls; + } + + public String getQuery() { + return query; + } + + @JsonPOJOBuilder(withPrefix = "with", buildMethodName = "build") + public static class Builder { + + private Double minValue; + private Double maxValue; + private boolean filterNulls = false; + private String query; + + public Builder() { + } + + public Builder withMinValue(Double minValue) { + this.minValue = minValue; + return this; + } + + public Builder withMaxValue(Double maxValue) { + this.maxValue = maxValue; + return this; + } + + public Builder withFilterNulls(boolean filterNulls) { + this.filterNulls = filterNulls; + return this; + } + + public Builder withQuery(String query) { + this.query = query; + return this; + } + + public static Builder from(FilteredTimeSeriesParameters params) { + // This NEEDS to include every field in the FilteredTimeSeriesRequestParameters + return new Builder() + .withMinValue(params.minValue) + .withMaxValue(params.maxValue) + .withFilterNulls(params.filterNulls) + .withQuery(params.query); + } + + public static Builder from(@NotNull Context ctx){ + Double minValue = ctx.queryParamAsClass(MIN_VALUE, Double.class).getOrDefault(null); + Double maxValue = ctx.queryParamAsClass(MAX_VALUE, Double.class).getOrDefault(null); + boolean filterNulls = ctx.queryParamAsClass(FILTER_NULLS, Boolean.class).getOrDefault(false); + String query = ctx.queryParamAsClass(QUERY, String.class).getOrDefault(null); + + return new Builder() + .withMinValue(minValue) + .withMaxValue(maxValue) + .withFilterNulls(filterNulls) + .withQuery(query) + ; + } + + public FilteredTimeSeriesParameters build() { + return new FilteredTimeSeriesParameters(this); + } + } +} diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java index 29be65bb2..eca99de09 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDao.java @@ -4,6 +4,7 @@ import cwms.cda.data.dto.Catalog; import cwms.cda.data.dto.RecentValue; import cwms.cda.data.dto.TimeSeries; +import cwms.cda.data.dto.filteredtimeseries.FilteredTimeSeries; import java.sql.Timestamp; import java.time.ZoneId; import java.time.ZonedDateTime; @@ -25,10 +26,31 @@ void store(TimeSeries timeSeries, boolean createAsLrts, void delete(String officeId, String tsId, TimeSeriesDeleteOptions options); + /** + * + * @param cursor + * @param pageSize + * @param names + * @param office + * @param unit + * @param begin + * @param end + * @param versionDate + * @param trim + * @param includeEntryDate + * @return requested TimeSeries + * @deprecated Use {@link #getTimeseries(String, int, TimeSeriesRequestParameters)} + * instead. Create a {@link TimeSeriesRequestParameters} instance and + * call that overload. + */ + @Deprecated TimeSeries getTimeseries(String cursor, int pageSize, String names, String office, String unit, ZonedDateTime begin, ZonedDateTime end, ZonedDateTime versionDate, boolean trim, boolean includeEntryDate); + TimeSeries getTimeseries(String cursor, int pageSize, TimeSeriesRequestParameters requestParameters); + FilteredTimeSeries getTimeseries(String page, int pageSize, TimeSeriesRequestParameters requestParameters, FilteredTimeSeriesParameters filterParams); + String getTimeseries(String format, String names, String office, String unit, String datum, ZonedDateTime begin, ZonedDateTime end, ZoneId timezone); diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java index dce0f3344..4f8d47b1b 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesDaoImpl.java @@ -1,11 +1,16 @@ package cwms.cda.data.dao; +import cwms.cda.data.dao.rsql.FieldResolver; +import cwms.cda.data.dao.rsql.MapFieldResolver; +import cwms.cda.data.dao.rsql.RSQLConditionBuilder; +import cwms.cda.data.dto.filteredtimeseries.FilteredTimeSeries; import cwms.cda.helpers.DateUtils; import static org.jooq.impl.DSL.asterisk; import static org.jooq.impl.DSL.countDistinct; import static org.jooq.impl.DSL.field; import static org.jooq.impl.DSL.max; import static org.jooq.impl.DSL.name; +import static org.jooq.impl.DSL.noCondition; import static org.jooq.impl.DSL.partitionBy; import static org.jooq.impl.DSL.select; import static org.jooq.impl.DSL.selectDistinct; @@ -59,6 +64,7 @@ import org.jetbrains.annotations.Nullable; import org.jooq.CommonTableExpression; import org.jooq.Condition; +import org.jooq.Cursor; import org.jooq.DSLContext; import org.jooq.Field; import org.jooq.Record; @@ -162,25 +168,71 @@ public String getTimeseries(String format, String names, String office, String u timezone.getId(), office); } - + /** + * Retrieves a TimeSeries from the database + * @param page an opaque token used for paging + * @param pageSize number of points to return in a page + * @param names the timeseries id + * @param office the office + * @param units the units to return + * @param beginTime the beginning of the time window + * @param endTime the end of the time window + * @param versionDate the requested version date or null + * @param shouldTrim should the beginning and end of the returned timeseries be trimmed + * @param includeEntryDate if the entry-date should be included in results + * @return TimeSeries containing the requested data + * @deprecated Use {@link #getTimeseries(String,int,TimeSeriesRequestParameters)} + * instead. Create a {@link TimeSeriesRequestParameters} instance and + * call that overload. + */ @Override + @Deprecated public TimeSeries getTimeseries(String page, int pageSize, String names, String office, - String units, - ZonedDateTime beginTime, ZonedDateTime endTime, + String units, + ZonedDateTime beginTime, ZonedDateTime endTime, ZonedDateTime versionDate, boolean shouldTrim, boolean includeEntryDate) { + TimeSeriesRequestParameters requestParameters = new TimeSeriesRequestParameters.Builder() + .withNames(names) + .withOffice(office) + .withUnits(units) + .withBeginTime(beginTime) + .withEndTime(endTime) + .withVersionDate(versionDate) + .withShouldTrim(shouldTrim) + .withIncludeEntryDate(includeEntryDate) + .build(); + return getTimeseries(page, pageSize, requestParameters); + } + + @Override + public TimeSeries getTimeseries(String page, int pageSize, TimeSeriesRequestParameters requestParameters) { + return getRequestedTimeSeries(page, pageSize,requestParameters, null); + } + + @Override + public FilteredTimeSeries getTimeseries(String page, int pageSize, TimeSeriesRequestParameters requestParameters, FilteredTimeSeriesParameters filterParams){ + TimeSeries ts = getRequestedTimeSeries(page, pageSize, requestParameters, filterParams); + FilteredTimeSeries fts = new FilteredTimeSeries(ts, filterParams); + fts.clearTimeSeriesPagination(); // we are wrapping the ts, it doesn't need to serialize its own page, nextPage etc. + return fts; + } + + protected TimeSeries getRequestedTimeSeries(String page, int pageSize, @NotNull TimeSeriesRequestParameters requestParameters, + @Nullable FilteredTimeSeriesParameters fp) { + + String names = requestParameters.getNames(); + String office = requestParameters.getOffice(); + String units = requestParameters.getUnits(); + ZonedDateTime beginTime = requestParameters.getBeginTime(); + ZonedDateTime endTime = requestParameters.getEndTime(); + ZonedDateTime versionDate = requestParameters.getVersionDate(); + boolean shouldTrim = requestParameters.isShouldTrim(); + boolean includeEntryDate = requestParameters.isIncludeEntryDate(); String cursor = null; Timestamp tsCursor = null; Integer total = null; - if (includeEntryDate) { - Record entryDateSupport = dsl.select(asterisk()).from(table("ALL_TYPES")) - .where(field("TYPE_NAME").eq("ZTSV_ENTRY_TYPE")) - .and(field("OWNER").eq("CWMS_20")).fetchOne(); - - if (entryDateSupport == null) { - throw new DataAccessException("Data entry date retrieval is not supported by this database"); - } - } + validateEntryDateSupport(includeEntryDate); if (page != null && !page.isEmpty()) { final String[] parts = CwmsDTOPaginated.decodeCursor(page); @@ -267,12 +319,13 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String Field dateTimeCol = field("DATE_TIME", Timestamp.class).as("DATE_TIME"); Field valueCol = field("VALUE", Double.class).as("VALUE"); Field qualityCol = field("QUALITY_CODE", Integer.class).as("QUALITY_CODE"); + Field dataEntryDate = field("DATA_ENTRY_DATE", Timestamp.class).as("data_entry_date"); Long beginTimeMilli = beginTime.toInstant().toEpochMilli(); Long endTimeMilli = endTime.toInstant().toEpochMilli(); String trim = formatBool(shouldTrim); - String startInclusive = "T"; - String endInclusive = "T"; + final String startInclusive = "T"; + final String endInclusive = "T"; String previous = "F"; String next = "F"; Long versionDateMilli = null; @@ -286,6 +339,17 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String Field tzName = AV_CWMS_TS_ID2.TIME_ZONE_ID; + Condition filterConditions = noCondition(); + if(fp != null) { + Map> nameToField = new LinkedHashMap<>(); + nameToField.put("value", valueCol); + nameToField.put("datetime", dateTimeCol); + nameToField.put("quality", qualityCol); + nameToField.put("data_entry_date", dataEntryDate); + FieldResolver resolver = new MapFieldResolver(nameToField); + filterConditions = getFilterCondition(fp, resolver); + } + Field totalField; if (total != null) { totalField = DSL.val(total).as("TOTAL"); @@ -294,7 +358,7 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String // Total is only an estimate, as it can change if fetching current data, // or the timeseries otherwise changes between queries. - SelectJoinStep> retrieveSelectCount = select( + SelectConditionStep> retrieveSelectCount = select( dateTimeCol, valueCol, qualityCol ).from(DSL.sql( "table(cwms_20.cwms_ts.retrieve_ts_out_tab(?,?," @@ -306,7 +370,9 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String endTimeMilli, trim, startInclusive, endInclusive, previous, next, versionDateMilli, maxVersion, valid.field("office_id", String.class) - )); + )) + .where(filterConditions) + ; totalField = DSL.selectCount().from(table(retrieveSelectCount)).asField("TOTAL"); } @@ -379,26 +445,26 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String Field qualityNormCol = CWMS_TS_PACKAGE.call_NORMALIZE_QUALITY( DSL.nvl(qualityCol, DSL.inline(5))).as("QUALITY_NORM"); - Field dataEntryDate = field("DATA_ENTRY_DATE", Timestamp.class).as("data_entry_date"); + TimeSeries retVal = null; if (pageSize != 0) { - SelectJoinStep> query2 = dsl.select( + SelectConditionStep> query2 = dsl.select( dateTimeCol, valueCol, qualityNormCol, dataEntryDate ) - .from(retrieveSelectData); + .from(retrieveSelectData) + .where(filterConditions); - SelectConditionStep> query = - dsl.select( - dateTimeCol, - valueCol, - qualityNormCol - ) - .from(retrieveSelectData) - .where(dateTimeCol + SelectConditionStep> query = dsl.select( + dateTimeCol, + valueCol, + qualityNormCol + ) + .from(retrieveSelectData) + .where(dateTimeCol .greaterOrEqual(CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( DSL.nvl(DSL.val(tsCursor == null ? null : tsCursor.toInstant().toEpochMilli()), @@ -406,38 +472,54 @@ public TimeSeries getTimeseries(String page, int pageSize, String names, String .and(dateTimeCol .lessOrEqual(CWMS_UTIL_PACKAGE.call_TO_TIMESTAMP__2( DSL.val(endTime.toInstant().toEpochMilli()))) - ); + ) + .and(filterConditions); if (pageSize > 0) { query.limit(DSL.val(pageSize + 1)); query2.limit(DSL.val(pageSize + 1)); } - if (includeEntryDate) { + if (requestParameters.isIncludeEntryDate()) { logger.fine(() -> query2.getSQL(ParamType.INLINED)); - final TimeSeries timeSeries = timeseries; - query2.forEach(tsRecord -> timeSeries.addValue( - tsRecord.getValue(dateTimeCol), - tsRecord.getValue(valueCol), - tsRecord.getValue(qualityNormCol).intValue(), - tsRecord.getValue(dataEntryDate) - )); - retVal = timeSeries; + try (Cursor> recCursor = query2.fetchLazy()) { + for(Record tsRecord: recCursor){ + timeseries.addValue( + tsRecord.getValue(dateTimeCol), + tsRecord.getValue(valueCol), + tsRecord.getValue(qualityNormCol).intValue(), + tsRecord.getValue(dataEntryDate)); + } + } } else { logger.fine(() -> query.getSQL(ParamType.INLINED)); - final TimeSeries finalTimeseries = timeseries; - query.forEach(tsRecord -> finalTimeseries.addValue( - tsRecord.getValue(dateTimeCol), - tsRecord.getValue(valueCol), - tsRecord.getValue(qualityNormCol).intValue() - )); - retVal = finalTimeseries; + try (Cursor> recCursor = query.fetchLazy()) { + for(Record tsRecord: recCursor){ + timeseries.addValue( + tsRecord.getValue(dateTimeCol), + tsRecord.getValue(valueCol), + tsRecord.getValue(qualityNormCol).intValue()); + } + } } + retVal = timeseries; } return retVal; } + private void validateEntryDateSupport(boolean includeEntryDate) { + if (includeEntryDate) { + Record entryDateSupport = dsl.select(asterisk()).from(table("ALL_TYPES")) + .where(field("TYPE_NAME").eq("ZTSV_ENTRY_TYPE")) + .and(field("OWNER").eq("CWMS_20")).fetchOne(); + + if (entryDateSupport == null) { + throw new DataAccessException("Data entry date retrieval is not supported by this database"); + } + } + } + private static String getVersionPart(ZonedDateTime versionDate) { if (versionDate != null) { return "cwms_20.cwms_util.to_timestamp(?)"; @@ -611,6 +693,34 @@ public Catalog getTimeSeriesCatalog(String page, int pageSize, CatalogRequestPar total, pageSize, entries, params); } + + @NotNull + private static Condition getFilterCondition( @Nullable FilteredTimeSeriesParameters ip, FieldResolver resolver) { + // In Filter case we want to skip certain points in time.... + Condition filterConditions = noCondition(); + if(ip != null) { + Field valueCol = resolver.resolve("value"); + if (ip.isFilterNulls()) { + filterConditions = filterConditions.and(valueCol.isNotNull()); + } + if (ip.getMaxValue() != null) { + filterConditions = filterConditions.and(valueCol.le(ip.getMaxValue())); + } + if (ip.getMinValue() != null) { + filterConditions = filterConditions.and(valueCol.ge(ip.getMinValue())); + } + + String query = ip.getQuery(); + if(query != null) { + RSQLConditionBuilder builder = RSQLConditionBuilder.create(resolver); + Condition condition = builder.buildCondition(query); + filterConditions = filterConditions.and(condition); + } + + } + return filterConditions; + } + private static @NotNull List buildPagingConditions(String cursorOffice, String cursorTsId) { List pagingConditions = new ArrayList<>(); @@ -1380,10 +1490,12 @@ public Builder withOverrideProtection(String overrideProtection) { return this; } + public DeleteOptions build() { return new DeleteOptions(this); } } } + } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesRequestParameters.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesRequestParameters.java new file mode 100644 index 000000000..527fbd8aa --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/TimeSeriesRequestParameters.java @@ -0,0 +1,128 @@ +package cwms.cda.data.dao; + +import java.time.ZonedDateTime; + +public class TimeSeriesRequestParameters { + private final String names; + private final String office; + private final String units; + private final ZonedDateTime beginTime; + private final ZonedDateTime endTime; + private final ZonedDateTime versionDate; + private final boolean shouldTrim; + private final boolean includeEntryDate; + + private TimeSeriesRequestParameters(Builder builder) { + this.names = builder.names; + this.office = builder.office; + this.units = builder.units; + this.beginTime = builder.beginTime; + this.endTime = builder.endTime; + this.versionDate = builder.versionDate; + this.shouldTrim = builder.shouldTrim; + this.includeEntryDate = builder.includeEntryDate; + } + + public String getNames() { + return names; + } + + public String getOffice() { + return office; + } + + public String getUnits() { + return units; + } + + public ZonedDateTime getBeginTime() { + return beginTime; + } + + public ZonedDateTime getEndTime() { + return endTime; + } + + public ZonedDateTime getVersionDate() { + return versionDate; + } + + public boolean isShouldTrim() { + return shouldTrim; + } + + public boolean isIncludeEntryDate() { + return includeEntryDate; + } + + public static class Builder { + private String names; + private String office; + private String units; + private ZonedDateTime beginTime; + private ZonedDateTime endTime; + private ZonedDateTime versionDate; + private boolean shouldTrim = true; + private boolean includeEntryDate = false; + + public Builder() { + } + + public Builder withNames(String names) { + this.names = names; + return this; + } + + public Builder withOffice(String office) { + this.office = office; + return this; + } + + public Builder withUnits(String units) { + this.units = units; + return this; + } + + public Builder withBeginTime(ZonedDateTime beginTime) { + this.beginTime = beginTime; + return this; + } + + public Builder withEndTime(ZonedDateTime endTime) { + this.endTime = endTime; + return this; + } + + public Builder withVersionDate(ZonedDateTime versionDate) { + this.versionDate = versionDate; + return this; + } + + public Builder withShouldTrim(boolean shouldTrim) { + this.shouldTrim = shouldTrim; + return this; + } + + public Builder withIncludeEntryDate(boolean includeEntryDate) { + this.includeEntryDate = includeEntryDate; + return this; + } + + public static Builder from(TimeSeriesRequestParameters params) { + // This NEEDS to include every field in the TimeSeriesRequestParameters + return new Builder() + .withNames(params.names) + .withOffice(params.office) + .withUnits(params.units) + .withBeginTime(params.beginTime) + .withEndTime(params.endTime) + .withVersionDate(params.versionDate) + .withShouldTrim(params.shouldTrim) + .withIncludeEntryDate(params.includeEntryDate); + } + + public TimeSeriesRequestParameters build() { + return new TimeSeriesRequestParameters(this); + } + } +} \ No newline at end of file diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/FieldResolver.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/FieldResolver.java new file mode 100644 index 000000000..96d778d67 --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/FieldResolver.java @@ -0,0 +1,8 @@ +package cwms.cda.data.dao.rsql; + +import org.jooq.Field; + +@FunctionalInterface +public interface FieldResolver { + Field resolve(String fieldName); +} diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/JooqFieldResolver.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/JooqFieldResolver.java new file mode 100644 index 000000000..f281c5c6c --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/JooqFieldResolver.java @@ -0,0 +1,73 @@ +package cwms.cda.data.dao.rsql; + +import org.jooq.Field; +import org.jooq.Table; + +import java.util.Arrays; +import java.util.Locale; + +/** + * Resolves a field name supplied by the client (e.g. "unit_id", "UNIT_ID") + * to the jOOQ {@link Field} generated for the given {@link Table}. + *

+ * The database returns fully qualified, quoted identifiers such as + * "CWMS_20"."AV_TSV_DQU"."UNIT_ID" + * This resolver: + * • removes quotes, + * • looks for "." and keeps everything that follows, + * • compares the result without regard to case. + */ +public class JooqFieldResolver implements FieldResolver { + + private final Table table; + private final String tableNameCanonical; + + public JooqFieldResolver(Table table) { + this.table = table; + // Canonical form (uppercase, no quotes) of the table name once, up front + this.tableNameCanonical = stripQuotes(table.getName()).toUpperCase(Locale.ROOT); + } + + @Override + public Field resolve(String fieldName) { + + Field field = table.field(fieldName); + + if (field == null) { + String wanted = canonicalColumn(fieldName); + + field = Arrays.stream(table.fields()) + .filter(f -> canonicalColumn(f.getName()).equals(wanted)) + .findFirst() + .orElseThrow(() -> + new IllegalArgumentException("Unknown field: " + fieldName)); + } + + return (Field) field; + } + + /* ----------------------------------------------------- helpers */ + + private String canonicalColumn(String rawIdentifier) { + String noQuotes = stripQuotes(rawIdentifier); + String upper = noQuotes.toUpperCase(Locale.ROOT); + + // Attempt to locate "." and use the remainder + String needle = tableNameCanonical + "."; + int idx = upper.lastIndexOf(needle); + String columnPart; + + if (idx >= 0) { + columnPart = upper.substring(idx + needle.length()); + } else { + // Fallback: keep text after the last dot + int lastDot = upper.lastIndexOf('.'); + columnPart = (lastDot >= 0) ? upper.substring(lastDot + 1) : upper; + } + return columnPart; + } + + private static String stripQuotes(String text) { + return text.replace("\"", ""); + } +} \ No newline at end of file diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/MapFieldResolver.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/MapFieldResolver.java new file mode 100644 index 000000000..4d869320f --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/MapFieldResolver.java @@ -0,0 +1,31 @@ +package cwms.cda.data.dao.rsql; + +import org.jooq.Field; +import java.util.Map; + +/** + * Resolves field names using a pre-built map of field names to jOOQ {@link Field}s. + */ +public class MapFieldResolver implements FieldResolver { + private final Map> nameToField; + + /** + * Creates a new MapFieldResolver with the given map of field names to fields. + * + * @param nameToField A map from field names to jOOQ Fields + */ + public MapFieldResolver(Map> nameToField) { + this.nameToField = nameToField; + } + + @Override + public Field resolve(String name) { + Field field = nameToField.get(name); + if (field == null) { + throw new IllegalArgumentException("Unknown field: " + name); + } + @SuppressWarnings("unchecked") + Field typedField = (Field) field; + return typedField; + } +} \ No newline at end of file diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/RSQLConditionBuilder.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/RSQLConditionBuilder.java new file mode 100644 index 000000000..487564341 --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/RSQLConditionBuilder.java @@ -0,0 +1,54 @@ +package cwms.cda.data.dao.rsql; + +import cz.jirutka.rsql.parser.RSQLParser; +import cz.jirutka.rsql.parser.ast.Node; +import org.jooq.Condition; + +/** + * A utility class for converting RSQL query strings into jOOQ Conditions. + * This class can be used from Dao code to parse RSQL query strings and convert them + * to jOOQ Conditions that can be used in database queries. + */ +public class RSQLConditionBuilder { + + private final FieldResolver fieldResolver; + + /** + * Creates a new RSQLConditionBuilder with the given field resolver. + * + * @param fieldResolver The resolver to use for converting field names to jOOQ Fields + */ + public RSQLConditionBuilder(FieldResolver fieldResolver) { + this.fieldResolver = fieldResolver; + } + + /** + * Converts an RSQL query string to a jOOQ Condition. + * + * @param rsqlQuery The RSQL query string to convert + * @return The equivalent jOOQ Condition + * @throws IllegalArgumentException if the query is invalid or contains unknown fields + */ + public Condition buildCondition(String rsqlQuery) { + if (rsqlQuery == null || rsqlQuery.trim().isEmpty()) { + throw new IllegalArgumentException("RSQL query cannot be null or empty"); + } + + // Parse the RSQL query string into a Node + Node rootNode = new RSQLParser().parse(rsqlQuery); + + // Use the RSQLJooqConditionVisitor to convert the Node to a Condition + RSQLJooqConditionVisitor visitor = new RSQLJooqConditionVisitor(fieldResolver); + return rootNode.accept(visitor); + } + + /** + * Static factory method to create a new RSQLConditionBuilder with the given field resolver. + * + * @param fieldResolver The resolver to use for converting field names to jOOQ Fields + * @return A new RSQLConditionBuilder + */ + public static RSQLConditionBuilder create(FieldResolver fieldResolver) { + return new RSQLConditionBuilder(fieldResolver); + } +} \ No newline at end of file diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/RSQLJooqConditionVisitor.java b/cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/RSQLJooqConditionVisitor.java new file mode 100644 index 000000000..b901a44dd --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/data/dao/rsql/RSQLJooqConditionVisitor.java @@ -0,0 +1,131 @@ +package cwms.cda.data.dao.rsql; + +import cwms.cda.helpers.DateUtils; +import cz.jirutka.rsql.parser.ast.AndNode; +import cz.jirutka.rsql.parser.ast.ComparisonNode; +import cz.jirutka.rsql.parser.ast.ComparisonOperator; +import cz.jirutka.rsql.parser.ast.OrNode; +import cz.jirutka.rsql.parser.ast.RSQLOperators; +import cz.jirutka.rsql.parser.ast.RSQLVisitor; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.impl.DSL; +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.time.ZonedDateTime; +import java.util.List; +import java.util.UUID; +import java.util.stream.Collectors; + +/** + * Inspired by ... + * + */ +public class RSQLJooqConditionVisitor implements RSQLVisitor { + + private final FieldResolver fieldResolver; + + public RSQLJooqConditionVisitor(FieldResolver fieldResolver) { + this.fieldResolver = fieldResolver; + } + + @Override + public Condition visit(AndNode node, Void param) { + return node.getChildren().stream() + .map(n -> n.accept(this, param)) + .reduce(DSL.noCondition(), Condition::and); + } + + @Override + public Condition visit(OrNode node, Void param) { + return node.getChildren().stream() + .map(n -> n.accept(this, param)) + .reduce(DSL.noCondition(), Condition::or); + } + + @Override + public Condition visit(ComparisonNode node, Void param) { + String selector = node.getSelector(); + ComparisonOperator op = node.getOperator(); + List arguments = node.getArguments(); + + Field field = fieldResolver.resolve(selector); + + // If not doing type conversion, just do this: Object value = arguments.get(0) + // else.... + String rawValue = arguments.isEmpty() ? null : arguments.get(0); + if (isNullLiteral(rawValue)) { + return buildNullCondition(field, op); + } + + Object value = convert(field, arguments.get(0)); + List values = convertList(field, arguments); + + return buildCondition(field, op, value, values); + } + + public static boolean isNullLiteral(String value) { + return value == null || "null".equalsIgnoreCase(value.trim()); + } + + public static Condition buildNullCondition(Field field, ComparisonOperator operator) { + if (RSQLOperators.EQUAL.equals(operator)) { + return field.isNull(); + } + if (RSQLOperators.NOT_EQUAL.equals(operator)) { + return field.isNotNull(); + } + throw new IllegalArgumentException( + "Operator " + operator + " is not valid with NULL literal"); + } + + public static Condition buildCondition(Field field, + ComparisonOperator operator, + Object value, + List values) { + + if (RSQLOperators.EQUAL.equals(operator)) return field.eq(value); + if (RSQLOperators.NOT_EQUAL.equals(operator)) return field.ne(value); + if (RSQLOperators.GREATER_THAN.equals(operator)) return field.gt(value); + if (RSQLOperators.GREATER_THAN_OR_EQUAL.equals(operator)) return field.ge(value); + if (RSQLOperators.LESS_THAN.equals(operator)) return field.lt(value); + if (RSQLOperators.LESS_THAN_OR_EQUAL.equals(operator)) return field.le(value); + if (RSQLOperators.IN.equals(operator)) return field.in(values); + if (RSQLOperators.NOT_IN.equals(operator)) return field.notIn(values); + + throw new IllegalArgumentException("Unknown operator: " + operator); + } + + + public static Object convert(Field field, String value) { + if (value == null || value.trim().isEmpty() || "null".equalsIgnoreCase(value.trim())) { + return null; + } + + Class type = field.getType(); + + if (type == String.class) return value; + if (type == Integer.class || type == int.class) return Integer.valueOf(value); + if (type == Long.class || type == long.class) return Long.valueOf(value); + if (type == Boolean.class || type == boolean.class) return Boolean.valueOf(value); + if (type == Double.class || type == double.class) return Double.valueOf(value); + if (type == BigDecimal.class) return new BigDecimal(value); + if (type == ZonedDateTime.class) return DateUtils.parseUserDate(value, "UTC"); + if (type == Timestamp.class) { + ZonedDateTime zdt = DateUtils.parseUserDate(value, "UTC"); + // return zdt; // we could just return zdt and let jOOQ do the zdt->ts + return Timestamp.from(zdt.toInstant()); + } + + if (type == UUID.class) return UUID.fromString(value); + + throw new IllegalArgumentException("Unsupported field type: " + type.getName()); + } + + public static List convertList(Field field, List values) { + return values.stream() + .map(value -> convert(field, value)) + .collect(Collectors.toList()); + } + +} diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java index 64778d37e..d1414527f 100644 --- a/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/TimeSeries.java @@ -186,6 +186,15 @@ public String getTimeZone() { return timeZone; } + /** + * Clears the pagination information (page, nextPage. total) for this TimeSeries. + */ + public void clearPagination() { + this.page = null; + this.nextPage = null; + this.total = null; + } + public ZonedDateTime getVersionDate() { return versionDate; } diff --git a/cwms-data-api/src/main/java/cwms/cda/data/dto/filteredtimeseries/FilteredTimeSeries.java b/cwms-data-api/src/main/java/cwms/cda/data/dto/filteredtimeseries/FilteredTimeSeries.java new file mode 100644 index 000000000..d04b97cf9 --- /dev/null +++ b/cwms-data-api/src/main/java/cwms/cda/data/dto/filteredtimeseries/FilteredTimeSeries.java @@ -0,0 +1,61 @@ +package cwms.cda.data.dto.filteredtimeseries; + +import com.fasterxml.jackson.annotation.JsonIgnoreProperties; +import com.fasterxml.jackson.annotation.JsonInclude; +import com.fasterxml.jackson.annotation.JsonProperty; +import com.fasterxml.jackson.annotation.JsonPropertyOrder; +import com.fasterxml.jackson.annotation.JsonRootName; +import com.fasterxml.jackson.databind.PropertyNamingStrategies; +import com.fasterxml.jackson.databind.annotation.JsonNaming; +import cwms.cda.data.dao.FilteredTimeSeriesParameters; +import cwms.cda.data.dto.CwmsDTOPaginated; +import cwms.cda.data.dto.TimeSeries; +import cwms.cda.formatters.Formats; +import cwms.cda.formatters.annotations.FormattableWith; +import cwms.cda.formatters.json.JsonV2; +import cwms.cda.formatters.xml.XMLv2; + +@JsonRootName("filtered-timeseries") +@JsonPropertyOrder(alphabetic = true) +@JsonInclude(JsonInclude.Include.NON_NULL) +@JsonNaming(PropertyNamingStrategies.KebabCaseStrategy.class) +@FormattableWith(contentType = Formats.JSONV2, formatter = JsonV2.class, aliases = {Formats.DEFAULT, Formats.JSON}) +@FormattableWith(contentType = Formats.XMLV2, formatter = XMLv2.class, aliases = {Formats.XML}) +@JsonIgnoreProperties(ignoreUnknown = true) +public class FilteredTimeSeries extends CwmsDTOPaginated { + + @JsonProperty("time-series") + private TimeSeries timeSeries; + + @JsonProperty("filter-parameters") + private FilteredTimeSeriesParameters filterParams; + + @SuppressWarnings("unused") // required so Jackson can deserialize + protected FilteredTimeSeries() { + super(); + } + + public FilteredTimeSeries(TimeSeries timeSeries, FilteredTimeSeriesParameters parameters) { + super(timeSeries.getPage(), timeSeries.getPageSize(), timeSeries.getTotal()); + this.timeSeries = timeSeries; + this.filterParams = parameters; + } + + public TimeSeries getTimeSeries() { + return timeSeries; + } + + public FilteredTimeSeriesParameters getFilterParams() { + return filterParams; + } + + /** + * Clears the pagination information (page and nextPage) in the contained TimeSeries object. + * This is useful when the pagination information should not be exposed or used. + */ + public void clearTimeSeriesPagination() { + if (timeSeries != null) { + timeSeries.clearPagination(); + } + } +} diff --git a/cwms-data-api/src/main/java/cwms/cda/helpers/DateUtils.java b/cwms-data-api/src/main/java/cwms/cda/helpers/DateUtils.java index 9c2799431..deb1d0902 100644 --- a/cwms-data-api/src/main/java/cwms/cda/helpers/DateUtils.java +++ b/cwms-data-api/src/main/java/cwms/cda/helpers/DateUtils.java @@ -1,6 +1,8 @@ package cwms.cda.helpers; import com.google.common.flogger.FluentLogger; +import org.jetbrains.annotations.NotNull; +import org.jetbrains.annotations.Nullable; import java.sql.Timestamp; import java.time.Duration; import java.time.Instant; @@ -41,18 +43,31 @@ private DateUtils() { // utility class } + @NotNull + public static ZonedDateTime parseUserDate(@NotNull String text) { + return parseUserDate(text, "UTC"); + } + /** * Parse a string into a ZonedDateTime. * @param text The string to parse * @param timezone The timezone to use if the string doesn't have one. * @return The parsed ZonedDateTime */ - public static ZonedDateTime parseUserDate(String text, String timezone) { + @NotNull + public static ZonedDateTime parseUserDate(@NotNull String text, @NotNull String timezone) { ZoneId tz = ZoneId.of(timezone); return parseUserDate(text, tz, ZonedDateTime.now(tz)); } - public static ZonedDateTime parseUserDate(String text, ZoneId tz, ZonedDateTime now) { + @NotNull + public static ZonedDateTime parseUserDate(@NotNull String text, @NotNull ZoneId tz){ + return parseUserDate(text, tz, ZonedDateTime.now(tz)); + } + + + @NotNull + public static ZonedDateTime parseUserDate(@NotNull String text, @NotNull ZoneId tz, @NotNull ZonedDateTime now) { if (text.startsWith("PT")) { return parseUserDuration(text, now); @@ -63,7 +78,8 @@ public static ZonedDateTime parseUserDate(String text, ZoneId tz, ZonedDateTime } } - private static ZonedDateTime parseFullDate(String text, ZoneId tz) { + @NotNull + private static ZonedDateTime parseFullDate(@NotNull String text, @NotNull ZoneId tz) { if (hasZone(text)) { return parseZonedDateTime(text); @@ -97,12 +113,13 @@ private static ZonedDateTime parseFullDate(String text, ZoneId tz) { * @param text The string to check * @return true if the string ends with a timezone indicator */ - public static boolean hasZone(String text) { + public static boolean hasZone(@NotNull String text) { return WITH_TZ_INFO.matcher(text).matches(); } - private static ZonedDateTime parseZonedDateTime(String text) { + @NotNull + private static ZonedDateTime parseZonedDateTime(@NotNull String text) { ZonedDateTime zdt; try { // Try to use the default ZonedDateTime format first. @@ -122,7 +139,8 @@ private static ZonedDateTime parseZonedDateTime(String text) { return zdt; } - private static Optional firstMatch(String text, String[] possibleDateFormats) { + @NotNull + private static Optional firstMatch(@NotNull String text, @Nullable String[] possibleDateFormats) { Optional retval = Optional.empty(); if (possibleDateFormats != null) { @@ -136,7 +154,8 @@ private static Optional firstMatch(String text, String[] possible return retval; } - public static Optional parseWithPattern(String text, String pattern) { + @NotNull + public static Optional parseWithPattern(@NotNull String text, @NotNull String pattern) { Optional retval = Optional.empty(); try { DateTimeFormatterBuilder builder = new DateTimeFormatterBuilder(); @@ -153,7 +172,8 @@ public static Optional parseWithPattern(String text, String patte return retval; } - private static DateTimeFormatterBuilder appendZoneId(DateTimeFormatterBuilder builder) { + @NotNull + private static DateTimeFormatterBuilder appendZoneId(@NotNull DateTimeFormatterBuilder builder) { return builder.optionalStart() // This is to allow for bracket zoneId like [Europe/Paris] .appendLiteral('[') .parseCaseSensitive() @@ -161,19 +181,22 @@ private static DateTimeFormatterBuilder appendZoneId(DateTimeFormatterBuilder bu .appendLiteral(']'); } - private static ZonedDateTime parserUserPeriod(String text, ZonedDateTime now) { + @NotNull + private static ZonedDateTime parserUserPeriod(@NotNull String text, @NotNull ZonedDateTime now) { Period period = Period.parse(text); return now.plusYears(period.getYears()) .plusMonths(period.getMonths()) .plusDays(period.getDays()); } - private static ZonedDateTime parseUserDuration(String text, ZonedDateTime now) { + @NotNull + private static ZonedDateTime parseUserDuration(@NotNull String text, @NotNull ZonedDateTime now) { Duration duration = Duration.parse(text); return now.plus(duration); } - public static ZonedDateTime toZdt(final Timestamp time) { + @Nullable + public static ZonedDateTime toZdt(@Nullable final Timestamp time) { if (time != null) { return ZonedDateTime.ofInstant(time.toInstant(), ZoneId.of("UTC")); } else { diff --git a/cwms-data-api/src/main/webapp/rsql.html b/cwms-data-api/src/main/webapp/rsql.html new file mode 100644 index 000000000..e12946088 --- /dev/null +++ b/cwms-data-api/src/main/webapp/rsql.html @@ -0,0 +1,118 @@ + + + + + + + CDA - CWMS Data API + + + + + + + + +
+ +
+
+
+
+
+

+ Filter Expression Language + +

+
+
+ The Filtered Time Series end point allows the user to supply a Filter Expression to reduce the amount of data returned. + This user-provided filter is not SQL but instead a super-set of the Feed Item Query Language (FIQL) + This capability is based on rsql-parser and inspired by this article. + A somewhat similar approach was used to parse the filter-expression into an AST and then to walk the tree and make use of jOOQ to build up a SQL Condition. + +

RSQL Syntax

+

RSQL is a query language for parametrized filtering of resources. It's based on FIQL (Feed Item Query Language) and provides a simple yet powerful syntax for filtering data.

+ +

Comparison

+

A comparison expression consists of three components: a selector, an operator, and a value

+ +

Selector

+

The selector is a reference to some field in the input data. Places where filter-expresions are used will specify the selectors that are available.

+

For TimeSeries the selectors are: value, datetime, quality, data_entry_date

+ + +

Basic Operators

+
    +
  • == Equal to
    • +
  • != Not equal to
    • +
  • > Greater than
    • +
  • >= Greater than or equal to
    • +
  • < Less than
    • +
  • <= Less than or equal to
    • +
  • =in= In a list of values
    • +
  • =out= Not in a list of values
    • +
+ +

Logical Operators

+
    +
  • ; or and Logical AND
    • +
  • , or or Logical OR
    • +
+ +

Special Values

+
    +
  • null - Represents a null value (e.g., value!=null)
    • +
  • String values with spaces should be enclosed in double quotes (e.g., name=="Example Name")
    • +
  • When a timestamp selector (such as datetime or data_entry_date) is encountered an attempt is made to convert the corresponding value into a Timestamp. + The same timestamp parsing methods used in other portions of CDA (such as parsing start/end parameters) are used in the filter-expression feature.
    • +
+ + +
+
+
+
+

+ Filter Expression Language examples + +

+
+
+ + The TimeSeries end-point can return a large amount of data. Using RSQL filter expressions allows you to narrow down the results based on specific criteria. Here are some examples: + +

Basic Filtering

+
    +
  • value==5.0 - Find time series points where value equal to 5
    • +
  • value>25 - Find time series with values greater than 25
    • +
  • value>=100 - Find time series with values greater than or equal to 100
    • +
  • value<50 - Find time series with values less than 50
    • +
  • value<=75 - Find time series with values less than or equal to 75
    • +
  • value!=null - Find time series with non-null values
    • +
+ +

Combining Conditions

+
    +
  • data_entry_date<=2021-04-05T00:00:00Z;value>25 - Find points where value is greater than 25 AND the data_entry_date is on or before April 4 2021
    • +
  • value==null or value==-901 - Find points where the value is null OR the value is equal to negative 901
    • +
+ +
+
+
+
+
+
+
+
+
+ + diff --git a/cwms-data-api/src/main/webapp/times.html b/cwms-data-api/src/main/webapp/times.html new file mode 100644 index 000000000..8a5ce6175 --- /dev/null +++ b/cwms-data-api/src/main/webapp/times.html @@ -0,0 +1,122 @@ + + + + + + + CDA - CWMS Data API + + + + + + + + +
+ +
+
+
+
+
+

+ Timestamps + +

+
+
+

Several web service end-points make use of timestamp fields to represent start and end times. The CWMS Data API provides flexible date/time parsing and supports several formats:

+ +
    +
  • Full ISO 8601 dates with timezone information
    • +
  • ISO 8601 dates without timezone information (using the fallback timezone)
    • +
  • Period strings (starting with "P")
    • +
  • Duration strings (starting with "PT")
    • +
+ +

There are still a few web-service end-points that pass user-provided timestamp string parameters directly to pl/sql functions. Those end-points are sometimes described as legacy or version 1 end-points. They often take a "format" parameter and do not use the accept header. In those cases the pl/sql parses the user string and a slightly different timestamp format must be used.

+
+
+
+
+

+ Time parsing examples + +

+
+
+

The following are examples of acceptable input strings for date/time parameters:

+ +

1. ISO 8601 Dates with Timezone Information

+

These formats include timezone information, so the fallback timezone is ignored:

+
    +
  • 2022-01-06T00:00:00Z - UTC timezone (Z)
    • +
  • 2022-01-06T07:00:01-07:00 - Offset timezone (-07:00)
    • +
  • 2022-01-06T07:00:02-0700 - Offset timezone without colon
    • +
  • 2021-06-10T13:00:23-07:00[PST8PDT] - Offset with named timezone
    • +
  • 2022-01-19T20:52:07+00:00[UTC] - Zero offset with named timezone
    • +
  • 2022-12-08T09:47:32-0800[America/Los_Angeles] - Offset with region timezone
    • +
+ +

2. ISO 8601 Dates without Timezone Information

+

When timezone information is not included, the fallback timezone is used:

+
    +
  • 2022-01-06T00:00:01 - Will be interpreted in the fallback timezone
    • +
  • 2021-04-05 - Date only (time will be set to midnight in the fallback timezone)
    • +
+

For example, if the fallback timezone is "US/Pacific" and you provide 2022-01-06T00:00:00, it will be interpreted as January 6, 2022, at midnight Pacific time.

+ +

3. Period Strings (Calendar-based)

+

Period strings start with "P" and represent calendar-based periods (days, months, years). They are relative to the current time:

+
    +
  • P-1D - 1 day ago
    • +
  • P-1M - 1 month ago
    • +
  • P-1Y - 1 year ago
    • +
  • P-1Y-2M-3D - 1 year, 2 months, and 3 days ago
    • +
+

Note: Periods respect calendar boundaries. For example, one month ago from March 30 is February 28 (in non-leap years).

+ +

4. Duration Strings (Time-based)

+

Duration strings start with "PT" and represent time-based durations (hours, minutes, seconds). They are relative to the current time:

+
    +
  • PT-24H - 24 hours ago
    • +
  • PT-25H-3M - 25 hours and 3 minutes ago
    • +
  • PT-1H-30M - 1 hour and 30 minutes ago
    • +
+

Note: Durations are exact time intervals. PT-24H is always exactly 24 hours, regardless of daylight saving time changes.

+ +

Timezone Handling

+

The API handles timezones in the following ways:

+
    +
  • If the input string includes timezone information, that timezone is used regardless of the fallback timezone.
    • +
  • If the input string does not include timezone information, the fallback timezone is used.
    • +
  • The fallback timezone comes from the "timezone" query parameter or is specified in the API call.
    • +
  • If no fallback timezone is specified, UTC is used as the default.
    • +
+ +

Error Handling

+

The API will throw a DateTimeException in the following cases:

+
    +
  • The input string cannot be parsed as a valid date/time, period, or duration.
    • +
  • The specified fallback timezone is invalid.
    • +
+ +
+
+
+
+
+
+
+
+
+ + diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java index 8987a6fbb..090b0b878 100644 --- a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesControllerTest.java @@ -1,24 +1,10 @@ package cwms.cda.api; import static org.junit.jupiter.api.Assertions.*; -import static org.mockito.ArgumentMatchers.eq; -import static org.mockito.ArgumentMatchers.isNotNull; -import static org.mockito.ArgumentMatchers.isNull; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.times; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; - -import com.codahale.metrics.MetricRegistry; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import cwms.cda.data.dao.TimeSeriesDao; + import cwms.cda.data.dto.TimeSeries; import cwms.cda.formatters.ContentType; import cwms.cda.formatters.Formats; -import cwms.cda.formatters.json.JsonV2; -import io.javalin.core.util.Header; -import io.javalin.http.Context; import java.io.ByteArrayInputStream; import java.io.InputStream; import java.nio.charset.StandardCharsets; @@ -26,103 +12,15 @@ import java.time.Duration; import java.time.Instant; import java.time.ZonedDateTime; -import java.util.LinkedHashMap; import java.util.List; -import java.util.Map; import java.util.TimeZone; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; import org.jetbrains.annotations.NotNull; -import org.jooq.DSLContext; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.ValueSource; class TimeSeriesControllerTest extends ControllerTest { - @Test - void testDaoMock() throws JsonProcessingException { - String officeId = "LRL"; - String tsId = "RYAN3.Stage.Inst.5Minutes.0.ZSTORE_TS_TEST"; - TimeSeries expected = buildTimeSeries(officeId, tsId); - - // build a mock dao that returns a pre-built ts when called a certain way - TimeSeriesDao dao = mock(TimeSeriesDao.class); - - // String cursor, - // int pageSize, - // String names, - // String office, - // String unit, - // ZonedDateTime begin, - // ZonedDateTime end, - // ZonedDateTime versionDate - // boolean trim - - when( - dao.getTimeseries(eq(""), eq(500), eq(tsId), eq(officeId), eq("EN"), - isNotNull(), isNotNull(), isNull(), eq(true), eq(false))).thenReturn(expected); - - - // build mock request and response - final HttpServletRequest request= mock(HttpServletRequest.class); - final HttpServletResponse response = mock(HttpServletResponse.class); - final Map map = new LinkedHashMap<>(); - - when(request.getAttribute("office-id")).thenReturn(officeId); - when(request.getAttribute("database")).thenReturn(null); - - when(request.getHeader(Header.ACCEPT)).thenReturn(Formats.JSONV2); - - Map urlParams = new LinkedHashMap<>(); - urlParams.put("office", officeId); - urlParams.put("name", tsId); - - String paramStr = buildParamStr(urlParams); - - when(request.getQueryString()).thenReturn(paramStr); - when(request.getRequestURL()).thenReturn(new StringBuffer( "http://127.0.0.1:7001/timeseries")); - - // build real context that uses the mock request/response - Context ctx = new Context(request, response, map); - - // Build a controller that doesn't actually talk to database - TimeSeriesController controller = new TimeSeriesController(new MetricRegistry()){ - @Override - protected DSLContext getDslContext(Context ctx) { - return null; - } - - @NotNull - @Override - protected TimeSeriesDao getTimeSeriesDao(DSLContext dsl) { - return dao; - } - }; - // make controller use our mock dao - - // Do a controller getAll with our context - controller.getAll(ctx); - - // Check that the controller accessed our mock dao in the expected way - verify(dao, times(1)). - getTimeseries(eq(""), eq(500), eq(tsId), eq(officeId), eq("EN"), - isNotNull(), isNotNull(), isNull(), eq(true), eq(false));// - - // Make sure controller thought it was happy - verify(response).setStatus(200); - // And make sure controller returned json - verify(response).setContentType(Formats.JSONV2); - - String result = ctx.resultString(); - assertNotNull(result); // MAke sure we got some sort of response - - // Turn json response back into a TimeSeries object - ObjectMapper om = JsonV2.buildObjectMapper(); - TimeSeries actual = om.readValue(result, TimeSeries.class); - - assertSimilar(expected, actual); - } private void assertSimilar(TimeSeries expected, TimeSeries actual) { // Make sure ts we got back resembles the fakeTS our mock dao was supposed to return. diff --git a/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesFilteredControllerTestIT.java b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesFilteredControllerTestIT.java new file mode 100644 index 000000000..14304f8fc --- /dev/null +++ b/cwms-data-api/src/test/java/cwms/cda/api/TimeSeriesFilteredControllerTestIT.java @@ -0,0 +1,382 @@ +package cwms.cda.api; + +import static io.restassured.RestAssured.given; +import static io.restassured.config.JsonConfig.jsonConfig; +import static org.hamcrest.Matchers.closeTo; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.nullValue; +import static org.junit.jupiter.api.Assertions.assertNotNull; + +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import cwms.cda.formatters.Formats; +import fixtures.TestAccounts; +import io.restassured.RestAssured; +import io.restassured.filter.log.LogDetail; +import io.restassured.path.json.config.JsonPathConfig; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import java.sql.SQLException; + +import javax.servlet.http.HttpServletResponse; + +import org.apache.commons.io.IOUtils; +import org.junit.jupiter.api.Tag; +import org.junit.jupiter.api.Test; + +@Tag("integration") +class TimeSeriesFilteredControllerTestIT extends DataApiTestIT { + + @Test + void test_filter_nulls() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/lrl/pseudo_reg_1hour.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + + JsonNode ts = mapper.readTree(tsData); + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; + String officeId = ts.get("office-id").asText(); + + try { + createLocation(location, true, officeId); + + TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; + + // inserting the time series + given() + .log().ifValidationFails(LogDetail.ALL,true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization",user.toHeaderValue()) + .queryParam(Controllers.OFFICE, officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // get it back without filter-nulls + given() + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL,true) + .accept(Formats.JSONV2) + .header("Authorization",user.toHeaderValue()) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT,"cfs") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN,"2023-01-11T12:00:00-00:00") + .queryParam(Controllers.END,"2023-01-11T13:00:00-00:00") + .queryParam(Controllers.FILTER_NULLS, false) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/filtered/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("time-series.values[0][1]", closeTo(500.0,0.0001)) + .body("time-series.values[1][1]", nullValue()) + .body("time-series.values[2][1]", nullValue()) + .body("time-series.values[3][1]", closeTo(600.0,0.0001)) + ; + + // get it back with filter-nulls + given() + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL,true) + .accept(Formats.JSONV2) + .header("Authorization",user.toHeaderValue()) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT,"cfs") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN,"2023-01-11T12:00:00-00:00") + .queryParam(Controllers.END,"2023-01-11T13:00:00-00:00") + .queryParam(Controllers.FILTER_NULLS, true) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/filtered/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("time-series.values[0][0]", equalTo(1673438400000L)) + .body("time-series.values[0][1]", closeTo(500.0,0.0001)) + .body("time-series.values[1][0]", equalTo(1673442000000L)) + .body("time-series.values[1][1]", closeTo(600.0,0.0001)) + .body("time-series.values.size()", equalTo(2)) + ; + } catch (SQLException ex) { + throw new RuntimeException("Unable to create location for TS", ex); + } + } + + @Test + void test_min_value() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/lrl/pseudo_reg_1hour.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + + JsonNode ts = mapper.readTree(tsData); + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; + String officeId = ts.get("office-id").asText(); + + try { + createLocation(location, true, officeId); + + TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; + + // inserting the time series + given() + .log().ifValidationFails(LogDetail.ALL,true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization",user.toHeaderValue()) + .queryParam(Controllers.OFFICE, officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // get it back with min-value + given() + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL,true) + .accept(Formats.JSONV2) + .header("Authorization",user.toHeaderValue()) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT,"cfs") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN,"2023-01-11T12:00:00-00:00") + .queryParam(Controllers.END,"2023-01-11T13:00:00-00:00") + .queryParam(Controllers.MIN_VALUE, 550.0) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/filtered/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("time-series.values[0][0]", equalTo(1673442000000L)) + .body("time-series.values[0][1]", closeTo(600.0,0.0001)) + .body("time-series.values.size()", equalTo(1)) + ; + } catch (SQLException ex) { + throw new RuntimeException("Unable to create location for TS", ex); + } + } + + @Test + void test_max_value() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/lrl/pseudo_reg_1hour.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + + JsonNode ts = mapper.readTree(tsData); + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; + String officeId = ts.get("office-id").asText(); + + try { + createLocation(location, true, officeId); + + TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; + + // inserting the time series + given() + .log().ifValidationFails(LogDetail.ALL,true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization",user.toHeaderValue()) + .queryParam(Controllers.OFFICE, officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // get it back with max-value + given() + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL,true) + .accept(Formats.JSONV2) + .header("Authorization",user.toHeaderValue()) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT,"cfs") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN,"2023-01-11T12:00:00-00:00") + .queryParam(Controllers.END,"2023-01-11T13:00:00-00:00") + .queryParam(Controllers.MAX_VALUE, 550.0) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/filtered") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("time-series.values[0][0]", equalTo(1673438400000L)) + .body("time-series.values[0][1]", closeTo(500.0,0.0001)) + .body("time-series.values.size()", equalTo(1)) + ; + } catch (SQLException ex) { + throw new RuntimeException("Unable to create location for TS", ex); + } + } + + @Test + void test_min_max_value_combined() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/lrl/pseudo_reg_1hour.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + + JsonNode ts = mapper.readTree(tsData); + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; + String officeId = ts.get("office-id").asText(); + + try { + createLocation(location, true, officeId); + + TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; + + // inserting the time series + given() + .log().ifValidationFails(LogDetail.ALL,true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization",user.toHeaderValue()) + .queryParam(Controllers.OFFICE, officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // get it back with min-value and max-value + given() + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL,true) + .accept(Formats.JSONV2) + .header("Authorization",user.toHeaderValue()) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT,"cfs") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN,"2023-01-11T12:00:00-00:00") + .queryParam(Controllers.END,"2023-01-11T13:00:00-00:00") + .queryParam(Controllers.MIN_VALUE, 450.0) + .queryParam(Controllers.MAX_VALUE, 550.0) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/filtered/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("time-series.values[0][0]", equalTo(1673438400000L)) + .body("time-series.values[0][1]", closeTo(500.0,0.0001)) + .body("time-series.values.size()", equalTo(1)) + ; + } catch (SQLException ex) { + throw new RuntimeException("Unable to create location for TS", ex); + } + } + + @Test + void test_all_filters_combined() throws Exception { + ObjectMapper mapper = new ObjectMapper(); + + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/api/lrl/pseudo_reg_1hour.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + + JsonNode ts = mapper.readTree(tsData); + String location = ts.get(Controllers.NAME).asText().split("\\.")[0]; + String officeId = ts.get("office-id").asText(); + + try { + createLocation(location, true, officeId); + + TestAccounts.KeyUser user = TestAccounts.KeyUser.SPK_NORMAL; + + // inserting the time series + given() + .log().ifValidationFails(LogDetail.ALL,true) + .accept(Formats.JSONV2) + .contentType(Formats.JSONV2) + .body(tsData) + .header("Authorization",user.toHeaderValue()) + .queryParam(Controllers.OFFICE, officeId) + .when() + .redirects().follow(true) + .redirects().max(3) + .post("/timeseries/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)); + + // get it back with filter-nulls, min-value, and max-value + given() + .config(RestAssured.config().jsonConfig(jsonConfig().numberReturnType(JsonPathConfig.NumberReturnType.DOUBLE))) + .log().ifValidationFails(LogDetail.ALL,true) + .accept(Formats.JSONV2) + .header("Authorization",user.toHeaderValue()) + .queryParam(Controllers.OFFICE, officeId) + .queryParam(Controllers.UNIT,"cfs") + .queryParam(Controllers.NAME, ts.get(Controllers.NAME).asText()) + .queryParam(Controllers.BEGIN,"2023-01-11T12:00:00-00:00") + .queryParam(Controllers.END,"2023-01-11T13:00:00-00:00") + .queryParam(Controllers.FILTER_NULLS, true) + .queryParam(Controllers.MIN_VALUE, 450.0) + .queryParam(Controllers.MAX_VALUE, 550.0) + .when() + .redirects().follow(true) + .redirects().max(3) + .get("/timeseries/filtered/") + .then() + .log().ifValidationFails(LogDetail.ALL,true) + .assertThat() + .statusCode(is(HttpServletResponse.SC_OK)) + .body("time-series.values[0][0]", equalTo(1673438400000L)) + .body("time-series.values[0][1]", closeTo(500.0,0.0001)) + .body("time-series.values.size()", equalTo(1)) + ; + } catch (SQLException ex) { + throw new RuntimeException("Unable to create location for TS", ex); + } + } + +} diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/rsql/MapFieldResolverTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/rsql/MapFieldResolverTest.java new file mode 100644 index 000000000..ae4c5eac6 --- /dev/null +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/rsql/MapFieldResolverTest.java @@ -0,0 +1,74 @@ +package cwms.cda.data.dao.rsql; + +import cz.jirutka.rsql.parser.RSQLParser; +import cz.jirutka.rsql.parser.ast.Node; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.Test; + +import java.util.HashMap; +import java.util.Map; + +import static org.junit.jupiter.api.Assertions.*; + +class MapFieldResolverTest { + + @Test + void testResolve() { + // Create a map of field names to fields + Map> nameToField = new HashMap<>(); + Field unitIdField = DSL.field("UNIT_ID", String.class); + Field valueField = DSL.field("VALUE", Integer.class); + + nameToField.put("unit_id", unitIdField); + nameToField.put("value", valueField); + + // Create the resolver + MapFieldResolver resolver = new MapFieldResolver(nameToField); + + // Test resolving existing fields + Field resolvedUnitId = resolver.resolve("unit_id"); + Field resolvedValue = resolver.resolve("value"); + + assertNotNull(resolvedUnitId); + assertNotNull(resolvedValue); + assertEquals("UNIT_ID", resolvedUnitId.getName()); + assertEquals("VALUE", resolvedValue.getName()); + + // Test resolving a non-existent field + Exception exception = assertThrows(IllegalArgumentException.class, () -> resolver.resolve("non_existent_field")); + + assertTrue(exception.getMessage().contains("Unknown field: non_existent_field")); + } + + @Test + void testWithRSQLJooqConditionVisitor() { + // Create a map of field names to fields + Map> nameToField = new HashMap<>(); + Field unitIdField = DSL.field("UNIT_ID", String.class); + Field valueField = DSL.field("VALUE", Integer.class); + + nameToField.put("unit_id", unitIdField); + nameToField.put("value", valueField); + + // Create the resolver + MapFieldResolver resolver = new MapFieldResolver(nameToField); + + // Parse an RSQL expression + Node root = new RSQLParser().parse("unit_id==EN;value>25"); + + // Use the resolver with RSQLJooqConditionVisitor + Condition testCondition = root.accept(new RSQLJooqConditionVisitor(resolver)); + + // Verify the condition + assertNotNull(testCondition); + String condStr = testCondition.toString(); + System.out.println("Condition string: " + condStr); + + // The actual format might be different, so we'll check for the field names and values + // rather than the exact format + assertTrue(condStr.contains("UNIT_ID") && condStr.contains("EN")); + assertTrue(condStr.contains("VALUE") && condStr.contains("25")); + } +} diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/rsql/RSQLConditionBuilderTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/rsql/RSQLConditionBuilderTest.java new file mode 100644 index 000000000..26db9e8b8 --- /dev/null +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/rsql/RSQLConditionBuilderTest.java @@ -0,0 +1,187 @@ +package cwms.cda.data.dao.rsql; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import org.jooq.Condition; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import usace.cwms.db.jooq.codegen.tables.AV_TSV_DQU; + +import java.util.HashMap; +import java.util.Map; +import java.util.TimeZone; + +class RSQLConditionBuilderTest { + + @BeforeAll + static void forceUtc() { + // When jOOQ converts the Timestamp to a str it seems to use the system default tz. + TimeZone.setDefault(TimeZone.getTimeZone("UTC")); + System.setProperty("user.timezone", "UTC"); + } + + @Test + void testBuildCondition() { + // Create a field resolver using the JooqFieldResolver + JooqFieldResolver resolver = new JooqFieldResolver(AV_TSV_DQU.AV_TSV_DQU); + RSQLConditionBuilder builder = new RSQLConditionBuilder(resolver); + + Condition condition = builder.buildCondition("unit_id==EN;value>25"); + + assertNotNull(condition); + String conditionString = condition.toString(); + + assertTrue(conditionString.contains("\"AV_TSV_DQU\".\"UNIT_ID\" = 'EN'")); + assertTrue(conditionString.contains("\"AV_TSV_DQU\".\"VALUE\" > 2.5E1")); + } + + @Test + void testBuildConditionWithMapFieldResolver() { + // Create a map of field names to fields + Map> fieldMap = new HashMap<>(); + fieldMap.put("unit_id", AV_TSV_DQU.AV_TSV_DQU.UNIT_ID); + fieldMap.put("value", AV_TSV_DQU.AV_TSV_DQU.VALUE); + + // Create a field resolver using the MapFieldResolver + MapFieldResolver resolver = new MapFieldResolver(fieldMap); + + // Create the RSQLConditionBuilder + RSQLConditionBuilder builder = new RSQLConditionBuilder(resolver); + + // Build a condition from an RSQL query + Condition condition = builder.buildCondition("unit_id==EN;value>25"); + + // Verify the condition is not null and contains the expected SQL + assertNotNull(condition); + String conditionString = condition.toString(); + + assertTrue(conditionString.contains("\"AV_TSV_DQU\".\"UNIT_ID\" = 'EN'")); + assertTrue(conditionString.contains("\"AV_TSV_DQU\".\"VALUE\" > 2.5E1")); + } + + @Test + void testBuildConditionWithFactoryMethod() { + // Create a field resolver using the JooqFieldResolver + JooqFieldResolver resolver = new JooqFieldResolver(AV_TSV_DQU.AV_TSV_DQU); + + // Create the RSQLConditionBuilder using the factory method + RSQLConditionBuilder builder = RSQLConditionBuilder.create(resolver); + + // Build a condition from an RSQL query + Condition condition = builder.buildCondition("unit_id==EN"); + + // Verify the condition is not null and contains the expected SQL + assertNotNull(condition); + String conditionString = condition.toString(); + assertTrue(conditionString.contains("\"UNIT_ID\" = 'EN'")); + } + + @Test + void testBuildConditionWithEmptyQuery() { + // Create a field resolver using the JooqFieldResolver + JooqFieldResolver resolver = new JooqFieldResolver(AV_TSV_DQU.AV_TSV_DQU); + + // Create the RSQLConditionBuilder + RSQLConditionBuilder builder = new RSQLConditionBuilder(resolver); + + // Verify that an empty query throws an IllegalArgumentException + assertThrows(IllegalArgumentException.class, () -> builder.buildCondition("")); + assertThrows(IllegalArgumentException.class, () -> builder.buildCondition(null)); + } + + @Test + void testBuildConditionWithInOperator() { + // Create a field resolver using the JooqFieldResolver + JooqFieldResolver resolver = new JooqFieldResolver(AV_TSV_DQU.AV_TSV_DQU); + + // Create the RSQLConditionBuilder + RSQLConditionBuilder builder = new RSQLConditionBuilder(resolver); + + // Build a condition from an RSQL query using IN operator + Condition condition = builder.buildCondition("unit_id=in=(EN,CFS,FEET)"); + + // Verify the condition is not null and contains the expected SQL + assertNotNull(condition); + String conditionString = condition.toString(); + + assertTrue(conditionString.contains("\"UNIT_ID\" in (")); + assertTrue(conditionString.contains("'EN'")); + assertTrue(conditionString.contains("'CFS'")); + assertTrue(conditionString.contains("'FEET'")); + } + + @Test + void testBuildConditionWithNotInOperator() { + // Create a field resolver using the JooqFieldResolver + JooqFieldResolver resolver = new JooqFieldResolver(AV_TSV_DQU.AV_TSV_DQU); + + // Create the RSQLConditionBuilder + RSQLConditionBuilder builder = new RSQLConditionBuilder(resolver); + + // Build a condition from an RSQL query using NOT IN operator + Condition condition = builder.buildCondition("unit_id=out=(EN,CFS)"); + + // Verify the condition is not null and contains the expected SQL + assertNotNull(condition); + String conditionString = condition.toString(); + + assertTrue(conditionString.contains("\"UNIT_ID\" not in (")); + assertTrue(conditionString.contains("'EN'")); + assertTrue(conditionString.contains("'CFS'")); + } + + @Test + void testBuildConditionWithVersionDate() { + // Create a field resolver using the JooqFieldResolver + JooqFieldResolver resolver = new JooqFieldResolver(AV_TSV_DQU.AV_TSV_DQU); + + // Create the RSQLConditionBuilder + RSQLConditionBuilder builder = new RSQLConditionBuilder(resolver); + + + // Build a condition from an RSQL query using VERSION_DATE + Condition condition = builder.buildCondition("version_date==2021-04-05T00:00:00Z"); + + // Verify the condition is not null and contains the expected SQL + assertNotNull(condition); + String conditionString = condition.toString(); + + assertTrue(conditionString.contains("\"AV_TSV_DQU\".\"VERSION_DATE\" = timestamp '2021-04-05 00:00:00.0'")); + } + + @Test + void testBuildConditionWithVersionDateRange() { + JooqFieldResolver resolver = new JooqFieldResolver(AV_TSV_DQU.AV_TSV_DQU); + RSQLConditionBuilder builder = new RSQLConditionBuilder(resolver); + Condition condition = builder.buildCondition("version_date>=2021-01-01T00:00:00Z;version_date<=2021-12-31T23:59:59Z"); + + // Verify the condition is not null and contains the expected SQL + assertNotNull(condition); + String conditionString = condition.toString(); + + assertTrue(conditionString.contains("\"AV_TSV_DQU\".\"VERSION_DATE\" >= timestamp '2021-01-01 00:00:00.0'")); + assertTrue(conditionString.contains("\"AV_TSV_DQU\".\"VERSION_DATE\" <= timestamp '2021-12-31 23:59:59.0'")); + } + + @Test + void testBuildConditionComparingFields() { + Map> fieldMap = new HashMap<>(); + fieldMap.put("version_date", AV_TSV_DQU.AV_TSV_DQU.VERSION_DATE); + fieldMap.put("data_entry_date", AV_TSV_DQU.AV_TSV_DQU.DATA_ENTRY_DATE); + fieldMap.put("date_time", AV_TSV_DQU.AV_TSV_DQU.DATE_TIME); + + MapFieldResolver mapResolver = new MapFieldResolver(fieldMap); + + RSQLConditionBuilder builder = new RSQLConditionBuilder(mapResolver); + + Condition condition = builder.buildCondition("version_date==2021-04-05T00:00:00Z;data_entry_date>=2021-04-01T00:00:00Z"); + + assertNotNull(condition); + String conditionString = condition.toString(); + + assertTrue(conditionString.contains("\"AV_TSV_DQU\".\"VERSION_DATE\" = timestamp '2021-04-05 00:00:00.0'")); + assertTrue(conditionString.contains("\"AV_TSV_DQU\".\"DATA_ENTRY_DATE\" >= timestamp '2021-04-01 00:00:00.0'")); + } +} diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/rsql/RSQLJooqConditionVisitorTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/rsql/RSQLJooqConditionVisitorTest.java new file mode 100644 index 000000000..e5fb8fde8 --- /dev/null +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/rsql/RSQLJooqConditionVisitorTest.java @@ -0,0 +1,325 @@ +package cwms.cda.data.dao.rsql; + +import cwms.cda.helpers.DateUtils; +import cz.jirutka.rsql.parser.RSQLParser; +import cz.jirutka.rsql.parser.ast.AndNode; +import cz.jirutka.rsql.parser.ast.ComparisonNode; +import cz.jirutka.rsql.parser.ast.ComparisonOperator; +import cz.jirutka.rsql.parser.ast.Node; +import cz.jirutka.rsql.parser.ast.OrNode; +import cz.jirutka.rsql.parser.ast.RSQLOperators; +import org.jooq.Condition; +import org.jooq.Field; +import org.jooq.impl.DSL; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + +import java.math.BigDecimal; +import java.sql.Timestamp; +import java.time.ZonedDateTime; +import java.util.Arrays; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.UUID; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.*; + +class RSQLJooqConditionVisitorTest { + + private MapFieldResolver fieldResolver; + private RSQLJooqConditionVisitor visitor; + private Map> fieldMap; + + @BeforeEach + void setUp() { + + fieldMap = new HashMap<>(); + fieldMap.put("string_field", DSL.field("STRING_FIELD", String.class)); + fieldMap.put("int_field", DSL.field("INT_FIELD", Integer.class)); + fieldMap.put("long_field", DSL.field("LONG_FIELD", Long.class)); + fieldMap.put("boolean_field", DSL.field("BOOLEAN_FIELD", Boolean.class)); + fieldMap.put("double_field", DSL.field("DOUBLE_FIELD", Double.class)); + fieldMap.put("decimal_field", DSL.field("DECIMAL_FIELD", BigDecimal.class)); + // Removed ZonedDateTime field as it's not supported in DEFAULT dialect + fieldMap.put("timestamp_field", DSL.field("TIMESTAMP_FIELD", Timestamp.class)); + fieldMap.put("uuid_field", DSL.field("UUID_FIELD", UUID.class)); + + fieldResolver = new MapFieldResolver(fieldMap); + visitor = new RSQLJooqConditionVisitor(fieldResolver); + } + + @Test + void testVisitAndNode() { + // Create an RSQL expression with AND operator + Node node = new RSQLParser().parse("string_field==value;int_field>10"); + + // The node should be an AndNode + assertTrue(node instanceof AndNode); + + // Visit the node + Condition condition = node.accept(visitor, null); + + // Verify the condition + assertNotNull(condition); + String conditionStr = condition.toString(); + + // Check that the condition contains both parts connected by AND + assertTrue(conditionStr.contains("STRING_FIELD") && conditionStr.contains("value")); + assertTrue(conditionStr.contains("INT_FIELD") && conditionStr.contains("10")); + assertTrue(conditionStr.contains("and")); + } + + @Test + void testVisitOrNode() { + // Create an RSQL expression with OR operator + Node node = new RSQLParser().parse("string_field==value,int_field>10"); + + // The node should be an OrNode + assertTrue(node instanceof OrNode); + + // Visit the node + Condition condition = node.accept(visitor, null); + + // Verify the condition + assertNotNull(condition); + String conditionStr = condition.toString(); + + // Check that the condition contains both parts connected by OR + assertTrue(conditionStr.contains("STRING_FIELD") && conditionStr.contains("value")); + assertTrue(conditionStr.contains("INT_FIELD") && conditionStr.contains("10")); + assertTrue(conditionStr.contains("or")); + } + + @Test + void testVisitComparisonNode() { + // Create a comparison node for testing + ComparisonOperator operator = RSQLOperators.EQUAL; + String selector = "string_field"; + List arguments = Collections.singletonList("test_value"); + ComparisonNode node = new ComparisonNode(operator, selector, arguments); + + // Visit the node + Condition condition = visitor.visit(node, null); + + // Verify the condition + assertNotNull(condition); + String conditionStr = condition.toString(); + + // Check that the condition contains the field and value + assertTrue(conditionStr.contains("STRING_FIELD") && conditionStr.contains("test_value")); + } + + @Test + void testIsNullLiteral() { + // Test with null value + assertTrue(RSQLJooqConditionVisitor.isNullLiteral(null)); + + // Test with "null" string + assertTrue(RSQLJooqConditionVisitor.isNullLiteral("null")); + + // Test with "NULL" string (case insensitive) + assertTrue(RSQLJooqConditionVisitor.isNullLiteral("NULL")); + + // Test with non-null value + assertFalse(RSQLJooqConditionVisitor.isNullLiteral("value")); + + // Test with double quoted null value + assertFalse(RSQLJooqConditionVisitor.isNullLiteral("\"null\"")); + + // Test with single quoted null value + assertFalse(RSQLJooqConditionVisitor.isNullLiteral("\'null\'")); + } + + @Test + void testBuildNullCondition() { + // Test with EQUAL operator + Field field = DSL.field("TEST_FIELD", Object.class); + Condition condition = RSQLJooqConditionVisitor.buildNullCondition(field, RSQLOperators.EQUAL); + + assertNotNull(condition); + String conditionStr = condition.toString(); + assertTrue(conditionStr.contains("TEST_FIELD is null")); + + // Test with NOT_EQUAL operator + condition = RSQLJooqConditionVisitor.buildNullCondition(field, RSQLOperators.NOT_EQUAL); + + assertNotNull(condition); + conditionStr = condition.toString(); + assertTrue(conditionStr.contains("TEST_FIELD is not null")); + + // Test with unsupported operator + assertThrows(IllegalArgumentException.class, + () -> RSQLJooqConditionVisitor.buildNullCondition(field, RSQLOperators.GREATER_THAN)); + } + + @Test + void testBuildCondition() { + Field field = DSL.field("TEST_FIELD", Object.class); + Object value = "test_value"; + List values = Arrays.asList("value1", "value2", "value3"); + + // Test with EQUAL operator + Condition condition = RSQLJooqConditionVisitor.buildCondition(field, RSQLOperators.EQUAL, value, values); + assertNotNull(condition); + String conditionStr = condition.toString(); + assertTrue(conditionStr.contains("TEST_FIELD = 'test_value'")); + + // Test with NOT_EQUAL operator + condition = RSQLJooqConditionVisitor.buildCondition(field, RSQLOperators.NOT_EQUAL, value, values); + assertNotNull(condition); + conditionStr = condition.toString(); + assertTrue(conditionStr.contains("TEST_FIELD <> 'test_value'")); + + // Test with GREATER_THAN operator + condition = RSQLJooqConditionVisitor.buildCondition(field, RSQLOperators.GREATER_THAN, value, values); + assertNotNull(condition); + conditionStr = condition.toString(); + assertTrue(conditionStr.contains("TEST_FIELD > 'test_value'")); + + // Test with GREATER_THAN_OR_EQUAL operator + condition = RSQLJooqConditionVisitor.buildCondition(field, RSQLOperators.GREATER_THAN_OR_EQUAL, value, values); + assertNotNull(condition); + conditionStr = condition.toString(); + assertTrue(conditionStr.contains("TEST_FIELD >= 'test_value'")); + + // Test with LESS_THAN operator + condition = RSQLJooqConditionVisitor.buildCondition(field, RSQLOperators.LESS_THAN, value, values); + assertNotNull(condition); + conditionStr = condition.toString(); + assertTrue(conditionStr.contains("TEST_FIELD < 'test_value'")); + + // Test with LESS_THAN_OR_EQUAL operator + condition = RSQLJooqConditionVisitor.buildCondition(field, RSQLOperators.LESS_THAN_OR_EQUAL, value, values); + assertNotNull(condition); + conditionStr = condition.toString(); + assertTrue(conditionStr.contains("TEST_FIELD <= 'test_value'")); + + // Test with IN operator + condition = RSQLJooqConditionVisitor.buildCondition(field, RSQLOperators.IN, value, values); + assertNotNull(condition); + conditionStr = condition.toString(); + System.out.println("[DEBUG_LOG] IN condition: " + conditionStr); + // The format of the IN condition might vary depending on the jOOQ version and dialect + // So we'll check for the field name and each value separately + assertTrue(conditionStr.contains("TEST_FIELD")); + assertTrue(conditionStr.toLowerCase().contains("in")); + assertTrue(conditionStr.contains("value1")); + assertTrue(conditionStr.contains("value2")); + assertTrue(conditionStr.contains("value3")); + + // Test with NOT_IN operator + condition = RSQLJooqConditionVisitor.buildCondition(field, RSQLOperators.NOT_IN, value, values); + assertNotNull(condition); + conditionStr = condition.toString(); + System.out.println("[DEBUG_LOG] NOT_IN condition: " + conditionStr); + // The format of the NOT IN condition might vary depending on the jOOQ version and dialect + // So we'll check for the field name and each value separately + assertTrue(conditionStr.contains("TEST_FIELD")); + assertTrue(conditionStr.toLowerCase().contains("not in")); + assertTrue(conditionStr.contains("value1")); + assertTrue(conditionStr.contains("value2")); + assertTrue(conditionStr.contains("value3")); + + // Test with unsupported operator + ComparisonOperator unsupportedOperator = new ComparisonOperator("=unsupported=", true); + assertThrows(IllegalArgumentException.class, + () -> RSQLJooqConditionVisitor.buildCondition(field, unsupportedOperator, value, values)); + } + + @ParameterizedTest + @MethodSource("provideFieldsAndValues") + void testConvert(Field field, String value, Class expectedType) { + Object result = RSQLJooqConditionVisitor.convert(field, value); + + if (value == null || value.trim().isEmpty() || "null".equalsIgnoreCase(value.trim())) { + assertNull(result); + } else { + assertNotNull(result); + assertTrue(expectedType.isInstance(result), + "Expected " + result + " to be instance of " + expectedType.getName()); + } + } + + @Test + void testConvertWithNullOrEmptyValue() { + Field field = DSL.field("TEST_FIELD", String.class); + + // Test with null value + assertNull(RSQLJooqConditionVisitor.convert(field, null)); + + // Test with empty value + assertNull(RSQLJooqConditionVisitor.convert(field, "")); + + // Test with "null" string + assertNull(RSQLJooqConditionVisitor.convert(field, "null")); + } + + @Test + void testConvertWithUnsupportedType() { + // Create a field with an unsupported type + Field field = DSL.field("TEST_FIELD", Object.class); + + // Test that convert throws IllegalArgumentException + assertThrows(IllegalArgumentException.class, + () -> RSQLJooqConditionVisitor.convert(field, "value")); + } + + @Test + void testConvertList() { + Field field = DSL.field("TEST_FIELD", String.class); + List values = Arrays.asList("value1", "value2", "value3"); + + List result = RSQLJooqConditionVisitor.convertList(field, values); + + assertNotNull(result); + assertEquals(3, result.size()); + assertEquals("value1", result.get(0)); + assertEquals("value2", result.get(1)); + assertEquals("value3", result.get(2)); + } + + + // Method source for parameterized test + private static Stream provideFieldsAndValues() { + return Stream.of( + Arguments.of(DSL.field("STRING_FIELD", String.class), "test_value", String.class), + Arguments.of(DSL.field("INT_FIELD", Integer.class), "42", Integer.class), + Arguments.of(DSL.field("LONG_FIELD", Long.class), "9223372036854775807", Long.class), + Arguments.of(DSL.field("BOOLEAN_FIELD", Boolean.class), "true", Boolean.class), + Arguments.of(DSL.field("DOUBLE_FIELD", Double.class), "3.14159", Double.class), + Arguments.of(DSL.field("DECIMAL_FIELD", BigDecimal.class), "123.456", BigDecimal.class), + + Arguments.of(DSL.field("TIMESTAMP_FIELD", Timestamp.class), "2023-01-01T00:00:00Z", Timestamp.class), + Arguments.of(DSL.field("UUID_FIELD", UUID.class), "123e4567-e89b-12d3-a456-426614174000", UUID.class), + Arguments.of(DSL.field("NULL_FIELD", String.class), null, null), + Arguments.of(DSL.field("EMPTY_FIELD", String.class), "", null), + Arguments.of(DSL.field("NULL_STRING_FIELD", String.class), "null", null) + ); + } + + @Test + void testConvertZonedDateTime() { + // Since we can't use DSL.field() with ZonedDateTime in DEFAULT dialect, + // and we can't easily mock the Field interface, we'll test the DateUtils.parseUserDate method directly, + // since that's what the convert method uses for ZonedDateTime conversion + + String dateStr = "2023-01-01T00:00:00Z"; + ZonedDateTime zdt = DateUtils.parseUserDate(dateStr, "UTC"); + + // Verify the parsed date + assertNotNull(zdt); + assertEquals(2023, zdt.getYear()); + assertEquals(1, zdt.getMonthValue()); + assertEquals(1, zdt.getDayOfMonth()); + assertEquals(0, zdt.getHour()); + assertEquals(0, zdt.getMinute()); + assertEquals(0, zdt.getSecond()); + assertEquals("Z", zdt.getZone().toString()); + } +} diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/rsql/RSQLParserTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/rsql/RSQLParserTest.java new file mode 100644 index 000000000..67dc0251f --- /dev/null +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/rsql/RSQLParserTest.java @@ -0,0 +1,52 @@ +package cwms.cda.data.dao.rsql; + +import cz.jirutka.rsql.parser.ast.Node; +import cz.jirutka.rsql.parser.RSQLParser; +import org.jooq.Condition; +import org.junit.jupiter.api.Test; +import usace.cwms.db.jooq.codegen.tables.AV_TSV_DQU; + +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class RSQLParserTest { + + @Test + void testParse(){ + Node root = new RSQLParser().parse("unit_id==EN;value>25"); + + JooqFieldResolver resolver = new JooqFieldResolver(AV_TSV_DQU.AV_TSV_DQU); + + Condition testCondition = root.accept(new RSQLJooqConditionVisitor(resolver)); + assertNotNull(testCondition); + String condStr = testCondition.toString(); + assertTrue( condStr.contains("\"UNIT_ID\" = 'EN'")); + + } + + @Test + void testParseQuote(){ + Node root = new RSQLParser().parse("unit_id==\"EN\";value>25"); + + JooqFieldResolver resolver = new JooqFieldResolver(AV_TSV_DQU.AV_TSV_DQU); + + Condition testCondition = root.accept(new RSQLJooqConditionVisitor(resolver)); + assertNotNull(testCondition); + String condStr = testCondition.toString(); + assertTrue( condStr.contains("\"UNIT_ID\" = 'EN'")); + + } + + @Test + void testParseNull(){ + Node root = new RSQLParser().parse("unit_id==EN;value!=null"); + + JooqFieldResolver resolver = new JooqFieldResolver(AV_TSV_DQU.AV_TSV_DQU); + + Condition testCondition = root.accept(new RSQLJooqConditionVisitor(resolver)); + assertNotNull(testCondition); + String condStr = testCondition.toString(); + System.out.println( condStr ); + assertTrue( condStr.contains("\"UNIT_ID\" = 'EN'")); + } +} diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dto/filteredtimeseries/FilteredTimeSeriesTest.java b/cwms-data-api/src/test/java/cwms/cda/data/dto/filteredtimeseries/FilteredTimeSeriesTest.java new file mode 100644 index 000000000..1bec8708c --- /dev/null +++ b/cwms-data-api/src/test/java/cwms/cda/data/dto/filteredtimeseries/FilteredTimeSeriesTest.java @@ -0,0 +1,164 @@ +package cwms.cda.data.dto.filteredtimeseries; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertNull; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import cwms.cda.data.dao.FilteredTimeSeriesParameters; +import cwms.cda.data.dto.TimeSeries; +import cwms.cda.formatters.ContentType; +import cwms.cda.formatters.Formats; +import cwms.cda.formatters.json.JsonV2; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.time.ZoneId; +import java.time.ZonedDateTime; +import java.io.IOException; +import java.io.InputStream; +import java.nio.charset.StandardCharsets; +import org.apache.commons.io.IOUtils; +import org.jetbrains.annotations.NotNull; +import org.junit.jupiter.api.Test; + +class FilteredTimeSeriesTest { + + @Test + void testConstructor() { + TimeSeries ts = buildTimeSeries(); + FilteredTimeSeriesParameters params = buildFilterParams(); + + FilteredTimeSeries filteredTs = new FilteredTimeSeries(ts, params); + + assertEquals(ts, filteredTs.getTimeSeries()); + assertEquals(params, filteredTs.getFilterParams()); + } + + @Test + void testJsonSerialization() throws JsonProcessingException { + TimeSeries ts = buildTimeSeries(); + FilteredTimeSeriesParameters params = buildFilterParams(); + FilteredTimeSeries filteredTs = new FilteredTimeSeries(ts, params); + + ObjectMapper om = buildObjectMapper(); + + String tsBody = om.writeValueAsString(filteredTs); + assertNotNull(tsBody); + + assertTrue(tsBody.contains("\"name\":\"Calhoun.Flow.Inst.~1Hour.0.cda-test\"")); + assertTrue(tsBody.contains("\"office-id\":\"SPK\"")); + assertTrue(tsBody.contains("\"units\":\"cfs\"")); + + + assertTrue(tsBody.contains("\"filter-parameters\"")); + assertTrue(tsBody.contains("\"min-value\":450.0")); + assertTrue(tsBody.contains("\"max-value\":550.0")); + assertTrue(tsBody.contains("\"filter-nulls\":true")); + } + + @Test + void testSerializerWithNulls() { + TimeSeries ts = buildTimeSeriesWithNulls(); + FilteredTimeSeriesParameters params = buildFilterParams(); + FilteredTimeSeries filteredTs = new FilteredTimeSeries(ts, params); + + String tsBody = Formats.format(new ContentType(Formats.JSONV2), filteredTs); + assertNotNull(tsBody); + } + + @Test + void testResourceFileExists() throws IOException { + InputStream resource = this.getClass().getResourceAsStream( + "/cwms/cda/data/dto/filteredtimeseries/filtered_timeseries_test.json"); + assertNotNull(resource); + String tsData = IOUtils.toString(resource, StandardCharsets.UTF_8); + assertNotNull(tsData); + + assertTrue(tsData.contains("name")); + assertTrue(tsData.contains("Calhoun.Flow.Inst.~1Hour.0.cda-test")); + assertTrue(tsData.contains("office-id")); + assertTrue(tsData.contains("SPK")); + assertTrue(tsData.contains("units")); + assertTrue(tsData.contains("cfs")); + + assertTrue(tsData.contains("filter-parameters")); + assertTrue(tsData.contains("min-value")); + assertTrue(tsData.contains("max-value")); + assertTrue(tsData.contains("filter-nulls")); + } + + @NotNull + private TimeSeries buildTimeSeries() { + String tsId = "Calhoun.Flow.Inst.~1Hour.0.cda-test"; + + ZonedDateTime start = ZonedDateTime.parse("2023-01-11T12:00:00Z"); + ZonedDateTime end = ZonedDateTime.parse("2023-01-11T13:00:00Z"); + ZonedDateTime versionDate = Instant.now().atZone(ZoneId.of("UTC")); + TimeSeries ts = new TimeSeries(null, -1, 0, tsId, "SPK", start, end, "cfs", Duration.ZERO, null, versionDate, null); + + + ts.addValue(Timestamp.from(Instant.ofEpochMilli(1673438400000L)), 500.0, 0); + ts.addValue(Timestamp.from(Instant.ofEpochMilli(1673442000000L)), 600.0, 0); + + return ts; + } + + private TimeSeries buildTimeSeriesWithNulls() { + String tsId = "Calhoun.Flow.Inst.~1Hour.0.cda-test"; + + ZonedDateTime start = ZonedDateTime.parse("2023-01-11T12:00:00Z"); + ZonedDateTime end = ZonedDateTime.parse("2023-01-11T13:00:00Z"); + ZonedDateTime versionDate = Instant.now().atZone(ZoneId.of("UTC")); + TimeSeries ts = new TimeSeries(null, -1, 0, tsId, "SPK", start, end, "cfs", Duration.ZERO, null, versionDate, null); + + ts.addValue(Timestamp.from(Instant.ofEpochMilli(1673438400000L)), 500.0, 0); + ts.addValue(Timestamp.from(Instant.ofEpochMilli(1673439400000L)), null, 5); + ts.addValue(Timestamp.from(Instant.ofEpochMilli(1673442000000L)), 600.0, 0); + + return ts; + } + + private FilteredTimeSeriesParameters buildFilterParams() { + return new FilteredTimeSeriesParameters.Builder() + .withMinValue(450.0) + .withMaxValue(550.0) + .withFilterNulls(true) + .build(); + } + + @NotNull + private ObjectMapper buildObjectMapper() { + return JsonV2.buildObjectMapper(); + } + + @Test + void testClearTimeSeriesPagination() { + // Create a TimeSeries with pagination information + String tsId = "Calhoun.Flow.Inst.~1Hour.0.cda-test"; + ZonedDateTime start = ZonedDateTime.parse("2023-01-11T12:00:00Z"); + ZonedDateTime end = ZonedDateTime.parse("2023-01-11T13:00:00Z"); + ZonedDateTime versionDate = Instant.now().atZone(ZoneId.of("UTC")); + + // Set page and pageSize to non-null values + String page = "testPage"; + int pageSize = 10; + TimeSeries ts = new TimeSeries(page, pageSize, 0, tsId, "SPK", start, end, "cfs", Duration.ZERO, null, versionDate, null); + + // Create a FilteredTimeSeries with this TimeSeries + FilteredTimeSeriesParameters params = buildFilterParams(); + FilteredTimeSeries filteredTs = new FilteredTimeSeries(ts, params); + + // Verify that the page is not null before clearing + assertNotNull(filteredTs.getTimeSeries().getPage()); + + // Call the clearTimeSeriesPagination method + filteredTs.clearTimeSeriesPagination(); + + // Verify that the page is now null + assertNull(filteredTs.getTimeSeries().getPage()); + assertNull(filteredTs.getTimeSeries().getNextPage()); + } +} diff --git a/cwms-data-api/src/test/java/cwms/cda/helpers/DateUtilsTest.java b/cwms-data-api/src/test/java/cwms/cda/helpers/DateUtilsTest.java index 01fbb5188..a1649555b 100644 --- a/cwms-data-api/src/test/java/cwms/cda/helpers/DateUtilsTest.java +++ b/cwms-data-api/src/test/java/cwms/cda/helpers/DateUtilsTest.java @@ -24,8 +24,7 @@ class DateUtilsTest { @ParameterizedTest @ArgumentsSource(FullDatesArguments.class) void test_iso_dates_from_user( String inputDate, ZoneId tz, ZonedDateTime expected){ - ZonedDateTime result = DateUtils.parseUserDate(inputDate, tz,null); // now not used in this case force npe if - // someone accidentally sets that up. + ZonedDateTime result = DateUtils.parseUserDate(inputDate, tz); // isEqual returns true if the instant is the same. They can have different zones. assertTrue(result.isEqual(expected), "Provided date input not correctly matched"); } @@ -33,7 +32,7 @@ void test_iso_dates_from_user( String inputDate, ZoneId tz, ZonedDateTime expect @ParameterizedTest @ArgumentsSource(DateWithoutZoneArguments.class) void test_iso_dates_without_zone_from_user( String inputDate, ZoneId tz, ZonedDateTime expected){ - ZonedDateTime result = DateUtils.parseUserDate(inputDate, tz,null); + ZonedDateTime result = DateUtils.parseUserDate(inputDate, tz); // This checks that the two times refer to the same instant on the time-line. assertTrue(result.isEqual(expected), "Provided date input not correctly matched"); // All the args in this test don't have a timezone, so the result should be in the provided fallback zone. @@ -70,7 +69,7 @@ void test_time_series_format_and_parse(){ DateTimeFormatter formatter = DateTimeFormatter.ofPattern(TimeSeries.ZONED_DATE_TIME_FORMAT); // FYI this pattern throws away milliseconds. String formatted = now.format(formatter); // looks like: 2022-12-08T09:27:14-0800[America/Los_Angeles] - ZonedDateTime parsed = DateUtils.parseUserDate(formatted, losAngeles, null); + ZonedDateTime parsed = DateUtils.parseUserDate(formatted, losAngeles); assertEquals(now, parsed, "Date parsed from TimeSeries format does not match original date"); } @@ -104,13 +103,11 @@ void test_has_time_zone(){ @Test void test_not_versioned(){ - Instant my_nv_inst = ZonedDateTime.of(1111,11,18,0,0,0,0, + Instant myNotVersionedInstant = ZonedDateTime.of(1111,11,18,0,0,0,0, ZoneId.of("UTC")).toInstant(); assertTrue(NumericalConstants.isNotVersioned(NumericalConstants.notVersioned())); - assertEquals(NumericalConstants.notVersioned(), my_nv_inst); -// assertTrue(NumericalConstants.isNotVersioned(my_nv_inst)); - + assertEquals(NumericalConstants.notVersioned(), myNotVersionedInstant); } @Test @@ -143,4 +140,26 @@ void test_DateTimeFormatter_iso_zoned_example(){ assertEquals(expected, zdt.toInstant()); } + @Test + void test_double_check_times_html_examples(){ + // I'm including some example strings in times.html helper file. Just want to double check that those EXACT examples do parse. + + /* +
  • 2022-01-06T00:00:00Z - UTC timezone (Z)
    • +
  • 2022-01-06T07:00:01-07:00 - Offset timezone (-07:00)
    • +
  • 2022-01-06T07:00:02-0700 - Offset timezone without colon
    • +
  • 2021-06-10T13:00:23-07:00[PST8PDT] - Offset with named timezone
    • +
  • 2022-01-19T20:52:07+00:00[UTC] - Zero offset with named timezone
    • +
  • 2022-12-08T09:47:32-0800[America/Los_Angeles] - Offset with region timezone
    • + */ + + assertNotNull(DateUtils.parseUserDate("2022-01-06T00:00:00Z")); + assertNotNull(DateUtils.parseUserDate("2022-01-06T07:00:01-07:00")); + assertNotNull(DateUtils.parseUserDate("2022-01-06T07:00:02-0700")); + assertNotNull(DateUtils.parseUserDate("2021-06-10T13:00:23-07:00[PST8PDT]")); + assertNotNull(DateUtils.parseUserDate("2022-01-19T20:52:07+00:00[UTC]")); + assertNotNull(DateUtils.parseUserDate("2022-12-08T09:47:32-0800[America/Los_Angeles]")); + + } + } diff --git a/cwms-data-api/src/test/resources/cwms/cda/data/dto/filteredtimeseries/filtered_timeseries_test.json b/cwms-data-api/src/test/resources/cwms/cda/data/dto/filteredtimeseries/filtered_timeseries_test.json new file mode 100644 index 000000000..6e4739870 --- /dev/null +++ b/cwms-data-api/src/test/resources/cwms/cda/data/dto/filteredtimeseries/filtered_timeseries_test.json @@ -0,0 +1,33 @@ +{ + "name": "Calhoun.Flow.Inst.~1Hour.0.cda-test", + "office-id": "SPK", + "units": "cfs", + "values": [ + [ + 1673438400000, + 500, + 0 + ], + [ + 1673439400000, + null, + 5 + ], + [ + 1673440400000, + -340282346638528859811704183484516925440, + 5 + ], + [ + 1673442000000, + 600, + 0 + ] + ], + "filter-parameters": { + "ascending": true, + "min-value": 450.0, + "max-value": 550.0, + "filter-nulls": true + } +} \ No newline at end of file