diff --git a/cwms-data-api/build.gradle b/cwms-data-api/build.gradle index 3cdd4d7fc..36bae6607 100644 --- a/cwms-data-api/build.gradle +++ b/cwms-data-api/build.gradle @@ -10,10 +10,11 @@ configurations { docker baseLibs tomcatLibs + jettyLibs } configurations.implementation { - exclude group: 'com.oracle.database.jdbc' +// exclude group: 'com.oracle.database.jdbc' } dependencies { @@ -39,7 +40,7 @@ dependencies { exclude group: "mil.army.usace.hec", module: "cwms-db-jooq-codegen" } - implementation(libs.cwms.db.jooq.codegen) { + implementation(libs.cwms.db.jooq.codegen) { exclude group: "com.oracle", module: "*" exclude group: "com.oracle.database.jdbc", module: "*" exclude group: "org.jooq.pro", module: "*" @@ -85,13 +86,13 @@ dependencies { implementation(libs.geojson.jackson) implementation(libs.javalin.core) { - exclude group: "org.eclipse.jetty" - exclude group: "org.eclipse.jetty.websocket" +// exclude group: "org.eclipse.jetty" +// exclude group: "org.eclipse.jetty.websocket" } implementation(libs.javalin.openapi) { - //exclude group: "org.eclipse.jetty" - exclude group: "org.eclipse.jetty.websocket" - exclude group: "jakarta.xml.bind", module: "*" +// //exclude group: "org.eclipse.jetty" +// exclude group: "org.eclipse.jetty.websocket" +// exclude group: "jakarta.xml.bind", module: "*" } implementation(libs.swagger.core) { exclude group: "jakarta.xml.bind", module: "*" @@ -123,8 +124,11 @@ dependencies { baseLibs(libs.oracle.jdbc.driver) - compileOnly(libs.oracle.jdbc.driver) - testRuntimeOnly(libs.oracle.jdbc.driver) + baseLibs 'com.oracle.database.jdbc:ojdbc8:21.9.0.0' + implementation 'com.oracle.database.jdbc:ojdbc8:19.3.0.0' + testImplementation 'com.oracle.database.jdbc:ojdbc8:19.3.0.0' + implementation(libs.oracle.jdbc.driver) + testImplementation(libs.oracle.jdbc.driver) testImplementation(libs.bundles.jackson) testImplementation(libs.rest.assured) @@ -135,8 +139,13 @@ dependencies { tomcatLibs(libs.google.flogger.api) tomcatLibs(libs.google.flogger.backend) + jettyLibs('org.eclipse.jetty:jetty-util:9.4.48.v20220622') + jettyLibs('org.eclipse.jetty:jetty-server:9.4.48.v20220622') + jettyLibs( 'org.eclipse.jetty:jetty-servlets:9.4.48.v20220622') + jettyLibs(libs.google.flogger.api) + jettyLibs(libs.google.flogger.backend) + testImplementation(libs.bundles.testcontainers) - testImplementation(libs.apache.commons.csv) testImplementation(libs.cwms.tomcat.auth) @@ -187,6 +196,9 @@ sourceSets { } main { runtimeClasspath += formatList.runtimeClasspath + resources { + srcDir 'src/main/webapp' // I don't think this did what I wanted. + } } test { runtimeClasspath += formatList.runtimeClasspath @@ -194,6 +206,10 @@ sourceSets { } test { + systemProperties += project.properties.findAll { k, v -> k.startsWith("RADAR") } + systemProperties += project.properties.findAll { k, v -> k.startsWith("CDA") } + systemProperties += project.properties.findAll { k, v -> k.startsWith("cwms") } + useJUnitPlatform() { excludeTags "integration" } @@ -224,6 +240,7 @@ task run(type: JavaExec) { classpath += configurations.baseLibs classpath += configurations.tomcatLibs classpath += sourceSets.test.output // to get the fixture + classpath += sourceSets.main.runtimeClasspath mainClass = "fixtures.TomcatServer" systemProperties += project.properties.findAll { k, v -> k.startsWith("RADAR") } @@ -235,15 +252,37 @@ task run(type: JavaExec) { jvmArgs += "-DTOMCAT_RESOURCES=$buildDir/tomcat/conf/context.xml" jvmArgs += "-Dorg.apache.tomcat.util.digester.PROPERTY_SOURCE=org.apache.tomcat.util.digester.EnvironmentPropertySource" jvmArgs += "-Dcatalina.base=$buildDir/tomcat" - // If you have the docker-compose environment up and are trying to run + // If you have the docker-compose environment up and are trying to run // CDA from run to debug uncomment the following lines. //jvmArgs += "-Dcwms.dataapi.access.providers=KeyAccessManager,CwmsAccessManager,OpenID" //jvmArgs += "-Dcwms.dataapi.access.openid.wellKnownUrl=https://auth.test:8444/auth/realms/cwms/.well-known/openid-configuration" //jvmArgs += "-Dcwms.dataapi.access.openid.issuer=https://auth.test:8444/auth/realms/cwms" } +task runApiServlet(type: JavaExec) { + dependsOn generateConfig + + classpath += configurations.baseLibs + classpath += configurations.jettyLibs + + classpath += sourceSets.test.output + classpath += sourceSets.main.runtimeClasspath + + mainClass = "cwms.cda.ApiServlet" + systemProperties += project.properties.findAll { k, v -> k.startsWith("RADAR") } + systemProperties += project.properties.findAll { k, v -> k.startsWith("CDA") } + systemProperties += project.properties.findAll { k, v -> k.startsWith("cwms") } + + jvmArgs += "-Djava.util.logging.config.file=$projectDir/logging.properties" + + jvmArgs += "-Dcwms.dataapi.access.providers=KeyAccessManager,CwmsAccessManager" + jvmArgs += "-Dcwms.dataapi.default.office=SPK" + jvmArgs += "-Dcwms.dataapi.external.root.dir.1=$buildDir\\..\\src\\main\\webapp" + jvmArgs += "-Dcwms.dataapi.external.root.dir.2=$buildDir\\extra" +} + task integrationTests(type: Test) { - dependsOn test +// dependsOn test dependsOn generateConfig dependsOn war diff --git a/cwms-data-api/logging.properties b/cwms-data-api/logging.properties index 5683fc81f..6ee4a4a6c 100644 --- a/cwms-data-api/logging.properties +++ b/cwms-data-api/logging.properties @@ -13,62 +13,16 @@ # See the License for the specific language governing permissions and # limitations under the License. -handlers = 1catalina.org.apache.juli.AsyncFileHandler, \ - 2localhost.org.apache.juli.AsyncFileHandler, java.util.logging.ConsoleHandler +handlers = \ + java.util.logging.ConsoleHandler -.handlers = 1catalina.org.apache.juli.AsyncFileHandler, java.util.logging.ConsoleHandler -############################################################ -# Handler specific properties. -# Describes specific configuration info for Handlers. -############################################################ - -1catalina.org.apache.juli.AsyncFileHandler.level = FINE -1catalina.org.apache.juli.AsyncFileHandler.directory = ${catalina.base}/logs -1catalina.org.apache.juli.AsyncFileHandler.prefix = catalina. -1catalina.org.apache.juli.AsyncFileHandler.maxDays = 4 -1catalina.org.apache.juli.AsyncFileHandler.encoding = UTF-8 - -2localhost.org.apache.juli.AsyncFileHandler.level = FINE -2localhost.org.apache.juli.AsyncFileHandler.directory = ${catalina.base}/logs -2localhost.org.apache.juli.AsyncFileHandler.prefix = localhost. -2localhost.org.apache.juli.AsyncFileHandler.maxDays = 4 -2localhost.org.apache.juli.AsyncFileHandler.encoding = UTF-8 java.util.logging.ConsoleHandler.level = INFO java.util.logging.ConsoleHandler.formatter = org.apache.juli.OneLineFormatter java.util.logging.ConsoleHandler.encoding = UTF-8 -############################################################ -# Facility specific properties. -# Provides extra control for each logger. -############################################################ - -org.apache.catalina.core.ContainerBase.[Catalina].[localhost].level = INFO -org.apache.catalina.core.ContainerBase.[Catalina].[localhost].handlers = 2localhost.org.apache.juli.AsyncFileHandler - -org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/cwms-data].level = FINE -org.apache.catalina.core.ContainerBase.[Catalina].[localhost].[/cwms-data].handlers = 2localhost.org.apache.juli.AsyncFileHandler - -# For example, set the org.apache.catalina.util.LifecycleBase logger to log -# each component that extends LifecycleBase changing state: -#org.apache.catalina.util.LifecycleBase.level = FINE - -# To see debug messages in TldLocationsCache, uncomment the following line: -#org.apache.jasper.compiler.TldLocationsCache.level = FINE - -# To see debug messages for HTTP/2 handling, uncomment the following line: -#org.apache.coyote.http2.level = FINE - -# To see debug messages for WebSocket handling, uncomment the following line: -#org.apache.tomcat.websocket.level = FINE - fixtures.level=FINE cwms.cda.datasource.level=FINE -org.apache.level=FINE -org.apache.catalina.realm.level=INFO -org.apache.catalina.realm.useParentHandlers=true -org.apache.catalina.authenticator.level=INFO -org.apache.catalina.authenticator.useParentHandlers=true diff --git a/cwms-data-api/src/main/java/cwms/cda/ApiServlet.java b/cwms-data-api/src/main/java/cwms/cda/ApiServlet.java index b9b1ed807..4ce611d74 100644 --- a/cwms-data-api/src/main/java/cwms/cda/ApiServlet.java +++ b/cwms-data-api/src/main/java/cwms/cda/ApiServlet.java @@ -24,6 +24,13 @@ package cwms.cda; +import static io.javalin.apibuilder.ApiBuilder.crud; +import static io.javalin.apibuilder.ApiBuilder.get; +import static io.javalin.apibuilder.ApiBuilder.prefixPath; +import static io.javalin.apibuilder.ApiBuilder.sse; +import static io.javalin.apibuilder.ApiBuilder.staticInstance; +import static io.javalin.apibuilder.ApiBuilder.ws; + import com.codahale.metrics.Meter; import com.codahale.metrics.MetricRegistry; import com.codahale.metrics.servlets.MetricsServlet; @@ -34,6 +41,7 @@ import cwms.cda.api.BasinController; import cwms.cda.api.BlobController; import cwms.cda.api.CatalogController; +import cwms.cda.api.ClobAsyncController; import cwms.cda.api.ClobController; import cwms.cda.api.Controllers; import cwms.cda.api.CountyController; @@ -43,6 +51,7 @@ import cwms.cda.api.LocationGroupController; import cwms.cda.api.OfficeController; import cwms.cda.api.ParametersController; +import cwms.cda.api.PingController; import cwms.cda.api.PoolController; import cwms.cda.api.RatingController; import cwms.cda.api.RatingMetadataController; @@ -66,7 +75,6 @@ import cwms.cda.api.errors.JsonFieldsException; import cwms.cda.api.errors.NotFoundException; import cwms.cda.data.dao.JooqDao; -import cwms.cda.formatters.Formats; import cwms.cda.formatters.FormattingException; import cwms.cda.security.CwmsAuthException; import cwms.cda.security.Role; @@ -83,6 +91,7 @@ import io.javalin.http.BadRequestResponse; import io.javalin.http.Handler; import io.javalin.http.JavalinServlet; +import io.javalin.http.staticfiles.Location; import io.javalin.plugin.openapi.OpenApiOptions; import io.javalin.plugin.openapi.OpenApiPlugin; import io.swagger.v3.oas.models.Components; @@ -91,35 +100,41 @@ import io.swagger.v3.oas.models.PathItem; import io.swagger.v3.oas.models.info.Info; import io.swagger.v3.oas.models.security.SecurityRequirement; -import java.util.concurrent.TimeUnit; -import org.apache.http.entity.ContentType; -import org.jetbrains.annotations.NotNull; -import org.owasp.html.HtmlPolicyBuilder; -import org.owasp.html.PolicyFactory; - -import javax.annotation.Resource; -import javax.management.ServiceNotFoundException; -import javax.servlet.ServletConfig; -import javax.servlet.ServletException; -import javax.servlet.annotation.WebServlet; -import javax.servlet.http.HttpServlet; -import javax.servlet.http.HttpServletRequest; -import javax.servlet.http.HttpServletResponse; -import javax.sql.DataSource; import java.io.IOException; import java.io.InputStream; import java.io.PrintWriter; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.sql.SQLException; import java.time.DateTimeException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; +import java.util.concurrent.TimeUnit; import java.util.jar.Manifest; - -import static io.javalin.apibuilder.ApiBuilder.*; +import javax.annotation.Resource; +import javax.management.ServiceNotFoundException; +import javax.servlet.ServletConfig; +import javax.servlet.ServletException; +import javax.servlet.annotation.WebServlet; +import javax.servlet.http.HttpServlet; +import javax.servlet.http.HttpServletRequest; +import javax.servlet.http.HttpServletResponse; +import javax.sql.DataSource; +import oracle.jdbc.pool.OracleDataSource; +import org.apache.http.entity.ContentType; +import org.eclipse.jetty.server.Connector; +import org.eclipse.jetty.server.Server; +import org.eclipse.jetty.server.ServerConnector; +import org.eclipse.jetty.util.log.Log; +import org.eclipse.jetty.util.log.StdErrLog; +import org.eclipse.jetty.util.thread.QueuedThreadPool; +import org.eclipse.jetty.webapp.WebAppContext; +import org.jetbrains.annotations.NotNull; +import org.owasp.html.HtmlPolicyBuilder; +import org.owasp.html.PolicyFactory; /** @@ -144,8 +159,13 @@ "/blobs/*", "/clobs/*", "/pools/*", - "/specified-levels/*" -}) + "/specified-levels/*", + "/sse/*", + "/ws/*", + "/ping/*" +}, + asyncSupported = true +) public class ApiServlet extends HttpServlet { public static final FluentLogger logger = FluentLogger.forEnclosingClass(); @@ -167,21 +187,27 @@ public class ApiServlet extends HttpServlet { public static final String DEFAULT_OFFICE_KEY = "cwms.dataapi.default.office"; public static final String DEFAULT_PROVIDER = "MultipleAccessManager"; - private MetricRegistry metrics; + private MetricRegistry metrics = new MetricRegistry(); private Meter totalRequests; private static final long serialVersionUID = 1L; - JavalinServlet javalin = null; + JavalinServlet javalinServlet = null; + public Javalin javalin = null; @Resource(name = "jdbc/CWMS3") DataSource cwms; + public ApiServlet() { + super(); + } @Override public void destroy() { - javalin.destroy(); + if(javalinServlet != null) { + javalinServlet.destroy(); + } } @Override @@ -189,32 +215,81 @@ public void init(ServletConfig config) throws ServletException { logger.atInfo().log("Initializing CWMS Data API Version: " + obtainFullVersion(config)); metrics = (MetricRegistry)config.getServletContext() .getAttribute(MetricsServlet.METRICS_REGISTRY); + + if(metrics == null){ + metrics = new MetricRegistry(); + config.getServletContext().setAttribute(MetricsServlet.METRICS_REGISTRY, metrics); + } + totalRequests = metrics.meter("cwms.dataapi.total_requests"); super.init(config); } @SuppressWarnings({"java:S125","java:S2095"}) // closed in destroy handler @Override - public void init() { + public void init() throws ServletException { + super.init(); JavalinValidation.register(UnitSystem.class, UnitSystem::systemFor); JavalinValidation.register(JooqDao.DeleteMethod.class, Controllers::getDeleteMethod); + + javalin = createJavalin(); + + javalinServlet = javalin.javalinServlet(); + + } + + public Javalin createJavalin() { + ObjectMapper om = new ObjectMapper(); om.setPropertyNamingStrategy(PropertyNamingStrategies.KEBAB_CASE); om.registerModule(new JavaTimeModule()); PolicyFactory sanitizer = new HtmlPolicyBuilder().disallowElements(" + + + diff --git a/cwms-data-api/src/main/webapp/ws_demo.html b/cwms-data-api/src/main/webapp/ws_demo.html new file mode 100644 index 000000000..db4ea082c --- /dev/null +++ b/cwms-data-api/src/main/webapp/ws_demo.html @@ -0,0 +1,23 @@ + + + + + Html5 WebSockets Example + + + + +

Html5 WebSockets Example.

+
+

Please input some text to Send :

+ + + + + + \ No newline at end of file diff --git a/cwms-data-api/src/main/webapp/ws_demo2.html b/cwms-data-api/src/main/webapp/ws_demo2.html new file mode 100644 index 000000000..a3170d910 --- /dev/null +++ b/cwms-data-api/src/main/webapp/ws_demo2.html @@ -0,0 +1,23 @@ + + + + + Html5 WebSockets Example + + + + +

Html5 WebSockets Example.

+
+

Please input some text to Send :

+ + + + + + \ No newline at end of file diff --git a/cwms-data-api/src/test/java/cwms/cda/data/dao/InsertClob.java b/cwms-data-api/src/test/java/cwms/cda/data/dao/InsertClob.java new file mode 100644 index 000000000..e0cc96ee8 --- /dev/null +++ b/cwms-data-api/src/test/java/cwms/cda/data/dao/InsertClob.java @@ -0,0 +1,122 @@ +package cwms.cda.data.dao; + +import static cwms.cda.data.dao.DaoTest.getDslContext; + +import java.io.File; +import java.io.FileNotFoundException; +import java.io.FileReader; +import java.io.IOException; +import java.io.Reader; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import org.jooq.DSLContext; +import usace.cwms.db.jooq.codegen.packages.CWMS_TEXT_PACKAGE; + +public class InsertClob { + + + public static final String OFFICE = "SPK"; + String clobId = "/TIME SERIES TEXT/6261044"; + String filePath = "C:\\Temp\\out.log"; // just a biggish text file. + +// @Test //not an actual test + void test_insert() throws SQLException, FileNotFoundException { + + long millis = System.currentTimeMillis(); + + DSLContext dsl = getDslContext(OFFICE); + + + try { + dsl.connection(c -> { + CWMS_TEXT_PACKAGE.call_STORE_TEXT( + dsl.configuration(), + "asdf", + clobId, + "this is a clob for a test", + ClobDao.getBoolean(true), + OFFICE); + }); + } catch (Exception e) { + System.out.println("Exception - This is fine. " +// + e.getMessage() + ); + // Don't really care if it inserted or already existed. Just needs to exist for this + // next bit. + } + + + // Get a binary stream from filepath + + File file = new File(filePath); + + assert(file.exists()); + assert(file.length() > 1000); + + dsl.connection(connection -> { + updateClobValue(connection, OFFICE, clobId, file); + }); + + + } + +// @Test // This isn't a real test - just a way to load a bunch of data... + void manual_connection() throws SQLException, ClassNotFoundException, IOException { + + String cdaJdbcUrl = System.getenv("CDA_JDBC_URL"); + if (cdaJdbcUrl == null) { + cdaJdbcUrl = System.getProperty("CDA_JDBC_URL"); + } + + String username = System.getenv("CDA_JDBC_USERNAME"); + if (username == null) { + username = System.getProperty("CDA_JDBC_USERNAME"); + } + + String password = System.getenv("CDA_JDBC_PASSWORD"); + if (password == null) { + password = System.getProperty("CDA_JDBC_PASSWORD"); + } + + + // File path and size + File file = new File(filePath); + System.out.println("File exists: " + file.exists() + " size: " + file.length()); + + // Load the Oracle JDBC driver + Class.forName("oracle.jdbc.driver.OracleDriver"); + + // Establish the database connection + Connection connection = DriverManager.getConnection(cdaJdbcUrl, username, password); + + updateClobValue(connection, OFFICE, clobId, file); // 42Mb took 4.4s + + connection.close(); + } + + private static void updateClobValue(Connection connection, String office, String clobId, + File file) throws SQLException, IOException { + + + String sql = "UPDATE CWMS_20.AV_CLOB\n" + + "SET VALUE = ?\n" + + "where CWMS_20.AV_CLOB.ID = ?\n" + + " and CWMS_20.AV_CLOB.OFFICE_CODE in (select AV_OFFICE.OFFICE_CODE from CWMS_20.AV_OFFICE where AV_OFFICE.OFFICE_ID = ?)"; + + Reader reader = new FileReader(file ); + + try(PreparedStatement preparedStatement = connection.prepareStatement(sql)) { + preparedStatement.setCharacterStream(1, reader); + preparedStatement.setString(2, clobId); + preparedStatement.setString(3, office); + System.out.println("Executing statement: "); + preparedStatement.executeUpdate(); + } + + System.out.println("CLOB data updated successfully with reader data."); + } + + +} diff --git a/jetty-logging.properties b/jetty-logging.properties new file mode 100644 index 000000000..1d5c4bdaa --- /dev/null +++ b/jetty-logging.properties @@ -0,0 +1,6 @@ +# Configure for System.err output +org.eclipse.jetty.util.log.class=org.eclipse.jetty.util.log.StdErrLog +# Configure StdErrLog to log all jetty namespace at default of WARN or above +org.eclipse.jetty.LEVEL=WARN +# Configure StdErrLog to log websocket specific namespace at DEBUG or above +org.eclipse.jetty.websocket.LEVEL=INFO \ No newline at end of file