Skip to content

Commit

Permalink
Add integration test for PostgreSQL metrics
Browse files Browse the repository at this point in the history
  • Loading branch information
easimon authored and shakuzen committed Jan 13, 2022
1 parent c033068 commit 48baa4f
Show file tree
Hide file tree
Showing 5 changed files with 265 additions and 49 deletions.
2 changes: 2 additions & 0 deletions dependencies.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -72,10 +72,12 @@ def VERSIONS = [
'org.mockito:mockito-core:3.+',
'org.mockito:mockito-inline:3.+',
'org.mongodb:mongodb-driver-sync:latest.release',
'org.postgresql:postgresql:latest.release',
'org.slf4j:slf4j-api:1.7.+',
'org.springframework:spring-context:latest.release',
'org.testcontainers:junit-jupiter:latest.release',
'org.testcontainers:kafka:latest.release',
'org.testcontainers:postgresql:latest.release',
'org.testcontainers:testcontainers:latest.release',
'ru.lanwen.wiremock:wiremock-junit5:latest.release',
'software.amazon.awssdk:cloudwatch:2.16.+'
Expand Down
4 changes: 4 additions & 0 deletions micrometer-core/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,10 @@ dependencies {
testImplementation 'org.testcontainers:testcontainers'
testImplementation 'org.testcontainers:junit-jupiter'
testImplementation 'org.testcontainers:kafka'

// Postgres Binder IT dependencies
testImplementation 'org.testcontainers:postgresql'
testImplementation 'org.postgresql:postgresql'
}

task shenandoahTest(type: Test) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
* @author Kristof Depypere
* @author Jon Schneider
* @author Johnny Lim
* @author Markus Dobel
* @since 1.1.0
*/
@NonNullApi
Expand Down Expand Up @@ -95,38 +96,38 @@ private static Tag createDbTag(String database) {

@Override
public void bindTo(MeterRegistry registry) {
Gauge.builder("postgres.size", postgresDataSource, dataSource -> getDatabaseSize())
Gauge.builder(Names.SIZE, postgresDataSource, dataSource -> getDatabaseSize())
.tags(tags)
.description("The database size")
.register(registry);
Gauge.builder("postgres.connections", postgresDataSource, dataSource -> getConnectionCount())
Gauge.builder(Names.CONNECTIONS, postgresDataSource, dataSource -> getConnectionCount())
.tags(tags)
.description("Number of active connections to the given db")
.register(registry);

// Hit ratio can be derived from dividing hits/reads
FunctionCounter.builder("postgres.blocks.hits", postgresDataSource,
dataSource -> resettableFunctionalCounter("postgres.blocks.hits", this::getBlockHits))
FunctionCounter.builder(Names.BLOCKS_HITS, postgresDataSource,
dataSource -> resettableFunctionalCounter(Names.BLOCKS_HITS, this::getBlockHits))
.tags(tags)
.description("Number of times disk blocks were found already in the buffer cache, so that a read was not necessary")
.register(registry);
FunctionCounter.builder("postgres.blocks.reads", postgresDataSource,
dataSource -> resettableFunctionalCounter("postgres.blocks.reads", this::getBlockReads))
FunctionCounter.builder(Names.BLOCKS_READS, postgresDataSource,
dataSource -> resettableFunctionalCounter(Names.BLOCKS_READS, this::getBlockReads))
.tags(tags)
.description("Number of disk blocks read in this database")
.register(registry);

FunctionCounter.builder("postgres.transactions", postgresDataSource,
dataSource -> resettableFunctionalCounter("postgres.transactions", this::getTransactionCount))
FunctionCounter.builder(Names.TRANSACTIONS, postgresDataSource,
dataSource -> resettableFunctionalCounter(Names.TRANSACTIONS, this::getTransactionCount))
.tags(tags)
.description("Total number of transactions executed (commits + rollbacks)")
.register(registry);
Gauge.builder("postgres.locks", postgresDataSource, dataSource -> getLockCount())
Gauge.builder(Names.LOCKS, postgresDataSource, dataSource -> getLockCount())
.tags(tags)
.description("Number of locks on the given db")
.register(registry);
FunctionCounter.builder("postgres.temp.writes", postgresDataSource,
dataSource -> resettableFunctionalCounter("postgres.temp.writes", this::getTempBytes))
FunctionCounter.builder(Names.TEMP_WRITES, postgresDataSource,
dataSource -> resettableFunctionalCounter(Names.TEMP_WRITES, this::getTempBytes))
.tags(tags)
.description("The total amount of temporary writes to disk to execute queries")
.baseUnit(BaseUnits.BYTES)
Expand All @@ -137,56 +138,56 @@ public void bindTo(MeterRegistry registry) {
}

private void registerRowCountMetrics(MeterRegistry registry) {
FunctionCounter.builder("postgres.rows.fetched", postgresDataSource,
dataSource -> resettableFunctionalCounter("postgres.rows.fetched", this::getReadCount))
FunctionCounter.builder(Names.ROWS_FETCHED, postgresDataSource,
dataSource -> resettableFunctionalCounter(Names.ROWS_FETCHED, this::getReadCount))
.tags(tags)
.description("Number of rows fetched from the db")
.register(registry);
FunctionCounter.builder("postgres.rows.inserted", postgresDataSource,
dataSource -> resettableFunctionalCounter("postgres.rows.inserted", this::getInsertCount))
FunctionCounter.builder(Names.ROWS_INSERTED, postgresDataSource,
dataSource -> resettableFunctionalCounter(Names.ROWS_INSERTED, this::getInsertCount))
.tags(tags)
.description("Number of rows inserted from the db")
.register(registry);
FunctionCounter.builder("postgres.rows.updated", postgresDataSource,
dataSource -> resettableFunctionalCounter("postgres.rows.updated", this::getUpdateCount))
FunctionCounter.builder(Names.ROWS_UPDATED, postgresDataSource,
dataSource -> resettableFunctionalCounter(Names.ROWS_UPDATED, this::getUpdateCount))
.tags(tags)
.description("Number of rows updated from the db")
.register(registry);
FunctionCounter.builder("postgres.rows.deleted", postgresDataSource,
dataSource -> resettableFunctionalCounter("postgres.rows.deleted", this::getDeleteCount))
FunctionCounter.builder(Names.ROWS_DELETED, postgresDataSource,
dataSource -> resettableFunctionalCounter(Names.ROWS_DELETED, this::getDeleteCount))
.tags(tags)
.description("Number of rows deleted from the db")
.register(registry);
Gauge.builder("postgres.rows.dead", postgresDataSource, dataSource -> getDeadTupleCount())
Gauge.builder(Names.ROWS_DEAD, postgresDataSource, dataSource -> getDeadTupleCount())
.tags(tags)
.description("Total number of dead rows in the current database")
.register(registry);
}

private void registerCheckpointMetrics(MeterRegistry registry) {
FunctionCounter.builder("postgres.checkpoints.timed", postgresDataSource,
dataSource -> resettableFunctionalCounter("postgres.checkpoints.timed", this::getTimedCheckpointsCount))
FunctionCounter.builder(Names.CHECKPOINTS_TIMED, postgresDataSource,
dataSource -> resettableFunctionalCounter(Names.CHECKPOINTS_TIMED, this::getTimedCheckpointsCount))
.tags(tags)
.description("Number of checkpoints timed")
.register(registry);
FunctionCounter.builder("postgres.checkpoints.requested", postgresDataSource,
dataSource -> resettableFunctionalCounter("postgres.checkpoints.requested", this::getRequestedCheckpointsCount))
FunctionCounter.builder(Names.CHECKPOINTS_REQUESTED, postgresDataSource,
dataSource -> resettableFunctionalCounter(Names.CHECKPOINTS_REQUESTED, this::getRequestedCheckpointsCount))
.tags(tags)
.description("Number of checkpoints requested")
.register(registry);

FunctionCounter.builder("postgres.buffers.checkpoint", postgresDataSource,
dataSource -> resettableFunctionalCounter("postgres.buffers.checkpoint", this::getBuffersCheckpoint))
FunctionCounter.builder(Names.BUFFERS_CHECKPOINT, postgresDataSource,
dataSource -> resettableFunctionalCounter(Names.BUFFERS_CHECKPOINT, this::getBuffersCheckpoint))
.tags(tags)
.description("Number of buffers written during checkpoints")
.register(registry);
FunctionCounter.builder("postgres.buffers.clean", postgresDataSource,
dataSource -> resettableFunctionalCounter("postgres.buffers.clean", this::getBuffersClean))
FunctionCounter.builder(Names.BUFFERS_CLEAN, postgresDataSource,
dataSource -> resettableFunctionalCounter(Names.BUFFERS_CLEAN, this::getBuffersClean))
.tags(tags)
.description("Number of buffers written by the background writer")
.register(registry);
FunctionCounter.builder("postgres.buffers.backend", postgresDataSource,
dataSource -> resettableFunctionalCounter("postgres.buffers.backend", this::getBuffersBackend))
FunctionCounter.builder(Names.BUFFERS_BACKEND, postgresDataSource,
dataSource -> resettableFunctionalCounter(Names.BUFFERS_BACKEND, this::getBuffersBackend))
.tags(tags)
.description("Number of buffers written directly by a backend")
.register(registry);
Expand Down Expand Up @@ -300,4 +301,34 @@ private static String getUserTableQuery(String statName) {
private static String getBgWriterQuery(String statName) {
return SELECT + statName + " FROM pg_stat_bgwriter";
}

static final class Names {
public static final String SIZE = of("size");
public static final String CONNECTIONS = of("connections");
public static final String BLOCKS_HITS = of("blocks.hits");
public static final String BLOCKS_READS = of("blocks.reads");
public static final String TRANSACTIONS = of("transactions");
public static final String LOCKS = of("locks");
public static final String TEMP_WRITES = of("temp.writes");

public static final String ROWS_FETCHED = of("rows.fetched");
public static final String ROWS_INSERTED = of("rows.inserted");
public static final String ROWS_UPDATED = of("rows.updated");
public static final String ROWS_DELETED = of("rows.deleted");
public static final String ROWS_DEAD = of("rows.dead");

public static final String CHECKPOINTS_TIMED = of("checkpoints.timed");
public static final String CHECKPOINTS_REQUESTED = of("checkpoints.requested");

public static final String BUFFERS_CHECKPOINT = of("buffers.checkpoint");
public static final String BUFFERS_CLEAN = of("buffers.clean");
public static final String BUFFERS_BACKEND = of("buffers.backend");

private static String of(String name) {
return "postgres." + name;
}

private Names() {
}
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,169 @@
/**
* Copyright 2019 VMware, Inc.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
* <p>
* https://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package io.micrometer.core.instrument.binder.db;

import io.micrometer.core.instrument.MeterRegistry;
import io.micrometer.core.instrument.Tags;
import io.micrometer.core.instrument.search.RequiredSearch;
import io.micrometer.core.instrument.simple.SimpleMeterRegistry;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Tag;
import org.junit.jupiter.api.Test;
import org.postgresql.ds.PGSimpleDataSource;
import org.testcontainers.containers.PostgreSQLContainer;
import org.testcontainers.junit.jupiter.Container;
import org.testcontainers.junit.jupiter.Testcontainers;
import org.testcontainers.utility.DockerImageName;

import javax.sql.DataSource;

import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.Arrays;
import java.util.List;

import static org.assertj.core.api.Assertions.*;

import static io.micrometer.core.instrument.binder.db.PostgreSQLDatabaseMetrics.Names.*;

/**
* @author Markus Dobel
*/
@Testcontainers
@Tag("docker")
public class PostgreSQLDatabaseMetricsIntegrationTest {

@Container
private final PostgreSQLContainer<?> postgres = new PostgreSQLContainer<>(getDockerImageName());
private final MeterRegistry registry = new SimpleMeterRegistry();

private DataSource dataSource;
private Tags tags;

// statistics are updated only every PGSTAT_STAT_INTERVAL, which is 500ms. Add a bit for stable tests.
private static final long PGSTAT_STAT_INTERVAL = 500L + 50L;

@BeforeEach
void setup() throws InterruptedException {
dataSource = createDatasource();
tags = Tags.of("database", postgres.getDatabaseName());

new PostgreSQLDatabaseMetrics(dataSource, postgres.getDatabaseName()).bindTo(registry);
}

@Test
void gaugesAreNotZero() throws Exception {
/* create noise to increment gauges */
executeSql(
"CREATE TABLE gauge_test_table (val varchar(255))",
"INSERT INTO gauge_test_table (val) VALUES ('foo')",
"UPDATE gauge_test_table SET val = 'bar'",
"SELECT * FROM gauge_test_table",
"DELETE FROM gauge_test_table"
);
Thread.sleep(PGSTAT_STAT_INTERVAL);

final List<String> GAUGES = Arrays.asList(
SIZE, CONNECTIONS, ROWS_DEAD, LOCKS
);

for (String name: GAUGES) {
assertThat(get(name).gauge().value())
.withFailMessage("Gauge " + name + " is zero.")
.isGreaterThan(0);
}
}

@Test
void countersAreNotZero() throws Exception {
/* create noise to increment counters */
executeSql(
"CREATE TABLE counter_test_table (val varchar(255))",
"INSERT INTO counter_test_table (val) VALUES ('foo')",
"UPDATE counter_test_table SET val = 'bar'",
"SELECT * FROM counter_test_table",
"DELETE FROM counter_test_table"
);
Thread.sleep(PGSTAT_STAT_INTERVAL);

final List<String> COUNTERS = Arrays.asList(
BLOCKS_HITS, BLOCKS_READS,
TRANSACTIONS,
ROWS_FETCHED, ROWS_INSERTED, ROWS_UPDATED, ROWS_DELETED,
BUFFERS_CHECKPOINT
);

/* the following counters are zero on a clean database and hard to increase reliably */
final List<String> ZERO_COUNTERS = Arrays.asList(
TEMP_WRITES,
CHECKPOINTS_TIMED, CHECKPOINTS_REQUESTED,
BUFFERS_CLEAN, BUFFERS_BACKEND
);

for (String name: COUNTERS) {
assertThat(get(name).functionCounter().count())
.withFailMessage("Counter " + name + " is zero.")
.isGreaterThan(0);
}
}

@Test
void deadTuplesGaugeIncreases() throws Exception {
final double deadRowsBefore = get(ROWS_DEAD).gauge().value();

executeSql(
"CREATE TABLE dead_tuples_test_table (val varchar(255))",
"INSERT INTO dead_tuples_test_table (val) VALUES ('foo')",
"UPDATE dead_tuples_test_table SET val = 'bar'"
);

// wait for stats to be updated
Thread.sleep(PGSTAT_STAT_INTERVAL);
assertThat(get(ROWS_DEAD).gauge().value()).isGreaterThan(deadRowsBefore);
}

private DataSource createDatasource() {
final PGSimpleDataSource result = new PGSimpleDataSource();
result.setURL(postgres.getJdbcUrl());
result.setUser(postgres.getUsername());
result.setPassword(postgres.getPassword());
result.setDatabaseName(postgres.getDatabaseName());
return result;
}

private void executeSql(String ... statements) throws SQLException {
try (final Connection connection = dataSource.getConnection()) {
executeSql(connection, statements);
}
}

private void executeSql(Connection connection, String ... statements) throws SQLException {
for (String sql: statements) {
try (final PreparedStatement stmt = connection.prepareStatement(sql)) {
stmt.execute();
}
}
}

private RequiredSearch get(final String name) {
return registry.get(name).tags(tags);
}

private static DockerImageName getDockerImageName() {
return DockerImageName.parse("postgres:9.6.21");
}
}
Loading

0 comments on commit 48baa4f

Please sign in to comment.