From 84fc3336c8c289b97b20d98dbcce82e7a2d2eef3 Mon Sep 17 00:00:00 2001 From: Asaf Levy Date: Thu, 18 May 2023 16:03:01 +0300 Subject: [PATCH 1/5] Add Max Allocated Storage metric for RDS --- README.md | 35 ++++++++++++++++++----------------- pkg/rds.go | 8 ++++++++ pkg/rds_test.go | 1 + 3 files changed, 27 insertions(+), 17 deletions(-) diff --git a/README.md b/README.md index 00d6008..511210e 100644 --- a/README.md +++ b/README.md @@ -6,23 +6,24 @@ This was made as a complement to [CloudWatch Exporter](https://github.com/promet ## Included metadata & metrics -| Service | Metric | Description | -|---------|-----------------------------|-----------------------------------------------------| -| RDS | allocatedstorage | The amount of allocated storage in GB | -| RDS | dbinstanceclass | The DB instance class (type) | -| RDS | dbinstancestatus | The instance status | -| RDS | engineversion | The DB engine type and version | -| RDS | pendingmaintenanceactions | The pending maintenance actions for a RDS instance | -| RDS | logs_amount | The amount of log files present in the RDS Instance | -| RDS | logsstorage_size_bytes | The amount of storage used by the log files nstance | -| VPC | vpcsperregion | Quota and usage of the VPCs per region | -| VPC | subnetspervpc | Quota and usage of subnets per VPC | -| VPC | interfacevpcendpointspervpc | Quota and usage of interface endpoints per VPC | -| VPC | routetablespervpc | Quota and usage of routetables per VPC | -| VPC | routesperroutetable | Quota and usage of the routes per routetable | -| VPC | ipv4blockspervpc | Quota and usage of ipv4 blocks per VPC | -| EC2 | transitgatewaysperregion | Quota and usage of transitgateways per region | -| Route53 | recordsperhostedzone | Quota and usage of resource records per Hosted Zone | +| Service | Metric | Description | +|---------|-----------------------------|---------------------------------------------------------------------------| +| RDS | allocatedstorage | The amount of allocated storage in GB | +| RDS | maxallocatedstorage | The amount of max allocated storage for RDS storage autoscaling in bytes. | +| RDS | dbinstanceclass | The DB instance class (type) | +| RDS | dbinstancestatus | The instance status | +| RDS | engineversion | The DB engine type and version | +| RDS | pendingmaintenanceactions | The pending maintenance actions for a RDS instance | +| RDS | logs_amount | The amount of log files present in the RDS Instance | +| RDS | logsstorage_size_bytes | The amount of storage used by the log files nstance | +| VPC | vpcsperregion | Quota and usage of the VPCs per region | +| VPC | subnetspervpc | Quota and usage of subnets per VPC | +| VPC | interfacevpcendpointspervpc | Quota and usage of interface endpoints per VPC | +| VPC | routetablespervpc | Quota and usage of routetables per VPC | +| VPC | routesperroutetable | Quota and usage of the routes per routetable | +| VPC | ipv4blockspervpc | Quota and usage of ipv4 blocks per VPC | +| EC2 | transitgatewaysperregion | Quota and usage of transitgateways per region | +| Route53 | recordsperhostedzone | Quota and usage of resource records per Hosted Zone | ## Running this software diff --git a/pkg/rds.go b/pkg/rds.go index 032c4d8..8daf666 100644 --- a/pkg/rds.go +++ b/pkg/rds.go @@ -264,6 +264,12 @@ var AllocatedStorage *prometheus.Desc = prometheus.NewDesc( []string{"aws_region", "dbinstance_identifier"}, nil, ) +var MaxAllocatedStorage *prometheus.Desc = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "", "rds_maxallocatedstorage"), + "The amount of max allocated storage for RDS storage autoscaling in bytes.", + []string{"aws_region", "dbinstance_identifier"}, + nil, +) var DBInstanceClass *prometheus.Desc = prometheus.NewDesc( prometheus.BuildFQName(namespace, "", "rds_dbinstanceclass"), "The DB instance class (type).", @@ -499,6 +505,7 @@ func (e *RDSExporter) addAllInstanceMetrics(sessionIndex int, instances []*rds.D e.cache.AddMetric(prometheus.MustNewConstMetric(MaxConnections, prometheus.GaugeValue, float64(maxConnections), e.getRegion(sessionIndex), *instance.DBInstanceIdentifier)) e.cache.AddMetric(prometheus.MustNewConstMetric(AllocatedStorage, prometheus.GaugeValue, float64(*instance.AllocatedStorage*1024*1024*1024), e.getRegion(sessionIndex), *instance.DBInstanceIdentifier)) + e.cache.AddMetric(prometheus.MustNewConstMetric(MaxAllocatedStorage, prometheus.GaugeValue, float64(*instance.MaxAllocatedStorage*1024*1024*1024), e.getRegion(sessionIndex), *instance.DBInstanceIdentifier)) e.cache.AddMetric(prometheus.MustNewConstMetric(DBInstanceStatus, prometheus.GaugeValue, 1, e.getRegion(sessionIndex), *instance.DBInstanceIdentifier, *instance.DBInstanceStatus)) e.cache.AddMetric(prometheus.MustNewConstMetric(EngineVersion, prometheus.GaugeValue, 1, e.getRegion(sessionIndex), *instance.DBInstanceIdentifier, *instance.Engine, *instance.EngineVersion)) e.cache.AddMetric(prometheus.MustNewConstMetric(DBInstanceClass, prometheus.GaugeValue, 1, e.getRegion(sessionIndex), *instance.DBInstanceIdentifier, *instance.DBInstanceClass)) @@ -551,6 +558,7 @@ func (e *RDSExporter) addAllPendingMaintenancesMetrics(ctx context.Context, sess // Describe is used by the Prometheus client to return a description of the metrics func (e *RDSExporter) Describe(ch chan<- *prometheus.Desc) { ch <- AllocatedStorage + ch <- MaxAllocatedStorage ch <- DBInstanceClass ch <- DBInstanceStatus ch <- EngineVersion diff --git a/pkg/rds_test.go b/pkg/rds_test.go index 50f3cd2..6506a73 100644 --- a/pkg/rds_test.go +++ b/pkg/rds_test.go @@ -25,6 +25,7 @@ func createTestDBInstances() []*rds.DBInstance { PubliclyAccessible: aws.Bool(false), StorageEncrypted: aws.Bool(false), AllocatedStorage: aws.Int64(1024), + MaxAllocatedStorage: aws.Int64(1024), DBInstanceStatus: aws.String("on fire"), Engine: aws.String("SQL"), EngineVersion: aws.String("1000"), From b6a94cdc0750713a53da9189110ffe4d5d165146 Mon Sep 17 00:00:00 2001 From: Asaf Levy Date: Thu, 18 May 2023 16:09:32 +0300 Subject: [PATCH 2/5] Change metrics array length --- pkg/rds_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/rds_test.go b/pkg/rds_test.go index 6506a73..8a89841 100644 --- a/pkg/rds_test.go +++ b/pkg/rds_test.go @@ -88,7 +88,7 @@ func TestAddAllInstanceMetrics(t *testing.T) { assert.Len(t, x.cache.GetAllMetrics(), 0) x.addAllInstanceMetrics(0, createTestDBInstances()) - assert.Len(t, x.cache.GetAllMetrics(), 9) + assert.Len(t, x.cache.GetAllMetrics(), 10) } func TestAddAllPendingMaintenancesMetrics(t *testing.T) { From 360e8d77d21a6759be89a5027b4ff54c4cea9d22 Mon Sep 17 00:00:00 2001 From: Asaf Levy Date: Tue, 18 Jul 2023 17:39:14 +0300 Subject: [PATCH 3/5] add missing mapping for db.m5d.large --- Dockerfile | 4 ++-- pkg/rds.go | 9 +++++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 894148d..9c9223c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,9 +1,9 @@ -FROM quay.io/app-sre/golang:1.18.5 as builder +FROM public.ecr.aws/docker/library/golang:1.18 as builder WORKDIR /build COPY . . RUN make build -FROM registry.access.redhat.com/ubi8-minimal +FROM public.ecr.aws/amazonlinux/amazonlinux:latest COPY --from=builder /build/aws-resource-exporter /bin/aws-resource-exporter EXPOSE 9115 diff --git a/pkg/rds.go b/pkg/rds.go index 8daf666..86fa133 100644 --- a/pkg/rds.go +++ b/pkg/rds.go @@ -123,6 +123,15 @@ var DBMaxConnections = map[string]map[string]int64{ "default.postgres13": 5000, "default.postgres14": 5000, }, + "db.m5d.large": { + "default": 830, + "default.postgres10": 830, + "default.postgres11": 830, + "default.postgres12": 830, + "default.postgres13": 830, + "default.postgres14": 830, + "default.postgres15": 830, + }, "db.r4.large": map[string]int64{ "default": 1301, "default.mysql5.7": 1301, From f4abebb13cb2bb4c210ae50648129b4e9acefb89 Mon Sep 17 00:00:00 2001 From: Asaf Levy Date: Sun, 6 Aug 2023 18:13:40 +0300 Subject: [PATCH 4/5] safely convert MaxAllocatedStorage incase of nil --- pkg/rds.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pkg/rds.go b/pkg/rds.go index 86fa133..37e7918 100644 --- a/pkg/rds.go +++ b/pkg/rds.go @@ -123,7 +123,7 @@ var DBMaxConnections = map[string]map[string]int64{ "default.postgres13": 5000, "default.postgres14": 5000, }, - "db.m5d.large": { + "db.m5d.large": map[string]int64{ "default": 830, "default.postgres10": 830, "default.postgres11": 830, @@ -510,11 +510,17 @@ func (e *RDSExporter) addAllInstanceMetrics(sessionIndex int, instances []*rds.D if instance.LatestRestorableTime != nil { restoreTime = float64(instance.LatestRestorableTime.Unix()) } + + var MaxAllocatedStorageValue = 0.0 + if instance.MaxAllocatedStorage != nil { + MaxAllocatedStorageValue = float64(*instance.MaxAllocatedStorage*1024*1024*1024) + } + e.cache.AddMetric(prometheus.MustNewConstMetric(LatestRestorableTime, prometheus.CounterValue, restoreTime, e.getRegion(sessionIndex), *instance.DBInstanceIdentifier)) e.cache.AddMetric(prometheus.MustNewConstMetric(MaxConnections, prometheus.GaugeValue, float64(maxConnections), e.getRegion(sessionIndex), *instance.DBInstanceIdentifier)) e.cache.AddMetric(prometheus.MustNewConstMetric(AllocatedStorage, prometheus.GaugeValue, float64(*instance.AllocatedStorage*1024*1024*1024), e.getRegion(sessionIndex), *instance.DBInstanceIdentifier)) - e.cache.AddMetric(prometheus.MustNewConstMetric(MaxAllocatedStorage, prometheus.GaugeValue, float64(*instance.MaxAllocatedStorage*1024*1024*1024), e.getRegion(sessionIndex), *instance.DBInstanceIdentifier)) + e.cache.AddMetric(prometheus.MustNewConstMetric(MaxAllocatedStorage, prometheus.GaugeValue, MaxAllocatedStorageValue, e.getRegion(sessionIndex), *instance.DBInstanceIdentifier)) e.cache.AddMetric(prometheus.MustNewConstMetric(DBInstanceStatus, prometheus.GaugeValue, 1, e.getRegion(sessionIndex), *instance.DBInstanceIdentifier, *instance.DBInstanceStatus)) e.cache.AddMetric(prometheus.MustNewConstMetric(EngineVersion, prometheus.GaugeValue, 1, e.getRegion(sessionIndex), *instance.DBInstanceIdentifier, *instance.Engine, *instance.EngineVersion)) e.cache.AddMetric(prometheus.MustNewConstMetric(DBInstanceClass, prometheus.GaugeValue, 1, e.getRegion(sessionIndex), *instance.DBInstanceIdentifier, *instance.DBInstanceClass)) From 23b3208ddbd3cf2d279f95eed359868eb261483a Mon Sep 17 00:00:00 2001 From: Asaf Levy Date: Sun, 6 Aug 2023 18:31:37 +0300 Subject: [PATCH 5/5] update go variable camelCase --- pkg/rds.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/rds.go b/pkg/rds.go index 37e7918..b75269f 100644 --- a/pkg/rds.go +++ b/pkg/rds.go @@ -511,16 +511,16 @@ func (e *RDSExporter) addAllInstanceMetrics(sessionIndex int, instances []*rds.D restoreTime = float64(instance.LatestRestorableTime.Unix()) } - var MaxAllocatedStorageValue = 0.0 + var maxAllocatedStorageValue = 0.0 if instance.MaxAllocatedStorage != nil { - MaxAllocatedStorageValue = float64(*instance.MaxAllocatedStorage*1024*1024*1024) + maxAllocatedStorageValue = float64(*instance.MaxAllocatedStorage*1024*1024*1024) } e.cache.AddMetric(prometheus.MustNewConstMetric(LatestRestorableTime, prometheus.CounterValue, restoreTime, e.getRegion(sessionIndex), *instance.DBInstanceIdentifier)) e.cache.AddMetric(prometheus.MustNewConstMetric(MaxConnections, prometheus.GaugeValue, float64(maxConnections), e.getRegion(sessionIndex), *instance.DBInstanceIdentifier)) e.cache.AddMetric(prometheus.MustNewConstMetric(AllocatedStorage, prometheus.GaugeValue, float64(*instance.AllocatedStorage*1024*1024*1024), e.getRegion(sessionIndex), *instance.DBInstanceIdentifier)) - e.cache.AddMetric(prometheus.MustNewConstMetric(MaxAllocatedStorage, prometheus.GaugeValue, MaxAllocatedStorageValue, e.getRegion(sessionIndex), *instance.DBInstanceIdentifier)) + e.cache.AddMetric(prometheus.MustNewConstMetric(MaxAllocatedStorage, prometheus.GaugeValue, maxAllocatedStorageValue, e.getRegion(sessionIndex), *instance.DBInstanceIdentifier)) e.cache.AddMetric(prometheus.MustNewConstMetric(DBInstanceStatus, prometheus.GaugeValue, 1, e.getRegion(sessionIndex), *instance.DBInstanceIdentifier, *instance.DBInstanceStatus)) e.cache.AddMetric(prometheus.MustNewConstMetric(EngineVersion, prometheus.GaugeValue, 1, e.getRegion(sessionIndex), *instance.DBInstanceIdentifier, *instance.Engine, *instance.EngineVersion)) e.cache.AddMetric(prometheus.MustNewConstMetric(DBInstanceClass, prometheus.GaugeValue, 1, e.getRegion(sessionIndex), *instance.DBInstanceIdentifier, *instance.DBInstanceClass))