Merge pull request #17 from totvslabs/disk

fix: better monitor disk stats
totvslabs · Dec 10, 2018 · 2d06a84 · 2d06a84
2 parents 57a14c7 + 34f5cf5
commit 2d06a84
Show file tree

Hide file tree

Showing 2 changed files with 96 additions and 21 deletions.
diff --git a/grafana/dashboard.jsonnet b/grafana/dashboard.jsonnet
@@ -245,6 +245,44 @@ dashboard.new(
       )
     )
   )
+  .addPanel(
+    graphPanel.new(
+      'Connections',
+      span=6,
+      legend_alignAsTable=true,
+      legend_rightSide=true,
+      legend_values=true,
+      legend_current=true,
+      legend_sort='current',
+      legend_sortDesc=true,
+      min=0,
+    )
+    .addTarget(
+      prometheus.target(
+        'couchbase_bucket_stats_curr_connections{bucket=~"$bucket",instance=~"$instance"}',
+        legendFormat='{{ bucket }}',
+      )
+    )
+  )
+  .addPanel(
+    graphPanel.new(
+      'Primary items total',
+      span=6,
+      legend_alignAsTable=true,
+      legend_rightSide=true,
+      legend_values=true,
+      legend_current=true,
+      legend_sort='current',
+      legend_sortDesc=true,
+      min=0,
+    )
+    .addTarget(
+      prometheus.target(
+        'couchbase_bucket_stats_curr_items{bucket=~"$bucket",instance=~"$instance"}',
+        legendFormat='{{ bucket }}',
+      )
+    )
+  )
 )
 .addRow(
   row.new(
@@ -284,7 +322,7 @@ dashboard.new(
   .addPanel(
     graphPanel.new(
       'Evictions',
-      span=4,
+      span=6,
       legend_alignAsTable=true,
       legend_rightSide=true,
       legend_values=true,
@@ -300,29 +338,10 @@ dashboard.new(
       )
     )
   )
-  .addPanel(
-    graphPanel.new(
-      'Disk Fetches',
-      span=4,
-      legend_alignAsTable=true,
-      legend_rightSide=true,
-      legend_values=true,
-      legend_current=true,
-      legend_sort='current',
-      legend_sortDesc=true,
-      min=0,
-    )
-    .addTarget(
-      prometheus.target(
-        'couchbase_bucket_stats_ep_bg_fetched{bucket=~"$bucket",instance=~"$instance"}',
-        legendFormat='{{ bucket }}',
-      )
-    )
-  )
   .addPanel(
     graphPanel.new(
       'Resident Ratio',
-      span=4,
+      span=6,
       legend_alignAsTable=true,
       legend_rightSide=true,
       legend_values=true,
@@ -426,6 +445,48 @@ dashboard.new(
     )
   )
 )
+.addRow(
+  row.new(
+    title='Disk',
+    collapse=true,
+  )
+  .addPanel(
+    graphPanel.new(
+      'Disk Fetches',
+      span=6,
+      legend_alignAsTable=true,
+      legend_rightSide=true,
+      legend_values=true,
+      legend_current=true,
+      legend_sort='current',
+      legend_sortDesc=true,
+    )
+    .addTarget(
+      prometheus.target(
+        'couchbase_bucket_basicstats_diskfetches{instance=~"$instance", bucket=~"$bucket"}',
+        legendFormat='{{ bucket }}',
+      )
+    )
+  )
+  .addPanel(
+    graphPanel.new(
+      'Disk Write Queue',
+      span=6,
+      legend_alignAsTable=true,
+      legend_rightSide=true,
+      legend_values=true,
+      legend_current=true,
+      legend_sort='current',
+      legend_sortDesc=true,
+    )
+    .addTarget(
+      prometheus.target(
+        'couchbase_bucket_stats_disk_write_queue{instance=~"$instance", bucket=~"$bucket"}',
+        legendFormat='{{ bucket }}',
+      )
+    )
+  )
+)
 .addRow(
   row.new(
     title='Compacting',

diff --git a/prometheus/rules/couchbase.rules.yml b/prometheus/rules/couchbase.rules.yml
@@ -134,3 +134,17 @@ groups:
       severity: warning
     annotations:
       summary: '{{ $labels.bucket }}: resident ratio is lower than 20%'
+  - alert: CouchbaseHighCacheMissRate
+    expr: couchbase_bucket_stats_ep_cache_miss_rate > 20
+    for: 1m
+    labels:
+      severity: warning
+    annotations:
+      summary: '{{ $labels.bucket }}: cache miss rate is high'
+  - alert: CouchbaseHighDiskWriteQueue
+    expr: couchbase_bucket_stats_disk_write_queue > 1000000
+    for: 1s
+    labels:
+      severity: critical
+    annotations:
+      summary: '{{ $labels.bucket }}: disk write queue is big (millions of items)'