From 458a9e4167e2ad79a5222ea8a37f23806aceb452 Mon Sep 17 00:00:00 2001 From: Oleg Zaytsev Date: Fri, 4 Mar 2022 09:13:44 +0100 Subject: [PATCH] Support object-typed datasource references See: https://github.com/grafana/grafana/pull/33817 Since Grafana v8.4.3 datasource references are not longer a string with the datasource name, but an object with the type and UID instead. This means that dashboards saved with Grafana v8.4.3 or higher are not longer parseable by this SDK. I replaced the `*string` datasource by a `DatasourceRef` struct, which has custom unmarshaling: it tries to unmarshal the new object, and if it fails, it will try to unmarshal data as string and put it into `LegacyName` instead. Signed-off-by: Oleg Zaytsev --- board.go | 70 ++++---- dashboard-unmarshal_test.go | 39 ++++- datasource.go | 38 +++++ panel.go | 18 +-- panel_test.go | 25 +-- ...h-object-datasource-ref-grafana-8.4.3.json | 153 ++++++++++++++++++ 6 files changed, 287 insertions(+), 56 deletions(-) create mode 100644 testdata/dashboard-with-object-datasource-ref-grafana-8.4.3.json diff --git a/board.go b/board.go index da8edf1f..2e8fdafc 100644 --- a/board.go +++ b/board.go @@ -79,24 +79,24 @@ type ( List []TemplateVar `json:"list"` } TemplateVar struct { - Name string `json:"name"` - Type string `json:"type"` - Auto bool `json:"auto,omitempty"` - AutoCount *int `json:"auto_count,omitempty"` - Datasource *string `json:"datasource"` - Refresh BoolInt `json:"refresh"` - Options []Option `json:"options"` - IncludeAll bool `json:"includeAll"` - AllFormat string `json:"allFormat"` - AllValue string `json:"allValue"` - Multi bool `json:"multi"` - MultiFormat string `json:"multiFormat"` - Query interface{} `json:"query"` - Regex string `json:"regex"` - Current Current `json:"current"` - Label string `json:"label"` - Hide uint8 `json:"hide"` - Sort int `json:"sort"` + Name string `json:"name"` + Type string `json:"type"` + Auto bool `json:"auto,omitempty"` + AutoCount *int `json:"auto_count,omitempty"` + Datasource *DatasourceRef `json:"datasource"` + Refresh BoolInt `json:"refresh"` + Options []Option `json:"options"` + IncludeAll bool `json:"includeAll"` + AllFormat string `json:"allFormat"` + AllValue string `json:"allValue"` + Multi bool `json:"multi"` + MultiFormat string `json:"multiFormat"` + Query interface{} `json:"query"` + Regex string `json:"regex"` + Current Current `json:"current"` + Label string `json:"label"` + Hide uint8 `json:"hide"` + Sort int `json:"sort"` } // for templateVar Option struct { @@ -111,23 +111,23 @@ type ( Value interface{} `json:"value"` // TODO select more precise type } Annotation struct { - Name string `json:"name"` - Datasource *string `json:"datasource"` - ShowLine bool `json:"showLine"` - IconColor string `json:"iconColor"` - LineColor string `json:"lineColor"` - IconSize uint `json:"iconSize"` - Enable bool `json:"enable"` - Query string `json:"query"` - Expr string `json:"expr"` - Step string `json:"step"` - TextField string `json:"textField"` - TextFormat string `json:"textFormat"` - TitleFormat string `json:"titleFormat"` - TagsField string `json:"tagsField"` - Tags []string `json:"tags"` - TagKeys string `json:"tagKeys"` - Type string `json:"type"` + Name string `json:"name"` + Datasource *DatasourceRef `json:"datasource"` + ShowLine bool `json:"showLine"` + IconColor string `json:"iconColor"` + LineColor string `json:"lineColor"` + IconSize uint `json:"iconSize"` + Enable bool `json:"enable"` + Query string `json:"query"` + Expr string `json:"expr"` + Step string `json:"step"` + TextField string `json:"textField"` + TextFormat string `json:"textFormat"` + TitleFormat string `json:"titleFormat"` + TagsField string `json:"tagsField"` + Tags []string `json:"tags"` + TagKeys string `json:"tagKeys"` + Type string `json:"type"` } // Link represents link to another dashboard or external weblink Link struct { diff --git a/dashboard-unmarshal_test.go b/dashboard-unmarshal_test.go index 5a622a46..1b6bd6e2 100644 --- a/dashboard-unmarshal_test.go +++ b/dashboard-unmarshal_test.go @@ -24,6 +24,9 @@ import ( "io/ioutil" "testing" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/grafana-tools/sdk" ) @@ -101,8 +104,8 @@ func TestUnmarshal_DashboardWithGraphWithTargets26(t *testing.T) { if panel.OfType != sdk.GraphType { t.Errorf("panel type should be %d (\"graph\") type but got %d", sdk.GraphType, panel.OfType) } - if *panel.Datasource != sdk.MixedSource { - t.Errorf("panel Datasource should be \"%s\" but got \"%s\"", sdk.MixedSource, *panel.Datasource) + if panel.Datasource.LegacyName != sdk.MixedSource { + t.Errorf("panel Datasource should have legacy name \"%s\" but got \"%s\"", sdk.MixedSource, panel.Datasource.LegacyName) } if len(panel.GraphPanel.Targets) != 2 { t.Errorf("panel has 2 targets but got %d", len(panel.GraphPanel.Targets)) @@ -188,3 +191,35 @@ func TestUnmarshal_DashboardWithMixedYaxes(t *testing.T) { t.Errorf("panel #1 has wrong max value: %f, expected: %f", max4.Value, 100.0) } } + +func TestUnmarshalDashboardWithObjectDatasourceRefs(t *testing.T) { + var board sdk.Board + raw, _ := ioutil.ReadFile("testdata/dashboard-with-object-datasource-ref-grafana-8.4.3.json") + + err := json.Unmarshal(raw, &board) + if err != nil { + t.Fatal(err) + } + + expectedPromDS := &sdk.DatasourceRef{Type: "prometheus", UID: "prom-ds-uid"} + expectedLogsDS := &sdk.DatasourceRef{Type: "logs", UID: "logs-ds-uid"} + expectedGraphiteDS := &sdk.DatasourceRef{Type: "graphite", UID: "graphite-ds-uid"} + + require.Lenf(t, board.Panels, 1, "there is 1 panel expected but got %d", len(board.Panels)) + + panel := board.Panels[0] + assert.Equal(t, expectedPromDS, panel.Datasource) + require.Equalf(t, sdk.TimeseriesType, panel.OfType, "expected panel to be timeseries panel type %d, got %d", sdk.TimeseriesType, panel.OfType) + require.Lenf(t, panel.TimeseriesPanel.Targets, 1, "there is 1 target expected but got %d", len(panel.TimeseriesPanel.Targets)) + + target := panel.TimeseriesPanel.Targets[0] + assert.Equal(t, expectedPromDS, target.Datasource) + + require.Lenf(t, board.Annotations.List, 1, "there is 1 annotation expected but got %d", len(board.Annotations.List)) + annotation := board.Annotations.List[0] + assert.Equal(t, expectedLogsDS, annotation.Datasource) + + require.Lenf(t, board.Templating.List, 1, "there is 1 template var expected but got %d", len(board.Templating.List)) + templating := board.Templating.List[0] + assert.Equal(t, expectedGraphiteDS, templating.Datasource) +} diff --git a/datasource.go b/datasource.go index fe582fd2..498ff47c 100644 --- a/datasource.go +++ b/datasource.go @@ -1,5 +1,9 @@ package sdk +import ( + "encoding/json" +) + /* Copyright 2016 Alexander I.Grafov Copyright 2016-2019 The Grafana SDK authors @@ -52,3 +56,37 @@ type DatasourceType struct { ServiceName string `json:"serviceName"` Type string `json:"type"` } + +// DatasourceRef is used to reference a datasource from panels, queries, etc. +type DatasourceRef struct { + // Type describes the type of the datasource, like "prometheus", "graphite", etc. + // Datasources of the same type should support same queries. + // If Type is empty in an unmarshaled DatasourceRef, check the LegacyName field. + Type string `json:"type"` + // UID is the uid of the specific datasource this references to. + UID string `json:"UID"` + // LegacyName is the old way of referencing a datasource by its name, replaced in Grafana v8.4.3 by Type and UID referencing. + // If datasource is encoded as a string, then it's unmarshaled into this LegacyName field (Type and UID will be empty). + // If LegacyName is not empty, then this DatasourceRef will be marshaled as a string, ignoring the values of Type and UID. + LegacyName string `json:"-"` +} + +func (ref DatasourceRef) MarshalJSON() ([]byte, error) { + if ref.LegacyName != "" { + return json.Marshal(ref.LegacyName) + } + type plain DatasourceRef + return json.Marshal(plain(ref)) +} + +func (ref *DatasourceRef) UnmarshalJSON(data []byte) error { + type plain DatasourceRef + err := json.Unmarshal(data, (*plain)(ref)) + if err != nil { + if err := json.Unmarshal(data, &ref.LegacyName); err == nil { + // We could check here if it's `-- Mixed --` and in that case set ref.Type="mixed". + return nil + } + } + return err +} diff --git a/panel.go b/panel.go index 552c058c..e8b951b2 100644 --- a/panel.go +++ b/panel.go @@ -66,9 +66,9 @@ type ( } panelType int8 CommonPanel struct { - Datasource *string `json:"datasource,omitempty"` // metrics - Editable bool `json:"editable"` - Error bool `json:"error"` + Datasource *DatasourceRef `json:"datasource,omitempty"` // metrics + Editable bool `json:"editable"` + Error bool `json:"error"` GridPos struct { H *int `json:"h,omitempty"` W *int `json:"w,omitempty"` @@ -549,9 +549,9 @@ type ( // for an any panel type Target struct { - RefID string `json:"refId"` - Datasource string `json:"datasource,omitempty"` - Hide bool `json:"hide,omitempty"` + RefID string `json:"refId"` + Datasource *DatasourceRef `json:"datasource,omitempty"` + Hide bool `json:"hide,omitempty"` // For PostgreSQL Table string `json:"table,omitempty"` @@ -942,7 +942,7 @@ func (p *Panel) RepeatDatasourcesForEachTarget(dsNames ...string) { for _, ds := range dsNames { newTarget := target newTarget.RefID = refID - newTarget.Datasource = ds + newTarget.Datasource = &DatasourceRef{LegacyName: ds} refID = incRefID(refID) *targets = append(*targets, newTarget) } @@ -973,13 +973,13 @@ func (p *Panel) RepeatTargetsForDatasources(dsNames ...string) { lenTargets := len(*targets) for i, name := range dsNames { if i < lenTargets { - (*targets)[i].Datasource = name + (*targets)[i].Datasource = &DatasourceRef{LegacyName: name} lastRefID = (*targets)[i].RefID } else { newTarget := (*targets)[i%lenTargets] lastRefID = incRefID(lastRefID) newTarget.RefID = lastRefID - newTarget.Datasource = name + newTarget.Datasource = &DatasourceRef{LegacyName: name} *targets = append(*targets, newTarget) } } diff --git a/panel_test.go b/panel_test.go index 138e2736..bbdb87c9 100644 --- a/panel_test.go +++ b/panel_test.go @@ -291,8 +291,9 @@ func TestNewTimeseries(t *testing.T) { func TestGraph_AddTarget(t *testing.T) { var target = sdk.Target{ RefID: "A", - Datasource: "Sample Source", - Expr: "sample request"} + Datasource: &sdk.DatasourceRef{LegacyName: "Sample Source"}, + Expr: "sample request", + } graph := sdk.NewGraph("") graph.AddTarget(&target) @@ -309,12 +310,14 @@ func TestGraph_SetTargetNew(t *testing.T) { var ( target1 = sdk.Target{ RefID: "A", - Datasource: "Sample Source 1", - Expr: "sample request 1"} + Datasource: &sdk.DatasourceRef{LegacyName: "Sample Source 1"}, + Expr: "sample request 1", + } target2 = sdk.Target{ RefID: "B", - Datasource: "Sample Source 2", - Expr: "sample request 2"} + Datasource: &sdk.DatasourceRef{LegacyName: "Sample Source 2"}, + Expr: "sample request 2", + } ) graph := sdk.NewGraph("") graph.AddTarget(&target1) @@ -336,12 +339,14 @@ func TestGraph_SetTargetUpdate(t *testing.T) { var ( target1 = sdk.Target{ RefID: "A", - Datasource: "Sample Source 1", - Expr: "sample request 1"} + Datasource: &sdk.DatasourceRef{LegacyName: "Sample Source 1"}, + Expr: "sample request 1", + } target2 = sdk.Target{ RefID: "A", - Datasource: "Sample Source 2", - Expr: "sample request 2"} + Datasource: &sdk.DatasourceRef{LegacyName: "Sample Source 2"}, + Expr: "sample request 2", + } ) graph := sdk.NewGraph("") graph.AddTarget(&target1) diff --git a/testdata/dashboard-with-object-datasource-ref-grafana-8.4.3.json b/testdata/dashboard-with-object-datasource-ref-grafana-8.4.3.json new file mode 100644 index 00000000..0f4c346c --- /dev/null +++ b/testdata/dashboard-with-object-datasource-ref-grafana-8.4.3.json @@ -0,0 +1,153 @@ +{ + "annotations": { + "list": [ + { + "datasource": { + "type": "logs", + "uid": "logs-ds-uid" + }, + "enable": true, + "expr": "count({instance=\"127.0.0.1\"}) > 1", + "iconColor": "red", + "name": "Something is happening", + "target": {} + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-ds-uid" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "prom-ds-uid" + }, + "editorMode": "code", + "expr": "go_build_info", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Panel Title", + "type": "timeseries" + } + ], + "schemaVersion": 35, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "isNone": true, + "selected": false, + "text": "None", + "value": "" + }, + "datasource": { + "type": "graphite", + "uid": "graphite-ds-uid" + }, + "definition": "foo", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "query0", + "options": [], + "query": "foo", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "New dashboard", + "version": 0, + "weekStart": "" +} \ No newline at end of file