[metricbeat] use deprecated kube-state-metrics config when existing (#…

…624)

This commit fix a bug introduced in #572 where Metricbeat deployment doesn't start after an upgrade from a previous version when using a custom metricbeatConfig.

BREAKING_CHANGES.md

@@ -70,6 +70,14 @@ Metricbeat is now using dedicated values for daemonset and deployment config.
 The old values are still working but are now deprecated. See [#572][] for more
 details.
 
+Warning: When upgrading Metricbeat while using custom `metricbeatConfig` value
+for `kube-state-metrics-metricbeat.yml`, Metricbeat deployment fails with
+`missing field accessing 'metricbeat.modules.0.hosts.0' (source:'metricbeat.yml')`.
+
+In this case `metricbeatConfig.kube-state-metrics-metricbeat.yml` value should
+be migrated to `deployment.metricbeatConfig.metricbeat.yml`. See [#623][] for
+more details.
+
 ## 6.8.9 - 2020/05/13
 
 See [7.7.0 Breaking changes](#770---20200513)
@@ -171,6 +179,7 @@ volumeClaimTemplate:
 [#572]: https://github.com/elastic/helm-charts/pull/572
 [#586]: https://github.com/elastic/helm-charts/pull/586
 [#621]: https://github.com/elastic/helm-charts/pull/621
+[#623]: https://github.com/elastic/helm-charts/pull/623
 [container input]: https://www.elastic.co/guide/en/beats/filebeat/7.7/filebeat-input-container.html
 [docker input]: https://www.elastic.co/guide/en/beats/filebeat/7.7/filebeat-input-docker.html
 [elastic helm repo]: https://helm.elastic.co

metricbeat/templates/deployment.yaml

@@ -72,6 +72,10 @@ spec:
         image: "{{ .Values.image }}:{{ .Values.imageTag }}"
         imagePullPolicy: "{{ .Values.imagePullPolicy }}"
         args:
+        {{- if index .Values "metricbeatConfig" "kube-state-metrics-metricbeat.yml" }}
+          - "-c"
+          - "/usr/share/metricbeat/kube-state-metrics-metricbeat.yml"
+        {{- end }}
           - "-e"
           - "-E"
           - "http.enabled=true"

metricbeat/tests/metricbeat_test.py

@@ -537,48 +537,77 @@ def test_adding_in_deprecated_metricbeat_config():
       nestedkey: value
     dot.notation: test
 
-  other-config.yml: |
+  kube-state-metrics-metricbeat.yml: |
     hello = world
 """
     r = helm_template(config)
     c = r["configmap"][name + "-config"]["data"]
 
     assert "metricbeat.yml" in c
-    assert "other-config.yml" in c
+    assert "kube-state-metrics-metricbeat.yml" in c
 
     assert "nestedkey: value" in c["metricbeat.yml"]
     assert "dot.notation: test" in c["metricbeat.yml"]
 
-    assert "hello = world" in c["other-config.yml"]
+    assert "hello = world" in c["kube-state-metrics-metricbeat.yml"]
 
-    d = r["daemonset"][name]["spec"]["template"]["spec"]
+    daemonset = r["daemonset"][name]["spec"]["template"]["spec"]
 
     assert {
         "configMap": {"name": name + "-config", "defaultMode": 0o600},
         "name": project + "-config",
-    } in d["volumes"]
+    } in daemonset["volumes"]
     assert {
         "mountPath": "/usr/share/metricbeat/metricbeat.yml",
         "name": project + "-config",
         "subPath": "metricbeat.yml",
         "readOnly": True,
-    } in d["containers"][0]["volumeMounts"]
+    } in daemonset["containers"][0]["volumeMounts"]
     assert {
-        "mountPath": "/usr/share/metricbeat/other-config.yml",
+        "mountPath": "/usr/share/metricbeat/kube-state-metrics-metricbeat.yml",
         "name": project + "-config",
-        "subPath": "other-config.yml",
+        "subPath": "kube-state-metrics-metricbeat.yml",
         "readOnly": True,
-    } in d["containers"][0]["volumeMounts"]
+    } in daemonset["containers"][0]["volumeMounts"]
 
     assert (
         "configChecksum"
         in r["daemonset"][name]["spec"]["template"]["metadata"]["annotations"]
     )
 
+    deployment = r["deployment"][name + "-metrics"]["spec"]["template"]["spec"]
+
+    assert {
+        "configMap": {"name": name + "-config", "defaultMode": 0o600},
+        "name": project + "-config",
+    } in deployment["volumes"]
+    assert {
+        "mountPath": "/usr/share/metricbeat/metricbeat.yml",
+        "name": project + "-config",
+        "subPath": "metricbeat.yml",
+        "readOnly": True,
+    } in deployment["containers"][0]["volumeMounts"]
+    assert {
+        "mountPath": "/usr/share/metricbeat/kube-state-metrics-metricbeat.yml",
+        "name": project + "-config",
+        "subPath": "kube-state-metrics-metricbeat.yml",
+        "readOnly": True,
+    } in deployment["containers"][0]["volumeMounts"]
+    assert ("/usr/share/metricbeat/kube-state-metrics-metricbeat.yml") in deployment[
+        "containers"
+    ][0]["args"]
+
+    assert (
+        "configChecksum"
+        in r["deployment"][name + "-metrics"]["spec"]["template"]["metadata"][
+            "annotations"
+        ]
+    )
+
 
 def test_adding_a_secret_mount():
     config = """
-daemonset:    
+daemonset:
   secretMounts:
     - name: elastic-certificates
       secretName: elastic-certificates-name
@@ -614,7 +643,7 @@ def test_adding_a_secret_mount():
     } not in r["deployment"][name + "-metrics"]["spec"]["template"]["spec"]["volumes"]
 
     config = """
-deployment:    
+deployment:
   secretMounts:
     - name: elastic-certificates
       secretName: elastic-certificates-name