Add new validations to manage object_type and metric_type fields

code/go/pkg/validator/validator_test.go

@@ -103,6 +103,18 @@ func TestValidateFile(t *testing.T) {
 			[]string{
 				`field 0.fields.4.type: 0.fields.4.type must be one of the following: "histogram", "aggregate_metric_double", "long", "integer", "short", "byte", "double", "float", "half_float", "scaled_float", "unsigned_long"`,
 				`field 0.fields.5: type is required`,
+				`field 0.fields.8.type: 0.fields.8.type must be one of the following: "histogram", "aggregate_metric_double", "long", "integer", "short", "byte", "double", "float", "half_float", "scaled_float", "unsigned_long"`,
+				"field \"example.agent.call_duration\" of type histogram can't be a dimension, allowed types for dimensions: constant_keyword, keyword, long, integer, short, byte, double, float, half_float, scaled_float, unsigned_long, ip",
+			},
+		},
+		"bad_metric_type_fields": {
+			"data_stream/example/fields/fields.yml",
+			[]string{
+				`field 0.fields.4.type: 0.fields.4.type must be one of the following: "histogram", "aggregate_metric_double", "long", "integer", "short", "byte", "double", "float", "half_float", "scaled_float", "unsigned_long"`,
+				`field 0.fields.5: type is required`,
+				`field 0.fields.6.type: 0.fields.6.type must be one of the following: "object"`,
+				`field 0.fields.7.object_type: 0.fields.7.object_type must be one of the following: "histogram", "long", "integer", "short", "byte", "double", "float", "half_float", "scaled_float", "unsigned_long"`,
+				`field 0.fields.8.type: 0.fields.8.type must be one of the following: "histogram", "aggregate_metric_double", "long", "integer", "short", "byte", "double", "float", "half_float", "scaled_float", "unsigned_long"`,
 				"field \"example.agent.call_duration\" of type histogram can't be a dimension, allowed types for dimensions: constant_keyword, keyword, long, integer, short, byte, double, float, half_float, scaled_float, unsigned_long, ip",
 			},
 		},

spec/changelog.yml

@@ -10,6 +10,9 @@
   - description: Add specific settings for data retention lifecycle configuration (for DLM).
     type: enhancement
     link: https://github.com/elastic/package-spec/pull/556
+  - description: Add new validation for usages of metric_type field
+    type: enhancement
+    link: https://github.com/elastic/package-spec/pull/527
 - version: 2.9.0
   changes:
   - description: Update minimum kibana version required for runtime fields

spec/integration/data_stream/fields/fields.spec.yml

@@ -328,6 +328,10 @@ spec:
           - object
           - short
           - text
+          - integer
+          - half_float
+          - scaled_float
+          - unsigned_long
 
       object_type_mapping_type:
         description: >
@@ -454,6 +458,9 @@ spec:
       - if:
           required:
             - metric_type
+          not:
+            required:
+              - object_type
         then:
           properties:
             type:
@@ -471,17 +478,58 @@ spec:
                 - unsigned_long
           required:
             - type
+      - if:
+          required:
+            - object_type
+        then:
+          properties:
+            type:
+              enum:
+                - object
+          required:
+            - type
+      - if:
+          properties:
+            type:
+              const: object
+          required:
+            - metric_type
+            - object_type
+        then:
+          properties:
+            type:
+              enum:
+                - object
+            object_type:
+              enum:
+                - histogram
+                - long
+                - integer
+                - short
+                - byte
+                - double
+                - float
+                - half_float
+                - scaled_float
+                - unsigned_long
+          required:
+            - type
+            - object_type
     required:
     - name
 
 # JSON patches for newer versions should be placed on top
 versions:
+  - before: 2.10.0
+    patch:
+      - op: remove
+        path: "/items/allOf/5" # removing if related to object_type and metric_type
   - before: 2.8.0
     patch:
       - op: remove
         path: "/items/properties/runtime"
       - op: remove
-        path: "/items/allOf/2"
+        path: "/items/allOf/2" # remove if related to runtime fields allowed types
   - before: 2.7.0
     patch:
       - op: remove
@@ -499,3 +547,7 @@ versions:
         value: true
       - op: remove
         path: "/items/properties/example"
+      - op: remove
+        # removing if related to ensure type is object if object_type is defined
+        # it should be /items/allOf/4 , but there is a JSON Patch removing second element for runtime fields in 2.8.0
+        path: "/items/allOf/3"

test/packages/bad_metric_type_fields/changelog.yml

@@ -0,0 +1,6 @@
+# newer versions go on top
+- version: "0.0.1"
+  changes:
+    - description: Initial draft of the package
+      type: enhancement
+      link: https://github.com/elastic/integrations/pull/1 # FIXME Replace with the real PR link

test/packages/bad_metric_type_fields/data_stream/example/agent/stream/stream.yml.hbs

@@ -0,0 +1,6 @@
+metricsets: ["sample_metricset"]
+hosts:
+{{#each hosts}}
+  - {{this}}
+{{/each}}
+period: {{period}}

test/packages/bad_metric_type_fields/data_stream/example/fields/base-fields.yml

@@ -0,0 +1,12 @@
+- name: data_stream.type
+  type: constant_keyword
+  description: Data stream type.
+- name: data_stream.dataset
+  type: constant_keyword
+  description: Data stream dataset.
+- name: data_stream.namespace
+  type: constant_keyword
+  description: Data stream namespace.
+- name: '@timestamp'
+  type: date
+  description: Event timestamp.

test/packages/bad_metric_type_fields/data_stream/example/fields/fields.yml

@@ -0,0 +1,32 @@
+- name: example
+  type: group
+  fields:
+  - name: agent.id
+    type: keyword
+    dimension: true
+  - name: agent.call_count
+    type: long
+    metric_type: counter
+  - name: agent.current_count
+    type: long
+    metric_type: gauge
+  - name: agent.call_duration
+    type: histogram
+    metric_type: gauge
+    dimension: true # This should fail, a histogram cannot be a dimension.
+  - name: no_valid_type
+    type: boolean
+    metric_type: gauge
+  - name: no_type
+    metric_type: gauge
+  - name: field_object_type_double
+    type: double
+    object_type: double
+    metric_type: gauge
+  - name: field_object_type_boolean
+    type: object
+    object_type: boolean
+    metric_type: gauge
+  - name: field_object
+    type: object
+    metric_type: gauge

test/packages/bad_metric_type_fields/data_stream/example/manifest.yml

@@ -0,0 +1,11 @@
+title: "Example data stream for time series"
+type: metrics
+streams:
+  - input: sample/metrics
+    title: Sample metrics
+    description: Collect sample metrics
+    vars:
+      - name: period
+        type: text
+        title: Period
+        default: 10s

test/packages/bad_metric_type_fields/docs/README.md

@@ -0,0 +1,5 @@
+# Package with fields using metric_type
+
+This is a new integration created using the [elastic-package](https://github.com/elastic/elastic-package) tool.
+
+Consider using the README template file `_dev/build/docs/README.md`to generate a list of exported fields or include a sample event.

test/packages/bad_metric_type_fields/manifest.yml

@@ -0,0 +1,32 @@
+format_version: 2.10.0
+name: bad_metric_type_fields
+title: "Package with fields using metric_type"
+version: 0.0.1
+description: "This is a package with fields using metric type"
+type: integration
+categories:
+  - custom
+  - monitoring
+conditions:
+  kibana.version: "^7.16.0"
+  elastic.subscription: basic
+screenshots:
+  - src: /img/sample-screenshot.png
+    title: Sample screenshot
+    size: 600x600
+    type: image/png
+icons:
+  - src: /img/sample-logo.svg
+    title: Sample logo
+    size: 32x32
+    type: image/svg+xml
+policy_templates:
+  - name: sample
+    title: Sample logs
+    description: Collect sample logs
+    inputs:
+      - type: logfile
+        title: Collect sample logs from instances
+        description: Collecting sample logs
+owner:
+  github: elastic/integrations-developer-experience

test/packages/bad_time_series/data_stream/example/fields/fields.yml

@@ -19,3 +19,14 @@
     metric_type: gauge
   - name: no_type
     metric_type: gauge
+  - name: field_object_type_double
+    type: double
+    object_type: double
+    metric_type: gauge
+  - name: field_object_type_boolean
+    type: object
+    object_type: boolean
+    metric_type: gauge
+  - name: field_object
+    type: object
+    metric_type: gauge

test/packages/good_v2/data_stream/foo/fields/some_fields.yml

@@ -72,3 +72,17 @@
   type: keyword
   runtime: |
     doc['message'].value().doSomething()
+- name: metric.a.*.gauge
+  type: object
+  object_type: integer
+  metric_type: gauge
+- name: metric.b.*.gauge
+  type: integer
+  metric_type: gauge
+- name: metric.c.*.gauge
+  type: aggregate_metric_double
+  metric_type: gauge
+  default_metric: max
+  metrics:
+    - min
+    - max