Skip to content

Commit

Permalink
Add service.node.configured_name to Intake API and transform into ser…
Browse files Browse the repository at this point in the history 
…vice.node.name for ES (#2746) (#2787)

Add service.node.configured_name to Intake API and add support for retrieving and indexing service.node.name in ES.

fixes #2696
  • Loading branch information
@simitt
simitt authored Oct 14, 2019
1 parent 229d823 commit 45ea5a9
Show file tree
Hide file tree
Showing 37 changed files with 396 additions and 84 deletions.
11 changes: 10 additions & 1 deletion _meta/fields.common.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,7 @@
- name: name
type: keyword
description: >
Immutable unique name of the service emitting this event.
Immutable name of the service emitting this event.
overwrite: true

- name: version
Expand All @@ -153,6 +153,15 @@
description: >
Service environment.
- name: node
type: group
fields:
- name: name
type: keyword
description: >
Unique meaningful name of the service node.
overwrite: true

- name: language
type: group
fields:
Expand Down
12 changes: 12 additions & 0 deletions beater/test_approved_es_documents/TestPublishIntegrationErrors.approved.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,6 +298,9 @@
"version": "1.2"
},
"name": "service1",
"node": {
"name": "node-xyz"
},
"runtime": {
"name": "node",
"version": "8.0.0"
Expand Down Expand Up @@ -407,6 +410,9 @@
"version": "8"
},
"name": "1234_service-12a3",
"node": {
"name": "node-abc"
},
"runtime": {
"name": "node",
"version": "8.0.0"
Expand Down Expand Up @@ -504,6 +510,9 @@
"version": "2.5"
},
"name": "service1",
"node": {
"name": "node-abc"
},
"runtime": {
"name": "cruby",
"version": "2.5"
Expand Down Expand Up @@ -603,6 +612,9 @@
"version": "8"
},
"name": "1234_service-12a3",
"node": {
"name": "node-abc"
},
"runtime": {
"name": "node",
"version": "8.0.0"
Expand Down
15 changes: 12 additions & 3 deletions beater/test_approved_es_documents/TestPublishIntegrationEvents.approved.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -63,7 +63,10 @@
"language": {
"name": "ecmascript"
},
"name": "1234_service-12a3"
"name": "1234_service-12a3",
"node": {
"name": "container-id"
}
},
"timestamp": {
"us": 1533827045999000
Expand Down Expand Up @@ -179,7 +182,10 @@
"language": {
"name": "ecmascript"
},
"name": "1234_service-12a3"
"name": "1234_service-12a3",
"node": {
"name": "container-id"
}
},
"timestamp": {
"us": 1535655207154000
Expand Down Expand Up @@ -264,7 +270,10 @@
"language": {
"name": "ecmascript"
},
"name": "1234_service-12a3"
"name": "1234_service-12a3",
"node": {
"name": "container-id"
}
}
}
]
Expand Down
10 changes: 8 additions & 2 deletions beater/test_approved_es_documents/TestPublishIntegrationMetricsets.approved.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,10 @@
"language": {
"name": "ecmascript"
},
"name": "1234_service-12a3"
"name": "1234_service-12a3",
"node": {
"name": "node-1"
}
},
"short_counter": 227,
"span": {
Expand Down Expand Up @@ -149,7 +152,10 @@
"language": {
"name": "ecmascript"
},
"name": "1234_service-12a3"
"name": "1234_service-12a3",
"node": {
"name": "node-1"
}
},
"user": {
"email": "user@mail.com",
Expand Down
9 changes: 9 additions & 0 deletions beater/test_approved_es_documents/TestPublishIntegrationTransactions.approved.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,9 @@
"version": "8"
},
"name": "1234_service-12a3",
"node": {
"name": "node-123"
},
"runtime": {
"name": "node",
"version": "8.0.0"
Expand Down Expand Up @@ -241,6 +244,9 @@
"version": "8"
},
"name": "1234_service-12a3",
"node": {
"name": "node-123"
},
"runtime": {
"name": "node",
"version": "7.0"
Expand Down Expand Up @@ -381,6 +387,9 @@
"version": "2.5"
},
"name": "service1",
"node": {
"name": "node-ABC"
},
"runtime": {
"name": "cruby",
"version": "2.5"
Expand Down
12 changes: 12 additions & 0 deletions docs/data/elasticsearch/generated/errors.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -281,6 +281,9 @@
"version": "1.2"
},
"name": "service1",
"node": {
"name": "node-xyz"
},
"runtime": {
"name": "node",
"version": "8.0.0"
Expand Down Expand Up @@ -373,6 +376,9 @@
"version": "8"
},
"name": "1234_service-12a3",
"node": {
"name": "node-abc"
},
"runtime": {
"name": "node",
"version": "8.0.0"
Expand Down Expand Up @@ -453,6 +459,9 @@
"version": "2.5"
},
"name": "service1",
"node": {
"name": "node-abc"
},
"runtime": {
"name": "cruby",
"version": "2.5"
Expand Down Expand Up @@ -535,6 +544,9 @@
"version": "8"
},
"name": "1234_service-12a3",
"node": {
"name": "node-abc"
},
"runtime": {
"name": "node",
"version": "8.0.0"
Expand Down
10 changes: 8 additions & 2 deletions docs/data/elasticsearch/generated/metricsets.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,10 @@
"language": {
"name": "ecmascript"
},
"name": "1234_service-12a3"
"name": "1234_service-12a3",
"node": {
"name": "node-1"
}
},
"short_counter": 227,
"span": {
Expand Down Expand Up @@ -115,7 +118,10 @@
"language": {
"name": "ecmascript"
},
"name": "1234_service-12a3"
"name": "1234_service-12a3",
"node": {
"name": "node-1"
}
},
"user": {
"email": "user@mail.com",
Expand Down
9 changes: 9 additions & 0 deletions docs/data/elasticsearch/generated/transactions.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,9 @@
"version": "8"
},
"name": "1234_service-12a3",
"node": {
"name": "node-123"
},
"runtime": {
"name": "node",
"version": "8.0.0"
Expand Down Expand Up @@ -207,6 +210,9 @@
"version": "8"
},
"name": "1234_service-12a3",
"node": {
"name": "node-123"
},
"runtime": {
"name": "node",
"version": "7.0"
Expand Down Expand Up @@ -330,6 +336,9 @@
"version": "2.5"
},
"name": "service1",
"node": {
"name": "node-ABC"
},
"runtime": {
"name": "cruby",
"version": "2.5"
Expand Down
13 changes: 12 additions & 1 deletion docs/fields.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -227,7 +227,7 @@ Service fields.
*`service.name`*::
+
--
Immutable unique name of the service emitting this event.
Immutable name of the service emitting this event.
type: keyword
Expand Down Expand Up @@ -255,6 +255,17 @@ type: keyword
--
*`service.node.name`*::
+
--
Unique meaningful name of the service node.
type: keyword
--
*`service.language.name`*::
+
--
Expand Down
10 changes: 10 additions & 0 deletions docs/spec/service.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,16 @@
"description": "Version of the service emitting this event",
"type": ["string", "null"],
"maxLength": 1024
},
"node": {
"description": "Unique meaningful name of the service node.",
"type": ["object", "null"],
"properties": {
"configured_name": {
"type": ["string", "null"],
"maxLength": 1024
}
}
}
}
}
2 changes: 1 addition & 1 deletion include/fields.go
View file

Large diffs are not rendered by default.

3 changes: 2 additions & 1 deletion model/error/event.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ var (
const (
processorName = "error"
errorDocType = "error"
emptyString = ""
)

var cachedModelSchema = validation.CreateSchema(schema.ModelSchema, processorName)
Expand Down Expand Up @@ -197,7 +198,7 @@ func (e *Event) Transform(tctx *transform.Context) []beat.Event {
utility.DeepUpdate(fields, "client", clientIP)
utility.DeepUpdate(fields, "source", clientIP)
utility.DeepUpdate(fields, "user_agent", e.User.UserAgentFields())
utility.DeepUpdate(fields, "service", e.Service.Fields())
utility.DeepUpdate(fields, "service", e.Service.Fields(emptyString, emptyString))
utility.DeepUpdate(fields, "agent", e.Service.AgentFields())
// merges with metadata labels, overrides conflicting keys
utility.DeepUpdate(fields, "labels", e.Labels.Fields())
Expand Down
18 changes: 16 additions & 2 deletions model/error/event_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -561,10 +561,11 @@ func TestEvents(t *testing.T) {
timestamp := time.Date(2019, 1, 3, 15, 17, 4, 908.596*1e6,
time.FixedZone("+0100", 3600))
timestampUs := timestamp.UnixNano() / 1000
serviceName, agentName, version := "myservice", "go", "1.0"
serviceName, agentName, agentVersion := "myservice", "go", "1.0"
service := metadata.Service{
Name: &serviceName, Agent: metadata.Agent{Name: &agentName, Version: &version},
Name: &serviceName, Agent: metadata.Agent{Name: &agentName, Version: &agentVersion},
}
serviceVersion := "1.2.3"
exMsg := "exception message"
trId := "945254c5-67a5-417e-8a4e-aa29efcbfb79"
sampledTrue, sampledFalse := true, false
Expand Down Expand Up @@ -677,6 +678,19 @@ func TestEvents(t *testing.T) {
},
Msg: "Payload with Event with Context.",
},
{
Transformable: &Event{Timestamp: timestamp, Service: &metadata.Service{Version: &serviceVersion}},
Output: common.MapStr{
"service": common.MapStr{"name": serviceName, "version": serviceVersion},
"labels": common.MapStr{"label": 101},
"agent": common.MapStr{"name": "go", "version": "1.0"},
"user": common.MapStr{"id": uid},
"error": common.MapStr{"grouping_key": "d41d8cd98f00b204e9800998ecf8427e"},
"processor": common.MapStr{"event": "error", "name": "error"},
"timestamp": common.MapStr{"us": timestampUs},
},
Msg: "Deep update service fields",
},
}

me := metadata.NewMetadata(&service, nil, nil, &metadata.User{Id: &uid}, metadataLabels)
Expand Down
10 changes: 10 additions & 0 deletions model/error/generated/schema/error.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -347,6 +347,16 @@ const ModelSchema = `{
"description": "Version of the service emitting this event",
"type": ["string", "null"],
"maxLength": 1024
},
"node": {
"description": "Unique meaningful name of the service node.",
"type": ["object", "null"],
"properties": {
"configured_name": {
"type": ["string", "null"],
"maxLength": 1024
}
}
}
}
}
Expand Down
10 changes: 10 additions & 0 deletions model/metadata/generated/schema/metadata.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -106,6 +106,16 @@ const ModelSchema = `{
"description": "Version of the service emitting this event",
"type": ["string", "null"],
"maxLength": 1024
},
"node": {
"description": "Unique meaningful name of the service node.",
"type": ["object", "null"],
"properties": {
"configured_name": {
"type": ["string", "null"],
"maxLength": 1024
}
}
}
},
"type": "object",
Expand Down
Loading

0 comments on commit 45ea5a9

Please sign in to comment.