From 2d0c7e64dceec7c2d9171493a94e1215cc9ed876 Mon Sep 17 00:00:00 2001 From: Robert Pocklington Date: Tue, 8 Mar 2022 14:00:18 +1100 Subject: [PATCH 1/2] output/csv: add ISO8601 time format flag --- output/csv/config.go | 10 +++++++++ output/csv/config_test.go | 13 +++++++++++ output/csv/consts.go | 40 ++++++++++++++++++++++++++++++++++ output/csv/output.go | 18 ++++++++++++--- output/csv/output_test.go | 46 ++++++++++++++++++++++++++++++++++++--- 5 files changed, 121 insertions(+), 6 deletions(-) create mode 100644 output/csv/consts.go diff --git a/output/csv/config.go b/output/csv/config.go index 8ab6f03d2e1..7a1c38fd67d 100644 --- a/output/csv/config.go +++ b/output/csv/config.go @@ -38,6 +38,7 @@ type Config struct { // Samples. FileName null.String `json:"file_name" envconfig:"K6_CSV_FILENAME"` SaveInterval types.NullDuration `json:"save_interval" envconfig:"K6_CSV_SAVE_INTERVAL"` + TimeFormat TimeFormat `json:"time_format" envconfig:"K6_CSV_TIME_FORMAT"` } // NewConfig creates a new Config instance with default values for some fields. @@ -45,6 +46,7 @@ func NewConfig() Config { return Config{ FileName: null.StringFrom("file.csv"), SaveInterval: types.NullDurationFrom(1 * time.Second), + TimeFormat: Unix, } } @@ -56,6 +58,7 @@ func (c Config) Apply(cfg Config) Config { if cfg.SaveInterval.Valid { c.SaveInterval = cfg.SaveInterval } + c.TimeFormat = cfg.TimeFormat return c } @@ -66,6 +69,7 @@ func ParseArg(arg string, logger *logrus.Logger) (Config, error) { if !strings.Contains(arg, "=") { c.FileName = null.StringFrom(arg) c.SaveInterval = types.NullDurationFrom(1 * time.Second) + c.TimeFormat = Unix return c, nil } @@ -89,6 +93,12 @@ func ParseArg(arg string, logger *logrus.Logger) (Config, error) { fallthrough case "fileName": c.FileName = null.StringFrom(r[1]) + case "timeFormat": + c.TimeFormat = TimeFormat(r[1]) + if !c.TimeFormat.IsValid() { + return c, fmt.Errorf("unknown value %q as argument for csv output timeFormat, expected 'unix' or 'rfc3399'", arg) + } + default: return c, fmt.Errorf("unknown key %q as argument for csv output", r[0]) } diff --git a/output/csv/config_test.go b/output/csv/config_test.go index cb53cd7c81e..a67f243fc57 100644 --- a/output/csv/config_test.go +++ b/output/csv/config_test.go @@ -39,6 +39,7 @@ func TestNewConfig(t *testing.T) { config := NewConfig() assert.Equal(t, "file.csv", config.FileName.String) assert.Equal(t, "1s", config.SaveInterval.String()) + assert.Equal(t, TimeFormat("unix"), config.TimeFormat) } func TestApply(t *testing.T) { @@ -46,23 +47,28 @@ func TestApply(t *testing.T) { { FileName: null.StringFrom(""), SaveInterval: types.NullDurationFrom(2 * time.Second), + TimeFormat: "unix", }, { FileName: null.StringFrom("newPath"), SaveInterval: types.NewNullDuration(time.Duration(1), false), + TimeFormat: "unix", }, } expected := []struct { FileName string SaveInterval string + TimeFormat TimeFormat }{ { FileName: "", SaveInterval: "2s", + TimeFormat: TimeFormat("unix"), }, { FileName: "newPath", SaveInterval: "1s", + TimeFormat: TimeFormat("unix"), }, } @@ -75,6 +81,7 @@ func TestApply(t *testing.T) { assert.Equal(t, expected.FileName, baseConfig.FileName.String) assert.Equal(t, expected.SaveInterval, baseConfig.SaveInterval.String()) + assert.Equal(t, expected.TimeFormat, baseConfig.TimeFormat) }) } } @@ -126,6 +133,12 @@ func TestParseArg(t *testing.T) { "filename=test.csv,save_interval=5s": { expectedErr: true, }, + "fileName=test.csv,timeFormat=rfc3399": { + config: Config{ + FileName: null.StringFrom("test.csv"), + TimeFormat: "rfc3399", + }, + }, } for arg, testCase := range cases { diff --git a/output/csv/consts.go b/output/csv/consts.go new file mode 100644 index 00000000000..0010dd5fbdf --- /dev/null +++ b/output/csv/consts.go @@ -0,0 +1,40 @@ +/* + * + * k6 - a next-generation load testing tool + * Copyright (C) 2016 Load Impact + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU Affero General Public License as + * published by the Free Software Foundation, either version 3 of the + * License, or (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU Affero General Public License for more details. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + * + */ + +// Package csv custom enum types used in config +package csv + +// TimeFormat custom enum type +type TimeFormat string + +// valid defined values for TimeFormat +const ( + Unix TimeFormat = "unix" + RFC3399 TimeFormat = "rfc3399" +) + +// IsValid validates TimeFormat +func (timeFormat TimeFormat) IsValid() bool { + switch timeFormat { + case Unix, RFC3399: + return true + } + return false +} diff --git a/output/csv/output.go b/output/csv/output.go index bfba60fba28..8a3a9e34796 100644 --- a/output/csv/output.go +++ b/output/csv/output.go @@ -54,6 +54,7 @@ type Output struct { ignoredTags []string row []string saveInterval time.Duration + timeFormat TimeFormat } // New Creates new instance of CSV output @@ -87,6 +88,7 @@ func newOutput(params output.Params) (*Output, error) { saveInterval := config.SaveInterval.TimeDuration() fname := config.FileName.String + timeFormat := config.TimeFormat if fname == "" || fname == "-" { stdoutWriter := csv.NewWriter(os.Stdout) @@ -97,6 +99,7 @@ func newOutput(params output.Params) (*Output, error) { csvWriter: stdoutWriter, row: make([]string, 3+len(resTags)+1), saveInterval: saveInterval, + timeFormat: timeFormat, closeFn: func() error { return nil }, logger: logger, params: params, @@ -114,6 +117,7 @@ func newOutput(params output.Params) (*Output, error) { ignoredTags: ignoredTags, row: make([]string, 3+len(resTags)+1), saveInterval: saveInterval, + timeFormat: timeFormat, logger: logger, params: params, } @@ -182,7 +186,7 @@ func (o *Output) flushMetrics() { for _, sc := range samples { for _, sample := range sc.GetSamples() { sample := sample - row := SampleToRow(&sample, o.resTags, o.ignoredTags, o.row) + row := SampleToRow(&sample, o.resTags, o.ignoredTags, o.row, o.timeFormat) err := o.csvWriter.Write(row) if err != nil { o.logger.WithField("filename", o.fname).Error("CSV: Error writing to file") @@ -200,9 +204,17 @@ func MakeHeader(tags []string) []string { } // SampleToRow converts sample into array of strings -func SampleToRow(sample *metrics.Sample, resTags []string, ignoredTags []string, row []string) []string { +func SampleToRow(sample *metrics.Sample, resTags []string, ignoredTags []string, row []string, + timeFormat TimeFormat, +) []string { row[0] = sample.Metric.Name - row[1] = fmt.Sprintf("%d", sample.Time.Unix()) + + if timeFormat == RFC3399 { + row[1] = fmt.Sprint(sample.Time.Format(time.RFC3339)) + } else { + row[1] = fmt.Sprintf("%d", sample.Time.Unix()) + } + row[2] = fmt.Sprintf("%f", sample.Value) sampleTags := sample.Tags.CloneTags() diff --git a/output/csv/output_test.go b/output/csv/output_test.go index d0fa3c13bcc..10c726e12d2 100644 --- a/output/csv/output_test.go +++ b/output/csv/output_test.go @@ -73,6 +73,7 @@ func TestSampleToRow(t *testing.T) { sample *metrics.Sample resTags []string ignoredTags []string + timeFormat string }{ { testname: "One res tag, one ignored tag, one extra tag", @@ -88,6 +89,7 @@ func TestSampleToRow(t *testing.T) { }, resTags: []string{"tag1"}, ignoredTags: []string{"tag2"}, + timeFormat: "", }, { testname: "Two res tags, three extra tags", @@ -105,9 +107,10 @@ func TestSampleToRow(t *testing.T) { }, resTags: []string{"tag1", "tag2"}, ignoredTags: []string{}, + timeFormat: "", }, { - testname: "Two res tags, two ignored", + testname: "Two res tags, two ignored, with RFC3399 timestamp", sample: &metrics.Sample{ Time: time.Unix(1562324644, 0), Metric: testMetric, @@ -123,6 +126,7 @@ func TestSampleToRow(t *testing.T) { }, resTags: []string{"tag1", "tag3"}, ignoredTags: []string{"tag4", "tag6"}, + timeFormat: "rfc3399", }, } @@ -158,7 +162,7 @@ func TestSampleToRow(t *testing.T) { { baseRow: []string{ "my_metric", - "1562324644", + "2019-07-05T11:04:04Z", "1.000000", "val1", "val3", @@ -173,10 +177,11 @@ func TestSampleToRow(t *testing.T) { for i := range testData { testname, sample := testData[i].testname, testData[i].sample resTags, ignoredTags := testData[i].resTags, testData[i].ignoredTags + timeFormat := TimeFormat(testData[i].timeFormat) expectedRow := expected[i] t.Run(testname, func(t *testing.T) { - row := SampleToRow(sample, resTags, ignoredTags, make([]string, 3+len(resTags)+1)) + row := SampleToRow(sample, resTags, ignoredTags, make([]string, 3+len(resTags)+1), timeFormat) for ind, cell := range expectedRow.baseRow { assert.Equal(t, cell, row[ind]) } @@ -225,6 +230,7 @@ func TestRun(t *testing.T) { samples []metrics.SampleContainer fileName string fileReaderFunc func(fileName string, fs afero.Fs) string + timeFormat string outputContent string }{ { @@ -253,6 +259,7 @@ func TestRun(t *testing.T) { }, fileName: "test", fileReaderFunc: readUnCompressedFile, + timeFormat: "", outputContent: "metric_name,timestamp,metric_value,check,error,extra_tags\n" + "my_metric,1562324643,1.000000,val1,val3,url=val2\n" + "my_metric,1562324644,1.000000,val1,val3,tag4=val4&url=val2\n", }, { @@ -281,8 +288,38 @@ func TestRun(t *testing.T) { }, fileName: "test.gz", fileReaderFunc: readCompressedFile, + timeFormat: "unix", outputContent: "metric_name,timestamp,metric_value,check,error,extra_tags\n" + "my_metric,1562324643,1.000000,val1,val3,url=val2\n" + "my_metric,1562324644,1.000000,val1,val3,name=val4&url=val2\n", }, + { + samples: []metrics.SampleContainer{ + metrics.Sample{ + Time: time.Unix(1562324644, 0), + Metric: testMetric, + Value: 1, + Tags: metrics.NewSampleTags(map[string]string{ + "check": "val1", + "url": "val2", + "error": "val3", + }), + }, + metrics.Sample{ + Time: time.Unix(1562324644, 0), + Metric: testMetric, + Value: 1, + Tags: metrics.NewSampleTags(map[string]string{ + "check": "val1", + "url": "val2", + "error": "val3", + "name": "val4", + }), + }, + }, + fileName: "test", + fileReaderFunc: readUnCompressedFile, + timeFormat: "rfc3399", + outputContent: "metric_name,timestamp,metric_value,check,error,extra_tags\n" + "my_metric,2019-07-05T11:04:04Z,1.000000,val1,val3,url=val2\n" + "my_metric,2019-07-05T11:04:04Z,1.000000,val1,val3,name=val4&url=val2\n", + }, } for _, data := range testData { @@ -295,6 +332,9 @@ func TestRun(t *testing.T) { SystemTags: metrics.NewSystemTagSet(metrics.TagError | metrics.TagCheck), }, }) + + output.timeFormat = TimeFormat(data.timeFormat) + require.NoError(t, err) require.NotNil(t, output) From d630effc8085626e0d1ea11a01e2f07ce937ed66 Mon Sep 17 00:00:00 2001 From: Robert Pocklington Date: Thu, 26 May 2022 22:39:19 +1000 Subject: [PATCH 2/2] Code review feedback --- output/csv/consts.go | 21 --------------------- output/csv/doc.go | 4 ++++ output/csv/output.go | 10 +++++----- 3 files changed, 9 insertions(+), 26 deletions(-) create mode 100644 output/csv/doc.go diff --git a/output/csv/consts.go b/output/csv/consts.go index 0010dd5fbdf..0bd9c1f9fb8 100644 --- a/output/csv/consts.go +++ b/output/csv/consts.go @@ -1,24 +1,3 @@ -/* - * - * k6 - a next-generation load testing tool - * Copyright (C) 2016 Load Impact - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU Affero General Public License as - * published by the Free Software Foundation, either version 3 of the - * License, or (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU Affero General Public License for more details. - * - * You should have received a copy of the GNU Affero General Public License - * along with this program. If not, see . - * - */ - -// Package csv custom enum types used in config package csv // TimeFormat custom enum type diff --git a/output/csv/doc.go b/output/csv/doc.go new file mode 100644 index 00000000000..323b4fdde1e --- /dev/null +++ b/output/csv/doc.go @@ -0,0 +1,4 @@ +/* +Package csv implements an output writing metrics in csv format +*/ +package csv diff --git a/output/csv/output.go b/output/csv/output.go index 8a3a9e34796..f6fb0e39740 100644 --- a/output/csv/output.go +++ b/output/csv/output.go @@ -27,6 +27,7 @@ import ( "fmt" "os" "sort" + "strconv" "strings" "sync" "time" @@ -88,7 +89,6 @@ func newOutput(params output.Params) (*Output, error) { saveInterval := config.SaveInterval.TimeDuration() fname := config.FileName.String - timeFormat := config.TimeFormat if fname == "" || fname == "-" { stdoutWriter := csv.NewWriter(os.Stdout) @@ -99,7 +99,7 @@ func newOutput(params output.Params) (*Output, error) { csvWriter: stdoutWriter, row: make([]string, 3+len(resTags)+1), saveInterval: saveInterval, - timeFormat: timeFormat, + timeFormat: config.TimeFormat, closeFn: func() error { return nil }, logger: logger, params: params, @@ -117,7 +117,7 @@ func newOutput(params output.Params) (*Output, error) { ignoredTags: ignoredTags, row: make([]string, 3+len(resTags)+1), saveInterval: saveInterval, - timeFormat: timeFormat, + timeFormat: config.TimeFormat, logger: logger, params: params, } @@ -210,9 +210,9 @@ func SampleToRow(sample *metrics.Sample, resTags []string, ignoredTags []string, row[0] = sample.Metric.Name if timeFormat == RFC3399 { - row[1] = fmt.Sprint(sample.Time.Format(time.RFC3339)) + row[1] = sample.Time.Format(time.RFC3339) } else { - row[1] = fmt.Sprintf("%d", sample.Time.Unix()) + row[1] = strconv.FormatInt(sample.Time.Unix(), 10) } row[2] = fmt.Sprintf("%f", sample.Value)