From 4e4d86ac6e175cbc015121fd7bc886bad41bb2c1 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 2 Nov 2021 11:42:06 -0700 Subject: [PATCH 1/2] Use uiSettings for csv separator and visual report timezone Signed-off-by: Joshua Li --- .../public/components/context_menu/context_menu.js | 6 +++--- .../public/components/main/main_utils.tsx | 10 ++-------- .../public/components/utils/settings_service.ts | 14 ++++++++++++++ .../server/routes/lib/createReport.ts | 3 +++ dashboards-reports/server/routes/report.ts | 7 +++++-- .../server/routes/utils/dataReportHelpers.ts | 4 ++-- .../server/routes/utils/savedSearchReportHelper.ts | 5 ++++- 7 files changed, 33 insertions(+), 16 deletions(-) diff --git a/dashboards-reports/public/components/context_menu/context_menu.js b/dashboards-reports/public/components/context_menu/context_menu.js index 6cf87e5d..30602ec4 100644 --- a/dashboards-reports/public/components/context_menu/context_menu.js +++ b/dashboards-reports/public/components/context_menu/context_menu.js @@ -105,9 +105,9 @@ const generateInContextReport = async ( }; fetch( - `../api/reporting/generateReport?timezone=${ - Intl.DateTimeFormat().resolvedOptions().timeZone - }&dateFormat=${uiSettingsService.get('dateFormat')}`, + `../api/reporting/generateReport?${new URLSearchParams( + uiSettingsService.getSearchParams() + )}`, { headers: { 'Content-Type': 'application/json', diff --git a/dashboards-reports/public/components/main/main_utils.tsx b/dashboards-reports/public/components/main/main_utils.tsx index 8ab80da3..5119a152 100644 --- a/dashboards-reports/public/components/main/main_utils.tsx +++ b/dashboards-reports/public/components/main/main_utils.tsx @@ -194,10 +194,7 @@ export const generateReportFromDefinitionId = async ( headers: { 'Content-Type': 'application/json', }, - query: { - timezone: Intl.DateTimeFormat().resolvedOptions().timeZone, - dateFormat: uiSettingsService.get('dateFormat'), - }, + query: uiSettingsService.getSearchParams(), }) .then(async (response: any) => { // for emailing a report, this API response doesn't have response body @@ -230,10 +227,7 @@ export const generateReportById = async ( ) => { await httpClient .get(`../api/reporting/generateReport/${reportId}`, { - query: { - timezone: Intl.DateTimeFormat().resolvedOptions().timeZone, - dateFormat: uiSettingsService.get('dateFormat'), - }, + query: uiSettingsService.getSearchParams(), }) .then(async (response) => { //TODO: duplicate code, extract to be a function that can reuse. e.g. handleResponse(response) diff --git a/dashboards-reports/public/components/utils/settings_service.ts b/dashboards-reports/public/components/utils/settings_service.ts index f1baf906..5df595f5 100644 --- a/dashboards-reports/public/components/utils/settings_service.ts +++ b/dashboards-reports/public/components/utils/settings_service.ts @@ -20,4 +20,18 @@ export const uiSettingsService = { get: (key: string, defaultOverride?: any) => { return uiSettings?.get(key, defaultOverride) || ''; }, + getSearchParams: function () { + const rawTimeZone = this.get('dateFormat:tz'); + const timezone = + !rawTimeZone || rawTimeZone === 'Browser' + ? Intl.DateTimeFormat().resolvedOptions().timeZone + : rawTimeZone; + const dateFormat = this.get('dateFormat'); + const csvSeparator = this.get('csv:separator'); + return { + timezone, + dateFormat, + csvSeparator, + }; + }, }; diff --git a/dashboards-reports/server/routes/lib/createReport.ts b/dashboards-reports/server/routes/lib/createReport.ts index 36709343..58a4b160 100644 --- a/dashboards-reports/server/routes/lib/createReport.ts +++ b/dashboards-reports/server/routes/lib/createReport.ts @@ -70,6 +70,8 @@ export const createReport = async ( const timezone = request.query.timezone; // @ts-ignore const dateFormat = request.query.dateFormat || DATA_REPORT_CONFIG.excelDateFormat; + // @ts-ignore + const csvSeparator = request.query.csvSeparator || ','; const { basePath, serverInfo: { protocol, port, hostname }, @@ -97,6 +99,7 @@ export const createReport = async ( report, opensearchClient, dateFormat, + csvSeparator, isScheduledTask ); } else { diff --git a/dashboards-reports/server/routes/report.ts b/dashboards-reports/server/routes/report.ts index be13d7ee..268a0234 100644 --- a/dashboards-reports/server/routes/report.ts +++ b/dashboards-reports/server/routes/report.ts @@ -58,6 +58,7 @@ export default function (router: IRouter, accessInfo: AccessInfoType) { query: schema.object({ timezone: schema.maybe(schema.string()), dateFormat: schema.maybe(schema.string()), + csvSeparator: schema.maybe(schema.string()), }), }, }, @@ -122,7 +123,8 @@ export default function (router: IRouter, accessInfo: AccessInfoType) { }), query: schema.object({ timezone: schema.string(), - dateFormat: schema.maybe(schema.string()), + dateFormat: schema.string(), + csvSeparator: schema.string(), }), }, }, @@ -188,7 +190,8 @@ export default function (router: IRouter, accessInfo: AccessInfoType) { }), query: schema.object({ timezone: schema.string(), - dateFormat: schema.maybe(schema.string()), + dateFormat: schema.string(), + csvSeparator: schema.string(), }), }, }, diff --git a/dashboards-reports/server/routes/utils/dataReportHelpers.ts b/dashboards-reports/server/routes/utils/dataReportHelpers.ts index 7a3b6d19..36c5d6b9 100644 --- a/dashboards-reports/server/routes/utils/dataReportHelpers.ts +++ b/dashboards-reports/server/routes/utils/dataReportHelpers.ts @@ -206,10 +206,10 @@ export const getOpenSearchData = (arrayHits, report, params, dateFormat: string) }; //Convert the data to Csv format -export const convertToCSV = async (dataset) => { +export const convertToCSV = async (dataset, csvSeparator) => { let convertedData: any = []; const options = { - delimiter: { field: ',', eol: '\n' }, + delimiter: { field: csvSeparator, eol: '\n' }, emptyFieldValue: ' ', }; await converter.json2csvAsync(dataset[0], options).then((csv) => { diff --git a/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts b/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts index 76a5a9cf..6f80d6c6 100644 --- a/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts +++ b/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts @@ -49,6 +49,7 @@ export async function createSavedSearchReport( report: any, client: ILegacyClusterClient | ILegacyScopedClusterClient, dateFormat: string, + csvSeparator: string, isScheduledTask: boolean = true ): Promise { const params = report.report_definition.report_params; @@ -60,6 +61,7 @@ export async function createSavedSearchReport( client, params.core_params, dateFormat, + csvSeparator, isScheduledTask ); @@ -144,6 +146,7 @@ async function generateReportData( client: ILegacyClusterClient | ILegacyScopedClusterClient, params: any, dateFormat: string, + csvSeparator: string, isScheduledTask: boolean ) { let opensearchData: any = {}; @@ -286,6 +289,6 @@ async function generateReportData( async function convertOpenSearchDataToCsv() { const dataset: any = []; dataset.push(getOpenSearchData(arrayHits, report, params, dateFormat)); - return await convertToCSV(dataset); + return await convertToCSV(dataset, csvSeparator); } } From ec256fd268798f4f8fdced12f901bff0ed5b4e77 Mon Sep 17 00:00:00 2001 From: Joshua Li Date: Tue, 2 Nov 2021 12:24:48 -0700 Subject: [PATCH 2/2] Update tests Signed-off-by: Joshua Li --- .../__tests__/savedSearchReportHelper.test.ts | 113 +++++++++++++++--- 1 file changed, 98 insertions(+), 15 deletions(-) diff --git a/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts b/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts index 6d3b7377..9f1d11c7 100644 --- a/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts +++ b/dashboards-reports/server/routes/utils/__tests__/savedSearchReportHelper.test.ts @@ -54,7 +54,7 @@ const input = { configIds: [], title: 'title', textDescription: 'text description', - htmlDescription: 'html description' + htmlDescription: 'html description', }, trigger: { trigger_type: 'On demand', @@ -78,7 +78,9 @@ describe('test create saved search report', () => { const client = mockOpenSearchClient(hits); const { timeCreated, fileName } = await createSavedSearchReport( input, - client + client, + 'MM/DD/YYYY h:mm:ss.SSS a', + ',' ); expect(fileName).toContain(`test report table order_`); }, 20000); @@ -86,14 +88,18 @@ describe('test create saved search report', () => { test('create report with expected file name extension', async () => { const csvReport = await createSavedSearchReport( input, - mockOpenSearchClient([]) + mockOpenSearchClient([]), + 'MM/DD/YYYY h:mm:ss.SSS a', + ',' ); expect(csvReport.fileName).toContain('.csv'); input.report_definition.report_params.core_params.report_format = 'xlsx'; const xlsxReport = await createSavedSearchReport( input, - mockOpenSearchClient([]) + mockOpenSearchClient([]), + 'MM/DD/YYYY h:mm:ss.SSS a', + ',' ); expect(xlsxReport.fileName).toContain('.xlsx'); }, 20000); @@ -101,7 +107,12 @@ describe('test create saved search report', () => { test('create report for empty data set', async () => { const hits: Array<{ _source: any }> = []; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + 'MM/DD/YYYY h:mm:ss.SSS a', + ',' + ); expect(dataUrl).toEqual(''); }, 20000); @@ -114,7 +125,12 @@ describe('test create saved search report', () => { hit({ category: 'c5', customer_gender: 'Male' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + 'MM/DD/YYYY h:mm:ss.SSS a', + ',' + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -141,7 +157,12 @@ describe('test create saved search report', () => { hit({ category: 'c11', customer_gender: 'Male' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + 'MM/DD/YYYY h:mm:ss.SSS a', + ',' + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -171,7 +192,12 @@ describe('test create saved search report', () => { hit({ category: 'c5', customer_gender: 'Male' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + 'MM/DD/YYYY h:mm:ss.SSS a', + ',' + ); expect(dataUrl).toEqual('category,customer_gender\n' + 'c1,Male'); }, 20000); @@ -193,7 +219,12 @@ describe('test create saved search report', () => { hit({ category: 'c10', customer_gender: 'Female' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + 'MM/DD/YYYY h:mm:ss.SSS a', + ',' + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -219,7 +250,12 @@ describe('test create saved search report', () => { hit({ category: 'c6', customer_gender: 'Female' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + 'MM/DD/YYYY h:mm:ss.SSS a', + ',' + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -239,7 +275,12 @@ describe('test create saved search report', () => { hit({ category: ',,c3', customer_gender: 'Male,,,' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + 'MM/DD/YYYY h:mm:ss.SSS a', + ',' + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -249,6 +290,28 @@ describe('test create saved search report', () => { ); }, 20000); + test('create report for data set with comma and custom separator', async () => { + const hits = [ + hit({ category: ',c1', customer_gender: 'Ma,le' }), + hit({ category: 'c2,', customer_gender: 'M,ale' }), + hit({ category: ',,c3', customer_gender: 'Male,,,' }), + ]; + const client = mockOpenSearchClient(hits); + const { dataUrl } = await createSavedSearchReport( + input, + client, + 'MM/DD/YYYY h:mm:ss.SSS a', + '|' + ); + + expect(dataUrl).toEqual( + 'category|customer_gender\n' + + ',c1|Ma,le\n' + + 'c2,|M,ale\n' + + ',,c3|Male,,,' + ); + }, 20000); + test('create report for data set with nested fields', async () => { const hits = [ hit({ @@ -265,7 +328,12 @@ describe('test create saved search report', () => { hits, '"geoip.country_iso_code", "geoip.city_name", "geoip.location"' ); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + 'MM/DD/YYYY h:mm:ss.SSS a', + ',' + ); expect(dataUrl).toEqual( 'geoip.country_iso_code,geoip.location.lon,geoip.location.lat,geoip.city_name\n' + @@ -283,7 +351,12 @@ describe('test create saved search report', () => { hit({ category: ',,,@c5', customer_gender: 'Male' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + 'MM/DD/YYYY h:mm:ss.SSS a', + ',' + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -307,7 +380,12 @@ describe('test create saved search report', () => { hit({ category: ',,,@c5', customer_gender: 'Male' }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + 'MM/DD/YYYY h:mm:ss.SSS a', + ',' + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + @@ -327,7 +405,12 @@ test('create report for data set contains null field value', async () => { hit({ category: 'c3', customer_gender: null }), ]; const client = mockOpenSearchClient(hits); - const { dataUrl } = await createSavedSearchReport(input, client); + const { dataUrl } = await createSavedSearchReport( + input, + client, + 'MM/DD/YYYY h:mm:ss.SSS a', + ',' + ); expect(dataUrl).toEqual( 'category,customer_gender\n' + 'c1,Ma\n' + 'c2,le\n' + 'c3, '