From 1451d13b8764b2f3b74234d2dcce5af07682251d Mon Sep 17 00:00:00 2001 From: Zhongnan Su Date: Wed, 3 Nov 2021 14:40:14 -0700 Subject: [PATCH] refactor logic for creating DSL from saved object using buildOpensearchQuery() (#213) --- dashboards-reports/.babelrc | 18 -- dashboards-reports/babel.config.js | 23 +++ .../server/routes/utils/dataReportHelpers.ts | 163 +++++++----------- .../routes/utils/savedSearchReportHelper.ts | 29 +--- 4 files changed, 96 insertions(+), 137 deletions(-) delete mode 100644 dashboards-reports/.babelrc create mode 100644 dashboards-reports/babel.config.js diff --git a/dashboards-reports/.babelrc b/dashboards-reports/.babelrc deleted file mode 100644 index e26ee76e..00000000 --- a/dashboards-reports/.babelrc +++ /dev/null @@ -1,18 +0,0 @@ -{ - "presets": [ - [ - "@babel/preset-env", - { - "targets": { "node": "10" } - } - ], - "@babel/preset-react", - "@babel/preset-typescript" - ], - "plugins": [ - "@babel/plugin-transform-modules-commonjs", - ["@babel/plugin-transform-runtime", { "regenerator": true }], - "@babel/plugin-proposal-class-properties", - "@babel/plugin-proposal-object-rest-spread" - ] -} diff --git a/dashboards-reports/babel.config.js b/dashboards-reports/babel.config.js new file mode 100644 index 00000000..ce57a641 --- /dev/null +++ b/dashboards-reports/babel.config.js @@ -0,0 +1,23 @@ +/* + * SPDX-License-Identifier: Apache-2.0 + * + * The OpenSearch Contributors require contributions made to + * this file be licensed under the Apache-2.0 license or a + * compatible open source license. + */ + +module.exports = { + presets: [ + require('@babel/preset-env', { + targets: { node: '10' }, + }), + require('@babel/preset-react'), + require('@babel/preset-typescript'), + ], + plugins: [ + require('@babel/plugin-proposal-class-properties'), + require('@babel/plugin-proposal-object-rest-spread'), + ['@babel/plugin-transform-modules-commonjs', { allowTopLevelThis: true }], + [require('@babel/plugin-transform-runtime'), { regenerator: true }], + ], +}; diff --git a/dashboards-reports/server/routes/utils/dataReportHelpers.ts b/dashboards-reports/server/routes/utils/dataReportHelpers.ts index a5ede76c..f313e2a8 100644 --- a/dashboards-reports/server/routes/utils/dataReportHelpers.ts +++ b/dashboards-reports/server/routes/utils/dataReportHelpers.ts @@ -29,6 +29,11 @@ import converter from 'json-2-csv'; import _ from 'lodash'; import moment from 'moment'; import { DATA_REPORT_CONFIG } from './constants'; +import { + buildOpenSearchQuery, + Filter, + Query, +} from '../../../../../src/plugins/data/common'; export var metaData = { saved_search_id: null, @@ -42,7 +47,7 @@ export var metaData = { fields_exist: false, selectedFields: [], paternName: null, - filters: [], + searchSourceJSON: [], dateFields: [], }; @@ -65,93 +70,20 @@ export const getSelectedFields = async (columns) => { // Build the OpenSearch query from the meta data // is_count is set to 1 if we building the count query but 0 if we building the fetch data query -export const buildQuery = (report, is_count) => { - let requestBody = esb.boolQuery(); - const filters = report._source.filters; - for (let item of JSON.parse(filters).filter) { - if (item.meta.disabled === false) { - switch (item.meta.negate) { - case false: - switch (item.meta.type) { - case 'phrase': - requestBody.must( - esb.matchPhraseQuery(item.meta.key, item.meta.params.query) - ); - break; - case 'exists': - requestBody.must(esb.existsQuery(item.meta.key)); - break; - case 'phrases': - if (item.meta.value.indexOf(',') > -1) { - const valueSplit = item.meta.value.split(', '); - for (const [key, incr] of valueSplit.entries()) { - requestBody.should(esb.matchPhraseQuery(item.meta.key, incr)); - } - } else { - requestBody.should( - esb.matchPhraseQuery(item.meta.key, item.meta.value) - ); - } - requestBody.minimumShouldMatch(1); - break; - case 'range': - const builder = esb.rangeQuery(item.meta.key); - if (item.meta.params.gte) builder.gte(item.meta.params.gte); - if (item.meta.params.lte) builder.lte(item.meta.params.lte); - if (item.meta.params.gt) builder.gt(item.meta.params.gt); - if (item.meta.params.lt) builder.lt(item.meta.params.lt); - requestBody.must(builder); - break; - } - break; - case true: - switch (item.meta.type) { - case 'phrase': - requestBody.mustNot( - esb.matchPhraseQuery(item.meta.key, item.meta.params.query) - ); - break; - case 'exists': - requestBody.mustNot(esb.existsQuery(item.meta.key)); - break; - case 'phrases': - let negatedBody = esb.boolQuery(); - if (item.meta.value.indexOf(',') > -1) { - const valueSplit = item.meta.value.split(', '); - for (const [key, incr] of valueSplit.entries()) { - negatedBody.should(esb.matchPhraseQuery(item.meta.key, incr)); - } - } else { - negatedBody.should( - esb.matchPhraseQuery(item.meta.key, item.meta.value) - ); - } - negatedBody.minimumShouldMatch(1); - requestBody.mustNot(negatedBody); - break; - case 'range': - const builder = esb.rangeQuery(item.meta.key); - if (item.meta.params.gte) builder.gte(item.meta.params.gte); - if (item.meta.params.lte) builder.lte(item.meta.params.lte); - if (item.meta.params.gt) builder.gt(item.meta.params.gt); - if (item.meta.params.lt) builder.lt(item.meta.params.lt); - requestBody.mustNot(builder); - break; - } - break; - } - } - } - //search part - let searchQuery = JSON.parse(filters) - .query.query.replace(/ and /g, ' AND ') - .replace(/ or /g, ' OR ') - .replace(/ not /g, ' NOT '); - if (searchQuery) { - requestBody.must(esb.queryStringQuery(searchQuery)); - } +export const buildRequestBody = (report: any, is_count: number) => { + let esbBoolQuery = esb.boolQuery(); + const searchSourceJSON = report._source.searchSourceJSON; + + const savedObjectQuery: Query = JSON.parse(searchSourceJSON).query; + const savedObjectFilter: Filter = JSON.parse(searchSourceJSON).filter; + const QueryFromSavedObject = buildOpenSearchQuery( + undefined, + savedObjectQuery, + savedObjectFilter + ); + // Add time range if (report._source.timeFieldName && report._source.timeFieldName.length > 0) { - requestBody.must( + esbBoolQuery.must( esb .rangeQuery(report._source.timeFieldName) .format('epoch_millis') @@ -160,28 +92,48 @@ export const buildQuery = (report, is_count) => { ); } if (is_count) { - return esb.requestBodySearch().query(requestBody); + return esb.requestBodySearch().query(esbBoolQuery); } - //Add the Sort to the query - let reqBody = esb.requestBodySearch().query(requestBody).version(true); + // Add sorting to the query + let esbSearchQuery = esb + .requestBodySearch() + .query(esbBoolQuery) + .version(true); if (report._source.sorting.length > 0) { const sortings: Sort[] = report._source.sorting.map((element: string[]) => { return esb.sort(element[0], element[1]); }); - reqBody.sorts(sortings); + esbSearchQuery.sorts(sortings); } - //get the selected fields only + // add selected fields to query if (report._source.fields_exist) { - reqBody.source({ includes: report._source.selectedFields }); + esbSearchQuery.source({ includes: report._source.selectedFields }); } - return reqBody; + // Add a customizer to merge queries to generate request body + let requestBody = _.mergeWith( + { query: QueryFromSavedObject }, + esbSearchQuery.toJSON(), + (objValue, srcValue) => { + if (_.isArray(objValue)) { + return objValue.concat(srcValue); + } + } + ); + + requestBody = addDocValueFields(report, requestBody); + return requestBody; }; // Fetch the data from OpenSearch -export const getOpenSearchData = (arrayHits, report, params, dateFormat: string) => { +export const getOpenSearchData = ( + arrayHits, + report, + params, + dateFormat: string +) => { let hits: any = []; for (let valueRes of arrayHits) { for (let data of valueRes.hits) { @@ -189,7 +141,9 @@ export const getOpenSearchData = (arrayHits, report, params, dateFormat: string) // get all the fields of type date and format them to excel format for (let dateType of report._source.dateFields) { if (data._source[dateType]) { - data._source[dateType] = moment(fields[dateType][0]).format(dateFormat); + data._source[dateType] = moment(fields[dateType][0]).format( + dateFormat + ); } } delete data['fields']; @@ -274,3 +228,20 @@ function sanitize(doc: any) { } return doc; } + +const addDocValueFields = (report: any, requestBody: any) => { + const docValues = []; + for (const dateType of report._source.dateFields) { + docValues.push({ + field: dateType, + format: 'date_hour_minute_second_fraction', + }); + } + // elastic-builder doesn't provide function to build docvalue_fields with format, + // this is a workaround which appends docvalues field to the request body. + requestBody = { + ...requestBody, + docvalue_fields: docValues, + }; + return requestBody; +}; diff --git a/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts b/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts index f42836f9..9f25ea74 100644 --- a/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts +++ b/dashboards-reports/server/routes/utils/savedSearchReportHelper.ts @@ -25,7 +25,7 @@ */ import { - buildQuery, + buildRequestBody, convertToCSV, getOpenSearchData, getSelectedFields, @@ -104,7 +104,7 @@ async function populateMetaData( metaData.sorting = ssInfos._source.search.sort; metaData.type = ssInfos._source.type; - metaData.filters = + metaData.searchSourceJSON = ssInfos._source.search.kibanaSavedObjectMeta.searchSourceJSON; // Get the list of selected columns in the saved search.Otherwise select all the fields under the _source @@ -112,7 +112,7 @@ async function populateMetaData( // Get index name for (const item of ssInfos._source.references) { - if (item.name === JSON.parse(metaData.filters).indexRefName) { + if (item.name === JSON.parse(metaData.searchSourceJSON).indexRefName) { // Get index-pattern information const indexPattern = await callCluster( client, @@ -163,7 +163,7 @@ async function generateReportData( return ''; } - const reqBody = buildRequestBody(buildQuery(report, 0)); + const reqBody = buildRequestBody(report, 0); logger.info( `[Reporting csv module] DSL request body: ${JSON.stringify(reqBody)}` ); @@ -200,13 +200,13 @@ async function generateReportData( // Build the OpenSearch Count query to count the size of result async function getOpenSearchDataSize() { - const countReq = buildQuery(report, 1); + const countReq = buildRequestBody(report, 1); return await callCluster( client, 'count', { index: indexPattern, - body: countReq.toJSON(), + body: countReq, }, isScheduledTask ); @@ -273,23 +273,6 @@ async function generateReportData( arrayHits.push(opensearchData.hits); } - function buildRequestBody(query: esb.RequestBodySearch) { - const docvalues = []; - for (const dateType of report._source.dateFields) { - docvalues.push({ - field: dateType, - format: 'date_hour_minute_second_fraction', - }); - } - - // elastic-builder doesn't provide function to build docvalue_fields with format, - // this is a workaround which appends docvalues field to the request body. - return { - ...query.toJSON(), - docvalue_fields: docvalues, - }; - } - // Parse OpenSearch data and convert to CSV async function convertOpenSearchDataToCsv() { const dataset: any = [];