From 008946a05b8d1d54add31a25cc52aba2a61448a8 Mon Sep 17 00:00:00 2001 From: Steffany Brown <30247553+steffnay@users.noreply.github.com> Date: Thu, 30 Jan 2020 09:50:56 -0800 Subject: [PATCH] feat(params): adds optional param types (#599) * fix(docs): publish new version of docs with getTables documented * draft * feat(namedParams): adds ability to provide optional named parameter types * feat(types): adds optional provided types for positional params * linted * refactor(params): refactors positional params and tests * chore: remove unnecessary test comments * refactor(getTypes): removes redundant providedType logic * refactor(params): adds tests and removes repeated null type logic * refactor: renames getType functions, adds throw if invalid types provided * adds tests * test: refactors tests * refactor: updates param types and tests * fixes test and updates providedType * adds interface for providedTypes and logic for nested structs * updates providedType type Co-authored-by: Benjamin E. Coe --- src/bigquery.ts | 178 +++++++++++++++++++++--- system-test/bigquery.ts | 20 +++ test/bigquery.ts | 301 +++++++++++++++++++++++++++++++++++----- 3 files changed, 447 insertions(+), 52 deletions(-) diff --git a/src/bigquery.ts b/src/bigquery.ts index 67a06240..d1a11e66 100644 --- a/src/bigquery.ts +++ b/src/bigquery.ts @@ -94,6 +94,7 @@ export type Query = JobRequest & { // tslint:disable-next-line no-any params?: any[] | {[param: string]: any}; dryRun?: boolean; + types?: string[] | string[][] | {[type: string]: string[]}; defaultDataset?: Dataset; job?: Job; maxResults?: number; @@ -155,6 +156,12 @@ export interface BigQueryDatetimeOptions { fractional?: string | number; } +export type ProvidedTypeArray = Array; + +export interface ProvidedTypeStruct { + [key: string]: string | ProvidedTypeArray | ProvidedTypeStruct; +} + export type QueryParameter = bigquery.IQueryParameter; export interface BigQueryOptions extends common.GoogleAuthOptions { @@ -731,6 +738,66 @@ export class BigQuery extends common.Service { return BigQuery.geography(value); } + /** + * Return a value's provided type. + * + * @private + * + * @throws {error} If the type provided is invalid. + * + * @see [Data Type]{@link https://cloud.google.com/bigquery/data-types} + * + * @param {*} providedType The type. + * @returns {string} The valid type provided. + */ + static getTypeDescriptorFromProvidedType_( + providedType: string | ProvidedTypeStruct | ProvidedTypeArray + ): ValueType { + // The list of types can be found in src/types.d.ts + const VALID_TYPES = [ + 'DATE', + 'DATETIME', + 'TIME', + 'TIMESTAMP', + 'BYTES', + 'NUMERIC', + 'BOOL', + 'INT64', + 'FLOAT64', + 'STRING', + 'GEOGRAPHY', + 'ARRAY', + 'STRUCT', + ]; + + if (is.array(providedType)) { + providedType = providedType as Array; + return { + type: 'ARRAY', + arrayType: BigQuery.getTypeDescriptorFromProvidedType_(providedType[0]), + }; + } else if (is.object(providedType)) { + return { + type: 'STRUCT', + structTypes: Object.keys(providedType).map(prop => { + return { + name: prop, + type: BigQuery.getTypeDescriptorFromProvidedType_( + (providedType as ProvidedTypeStruct)[prop] + ), + }; + }), + }; + } + + providedType = (providedType as string).toUpperCase(); + if (!VALID_TYPES.includes(providedType)) { + throw new Error(`Invalid type provided: "${providedType}"`); + } + + return {type: providedType.toUpperCase()}; + } + /** * Detect a value's type. * @@ -743,10 +810,16 @@ export class BigQuery extends common.Service { * @param {*} value The value. * @returns {string} The type detected from the value. */ - // tslint:disable-next-line no-any - static getType_(value: any): ValueType { + static getTypeDescriptorFromValue_( + // tslint:disable-next-line: no-any + value: any + ): ValueType { let typeName; + if (value === null) { + throw new Error('Type must be provided for null values.'); + } + if (value instanceof BigQueryDate) { typeName = 'DATE'; } else if (value instanceof BigQueryDatetime) { @@ -760,9 +833,12 @@ export class BigQuery extends common.Service { } else if (value instanceof Big) { typeName = 'NUMERIC'; } else if (is.array(value)) { + if (value.length === 0) { + throw new Error('Type must be provided for empty array.'); + } return { type: 'ARRAY', - arrayType: BigQuery.getType_(value[0]), + arrayType: BigQuery.getTypeDescriptorFromValue_(value[0]), }; } else if (is.boolean(value)) { typeName = 'BOOL'; @@ -774,7 +850,7 @@ export class BigQuery extends common.Service { structTypes: Object.keys(value).map(prop => { return { name: prop, - type: BigQuery.getType_(value[prop]), + type: BigQuery.getTypeDescriptorFromValue_(value[prop]), }; }), }; @@ -806,23 +882,36 @@ export class BigQuery extends common.Service { * @param {*} value The value. * @returns {object} A properly-formed `queryParameter` object. */ - // tslint:disable-next-line no-any - static valueToQueryParameter_(value: any) { + static valueToQueryParameter_( + // tslint:disable-next-line: no-any + value: any, + providedType?: string | ProvidedTypeStruct | ProvidedTypeArray + ) { if (is.date(value)) { value = BigQuery.timestamp(value as Date); } - const parameterType = BigQuery.getType_(value); + let parameterType: bigquery.IQueryParameterType; + if (providedType) { + parameterType = BigQuery.getTypeDescriptorFromProvidedType_(providedType); + } else { + parameterType = BigQuery.getTypeDescriptorFromValue_(value); + } const queryParameter: QueryParameter = {parameterType, parameterValue: {}}; const typeName = queryParameter!.parameterType!.type!; - if (typeName === 'ARRAY') { queryParameter.parameterValue!.arrayValues = (value as Array<{}>).map( itemValue => { const value = getValue(itemValue, parameterType.arrayType!); if (is.object(value) || is.array(value)) { - return BigQuery.valueToQueryParameter_(value).parameterValue!; + if (is.array(providedType)) { + providedType = providedType as []; + return BigQuery.valueToQueryParameter_(value, providedType[0]) + .parameterValue!; + } else { + return BigQuery.valueToQueryParameter_(value).parameterValue!; + } } return {value} as bigquery.IQueryParameterValue; } @@ -830,9 +919,15 @@ export class BigQuery extends common.Service { } else if (typeName === 'STRUCT') { queryParameter.parameterValue!.structValues = Object.keys(value).reduce( (structValues, prop) => { - const nestedQueryParameter = BigQuery.valueToQueryParameter_( - value[prop] - ); + let nestedQueryParameter; + if (providedType) { + nestedQueryParameter = BigQuery.valueToQueryParameter_( + value[prop], + (providedType as ProvidedTypeStruct)[prop] + ); + } else { + nestedQueryParameter = BigQuery.valueToQueryParameter_(value[prop]); + } // tslint:disable-next-line no-any (structValues as any)[prop] = nestedQueryParameter.parameterValue; return structValues; @@ -1055,18 +1150,61 @@ export class BigQuery extends common.Service { query.queryParameters = []; // tslint:disable-next-line forin - for (const namedParamater in query.params) { - const value = query.params[namedParamater]; - const queryParameter = BigQuery.valueToQueryParameter_(value); - queryParameter.name = namedParamater; + for (const namedParameter in query.params) { + const value = query.params[namedParameter]; + let queryParameter; + + if (query.types) { + if (!is.object(query.types)) { + throw new Error( + 'Provided types must match the value type passed to `params`' + ); + } + + if (query.types[namedParameter]) { + queryParameter = BigQuery.valueToQueryParameter_( + value, + query.types[namedParameter] + ); + } else { + throw new Error( + `Type not provided for parameter: ${namedParameter}` + ); + } + } else { + queryParameter = BigQuery.valueToQueryParameter_(value); + } + + queryParameter.name = namedParameter; query.queryParameters.push(queryParameter); } } else { - query.queryParameters = query.params.map( - BigQuery.valueToQueryParameter_ - ); - } + query.queryParameters = []; + + if (query.types) { + if (!is.array(query.types)) { + throw new Error( + 'Provided types must match the value type passed to `params`' + ); + } + if (query.params.length !== query.types.length) { + throw new Error('Incorrect number of parameter types provided.'); + } + query.params.forEach((value: {}, i: number) => { + const queryParameter = BigQuery.valueToQueryParameter_( + value, + query.types[i] + ); + query.queryParameters.push(queryParameter); + }); + } else { + query.params.forEach((value: {}) => { + const queryParameter = BigQuery.valueToQueryParameter_(value); + query.queryParameters.push(queryParameter); + }); + } + } delete query.params; } diff --git a/system-test/bigquery.ts b/system-test/bigquery.ts index b49f155b..6811db94 100644 --- a/system-test/bigquery.ts +++ b/system-test/bigquery.ts @@ -1109,6 +1109,15 @@ describe('BigQuery', () => { ); }); + it('should work with empty arrays', async () => { + const [rows] = await bigquery.query({ + query: 'SELECT * FROM UNNEST (?)', + params: [[]], + types: [['INT64']], + }); + assert.strictEqual(rows.length, 0); + }); + it('should work with structs', done => { bigquery.query( { @@ -1344,6 +1353,17 @@ describe('BigQuery', () => { ); }); + it('should work with empty arrays', async () => { + const [rows] = await bigquery.query({ + query: 'SELECT * FROM UNNEST (@nums)', + params: { + nums: [], + }, + types: {nums: ['INT64']}, + }); + assert.strictEqual(rows.length, 0); + }); + it('should work with structs', done => { bigquery.query( { diff --git a/test/bigquery.ts b/test/bigquery.ts index 6fd8e3e8..b2a5ddc6 100644 --- a/test/bigquery.ts +++ b/test/bigquery.ts @@ -658,22 +658,49 @@ describe('BigQuery', () => { }); }); - describe('getType_', () => { + describe('getTypeDescriptorFromValue_', () => { it('should return correct types', () => { - assert.strictEqual(BigQuery.getType_(bq.date()).type, 'DATE'); - assert.strictEqual(BigQuery.getType_(bq.datetime('')).type, 'DATETIME'); - assert.strictEqual(BigQuery.getType_(bq.time()).type, 'TIME'); - assert.strictEqual(BigQuery.getType_(bq.timestamp(0)).type, 'TIMESTAMP'); - assert.strictEqual(BigQuery.getType_(Buffer.alloc(2)).type, 'BYTES'); - assert.strictEqual(BigQuery.getType_(true).type, 'BOOL'); - assert.strictEqual(BigQuery.getType_(8).type, 'INT64'); - assert.strictEqual(BigQuery.getType_(8.1).type, 'FLOAT64'); - assert.strictEqual(BigQuery.getType_('hi').type, 'STRING'); - assert.strictEqual(BigQuery.getType_(new Big('1.1')).type, 'NUMERIC'); + assert.strictEqual( + BigQuery.getTypeDescriptorFromValue_(bq.date()).type, + 'DATE' + ); + assert.strictEqual( + BigQuery.getTypeDescriptorFromValue_(bq.datetime('')).type, + 'DATETIME' + ); + assert.strictEqual( + BigQuery.getTypeDescriptorFromValue_(bq.time()).type, + 'TIME' + ); + assert.strictEqual( + BigQuery.getTypeDescriptorFromValue_(bq.timestamp(0)).type, + 'TIMESTAMP' + ); + assert.strictEqual( + BigQuery.getTypeDescriptorFromValue_(Buffer.alloc(2)).type, + 'BYTES' + ); + assert.strictEqual( + BigQuery.getTypeDescriptorFromValue_(true).type, + 'BOOL' + ); + assert.strictEqual(BigQuery.getTypeDescriptorFromValue_(8).type, 'INT64'); + assert.strictEqual( + BigQuery.getTypeDescriptorFromValue_(8.1).type, + 'FLOAT64' + ); + assert.strictEqual( + BigQuery.getTypeDescriptorFromValue_('hi').type, + 'STRING' + ); + assert.strictEqual( + BigQuery.getTypeDescriptorFromValue_(new Big('1.1')).type, + 'NUMERIC' + ); }); it('should return correct type for an array', () => { - const type = BigQuery.getType_([1]); + const type = BigQuery.getTypeDescriptorFromValue_([1]); assert.deepStrictEqual(type, { type: 'ARRAY', @@ -684,7 +711,7 @@ describe('BigQuery', () => { }); it('should return correct type for a struct', () => { - const type = BigQuery.getType_({prop: 1}); + const type = BigQuery.getTypeDescriptorFromValue_({prop: 1}); assert.deepStrictEqual(type, { type: 'STRUCT', @@ -708,24 +735,102 @@ describe('BigQuery', () => { ); assert.throws(() => { - BigQuery.getType_(undefined); + BigQuery.getTypeDescriptorFromValue_(undefined); + }, expectedError); + }); + + it('should throw with an empty array', () => { + assert.throws(() => { + BigQuery.getTypeDescriptorFromValue_([]); + }, /Type must be provided for empty array./); + }); + + it('should throw with an null value', () => { + const expectedError = new RegExp( + 'Type must be provided for null values.' + ); + + assert.throws(() => { + BigQuery.getTypeDescriptorFromValue_(null); }, expectedError); }); }); + describe('getTypeDescriptorFromProvidedType_', () => { + it('should return correct type for an array', () => { + const type = BigQuery.getTypeDescriptorFromProvidedType_(['INT64']); + + assert.deepStrictEqual(type, { + type: 'ARRAY', + arrayType: { + type: 'INT64', + }, + }); + }); + + it('should return correct type for a struct', () => { + const type = BigQuery.getTypeDescriptorFromProvidedType_({prop: 'INT64'}); + + assert.deepStrictEqual(type, { + type: 'STRUCT', + structTypes: [ + { + name: 'prop', + type: { + type: 'INT64', + }, + }, + ], + }); + }); + + it('should throw for invalid provided type', () => { + const INVALID_TYPE = 'invalid'; + + assert.throws(() => { + BigQuery.getTypeDescriptorFromProvidedType_(INVALID_TYPE); + }, /Invalid type provided:/); + }); + }); + describe('valueToQueryParameter_', () => { it('should get the type', done => { const value = {}; - sandbox.stub(BigQuery, 'getType_').callsFake(value_ => { - assert.strictEqual(value_, value); - setImmediate(done); - return { - type: '', - }; - }); + sandbox + .stub(BigQuery, 'getTypeDescriptorFromValue_') + .callsFake(value_ => { + assert.strictEqual(value_, value); + setImmediate(done); + return { + type: '', + }; + }); - BigQuery.valueToQueryParameter_(value); + const queryParameter = BigQuery.valueToQueryParameter_(value); + assert.strictEqual(queryParameter.parameterValue.value, value); + }); + + it('should get the provided type', done => { + const value = {}; + const providedType = 'STRUCT'; + + sandbox + .stub(BigQuery, 'getTypeDescriptorFromProvidedType_') + .callsFake(providedType_ => { + assert.strictEqual(providedType_, providedType); + setImmediate(done); + return { + type: '', + }; + }); + + const queryParameter = BigQuery.valueToQueryParameter_( + value, + providedType + ); + + assert.strictEqual(queryParameter.parameterValue.value, value); }); it('should format a Date', () => { @@ -739,7 +844,7 @@ describe('BigQuery', () => { }; }); - sandbox.stub(BigQuery, 'getType_').returns({ + sandbox.stub(BigQuery, 'getTypeDescriptorFromValue_').returns({ type: 'TIMESTAMP', }); @@ -752,7 +857,7 @@ describe('BigQuery', () => { value: 'value', }; - sandbox.stub(BigQuery, 'getType_').returns({ + sandbox.stub(BigQuery, 'getTypeDescriptorFromValue_').returns({ type: 'DATETIME', }); @@ -767,7 +872,7 @@ describe('BigQuery', () => { }, ]; - sandbox.stub(BigQuery, 'getType_').returns({ + sandbox.stub(BigQuery, 'getTypeDescriptorFromValue_').returns({ type: 'ARRAY', arrayType: {type: 'DATETIME'}, }); @@ -781,7 +886,7 @@ describe('BigQuery', () => { value: 'value', }; - sandbox.stub(BigQuery, 'getType_').returns({ + sandbox.stub(BigQuery, 'getTypeDescriptorFromValue_').returns({ type: 'TIME', }); @@ -796,7 +901,7 @@ describe('BigQuery', () => { }, ]; - sandbox.stub(BigQuery, 'getType_').returns({ + sandbox.stub(BigQuery, 'getTypeDescriptorFromValue_').returns({ type: 'ARRAY', arrayType: {type: 'TIME'}, }); @@ -808,7 +913,7 @@ describe('BigQuery', () => { it('should format an array', () => { const array = [1]; - sandbox.stub(BigQuery, 'getType_').returns({ + sandbox.stub(BigQuery, 'getTypeDescriptorFromValue_').returns({ type: 'ARRAY', arrayType: {type: 'INT64'}, }); @@ -822,6 +927,34 @@ describe('BigQuery', () => { ]); }); + it('should format an array with provided type', () => { + const array = [[1]]; + const providedType = [['INT64']]; + + sandbox.stub(BigQuery, 'getTypeDescriptorFromProvidedType_').returns({ + type: 'ARRAY', + arrayType: { + type: 'ARRAY', + arrayType: {type: 'INT64'}, + }, + }); + + const queryParameter = BigQuery.valueToQueryParameter_( + array, + providedType + ); + const arrayValues = queryParameter.parameterValue.arrayValues; + assert.deepStrictEqual(arrayValues, [ + { + arrayValues: [ + { + value: array[0][0], + }, + ], + }, + ]); + }); + it('should format a struct', () => { const struct = { key: 'value', @@ -829,7 +962,7 @@ describe('BigQuery', () => { const expectedParameterValue = {}; - sandbox.stub(BigQuery, 'getType_').callsFake(() => { + sandbox.stub(BigQuery, 'getTypeDescriptorFromValue_').callsFake(() => { sandbox.stub(BigQuery, 'valueToQueryParameter_').callsFake(value => { assert.strictEqual(value, struct.key); return { @@ -875,7 +1008,7 @@ describe('BigQuery', () => { it('should format all other types', () => { const typeName = 'ANY-TYPE'; - sandbox.stub(BigQuery, 'getType_').returns({ + sandbox.stub(BigQuery, 'getTypeDescriptorFromValue_').returns({ type: typeName, }); assert.deepStrictEqual(BigQuery.valueToQueryParameter_(8), { @@ -1235,6 +1368,10 @@ describe('BigQuery', () => { const POSITIONAL_PARAMS = ['value']; + const NAMED_TYPES = {key: 'STRING'}; + + const POSITIONAL_TYPES = ['STRING']; + it('should delete the params option', done => { bq.createJob = (reqOpts: JobOptions) => { // tslint:disable-next-line no-any @@ -1268,7 +1405,7 @@ describe('BigQuery', () => { ); }); - it('should get set the correct query parameters', done => { + it('should set the correct query parameters', done => { const queryParameter = {}; BigQuery.valueToQueryParameter_ = (value: {}) => { @@ -1291,6 +1428,44 @@ describe('BigQuery', () => { assert.ifError ); }); + + it('should allow for optional parameter types', () => { + bq.createJob = (reqOpts: JobOptions) => { + // tslint:disable-next-line no-any + assert.strictEqual((reqOpts as any).params, undefined); + }; + + bq.createQueryJob( + { + query: QUERY_STRING, + params: NAMED_PARAMS, + types: NAMED_TYPES, + }, + assert.ifError + ); + }); + + it('should throw for invalid type structure provided', () => { + assert.throws(() => { + bq.createQueryJob({ + query: QUERY_STRING, + params: NAMED_PARAMS, + types: POSITIONAL_TYPES, + }); + }, /Provided types must match the value type passed to `params`/); + }); + + it('should throw if named param not present in provided types', () => { + const INVALID_TYPES = {other: 'string'}; + + assert.throws(() => { + bq.createQueryJob({ + query: QUERY_STRING, + params: NAMED_PARAMS, + types: INVALID_TYPES, + }); + }, /Type not provided for parameter: key/); + }); }); describe('positional', () => { @@ -1310,7 +1485,7 @@ describe('BigQuery', () => { ); }); - it('should get set the correct query parameters', done => { + it('should set the correct query parameters', done => { const queryParameter = {}; BigQuery.valueToQueryParameter_ = (value: {}) => { @@ -1332,6 +1507,68 @@ describe('BigQuery', () => { assert.ifError ); }); + + it('should convert value and type to query parameter', done => { + const fakeQueryParameter = {fake: 'query parameter'}; + + bq.createJob = (reqOpts: JobOptions) => { + const queryParameters = reqOpts.configuration!.query! + .queryParameters; + assert.deepStrictEqual(queryParameters, [fakeQueryParameter]); + done(); + }; + + sandbox + .stub(BigQuery, 'valueToQueryParameter_') + .callsFake((value, type) => { + assert.strictEqual(value, POSITIONAL_PARAMS[0]); + assert.strictEqual(type, POSITIONAL_TYPES[0]); + return fakeQueryParameter; + }); + + bq.createQueryJob({ + query: QUERY_STRING, + params: POSITIONAL_PARAMS, + types: POSITIONAL_TYPES, + }); + }); + + it('should allow for optional parameter types', () => { + bq.createJob = (reqOpts: JobOptions) => { + // tslint:disable-next-line no-any + assert.strictEqual((reqOpts as any).params, undefined); + }; + + bq.createQueryJob( + { + query: QUERY_STRING, + params: POSITIONAL_PARAMS, + types: POSITIONAL_TYPES, + }, + assert.ifError + ); + }); + + it('should throw for invalid type structure provided for positional params', () => { + assert.throws(() => { + bq.createQueryJob({ + query: QUERY_STRING, + params: POSITIONAL_PARAMS, + types: NAMED_TYPES, + }); + }, /Provided types must match the value type passed to `params`/); + }); + + it('should throw for incorrect number of types provided for positional params', () => { + const ADDITIONAL_TYPES = ['string', 'string']; + assert.throws(() => { + bq.createQueryJob({ + query: QUERY_STRING, + params: POSITIONAL_PARAMS, + types: ADDITIONAL_TYPES, + }); + }, /Incorrect number of parameter types provided./); + }); }); });