From ac6284b6d5eefeee2beaac55761c502e36216eba Mon Sep 17 00:00:00 2001 From: Weijun Huang Date: Thu, 17 Aug 2023 13:32:13 +0800 Subject: [PATCH 01/14] feat: array-empty --- datafusion/expr/src/built_in_function.rs | 6 +++ .../physical-expr/src/array_expressions.rs | 42 +++++++++++++++++++ datafusion/physical-expr/src/functions.rs | 3 ++ datafusion/proto/src/generated/prost.rs | 3 ++ .../proto/src/logical_plan/from_proto.rs | 1 + datafusion/proto/src/logical_plan/to_proto.rs | 1 + datafusion/sqllogictest/test_files/array.slt | 10 +++++ 7 files changed, 66 insertions(+) diff --git a/datafusion/expr/src/built_in_function.rs b/datafusion/expr/src/built_in_function.rs index 35c117a12ca5..62dbd332bcd4 100644 --- a/datafusion/expr/src/built_in_function.rs +++ b/datafusion/expr/src/built_in_function.rs @@ -138,6 +138,8 @@ pub enum BuiltinScalarFunction { ArrayDims, /// array_element ArrayElement, + /// array_empty + ArrayEmpty, /// array_length ArrayLength, /// array_ndims @@ -360,6 +362,7 @@ impl BuiltinScalarFunction { BuiltinScalarFunction::Trunc => Volatility::Immutable, BuiltinScalarFunction::ArrayAppend => Volatility::Immutable, BuiltinScalarFunction::ArrayConcat => Volatility::Immutable, + BuiltinScalarFunction::ArrayEmpty => Volatility::Immutable, BuiltinScalarFunction::ArrayHasAll => Volatility::Immutable, BuiltinScalarFunction::ArrayHasAny => Volatility::Immutable, BuiltinScalarFunction::ArrayHas => Volatility::Immutable, @@ -548,6 +551,7 @@ impl BuiltinScalarFunction { "The {self} function can only accept list as the first argument" ), }, + BuiltinScalarFunction::ArrayEmpty => Ok(UInt8), BuiltinScalarFunction::ArrayLength => Ok(UInt64), BuiltinScalarFunction::ArrayNdims => Ok(UInt64), BuiltinScalarFunction::ArrayPosition => Ok(UInt64), @@ -832,6 +836,7 @@ impl BuiltinScalarFunction { Signature::variadic_any(self.volatility()) } BuiltinScalarFunction::ArrayDims => Signature::any(1, self.volatility()), + BuiltinScalarFunction::ArrayEmpty => Signature::any(1, self.volatility()), BuiltinScalarFunction::ArrayElement => Signature::any(2, self.volatility()), BuiltinScalarFunction::Flatten => Signature::any(1, self.volatility()), BuiltinScalarFunction::ArrayHasAll @@ -1322,6 +1327,7 @@ fn aliases(func: &BuiltinScalarFunction) -> &'static [&'static str] { &["array_concat", "array_cat", "list_concat", "list_cat"] } BuiltinScalarFunction::ArrayDims => &["array_dims", "list_dims"], + BuiltinScalarFunction::ArrayEmpty => &["empty"], BuiltinScalarFunction::ArrayElement => &[ "array_element", "array_extract", diff --git a/datafusion/physical-expr/src/array_expressions.rs b/datafusion/physical-expr/src/array_expressions.rs index 3136383c31f4..f3b25308298e 100644 --- a/datafusion/physical-expr/src/array_expressions.rs +++ b/datafusion/physical-expr/src/array_expressions.rs @@ -986,6 +986,36 @@ macro_rules! general_repeat_list { }}; } +/// Array_empty SQL function +pub fn array_empty(args: &[ArrayRef]) -> Result { + let array = as_list_array(&args[0])?; + let mut builder = UInt8Array::builder(1); + + for arr in array.iter() { + if let Some(arr) = arr { + let res = match arr.data_type() { + DataType::List(_) => { + let arr = downcast_arg!(arr, ListArray); + let mut res = 1; + for i in 0..arr.len() { + if arr.value_length(i) > 0 { + res = 0; + break; + } + } + res + } + DataType::Null => 1, + _ => 0, + }; + builder.append_value(res); + } else { + builder.append_value(1); + } + } + Ok(Arc::new(builder.finish())) +} + /// Array_repeat SQL function pub fn array_repeat(args: &[ArrayRef]) -> Result { let element = &args[0]; @@ -2988,6 +3018,18 @@ mod tests { ); } + #[test] + fn test_array_empty() { + // array_dims([[1, 2, 3, 4], [5, 6, 7, 8]]) = [2, 4] + let list_array = return_nested_array().into_array(1); + + let array = array_empty(&[list_array]) + .expect("failed to initialize function array_empty"); + let result = array.as_any().downcast_ref::(); + + assert_eq!(&[1], result.unwrap().values()) + } + #[test] fn test_nested_array_dims() { // array_dims([[1, 2, 3, 4], [5, 6, 7, 8]]) = [2, 4] diff --git a/datafusion/physical-expr/src/functions.rs b/datafusion/physical-expr/src/functions.rs index ab7601f7d779..746ad76d13e5 100644 --- a/datafusion/physical-expr/src/functions.rs +++ b/datafusion/physical-expr/src/functions.rs @@ -425,6 +425,9 @@ pub fn create_physical_fun( BuiltinScalarFunction::ArrayConcat => { Arc::new(|args| make_scalar_function(array_expressions::array_concat)(args)) } + BuiltinScalarFunction::ArrayEmpty => { + Arc::new(|args| make_scalar_function(array_expressions::array_empty)(args)) + } BuiltinScalarFunction::ArrayHasAll => { Arc::new(|args| make_scalar_function(array_expressions::array_has_all)(args)) } diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs index 6cf402fe66e9..afb4e2eb59b4 100644 --- a/datafusion/proto/src/generated/prost.rs +++ b/datafusion/proto/src/generated/prost.rs @@ -2377,6 +2377,7 @@ pub enum ScalarFunction { Flatten = 112, Isnan = 113, Iszero = 114, + ArrayEmpty = 115, } impl ScalarFunction { /// String value of the enum field names used in the ProtoBuf definition. @@ -2474,6 +2475,7 @@ impl ScalarFunction { ScalarFunction::ArrayAppend => "ArrayAppend", ScalarFunction::ArrayConcat => "ArrayConcat", ScalarFunction::ArrayDims => "ArrayDims", + ScalarFunction::ArrayEmpty => "ArrayEmpty", ScalarFunction::ArrayRepeat => "ArrayRepeat", ScalarFunction::ArrayLength => "ArrayLength", ScalarFunction::ArrayNdims => "ArrayNdims", @@ -2594,6 +2596,7 @@ impl ScalarFunction { "ArrayAppend" => Some(Self::ArrayAppend), "ArrayConcat" => Some(Self::ArrayConcat), "ArrayDims" => Some(Self::ArrayDims), + "ArrayEmpty" => Some(Self::ArrayEmpty), "ArrayRepeat" => Some(Self::ArrayRepeat), "ArrayLength" => Some(Self::ArrayLength), "ArrayNdims" => Some(Self::ArrayNdims), diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rs index 43f1d44b7ded..ea3d344e0bcb 100644 --- a/datafusion/proto/src/logical_plan/from_proto.rs +++ b/datafusion/proto/src/logical_plan/from_proto.rs @@ -452,6 +452,7 @@ impl From<&protobuf::ScalarFunction> for BuiltinScalarFunction { ScalarFunction::ToTimestamp => Self::ToTimestamp, ScalarFunction::ArrayAppend => Self::ArrayAppend, ScalarFunction::ArrayConcat => Self::ArrayConcat, + ScalarFunction::ArrayEmpty => Self::ArrayEmpty, ScalarFunction::ArrayHasAll => Self::ArrayHasAll, ScalarFunction::ArrayHasAny => Self::ArrayHasAny, ScalarFunction::ArrayHas => Self::ArrayHas, diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rs index cb3296438165..82df53af92c3 100644 --- a/datafusion/proto/src/logical_plan/to_proto.rs +++ b/datafusion/proto/src/logical_plan/to_proto.rs @@ -1451,6 +1451,7 @@ impl TryFrom<&BuiltinScalarFunction> for protobuf::ScalarFunction { BuiltinScalarFunction::ToTimestamp => Self::ToTimestamp, BuiltinScalarFunction::ArrayAppend => Self::ArrayAppend, BuiltinScalarFunction::ArrayConcat => Self::ArrayConcat, + BuiltinScalarFunction::ArrayEmpty => Self::ArrayEmpty, BuiltinScalarFunction::ArrayHasAll => Self::ArrayHasAll, BuiltinScalarFunction::ArrayHasAny => Self::ArrayHasAny, BuiltinScalarFunction::ArrayHas => Self::ArrayHas, diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt index eb949c4f8693..59bc035c3410 100644 --- a/datafusion/sqllogictest/test_files/array.slt +++ b/datafusion/sqllogictest/test_files/array.slt @@ -2363,6 +2363,16 @@ from flatten_table; [1, 2, 3] [1, 2, 3, 4, 5, 6] [1, 2, 3] [1.0, 2.1, 2.2, 3.2, 3.3, 3.4] [1, 2, 3, 4, 5, 6] [8] [1, 2, 3] [1.0, 2.0, 3.0, 4.0, 5.0, 6.0] +query I +select empty(make_array(1)); +---- +0 + +query I +select empty(make_array()); +---- +1 + ### Delete tables statement ok From 25327751933b84560418910e43cdba2c0ee2d8a2 Mon Sep 17 00:00:00 2001 From: Weijun Huang Date: Thu, 17 Aug 2023 15:10:49 +0800 Subject: [PATCH 02/14] add definition in pbjson --- datafusion/proto/src/generated/pbjson.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs index f1a9e9c7bb74..0623b7653140 100644 --- a/datafusion/proto/src/generated/pbjson.rs +++ b/datafusion/proto/src/generated/pbjson.rs @@ -18921,6 +18921,7 @@ impl serde::Serialize for ScalarFunction { Self::ArrayAppend => "ArrayAppend", Self::ArrayConcat => "ArrayConcat", Self::ArrayDims => "ArrayDims", + Self::ArrayEmpty => "ArrayEmpty", Self::ArrayRepeat => "ArrayRepeat", Self::ArrayLength => "ArrayLength", Self::ArrayNdims => "ArrayNdims", From f20002d15837d151ef5400f641b196007c172dfe Mon Sep 17 00:00:00 2001 From: Weijun Huang Date: Thu, 17 Aug 2023 15:26:11 +0800 Subject: [PATCH 03/14] add definition --- datafusion/expr/src/expr_fn.rs | 6 ++++++ datafusion/proto/proto/datafusion.proto | 1 + datafusion/proto/src/generated/pbjson.rs | 4 +++- datafusion/proto/src/generated/prost.rs | 4 ++-- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/datafusion/expr/src/expr_fn.rs b/datafusion/expr/src/expr_fn.rs index 3ca6aa5d6e95..a6364dc9fce1 100644 --- a/datafusion/expr/src/expr_fn.rs +++ b/datafusion/expr/src/expr_fn.rs @@ -552,6 +552,12 @@ scalar_expr!( first_array second_array, "Returns true, if the element appears in the first array, otherwise false." ); +scalar_expr!( + ArrayEmpty, + array_empty, + array, + "returns 1 for an empty array or 0 for a non-empty array." +); scalar_expr!( ArrayHasAll, array_has_all, diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto index e4ef7b1bd448..f31a593ad5fe 100644 --- a/datafusion/proto/proto/datafusion.proto +++ b/datafusion/proto/proto/datafusion.proto @@ -597,6 +597,7 @@ enum ScalarFunction { Flatten = 112; Isnan = 113; Iszero = 114; + ArrayEmpty = 115; } message ScalarFunctionNode { diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs index 0623b7653140..7d1a18349c14 100644 --- a/datafusion/proto/src/generated/pbjson.rs +++ b/datafusion/proto/src/generated/pbjson.rs @@ -18921,7 +18921,6 @@ impl serde::Serialize for ScalarFunction { Self::ArrayAppend => "ArrayAppend", Self::ArrayConcat => "ArrayConcat", Self::ArrayDims => "ArrayDims", - Self::ArrayEmpty => "ArrayEmpty", Self::ArrayRepeat => "ArrayRepeat", Self::ArrayLength => "ArrayLength", Self::ArrayNdims => "ArrayNdims", @@ -18948,6 +18947,7 @@ impl serde::Serialize for ScalarFunction { Self::Flatten => "Flatten", Self::Isnan => "Isnan", Self::Iszero => "Iszero", + Self::ArrayEmpty => "ArrayEmpty", }; serializer.serialize_str(variant) } @@ -19074,6 +19074,7 @@ impl<'de> serde::Deserialize<'de> for ScalarFunction { "Flatten", "Isnan", "Iszero", + "ArrayEmpty", ]; struct GeneratedVisitor; @@ -19231,6 +19232,7 @@ impl<'de> serde::Deserialize<'de> for ScalarFunction { "Flatten" => Ok(ScalarFunction::Flatten), "Isnan" => Ok(ScalarFunction::Isnan), "Iszero" => Ok(ScalarFunction::Iszero), + "ArrayEmpty" => Ok(ScalarFunction::ArrayEmpty), _ => Err(serde::de::Error::unknown_variant(value, FIELDS)), } } diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs index afb4e2eb59b4..fc55b7e23af2 100644 --- a/datafusion/proto/src/generated/prost.rs +++ b/datafusion/proto/src/generated/prost.rs @@ -2475,7 +2475,6 @@ impl ScalarFunction { ScalarFunction::ArrayAppend => "ArrayAppend", ScalarFunction::ArrayConcat => "ArrayConcat", ScalarFunction::ArrayDims => "ArrayDims", - ScalarFunction::ArrayEmpty => "ArrayEmpty", ScalarFunction::ArrayRepeat => "ArrayRepeat", ScalarFunction::ArrayLength => "ArrayLength", ScalarFunction::ArrayNdims => "ArrayNdims", @@ -2502,6 +2501,7 @@ impl ScalarFunction { ScalarFunction::Flatten => "Flatten", ScalarFunction::Isnan => "Isnan", ScalarFunction::Iszero => "Iszero", + ScalarFunction::ArrayEmpty => "ArrayEmpty", } } /// Creates an enum from field names used in the ProtoBuf definition. @@ -2596,7 +2596,6 @@ impl ScalarFunction { "ArrayAppend" => Some(Self::ArrayAppend), "ArrayConcat" => Some(Self::ArrayConcat), "ArrayDims" => Some(Self::ArrayDims), - "ArrayEmpty" => Some(Self::ArrayEmpty), "ArrayRepeat" => Some(Self::ArrayRepeat), "ArrayLength" => Some(Self::ArrayLength), "ArrayNdims" => Some(Self::ArrayNdims), @@ -2623,6 +2622,7 @@ impl ScalarFunction { "Flatten" => Some(Self::Flatten), "Isnan" => Some(Self::Isnan), "Iszero" => Some(Self::Iszero), + "ArrayEmpty" => Some(Self::ArrayEmpty), _ => None, } } From 817a506a9f5d45515b54111e77140b7c5813fb52 Mon Sep 17 00:00:00 2001 From: Weijun Huang Date: Thu, 17 Aug 2023 15:48:18 +0800 Subject: [PATCH 04/14] remove useless tests --- datafusion/physical-expr/src/array_expressions.rs | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/datafusion/physical-expr/src/array_expressions.rs b/datafusion/physical-expr/src/array_expressions.rs index f3b25308298e..ce541c506bd9 100644 --- a/datafusion/physical-expr/src/array_expressions.rs +++ b/datafusion/physical-expr/src/array_expressions.rs @@ -3018,18 +3018,6 @@ mod tests { ); } - #[test] - fn test_array_empty() { - // array_dims([[1, 2, 3, 4], [5, 6, 7, 8]]) = [2, 4] - let list_array = return_nested_array().into_array(1); - - let array = array_empty(&[list_array]) - .expect("failed to initialize function array_empty"); - let result = array.as_any().downcast_ref::(); - - assert_eq!(&[1], result.unwrap().values()) - } - #[test] fn test_nested_array_dims() { // array_dims([[1, 2, 3, 4], [5, 6, 7, 8]]) = [2, 4] From 16e174974a0f23fcfe00bdad35cfad07e634b3f2 Mon Sep 17 00:00:00 2001 From: Weijun Huang Date: Fri, 18 Aug 2023 06:38:27 +0800 Subject: [PATCH 05/14] add the function in proto --- datafusion/proto/src/logical_plan/from_proto.rs | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rs index ea3d344e0bcb..285b69a5499e 100644 --- a/datafusion/proto/src/logical_plan/from_proto.rs +++ b/datafusion/proto/src/logical_plan/from_proto.rs @@ -34,7 +34,6 @@ use datafusion_common::{ Column, DFField, DFSchema, DFSchemaRef, DataFusionError, OwnedTableReference, Result, ScalarValue, }; -use datafusion_expr::expr::{Alias, Placeholder}; use datafusion_expr::{ abs, acos, acosh, array, array_append, array_concat, array_dims, array_element, array_has, array_has_all, array_has_any, array_length, array_ndims, array_position, @@ -59,6 +58,10 @@ use datafusion_expr::{ JoinConstraint, JoinType, Like, Operator, TryCast, WindowFrame, WindowFrameBound, WindowFrameUnits, }; +use datafusion_expr::{ + array_empty, + expr::{Alias, Placeholder}, +}; use std::sync::Arc; #[derive(Debug)] @@ -1356,6 +1359,9 @@ pub fn parse_expr( parse_expr(&args[0], registry)?, parse_expr(&args[1], registry)?, )), + ScalarFunction::ArrayEmpty => { + Ok(array_empty(parse_expr(&args[0], registry)?)) + } ScalarFunction::ArrayNdims => { Ok(array_ndims(parse_expr(&args[0], registry)?)) } From cfbee169c04589fbcb1066db87f8edb0c3b42644 Mon Sep 17 00:00:00 2001 From: Weijun Huang Date: Fri, 18 Aug 2023 06:47:12 +0800 Subject: [PATCH 06/14] add doc --- .../source/user-guide/sql/scalar_functions.md | 24 +++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index 6dbe5c05f6d1..ec7712d9b4bf 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -1495,6 +1495,7 @@ from_unixtime(expression) - [array_slice](#array_slice) - [array_to_string](#array_to_string) - [cardinality](#cardinality) +- [empty](#empty) - [list_append](#list_append) - [list_cat](#list_cat) - [list_concat](#list_concat) @@ -1693,6 +1694,8 @@ array_element(array, index) - list_element - list_extract +### `array_empty` + ### `array_extract` _Alias of [array_element](#array_element)._ @@ -2188,6 +2191,27 @@ cardinality(array) +--------------------------------------+ ``` +### `empty` +Returns 1 for an empty array or 0 for a non-empty array. + +```empty(array)``` + +#### Arguments + +- **array**: Array expression. + Can be a constant, column, or function, and any combination of array operators. + +#### Example + +``` +❯ select empty([1]); ++------------------+ +| empty(List([1])) | ++------------------+ +| 0 | ++------------------+ +``` + ### `list_append` _Alias of [array_append](#array_append)._ From daf5907f6627e513cdc5c9866bfb184da47a3ae4 Mon Sep 17 00:00:00 2001 From: Weijun Huang Date: Fri, 18 Aug 2023 07:11:07 +0800 Subject: [PATCH 07/14] refactor --- datafusion/expr/src/built_in_function.rs | 4 +- .../physical-expr/src/array_expressions.rs | 60 +++++++++++-------- datafusion/sqllogictest/test_files/array.slt | 21 +++++-- 3 files changed, 54 insertions(+), 31 deletions(-) diff --git a/datafusion/expr/src/built_in_function.rs b/datafusion/expr/src/built_in_function.rs index 62dbd332bcd4..17066f6aea27 100644 --- a/datafusion/expr/src/built_in_function.rs +++ b/datafusion/expr/src/built_in_function.rs @@ -541,7 +541,8 @@ impl BuiltinScalarFunction { } BuiltinScalarFunction::ArrayHasAll | BuiltinScalarFunction::ArrayHasAny - | BuiltinScalarFunction::ArrayHas => Ok(Boolean), + | BuiltinScalarFunction::ArrayHas + | BuiltinScalarFunction::ArrayEmpty => Ok(Boolean), BuiltinScalarFunction::ArrayDims => { Ok(List(Arc::new(Field::new("item", UInt64, true)))) } @@ -551,7 +552,6 @@ impl BuiltinScalarFunction { "The {self} function can only accept list as the first argument" ), }, - BuiltinScalarFunction::ArrayEmpty => Ok(UInt8), BuiltinScalarFunction::ArrayLength => Ok(UInt64), BuiltinScalarFunction::ArrayNdims => Ok(UInt64), BuiltinScalarFunction::ArrayPosition => Ok(UInt64), diff --git a/datafusion/physical-expr/src/array_expressions.rs b/datafusion/physical-expr/src/array_expressions.rs index ce541c506bd9..8f7b94927019 100644 --- a/datafusion/physical-expr/src/array_expressions.rs +++ b/datafusion/physical-expr/src/array_expressions.rs @@ -989,31 +989,41 @@ macro_rules! general_repeat_list { /// Array_empty SQL function pub fn array_empty(args: &[ArrayRef]) -> Result { let array = as_list_array(&args[0])?; - let mut builder = UInt8Array::builder(1); - - for arr in array.iter() { - if let Some(arr) = arr { - let res = match arr.data_type() { - DataType::List(_) => { - let arr = downcast_arg!(arr, ListArray); - let mut res = 1; - for i in 0..arr.len() { - if arr.value_length(i) > 0 { - res = 0; - break; - } - } - res - } - DataType::Null => 1, - _ => 0, - }; - builder.append_value(res); - } else { - builder.append_value(1); - } - } - Ok(Arc::new(builder.finish())) + let builder = array + .iter() + .map(|arr| { + if let Some(arr) = arr { + Some(arr.is_empty()) + } else { + Some(true) + } + }) + .collect::(); + // let mut builder = UInt8Array::builder(1); + + // for arr in array.iter() { + // if let Some(arr) = arr { + // let res = match arr.data_type() { + // DataType::List(_) => { + // let arr = downcast_arg!(arr, ListArray); + // let mut res = 1; + // for i in 0..arr.len() { + // if arr.value_length(i) > 0 { + // res = 0; + // break; + // } + // } + // res + // } + // DataType::Null => 1, + // _ => 0, + // }; + // builder.append_value(res); + // } else { + // builder.append_value(1); + // } + // } + Ok(Arc::new(builder)) } /// Array_repeat SQL function diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt index 59bc035c3410..310b2abfc4a5 100644 --- a/datafusion/sqllogictest/test_files/array.slt +++ b/datafusion/sqllogictest/test_files/array.slt @@ -2363,15 +2363,28 @@ from flatten_table; [1, 2, 3] [1, 2, 3, 4, 5, 6] [1, 2, 3] [1.0, 2.1, 2.2, 3.2, 3.3, 3.4] [1, 2, 3, 4, 5, 6] [8] [1, 2, 3] [1.0, 2.0, 3.0, 4.0, 5.0, 6.0] -query I +# empty scalar function #1 +query B select empty(make_array(1)); ---- -0 +false -query I +# empty scalar function #2 +query B select empty(make_array()); ---- -1 +true + +query B +select empty(column1) from arrays; +---- +false +false +false +false +true +false +false ### Delete tables From 2424ca753705147f9d9c02a72c1cdad3e583df94 Mon Sep 17 00:00:00 2001 From: Weijun Huang Date: Fri, 18 Aug 2023 16:36:47 +0800 Subject: [PATCH 08/14] format --- docs/source/user-guide/sql/scalar_functions.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index ec7712d9b4bf..1e5424e71fab 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -2192,9 +2192,11 @@ cardinality(array) ``` ### `empty` -Returns 1 for an empty array or 0 for a non-empty array. -```empty(array)``` +Returns 1 for an empty array or 0 for a non-empty array. +``` +empty(array) +``` #### Arguments From 89a523ced7d5507e790b4a0878f2257418c8aec0 Mon Sep 17 00:00:00 2001 From: Weijun Huang Date: Fri, 18 Aug 2023 16:37:19 +0800 Subject: [PATCH 09/14] remove useless code --- .../physical-expr/src/array_expressions.rs | 24 ------------------- 1 file changed, 24 deletions(-) diff --git a/datafusion/physical-expr/src/array_expressions.rs b/datafusion/physical-expr/src/array_expressions.rs index 8f7b94927019..510764b005a5 100644 --- a/datafusion/physical-expr/src/array_expressions.rs +++ b/datafusion/physical-expr/src/array_expressions.rs @@ -999,30 +999,6 @@ pub fn array_empty(args: &[ArrayRef]) -> Result { } }) .collect::(); - // let mut builder = UInt8Array::builder(1); - - // for arr in array.iter() { - // if let Some(arr) = arr { - // let res = match arr.data_type() { - // DataType::List(_) => { - // let arr = downcast_arg!(arr, ListArray); - // let mut res = 1; - // for i in 0..arr.len() { - // if arr.value_length(i) > 0 { - // res = 0; - // break; - // } - // } - // res - // } - // DataType::Null => 1, - // _ => 0, - // }; - // builder.append_value(res); - // } else { - // builder.append_value(1); - // } - // } Ok(Arc::new(builder)) } From 9bc126e13e82cf60fa21abe13c24fe12bdafe79d Mon Sep 17 00:00:00 2001 From: Weijun Huang Date: Fri, 18 Aug 2023 16:40:04 +0800 Subject: [PATCH 10/14] fix format --- docs/source/user-guide/sql/scalar_functions.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index 1e5424e71fab..c6e2f5ddd828 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -2194,6 +2194,7 @@ cardinality(array) ### `empty` Returns 1 for an empty array or 0 for a non-empty array. + ``` empty(array) ``` From 8e5f079ba0148e159dfa50fad817e19365d4bf61 Mon Sep 17 00:00:00 2001 From: Weijun Huang Date: Mon, 21 Aug 2023 23:31:19 +0800 Subject: [PATCH 11/14] support NULL --- datafusion/physical-expr/src/array_expressions.rs | 7 ++++++- datafusion/sqllogictest/test_files/array.slt | 13 +++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/datafusion/physical-expr/src/array_expressions.rs b/datafusion/physical-expr/src/array_expressions.rs index 510764b005a5..df938b3265a7 100644 --- a/datafusion/physical-expr/src/array_expressions.rs +++ b/datafusion/physical-expr/src/array_expressions.rs @@ -988,17 +988,22 @@ macro_rules! general_repeat_list { /// Array_empty SQL function pub fn array_empty(args: &[ArrayRef]) -> Result { + if args[0].as_any().downcast_ref::().is_some() { + return Ok(args[0].clone()); + } + let array = as_list_array(&args[0])?; let builder = array .iter() .map(|arr| { if let Some(arr) = arr { - Some(arr.is_empty()) + Some(arr.len() == arr.null_count()) } else { Some(true) } }) .collect::(); + println!("\n"); Ok(Arc::new(builder)) } diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt index 310b2abfc4a5..a419ae22c2da 100644 --- a/datafusion/sqllogictest/test_files/array.slt +++ b/datafusion/sqllogictest/test_files/array.slt @@ -2375,6 +2375,19 @@ select empty(make_array()); ---- true +# empty scalar function #3 +query B +select empty(make_array(NULL)); +---- +true + +# empty scalar function #4 +query B +select empty(NULL); +---- +NULL + +# empty scalar function #5 query B select empty(column1) from arrays; ---- From 8df5ba2a38dc32bc8a271ba03aaf9e316153cd20 Mon Sep 17 00:00:00 2001 From: Weijun Huang Date: Tue, 22 Aug 2023 08:16:29 +0800 Subject: [PATCH 12/14] remove redundant code --- datafusion/physical-expr/src/array_expressions.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/datafusion/physical-expr/src/array_expressions.rs b/datafusion/physical-expr/src/array_expressions.rs index df938b3265a7..46afc30a8879 100644 --- a/datafusion/physical-expr/src/array_expressions.rs +++ b/datafusion/physical-expr/src/array_expressions.rs @@ -1003,7 +1003,6 @@ pub fn array_empty(args: &[ArrayRef]) -> Result { } }) .collect::(); - println!("\n"); Ok(Arc::new(builder)) } From c6f51440bc57f3a386f11f86c781658d47773a70 Mon Sep 17 00:00:00 2001 From: Weijun Huang Date: Tue, 22 Aug 2023 21:50:44 +0800 Subject: [PATCH 13/14] support NULL --- datafusion/physical-expr/src/array_expressions.rs | 3 ++- datafusion/sqllogictest/test_files/array.slt | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/datafusion/physical-expr/src/array_expressions.rs b/datafusion/physical-expr/src/array_expressions.rs index 46afc30a8879..916f2d218586 100644 --- a/datafusion/physical-expr/src/array_expressions.rs +++ b/datafusion/physical-expr/src/array_expressions.rs @@ -988,6 +988,7 @@ macro_rules! general_repeat_list { /// Array_empty SQL function pub fn array_empty(args: &[ArrayRef]) -> Result { + println!("args[0]: {:?}", &args[0]); if args[0].as_any().downcast_ref::().is_some() { return Ok(args[0].clone()); } @@ -999,7 +1000,7 @@ pub fn array_empty(args: &[ArrayRef]) -> Result { if let Some(arr) = arr { Some(arr.len() == arr.null_count()) } else { - Some(true) + None } }) .collect::(); diff --git a/datafusion/sqllogictest/test_files/array.slt b/datafusion/sqllogictest/test_files/array.slt index a419ae22c2da..bd16072b29d7 100644 --- a/datafusion/sqllogictest/test_files/array.slt +++ b/datafusion/sqllogictest/test_files/array.slt @@ -2395,7 +2395,7 @@ false false false false -true +NULL false false From f8176f3531c3513f3abe5bd5d7a0340c8ecca724 Mon Sep 17 00:00:00 2001 From: Weijun Huang Date: Tue, 22 Aug 2023 22:56:21 +0800 Subject: [PATCH 14/14] fix clippy --- datafusion/physical-expr/src/array_expressions.rs | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/datafusion/physical-expr/src/array_expressions.rs b/datafusion/physical-expr/src/array_expressions.rs index 916f2d218586..49c136f3fb11 100644 --- a/datafusion/physical-expr/src/array_expressions.rs +++ b/datafusion/physical-expr/src/array_expressions.rs @@ -996,13 +996,7 @@ pub fn array_empty(args: &[ArrayRef]) -> Result { let array = as_list_array(&args[0])?; let builder = array .iter() - .map(|arr| { - if let Some(arr) = arr { - Some(arr.len() == arr.null_count()) - } else { - None - } - }) + .map(|arr| arr.map(|arr| arr.len() == arr.null_count())) .collect::(); Ok(Arc::new(builder)) }