diff --git a/presto-common/src/main/java/com/facebook/presto/common/InvalidTypeDefinitionException.java b/presto-common/src/main/java/com/facebook/presto/common/InvalidTypeDefinitionException.java new file mode 100644 index 0000000000000..b3b417ef4e598 --- /dev/null +++ b/presto-common/src/main/java/com/facebook/presto/common/InvalidTypeDefinitionException.java @@ -0,0 +1,39 @@ +/* + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.facebook.presto.common; + +public class InvalidTypeDefinitionException + extends RuntimeException +{ + public InvalidTypeDefinitionException(String message) + { + this(message, null); + } + + public InvalidTypeDefinitionException(Throwable throwable) + { + this(null, throwable); + } + + public InvalidTypeDefinitionException(String message, Throwable cause) + { + super(message, cause); + } + + @Override + public String getMessage() + { + return super.getMessage(); + } +} diff --git a/presto-common/src/main/java/com/facebook/presto/common/function/SqlFunctionProperties.java b/presto-common/src/main/java/com/facebook/presto/common/function/SqlFunctionProperties.java index ce348dbbeac11..17dc78482061b 100644 --- a/presto-common/src/main/java/com/facebook/presto/common/function/SqlFunctionProperties.java +++ b/presto-common/src/main/java/com/facebook/presto/common/function/SqlFunctionProperties.java @@ -34,6 +34,7 @@ public class SqlFunctionProperties private final Locale sessionLocale; private final String sessionUser; private final boolean fieldNamesInJsonCastEnabled; + private final boolean legacyJsonCast; private final Map extraCredentials; private SqlFunctionProperties( @@ -46,6 +47,7 @@ private SqlFunctionProperties( Locale sessionLocale, String sessionUser, boolean fieldNamesInJsonCastEnabled, + boolean legacyJsonCast, Map extraCredentials) { this.parseDecimalLiteralAsDouble = parseDecimalLiteralAsDouble; @@ -57,6 +59,7 @@ private SqlFunctionProperties( this.sessionLocale = requireNonNull(sessionLocale, "sessionLocale is null"); this.sessionUser = requireNonNull(sessionUser, "sessionUser is null"); this.fieldNamesInJsonCastEnabled = fieldNamesInJsonCastEnabled; + this.legacyJsonCast = legacyJsonCast; this.extraCredentials = requireNonNull(extraCredentials, "extraCredentials is null"); } @@ -111,6 +114,11 @@ public boolean isFieldNamesInJsonCastEnabled() return fieldNamesInJsonCastEnabled; } + public boolean isLegacyJsonCast() + { + return legacyJsonCast; + } + @Override public boolean equals(Object o) { @@ -129,13 +137,16 @@ public boolean equals(Object o) Objects.equals(sessionStartTime, that.sessionStartTime) && Objects.equals(sessionLocale, that.sessionLocale) && Objects.equals(sessionUser, that.sessionUser) && - Objects.equals(extraCredentials, that.extraCredentials); + Objects.equals(extraCredentials, that.extraCredentials) && + Objects.equals(legacyJsonCast, that.legacyJsonCast); } @Override public int hashCode() { - return Objects.hash(parseDecimalLiteralAsDouble, legacyRowFieldOrdinalAccessEnabled, timeZoneKey, legacyTimestamp, legacyMapSubscript, sessionStartTime, sessionLocale, sessionUser, extraCredentials); + return Objects.hash(parseDecimalLiteralAsDouble, legacyRowFieldOrdinalAccessEnabled, timeZoneKey, + legacyTimestamp, legacyMapSubscript, sessionStartTime, sessionLocale, sessionUser, + extraCredentials, legacyJsonCast); } public static Builder builder() @@ -154,6 +165,7 @@ public static class Builder private Locale sessionLocale; private String sessionUser; private boolean fieldNamesInJsonCastEnabled; + private boolean legacyJsonCast; private Map extraCredentials = emptyMap(); private Builder() {} @@ -218,9 +230,26 @@ public Builder setFieldNamesInJsonCastEnabled(boolean fieldNamesInJsonCastEnable return this; } + public Builder setLegacyJsonCast(boolean legacyJsonCast) + { + this.legacyJsonCast = legacyJsonCast; + return this; + } + public SqlFunctionProperties build() { - return new SqlFunctionProperties(parseDecimalLiteralAsDouble, legacyRowFieldOrdinalAccessEnabled, timeZoneKey, legacyTimestamp, legacyMapSubscript, sessionStartTime, sessionLocale, sessionUser, fieldNamesInJsonCastEnabled, extraCredentials); + return new SqlFunctionProperties( + parseDecimalLiteralAsDouble, + legacyRowFieldOrdinalAccessEnabled, + timeZoneKey, + legacyTimestamp, + legacyMapSubscript, + sessionStartTime, + sessionLocale, + sessionUser, + fieldNamesInJsonCastEnabled, + legacyJsonCast, + extraCredentials); } } } diff --git a/presto-common/src/main/java/com/facebook/presto/common/type/RowType.java b/presto-common/src/main/java/com/facebook/presto/common/type/RowType.java index 0740f0ebf6b6b..347665bba084e 100644 --- a/presto-common/src/main/java/com/facebook/presto/common/type/RowType.java +++ b/presto-common/src/main/java/com/facebook/presto/common/type/RowType.java @@ -83,6 +83,11 @@ public static Field field(Type type) return new Field(Optional.empty(), type); } + public static Field field(String name, Type type, boolean delimited) + { + return new Field(Optional.of(name), type, delimited); + } + private static TypeSignature makeSignature(List fields) { int size = fields.size(); diff --git a/presto-common/src/main/java/com/facebook/presto/common/type/TypeSignature.java b/presto-common/src/main/java/com/facebook/presto/common/type/TypeSignature.java index 5474b2dc0f5ae..4a6649873436f 100644 --- a/presto-common/src/main/java/com/facebook/presto/common/type/TypeSignature.java +++ b/presto-common/src/main/java/com/facebook/presto/common/type/TypeSignature.java @@ -16,6 +16,7 @@ import com.facebook.drift.annotations.ThriftConstructor; import com.facebook.drift.annotations.ThriftField; import com.facebook.drift.annotations.ThriftStruct; +import com.facebook.presto.common.InvalidTypeDefinitionException; import com.facebook.presto.common.QualifiedObjectName; import com.facebook.presto.common.type.BigintEnumType.LongEnumMap; import com.facebook.presto.common.type.VarcharEnumType.VarcharEnumMap; @@ -510,6 +511,7 @@ private static TypeSignature parseRowTypeSignature(String signature, Set List fields = new ArrayList<>(); + Set distinctFieldNames = new HashSet<>(); for (int i = StandardTypes.ROW.length() + 1; i < signature.length(); i++) { char c = signature.charAt(i); switch (state) { @@ -556,13 +558,19 @@ else if (c == ')' && bracketLevel > 1) { } else if (c == ')') { verify(tokenStart >= 0, "Expect tokenStart to be non-negative"); - fields.add(parseTypeOrNamedType(signature.substring(tokenStart, i).trim(), literalParameters)); + TypeSignatureParameter parameter = parseTypeOrNamedType(signature.substring(tokenStart, i).trim(), literalParameters); + parameter.getNamedTypeSignature().getName() + .ifPresent(fieldName -> checkDuplicateAndAdd(distinctFieldNames, fieldName)); + fields.add(parameter); tokenStart = -1; state = RowTypeSignatureParsingState.FINISHED; } else if (c == ',' && bracketLevel == 1) { verify(tokenStart >= 0, "Expect tokenStart to be non-negative"); - fields.add(parseTypeOrNamedType(signature.substring(tokenStart, i).trim(), literalParameters)); + TypeSignatureParameter parameter = parseTypeOrNamedType(signature.substring(tokenStart, i).trim(), literalParameters); + parameter.getNamedTypeSignature().getName() + .ifPresent(fieldName -> checkDuplicateAndAdd(distinctFieldNames, fieldName)); + fields.add(parameter); tokenStart = -1; state = RowTypeSignatureParsingState.START_OF_FIELD; } @@ -578,6 +586,7 @@ else if (c == ')' && bracketLevel > 1) { else if (c == ')') { verify(tokenStart >= 0, "Expect tokenStart to be non-negative"); verify(delimitedColumnName != null, "Expect delimitedColumnName to be non-null"); + checkDuplicateAndAdd(distinctFieldNames, delimitedColumnName); fields.add(TypeSignatureParameter.of(new NamedTypeSignature( Optional.of(new RowFieldName(delimitedColumnName, true)), parseTypeSignature(signature.substring(tokenStart, i).trim(), literalParameters)))); @@ -588,6 +597,7 @@ else if (c == ')') { else if (c == ',' && bracketLevel == 1) { verify(tokenStart >= 0, "Expect tokenStart to be non-negative"); verify(delimitedColumnName != null, "Expect delimitedColumnName to be non-null"); + checkDuplicateAndAdd(distinctFieldNames, delimitedColumnName); fields.add(TypeSignatureParameter.of(new NamedTypeSignature( Optional.of(new RowFieldName(delimitedColumnName, true)), parseTypeSignature(signature.substring(tokenStart, i).trim(), literalParameters)))); @@ -609,6 +619,13 @@ else if (c == ',' && bracketLevel == 1) { return new TypeSignature(signature.substring(0, StandardTypes.ROW.length()), fields); } + private static void checkDuplicateAndAdd(Set fieldNames, String fieldName) + { + if (!fieldNames.add(fieldName)) { + throw new InvalidTypeDefinitionException("Duplicate field: " + fieldName); + } + } + private static TypeSignatureParameter parseTypeOrNamedType(String typeOrNamedType, Set literalParameters) { int split = typeOrNamedType.indexOf(' '); diff --git a/presto-docs/src/main/sphinx/admin/properties.rst b/presto-docs/src/main/sphinx/admin/properties.rst index 1b2d5833f6cb2..5dcf4e700b84d 100644 --- a/presto-docs/src/main/sphinx/admin/properties.rst +++ b/presto-docs/src/main/sphinx/admin/properties.rst @@ -989,3 +989,16 @@ Logging Properties * **Default value:** ``100MB`` The maximum file size for the log file of the HTTP server. + + +Legacy Compatible Properties +------------------------------ + +``legacy_json_cast`` +^^^^^^^^^^^^^^^^^^^^^ + + * **Type:** ``boolean`` + * **Default value:** ``false`` + + When casting from ``JSON`` to ``ROW``, by default the case of double quoted field names in RowType is strictly enforced when matching, and unquoted field name in RowType is case-insensitive when matching. + To ignore the case of field names in RowType when casting from ``JSON`` to ``ROW`` (for legacy support), set config property ``legacy_json_cast`` to ``true``. After setting the property, the matching would be always case-insensitive. diff --git a/presto-docs/src/main/sphinx/functions/json.rst b/presto-docs/src/main/sphinx/functions/json.rst index 28be7c0d90064..b467b9a8f3e65 100644 --- a/presto-docs/src/main/sphinx/functions/json.rst +++ b/presto-docs/src/main/sphinx/functions/json.rst @@ -71,6 +71,31 @@ Cast from JSON SELECT CAST(JSON '{"k1": [1, 23], "k2": 456}' AS MAP(VARCHAR, JSON)); -- {k1 = JSON '[1,23]', k2 = JSON '456'} SELECT CAST(JSON '[null]' AS ARRAY(JSON)); -- [JSON 'null'] +.. note:: + + When casting from ``JSON`` to ``ROW``, by default the case of double quoted field names + in RowType is strictly enforced when matching, and unquoted field name in RowType is + case-insensitive when matching. For example:: + + SELECT CAST(JSON '{"v1":123,"V2":"abc","v3":true}' AS ROW(v1 BIGINT, v2 VARCHAR, v3 BOOLEAN)); -- {v1=123, v2=abc, v3=true} + SELECT CAST(JSON '{"v1":123,"V2":"abc","v3":true}' AS ROW(v1 BIGINT, "V2" VARCHAR, v3 BOOLEAN)); -- {v1=123, V2=abc, v3=true} + SELECT CAST(JSON '{"v1":123,"V2":"abc","v3":true}' AS ROW(v1 BIGINT, "v2" VARCHAR, v3 BOOLEAN)); -- {v1=123, v2=null, v3=true} + SELECT CAST(JSON '{"v1":123,"V2":"abc", "v2":"abc2","v3":true}' AS ROW(v1 BIGINT, v2 VARCHAR, "V2" VARCHAR, v3 BOOLEAN)); -- {v1=123, v2=abc2, V2=abc, v3=true} + + If the name of a field does not match (including case sensitivity), the value is null. + + To ignore the case of field names in RowType when casting from ``JSON`` to ``ROW`` + (for example, for legacy support) set config property ``legacy_json_cast`` to ``true`` + in the coordinator and the worker's `config properties <../admin/properties.html#legacy-compatible-properties>`_. + After setting the property, the matching would be case-insensitive as the following example:: + + SELECT CAST(JSON '{"v1":123,"V2":"abc","v3":true}' AS ROW(v1 BIGINT, v2 VARCHAR, v3 BOOLEAN)); -- {v1=123, v2=abc, v3=true} + SELECT CAST(JSON '{"v1":123,"V2":"abc","v3":true}' AS ROW(v1 BIGINT, "V2" VARCHAR, "V3" BOOLEAN)); -- {v1=123, V2=abc, V3=true} + + The following statement returns an error due to duplicate field names in RowType:: + + SELECT CAST(JSON '{"v1":123,"V2":"abc","v2":"abc2","v3":true}' AS ROW(v1 BIGINT, "V2" VARCHAR, v2 VARCHAR, "V3" BOOLEAN)); + .. note:: When casting from ``JSON`` to ``ROW``, both JSON array and JSON object are supported. JSON Functions diff --git a/presto-main/src/main/java/com/facebook/presto/Session.java b/presto-main/src/main/java/com/facebook/presto/Session.java index 6d343e9afda36..d775c7368bb55 100644 --- a/presto-main/src/main/java/com/facebook/presto/Session.java +++ b/presto-main/src/main/java/com/facebook/presto/Session.java @@ -52,6 +52,7 @@ import java.util.TimeZone; import java.util.stream.Collectors; +import static com.facebook.presto.SystemSessionProperties.LEGACY_JSON_CAST; import static com.facebook.presto.SystemSessionProperties.isFieldNameInJsonCastEnabled; import static com.facebook.presto.SystemSessionProperties.isLegacyMapSubscript; import static com.facebook.presto.SystemSessionProperties.isLegacyRowFieldOrdinalAccessEnabled; @@ -496,6 +497,7 @@ public ConnectorSession toConnectorSession() public SqlFunctionProperties getSqlFunctionProperties() { + boolean legacyJsonCast = this.sessionPropertyManager.decodeSystemPropertyValue(LEGACY_JSON_CAST, null, Boolean.class); return SqlFunctionProperties.builder() .setTimeZoneKey(timeZoneKey) .setLegacyRowFieldOrdinalAccessEnabled(isLegacyRowFieldOrdinalAccessEnabled(this)) @@ -506,6 +508,7 @@ public SqlFunctionProperties getSqlFunctionProperties() .setSessionLocale(getLocale()) .setSessionUser(getUser()) .setFieldNamesInJsonCastEnabled(isFieldNameInJsonCastEnabled(this)) + .setLegacyJsonCast(legacyJsonCast) .setExtraCredentials(identity.getExtraCredentials()) .build(); } diff --git a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java index 08cbb5b149014..b722b19aff2a8 100644 --- a/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java +++ b/presto-main/src/main/java/com/facebook/presto/SystemSessionProperties.java @@ -312,6 +312,7 @@ public final class SystemSessionProperties public static final String ADD_PARTIAL_NODE_FOR_ROW_NUMBER_WITH_LIMIT = "add_partial_node_for_row_number_with_limit"; public static final String REWRITE_CASE_TO_MAP_ENABLED = "rewrite_case_to_map_enabled"; public static final String FIELD_NAMES_IN_JSON_CAST_ENABLED = "field_names_in_json_cast_enabled"; + public static final String LEGACY_JSON_CAST = "legacy_json_cast"; public static final String PULL_EXPRESSION_FROM_LAMBDA_ENABLED = "pull_expression_from_lambda_enabled"; public static final String REWRITE_CONSTANT_ARRAY_CONTAINS_TO_IN_EXPRESSION = "rewrite_constant_array_contains_to_in_expression"; public static final String INFER_INEQUALITY_PREDICATES = "infer_inequality_predicates"; @@ -1754,6 +1755,11 @@ public SystemSessionProperties( "Include field names in json output when casting rows", featuresConfig.isFieldNamesInJsonCastEnabled(), false), + booleanProperty( + LEGACY_JSON_CAST, + "Keep the legacy json cast behavior, do not reserve the case for field names when casting to row type", + featuresConfig.isLegacyJsonCast(), + false), booleanProperty( OPTIMIZE_JOIN_PROBE_FOR_EMPTY_BUILD_RUNTIME, "Optimize join probe at runtime if build side is empty", diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToArrayCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToArrayCast.java index 3b520dfc8c0c7..fbffb1d1ef6b4 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToArrayCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToArrayCast.java @@ -95,7 +95,7 @@ public static Block toArray(ArrayType arrayType, BlockBuilderAppender elementApp } BlockBuilder blockBuilder = arrayType.getElementType().createBlockBuilder(null, 20); while (jsonParser.nextToken() != JsonToken.END_ARRAY) { - elementAppender.append(jsonParser, blockBuilder); + elementAppender.append(jsonParser, blockBuilder, properties); } if (jsonParser.nextToken() != null) { throw new JsonCastException(format("Unexpected trailing token: %s", jsonParser.getText())); diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToMapCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToMapCast.java index 909d3d430d55f..54ad54a11bc06 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToMapCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToMapCast.java @@ -103,9 +103,9 @@ public static Block toMap(MapType mapType, BlockBuilderAppender keyAppender, Blo HashTable hashTable = new HashTable(mapType.getKeyType(), singleMapBlockBuilder); int position = 0; while (jsonParser.nextToken() != JsonToken.END_OBJECT) { - keyAppender.append(jsonParser, singleMapBlockBuilder); + keyAppender.append(jsonParser, singleMapBlockBuilder, properties); jsonParser.nextToken(); - valueAppender.append(jsonParser, singleMapBlockBuilder); + valueAppender.append(jsonParser, singleMapBlockBuilder, properties); // Duplicate key detection is required even if the JSON is valid. // For example: CAST(JSON '{"1": 1, "01": 2}' AS MAP). diff --git a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToRowCast.java b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToRowCast.java index ab7c0d37e681e..bb87a4285528b 100644 --- a/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToRowCast.java +++ b/presto-main/src/main/java/com/facebook/presto/operator/scalar/JsonToRowCast.java @@ -35,8 +35,6 @@ import java.lang.invoke.MethodHandle; import java.util.List; -import java.util.Map; -import java.util.Optional; import static com.facebook.presto.common.type.TypeSignature.parseTypeSignature; import static com.facebook.presto.operator.scalar.ScalarFunctionImplementationChoice.ArgumentProperty.valueTypeArgumentProperty; @@ -48,7 +46,6 @@ import static com.facebook.presto.util.JsonUtil.JSON_FACTORY; import static com.facebook.presto.util.JsonUtil.canCastFromJson; import static com.facebook.presto.util.JsonUtil.createJsonParser; -import static com.facebook.presto.util.JsonUtil.getFieldNameToIndex; import static com.facebook.presto.util.JsonUtil.parseJsonToSingleRowBlock; import static com.facebook.presto.util.JsonUtil.truncateIfNecessaryForErrorMessage; import static com.facebook.presto.util.Reflection.methodHandle; @@ -61,7 +58,7 @@ public class JsonToRowCast extends SqlOperator { public static final JsonToRowCast JSON_TO_ROW = new JsonToRowCast(); - private static final MethodHandle METHOD_HANDLE = methodHandle(JsonToRowCast.class, "toRow", RowType.class, BlockBuilderAppender[].class, Optional.class, SqlFunctionProperties.class, Slice.class); + private static final MethodHandle METHOD_HANDLE = methodHandle(JsonToRowCast.class, "toRow", RowType.class, BlockBuilderAppender[].class, SqlFunctionProperties.class, Slice.class); private JsonToRowCast() { @@ -83,7 +80,7 @@ public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariab BlockBuilderAppender[] fieldAppenders = rowFields.stream() .map(rowField -> createBlockBuilderAppender(rowField.getType())) .toArray(BlockBuilderAppender[]::new); - MethodHandle methodHandle = METHOD_HANDLE.bindTo(rowType).bindTo(fieldAppenders).bindTo(getFieldNameToIndex(rowFields)); + MethodHandle methodHandle = METHOD_HANDLE.bindTo(rowType).bindTo(fieldAppenders); return new BuiltInScalarFunctionImplementation( true, ImmutableList.of(valueTypeArgumentProperty(RETURN_NULL_ON_NULL)), @@ -94,7 +91,6 @@ public BuiltInScalarFunctionImplementation specialize(BoundVariables boundVariab public static Block toRow( RowType rowType, BlockBuilderAppender[] fieldAppenders, - Optional> fieldNameToIndex, SqlFunctionProperties properties, Slice json) { @@ -113,7 +109,8 @@ public static Block toRow( jsonParser, (SingleRowBlockWriter) rowBlockBuilder.beginBlockEntry(), fieldAppenders, - fieldNameToIndex); + rowType, + properties); rowBlockBuilder.closeEntry(); if (jsonParser.nextToken() != null) { diff --git a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java index f7e4fcfe6a7e0..5546873c6de91 100644 --- a/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java +++ b/presto-main/src/main/java/com/facebook/presto/sql/analyzer/FeaturesConfig.java @@ -277,6 +277,7 @@ public class FeaturesConfig private boolean useDefaultsForCorrelatedAggregationPushdownThroughOuterJoins = true; private boolean mergeDuplicateAggregationsEnabled = true; private boolean fieldNamesInJsonCastEnabled; + private boolean legacyJsonCast; private boolean mergeAggregationsWithAndWithoutFilter; private boolean simplifyPlanWithEmptyInput = true; private PushDownFilterThroughCrossJoinStrategy pushDownFilterExpressionEvaluationThroughCrossJoin = PushDownFilterThroughCrossJoinStrategy.REWRITTEN_TO_INNER_JOIN; @@ -654,6 +655,18 @@ public FeaturesConfig setFieldNamesInJsonCastEnabled(boolean fieldNamesInJsonCas return this; } + public boolean isLegacyJsonCast() + { + return legacyJsonCast; + } + + @Config("legacy-json-cast") + public FeaturesConfig setLegacyJsonCast(boolean legacyJsonCast) + { + this.legacyJsonCast = legacyJsonCast; + return this; + } + @Config("reduce-agg-for-complex-types-enabled") public FeaturesConfig setReduceAggForComplexTypesEnabled(boolean reduceAggForComplexTypesEnabled) { diff --git a/presto-main/src/main/java/com/facebook/presto/util/Failures.java b/presto-main/src/main/java/com/facebook/presto/util/Failures.java index de44e9fc3b7b2..0d8ece16f113e 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/Failures.java +++ b/presto-main/src/main/java/com/facebook/presto/util/Failures.java @@ -16,6 +16,7 @@ import com.facebook.presto.ExceededMemoryLimitException; import com.facebook.presto.client.ErrorLocation; import com.facebook.presto.common.ErrorCode; +import com.facebook.presto.common.InvalidTypeDefinitionException; import com.facebook.presto.execution.ExecutionFailureInfo; import com.facebook.presto.execution.Failure; import com.facebook.presto.spi.ErrorCause; @@ -40,6 +41,7 @@ import static com.facebook.presto.spi.ErrorCause.UNKNOWN; import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_TYPE_DEFINITION; import static com.facebook.presto.spi.StandardErrorCode.SLICE_TOO_LARGE; import static com.facebook.presto.spi.StandardErrorCode.SYNTAX_ERROR; import static com.google.common.base.Functions.toStringFunction; @@ -171,6 +173,10 @@ private static ErrorCode toErrorCode(Throwable throwable) return SLICE_TOO_LARGE.toErrorCode(); } + if (throwable instanceof InvalidTypeDefinitionException) { + return INVALID_TYPE_DEFINITION.toErrorCode(); + } + if (throwable instanceof PrestoException) { return ((PrestoException) throwable).getErrorCode(); } diff --git a/presto-main/src/main/java/com/facebook/presto/util/JsonUtil.java b/presto-main/src/main/java/com/facebook/presto/util/JsonUtil.java index edc9765cdd0c5..4b48dcbd8fff8 100644 --- a/presto-main/src/main/java/com/facebook/presto/util/JsonUtil.java +++ b/presto-main/src/main/java/com/facebook/presto/util/JsonUtil.java @@ -54,7 +54,6 @@ import java.util.Arrays; import java.util.HashMap; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Optional; import java.util.TreeMap; @@ -96,6 +95,7 @@ import static java.lang.String.format; import static java.math.RoundingMode.HALF_UP; import static java.nio.charset.StandardCharsets.UTF_8; +import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; public final class JsonUtil @@ -914,7 +914,7 @@ private static BigDecimal currentTokenAsJavaDecimal(JsonParser parser, int preci // given a JSON parser, write to the BlockBuilder public interface BlockBuilderAppender { - void append(JsonParser parser, BlockBuilder blockBuilder) + void append(JsonParser parser, BlockBuilder blockBuilder, SqlFunctionProperties sqlFunctionProperties) throws IOException; static BlockBuilderAppender createBlockBuilderAppender(Type type) @@ -955,7 +955,7 @@ static BlockBuilderAppender createBlockBuilderAppender(Type type) case StandardTypes.VARCHAR: return new VarcharBlockBuilderAppender(type); case StandardTypes.JSON: - return (parser, blockBuilder) -> { + return (parser, blockBuilder, sqlFunctionProperties) -> { String json = OBJECT_MAPPED_UNORDERED.writeValueAsString(parser.readValueAsTree()); JSON.writeSlice(blockBuilder, Slices.utf8Slice(json)); }; @@ -974,7 +974,7 @@ static BlockBuilderAppender createBlockBuilderAppender(Type type) for (int i = 0; i < fieldAppenders.length; i++) { fieldAppenders[i] = createBlockBuilderAppender(rowFields.get(i).getType()); } - return new RowBlockBuilderAppender(fieldAppenders, getFieldNameToIndex(rowFields)); + return new RowBlockBuilderAppender(fieldAppenders, rowType); default: throw new PrestoException(INVALID_FUNCTION_ARGUMENT, format("Unsupported type: %s", type)); } @@ -985,7 +985,7 @@ private static class BooleanBlockBuilderAppender implements BlockBuilderAppender { @Override - public void append(JsonParser parser, BlockBuilder blockBuilder) + public void append(JsonParser parser, BlockBuilder blockBuilder, SqlFunctionProperties sqlFunctionProperties) throws IOException { Boolean result = currentTokenAsBoolean(parser); @@ -1002,7 +1002,7 @@ private static class TinyintBlockBuilderAppender implements BlockBuilderAppender { @Override - public void append(JsonParser parser, BlockBuilder blockBuilder) + public void append(JsonParser parser, BlockBuilder blockBuilder, SqlFunctionProperties sqlFunctionProperties) throws IOException { Long result = currentTokenAsTinyint(parser); @@ -1019,7 +1019,7 @@ private static class SmallintBlockBuilderAppender implements BlockBuilderAppender { @Override - public void append(JsonParser parser, BlockBuilder blockBuilder) + public void append(JsonParser parser, BlockBuilder blockBuilder, SqlFunctionProperties sqlFunctionProperties) throws IOException { Long result = currentTokenAsInteger(parser); @@ -1036,7 +1036,7 @@ private static class IntegerBlockBuilderAppender implements BlockBuilderAppender { @Override - public void append(JsonParser parser, BlockBuilder blockBuilder) + public void append(JsonParser parser, BlockBuilder blockBuilder, SqlFunctionProperties sqlFunctionProperties) throws IOException { Long result = currentTokenAsInteger(parser); @@ -1053,7 +1053,7 @@ private static class BigintBlockBuilderAppender implements BlockBuilderAppender { @Override - public void append(JsonParser parser, BlockBuilder blockBuilder) + public void append(JsonParser parser, BlockBuilder blockBuilder, SqlFunctionProperties sqlFunctionProperties) throws IOException { Long result = currentTokenAsBigint(parser); @@ -1070,7 +1070,7 @@ private static class RealBlockBuilderAppender implements BlockBuilderAppender { @Override - public void append(JsonParser parser, BlockBuilder blockBuilder) + public void append(JsonParser parser, BlockBuilder blockBuilder, SqlFunctionProperties sqlFunctionProperties) throws IOException { Long result = currentTokenAsReal(parser); @@ -1087,7 +1087,7 @@ private static class DoubleBlockBuilderAppender implements BlockBuilderAppender { @Override - public void append(JsonParser parser, BlockBuilder blockBuilder) + public void append(JsonParser parser, BlockBuilder blockBuilder, SqlFunctionProperties sqlFunctionProperties) throws IOException { Double result = currentTokenAsDouble(parser); @@ -1111,7 +1111,7 @@ private static class ShortDecimalBlockBuilderAppender } @Override - public void append(JsonParser parser, BlockBuilder blockBuilder) + public void append(JsonParser parser, BlockBuilder blockBuilder, SqlFunctionProperties sqlFunctionProperties) throws IOException { Long result = currentTokenAsShortDecimal(parser, type.getPrecision(), type.getScale()); @@ -1136,7 +1136,7 @@ private static class LongDecimalBlockBuilderAppender } @Override - public void append(JsonParser parser, BlockBuilder blockBuilder) + public void append(JsonParser parser, BlockBuilder blockBuilder, SqlFunctionProperties sqlFunctionProperties) throws IOException { Slice result = currentTokenAsLongDecimal(parser, type.getPrecision(), type.getScale()); @@ -1161,7 +1161,7 @@ private static class VarcharBlockBuilderAppender } @Override - public void append(JsonParser parser, BlockBuilder blockBuilder) + public void append(JsonParser parser, BlockBuilder blockBuilder, SqlFunctionProperties sqlFunctionProperties) throws IOException { Slice result = currentTokenAsVarchar(parser); @@ -1185,7 +1185,7 @@ private static class ArrayBlockBuilderAppender } @Override - public void append(JsonParser parser, BlockBuilder blockBuilder) + public void append(JsonParser parser, BlockBuilder blockBuilder, SqlFunctionProperties sqlFunctionProperties) throws IOException { if (parser.getCurrentToken() == JsonToken.VALUE_NULL) { @@ -1198,7 +1198,7 @@ public void append(JsonParser parser, BlockBuilder blockBuilder) } BlockBuilder entryBuilder = blockBuilder.beginBlockEntry(); while (parser.nextToken() != END_ARRAY) { - elementAppender.append(parser, entryBuilder); + elementAppender.append(parser, entryBuilder, sqlFunctionProperties); } blockBuilder.closeEntry(); } @@ -1219,7 +1219,7 @@ private static class MapBlockBuilderAppender } @Override - public void append(JsonParser parser, BlockBuilder blockBuilder) + public void append(JsonParser parser, BlockBuilder blockBuilder, SqlFunctionProperties sqlFunctionProperties) throws IOException { if (parser.getCurrentToken() == JsonToken.VALUE_NULL) { @@ -1234,9 +1234,9 @@ public void append(JsonParser parser, BlockBuilder blockBuilder) HashTable entryBuilderHashTable = new HashTable(keyType, entryBuilder); int position = 0; while (parser.nextToken() != END_OBJECT) { - keyAppender.append(parser, entryBuilder); + keyAppender.append(parser, entryBuilder, sqlFunctionProperties); parser.nextToken(); - valueAppender.append(parser, entryBuilder); + valueAppender.append(parser, entryBuilder, sqlFunctionProperties); if (!entryBuilderHashTable.addIfAbsent(position)) { throw new JsonCastException("Duplicate keys are not allowed"); } @@ -1250,16 +1250,18 @@ private static class RowBlockBuilderAppender implements BlockBuilderAppender { final BlockBuilderAppender[] fieldAppenders; - final Optional> fieldNameToIndex; + final RowType rowType; - RowBlockBuilderAppender(BlockBuilderAppender[] fieldAppenders, Optional> fieldNameToIndex) + RowBlockBuilderAppender( + BlockBuilderAppender[] fieldAppenders, + RowType rowType) { this.fieldAppenders = fieldAppenders; - this.fieldNameToIndex = fieldNameToIndex; + this.rowType = rowType; } @Override - public void append(JsonParser parser, BlockBuilder blockBuilder) + public void append(JsonParser parser, BlockBuilder blockBuilder, SqlFunctionProperties sqlFunctionProperties) throws IOException { if (parser.getCurrentToken() == JsonToken.VALUE_NULL) { @@ -1275,20 +1277,21 @@ public void append(JsonParser parser, BlockBuilder blockBuilder) parser, (SingleRowBlockWriter) blockBuilder.beginBlockEntry(), fieldAppenders, - fieldNameToIndex); + rowType, + sqlFunctionProperties); blockBuilder.closeEntry(); } } - public static Optional> getFieldNameToIndex(List rowFields) + public static Optional> getFieldToIndex(List rowFields) { if (!rowFields.get(0).getName().isPresent()) { return Optional.empty(); } - Map fieldNameToIndex = new HashMap<>(rowFields.size()); + Map fieldNameToIndex = new HashMap<>(rowFields.size()); for (int i = 0; i < rowFields.size(); i++) { - fieldNameToIndex.put(rowFields.get(i).getName().get(), i); + fieldNameToIndex.put(rowFields.get(i), i); } return Optional.of(fieldNameToIndex); } @@ -1300,13 +1303,14 @@ public static void parseJsonToSingleRowBlock( JsonParser parser, SingleRowBlockWriter singleRowBlockWriter, BlockBuilderAppender[] fieldAppenders, - Optional> fieldNameToIndex) + RowType rowType, + SqlFunctionProperties sqlFunctionProperties) throws IOException { if (parser.getCurrentToken() == START_ARRAY) { for (int i = 0; i < fieldAppenders.length; i++) { parser.nextToken(); - fieldAppenders[i].append(parser, singleRowBlockWriter); + fieldAppenders[i].append(parser, singleRowBlockWriter, sqlFunctionProperties); } if (parser.nextToken() != JsonToken.END_ARRAY) { throw new JsonCastException(format("Expected json array ending, but got %s", parser.getText())); @@ -1314,18 +1318,47 @@ public static void parseJsonToSingleRowBlock( } else { verify(parser.getCurrentToken() == START_OBJECT); - if (!fieldNameToIndex.isPresent()) { + Optional> fieldToIndex = getFieldToIndex(rowType.getFields()); + if (!fieldToIndex.isPresent()) { throw new JsonCastException("Cannot cast a JSON object to anonymous row type. Input must be a JSON array."); } boolean[] fieldWritten = new boolean[fieldAppenders.length]; int numFieldsWritten = 0; + Map caseSensitiveWhenMatching = new HashMap<>(); + Map caseInsensitiveWhenMatching = new HashMap<>(); + if (sqlFunctionProperties.isLegacyJsonCast()) { + fieldToIndex.get().entrySet().stream() + .forEach(entry -> caseInsensitiveWhenMatching.put( + entry.getKey().getName().get().toLowerCase(ENGLISH), + entry.getValue())); + } + else { + fieldToIndex.get().entrySet().stream() + .forEach(entry -> { + if (entry.getKey().isDelimited()) { + caseSensitiveWhenMatching.put( + entry.getKey().getName().get(), + entry.getValue()); + } + else { + caseInsensitiveWhenMatching.put( + entry.getKey().getName().get().toLowerCase(ENGLISH), + entry.getValue()); + } + }); + } while (parser.nextToken() != JsonToken.END_OBJECT) { if (parser.currentToken() != FIELD_NAME) { throw new JsonCastException(format("Expected a json field name, but got %s", parser.getText())); } - String fieldName = parser.getText().toLowerCase(Locale.ENGLISH); - Integer fieldIndex = fieldNameToIndex.get().get(fieldName); + + String fieldName = parser.getText(); + Integer fieldIndex = caseSensitiveWhenMatching.get(fieldName); + if (fieldIndex == null) { + fieldIndex = caseInsensitiveWhenMatching.get(fieldName.toLowerCase(ENGLISH)); + } + parser.nextToken(); if (fieldIndex != null) { if (fieldWritten[fieldIndex]) { @@ -1333,7 +1366,9 @@ public static void parseJsonToSingleRowBlock( } fieldWritten[fieldIndex] = true; numFieldsWritten++; - fieldAppenders[fieldIndex].append(parser, singleRowBlockWriter.getFieldBlockBuilder(fieldIndex)); + fieldAppenders[fieldIndex].append(parser, + singleRowBlockWriter.getFieldBlockBuilder(fieldIndex), + sqlFunctionProperties); } else { parser.skipChildren(); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java index d25bde5ba8f73..ba4db0e159ae7 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/AbstractTestFunctions.java @@ -139,13 +139,6 @@ protected void assertDecimalFunction(String statement, SqlDecimal expectedResult expectedResult); } - // this is not safe as it catches all RuntimeExceptions - @Deprecated - protected void assertInvalidFunction(String projection) - { - functionAssertions.assertInvalidFunction(projection); - } - protected void assertInvalidFunction(String projection, StandardErrorCode errorCode, String messagePattern) { functionAssertions.assertInvalidFunction(projection, errorCode, messagePattern); @@ -181,6 +174,11 @@ protected void assertNumericOverflow(String projection, String message) functionAssertions.assertNumericOverflow(projection, message); } + protected void assertInvalidTypeDefinition(String projection, String message) + { + functionAssertions.assertInvalidTypeDefinition(projection, message); + } + protected void assertInvalidCast(String projection) { functionAssertions.assertInvalidCast(projection); diff --git a/presto-main/src/test/java/com/facebook/presto/operator/scalar/FunctionAssertions.java b/presto-main/src/test/java/com/facebook/presto/operator/scalar/FunctionAssertions.java index bf4a2ed33fe91..36f27421f0660 100644 --- a/presto-main/src/test/java/com/facebook/presto/operator/scalar/FunctionAssertions.java +++ b/presto-main/src/test/java/com/facebook/presto/operator/scalar/FunctionAssertions.java @@ -14,6 +14,7 @@ package com.facebook.presto.operator.scalar; import com.facebook.presto.Session; +import com.facebook.presto.common.InvalidTypeDefinitionException; import com.facebook.presto.common.Page; import com.facebook.presto.common.PageBuilder; import com.facebook.presto.common.Utils; @@ -133,6 +134,7 @@ import static com.facebook.presto.memory.context.AggregatedMemoryContext.newSimpleAggregatedMemoryContext; import static com.facebook.presto.spi.StandardErrorCode.INVALID_CAST_ARGUMENT; import static com.facebook.presto.spi.StandardErrorCode.INVALID_FUNCTION_ARGUMENT; +import static com.facebook.presto.spi.StandardErrorCode.INVALID_TYPE_DEFINITION; import static com.facebook.presto.spi.StandardErrorCode.NUMERIC_VALUE_OUT_OF_RANGE; import static com.facebook.presto.spi.schedule.NodeSelectionStrategy.HARD_AFFINITY; import static com.facebook.presto.sql.ExpressionUtils.rewriteIdentifiersToSymbolReferences; @@ -143,6 +145,7 @@ import static com.facebook.presto.sql.relational.SqlToRowExpressionTranslator.translate; import static com.facebook.presto.testing.TestingTaskContext.createTaskContext; import static com.facebook.presto.util.AnalyzerUtil.createParsingOptions; +import static com.facebook.presto.util.Failures.toFailure; import static io.airlift.slice.SizeOf.sizeOf; import static io.airlift.units.DataSize.Unit.BYTE; import static java.lang.String.format; @@ -221,12 +224,22 @@ public FunctionAssertions(Session session) public FunctionAssertions(Session session, FeaturesConfig featuresConfig) { - this.session = requireNonNull(session, "session is null"); + this(session, featuresConfig, false); + } + + public FunctionAssertions(Session session, FeaturesConfig featuresConfig, boolean refreshSession) + { + requireNonNull(session, "session is null"); runner = new LocalQueryRunner(session, featuresConfig); + if (refreshSession) { + this.session = runner.getDefaultSession(); + } + else { + this.session = session; + } metadata = runner.getMetadata(); compiler = runner.getExpressionCompiler(); } - public FunctionAndTypeManager getFunctionAndTypeManager() { return runner.getFunctionAndTypeManager(); @@ -341,19 +354,6 @@ private Object selectUniqueValue(String projection, Type expectedType, Session s return Iterables.getOnlyElement(resultSet); } - // this is not safe as it catches all RuntimeExceptions - @Deprecated - public void assertInvalidFunction(String projection) - { - try { - Object value = evaluateInvalid(projection); - fail(format("Expected to throw but got %s", value)); - } - catch (RuntimeException e) { - // Expected - } - } - public void assertInvalidFunction(String projection, StandardErrorCode errorCode, String messagePattern) { try { @@ -412,6 +412,24 @@ public void assertInvalidFunction(String projection, SemanticErrorCode expectedE } } + public void assertInvalidTypeDefinition(String projection, String message) + { + try { + Object value = evaluateInvalid(projection); + fail("Expected to throw an INVALID_CAST_ARGUMENT exception, but got " + value); + } + catch (InvalidTypeDefinitionException e) { + try { + assertEquals(toFailure(e).getErrorCode(), INVALID_TYPE_DEFINITION.toErrorCode()); + assertEquals(e.getMessage(), message); + } + catch (Throwable failure) { + failure.addSuppressed(e); + throw failure; + } + } + } + public void assertInvalidFunction(String projection, ErrorCodeSupplier expectedErrorCode) { try { diff --git a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java index 1f88b6d878cc6..e6c195ed571b4 100644 --- a/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java +++ b/presto-main/src/test/java/com/facebook/presto/sql/analyzer/TestFeaturesConfig.java @@ -270,7 +270,8 @@ public void testDefaults() .setDefaultWriterReplicationCoefficient(3.0) .setDefaultViewSecurityMode(DEFINER) .setCteHeuristicReplicationThreshold(4) - .setUseHistograms(false)); + .setUseHistograms(false) + .setLegacyJsonCast(false)); } @Test @@ -454,6 +455,7 @@ public void testExplicitPropertyMappings() .put("optimizer.push-aggregation-below-join-byte-reduction-threshold", "0.9") .put("optimizer.prefilter-for-groupby-limit", "true") .put("field-names-in-json-cast-enabled", "true") + .put("legacy-json-cast", "true") .put("optimizer.optimize-probe-for-empty-build-runtime", "true") .put("optimizer.use-defaults-for-correlated-aggregation-pushdown-through-outer-joins", "false") .put("optimizer.merge-duplicate-aggregations", "false") @@ -696,7 +698,8 @@ public void testExplicitPropertyMappings() .setDefaultWriterReplicationCoefficient(5.0) .setDefaultViewSecurityMode(INVOKER) .setCteHeuristicReplicationThreshold(2) - .setUseHistograms(true); + .setUseHistograms(true) + .setLegacyJsonCast(true); assertFullMapping(properties, expected); } diff --git a/presto-main/src/test/java/com/facebook/presto/type/TestRowOperators.java b/presto-main/src/test/java/com/facebook/presto/type/TestRowOperators.java index 1db892b85df03..c62607bf2159d 100644 --- a/presto-main/src/test/java/com/facebook/presto/type/TestRowOperators.java +++ b/presto-main/src/test/java/com/facebook/presto/type/TestRowOperators.java @@ -19,6 +19,7 @@ import com.facebook.presto.common.type.RowType; import com.facebook.presto.common.type.StandardTypes; import com.facebook.presto.common.type.Type; +import com.facebook.presto.common.type.VarcharType; import com.facebook.presto.operator.scalar.AbstractTestFunctions; import com.facebook.presto.operator.scalar.FunctionAssertions; import com.facebook.presto.spi.StandardErrorCode; @@ -74,6 +75,7 @@ public class TestRowOperators { private static FunctionAssertions legacyRowFieldOrdinalAccess; private static FunctionAssertions fieldNameInJsonCastEnabled; + private static FunctionAssertions legacyJsonCastEnabled; @BeforeClass public void setUp() @@ -89,6 +91,10 @@ public void setUp() .setSystemProperty(FIELD_NAMES_IN_JSON_CAST_ENABLED, "true") .build(), new FeaturesConfig()); + + FeaturesConfig featuresConfig = new FeaturesConfig() + .setLegacyJsonCast(true); + legacyJsonCastEnabled = new FunctionAssertions(session, featuresConfig, true); } @AfterClass(alwaysRun = true) @@ -98,6 +104,8 @@ public final void tearDown() legacyRowFieldOrdinalAccess = null; fieldNameInJsonCastEnabled.close(); fieldNameInJsonCastEnabled = null; + legacyJsonCastEnabled.close(); + legacyJsonCastEnabled = null; } @ScalarFunction @@ -404,7 +412,8 @@ public void testJsonToRow() asList(1L, 2L, null, 3L), null, asList(1L, 2L, 3L, null), null)); - assertFunction("CAST(JSON '{" + + // legacy json cast would not reserve the case of field name in json when casting to row type + legacyJsonCastEnabled.assertFunction("CAST(JSON '{" + "\"array2\": [1, 2, null, 3], " + "\"array1\": [], " + "\"array3\": null, " + @@ -440,7 +449,89 @@ public void testJsonToRow() asList(1L, 2L, null, 3L), null, null, asList(1L, 2L, 3L, null))); + // without legacy json cast, we would reserve the case of field name in json when casting to row type + assertFunction("CAST(JSON '{" + + "\"array2\": [1, 2, null, 3], " + + "\"array1\": [], " + + "\"array3\": null, " + + "\"map3\": {\"a\": 1, \"b\": 2, \"none\": null, \"Three\": 3}, " + + "\"map1\": {}, " + + "\"map2\": null, " + + "\"rowAsJsonArray1\": [1, 2, null, 3], " + + "\"rowAsJsonArray2\": null, " + + "\"rowAsJsonObject2\": {\"a\": 1, \"b\": 2, \"none\": null, \"Three\": 3}, " + + "\"rowAsJsonObject1\": null}' " + + "AS ROW(array1 array, array2 ARRAY, array3 ARRAY, " + + "map1 MAP, map2 map, map3 MAP, " + + "\"rowAsJsonArray1\" row(BIGINT, bigint, BIGINT, BIGINT), \"rowAsJsonArray2\" ROW(BIGINT)," + + "\"rowAsJsonObject1\" ROW(nothing BIGINT), \"rowAsJsonObject2\" ROW(a BIGINT, b BIGINT, \"Three\" BIGINT, none BIGINT)))", + RowType.from(ImmutableList.of( + RowType.field("array1", new ArrayType(BIGINT)), + RowType.field("array2", new ArrayType(BIGINT)), + RowType.field("array3", new ArrayType(BIGINT)), + RowType.field("map1", mapType(VARCHAR, BIGINT)), + RowType.field("map2", mapType(VARCHAR, BIGINT)), + RowType.field("map3", mapType(VARCHAR, BIGINT)), + RowType.field("rowAsJsonArray1", RowType.anonymous(ImmutableList.of(BIGINT, BIGINT, BIGINT, BIGINT)), true), + RowType.field("rowAsJsonArray2", RowType.anonymous(ImmutableList.of(BIGINT)), true), + RowType.field("rowAsJsonObject1", RowType.from(ImmutableList.of(RowType.field("nothing", BIGINT))), true), + RowType.field("rowAsJsonObject2", RowType.from(ImmutableList.of( + RowType.field("a", BIGINT), + RowType.field("b", BIGINT), + RowType.field("Three", BIGINT, true), + RowType.field("none", BIGINT))), true))), + asList( + emptyList(), asList(1L, 2L, null, 3L), null, + ImmutableMap.of(), null, asMap(ImmutableList.of("a", "b", "none", "Three"), asList(1L, 2L, null, 3L)), + asList(1L, 2L, null, 3L), null, + null, asList(1L, 2L, 3L, null))); + + // legacy json cast would not reserve the case of field name in json when casting to row type + legacyJsonCastEnabled.assertFunction("CAST(json_extract('{\"1\":[{\"name\": \"John\", \"AGE\": 30}]}', '$') AS MAP>)", + mapType(BIGINT, new ArrayType(RowType.from(ImmutableList.of( + RowType.field("name", VARCHAR), + RowType.field("age", VARCHAR))))), + asMap(ImmutableList.of(1L), asList(asList(asList("John", "30"))))); + + // without legacy json cast, we would reserve the case of field name in json when casting to row type + assertFunction("CAST(json_extract('{\"1\":[{\"name\": \"John\", \"AGE\": 30}]}', '$') AS MAP>)", + mapType(BIGINT, new ArrayType(RowType.from(ImmutableList.of( + RowType.field("name", VARCHAR), + RowType.field("AGE", VARCHAR, true))))), + asMap(ImmutableList.of(1L), asList(asList(asList("John", "30"))))); + + assertFunction("CAST(json_extract('{\"1\":[{\"name\": \"John\", \"AGE\": 30}]}', '$') AS MAP>)", + mapType(BIGINT, new ArrayType(RowType.from(ImmutableList.of( + RowType.field("name", VARCHAR), + RowType.field("age", VARCHAR, true))))), + asMap(ImmutableList.of(1L), asList(asList(asList("John", null))))); + + assertFunction("CAST(json_extract('{\"1\":[{\"name\": \"John\", \"AGE\": 30}]}', '$') AS MAP>)", + mapType(BIGINT, new ArrayType(RowType.from(ImmutableList.of( + RowType.field("name", VARCHAR), + RowType.field("age", VARCHAR))))), + asMap(ImmutableList.of(1L), asList(asList(asList("John", "30"))))); + + assertFunction("CAST(json_extract('{\"1\":[{\"key1\": \"John\", \"KEY1\": \"Johnny\"}]}', '$') AS MAP>)", + mapType(BIGINT, new ArrayType(RowType.from(ImmutableList.of( + RowType.field("key1", VARCHAR, true), + RowType.field("KEY1", VARCHAR, true))))), + asMap(ImmutableList.of(1L), asList(asList(asList("John", "Johnny"))))); + + assertFunction("CAST(json_extract('{\"1\":[{\"key1\": \"John\", \"KEY1\": \"Johnny\"}]}', '$') AS MAP>)", + mapType(BIGINT, new ArrayType(RowType.from(ImmutableList.of( + RowType.field("key1", VARCHAR), + RowType.field("KEY1", VARCHAR, true))))), + asMap(ImmutableList.of(1L), asList(asList(asList("John", "Johnny"))))); + + // invalid type definition + assertInvalidTypeDefinition("CAST(json_extract('{\"1\":[{\"key1\": \"John\", \"KEY1\": \"Johnny\"}]}', '$') AS MAP>)", + "Duplicate field: key1"); + // invalid cast + assertInvalidCast("CAST(json_extract('{\"1\":[{\"key1\": \"John\", \"KEY1\":\"Johnny\"}]}', '$') AS MAP>)", + "Cannot cast to map(bigint,array(row(key1 varchar))). Duplicate field: KEY1\n" + + "{\"1\":[{\"key1\":\"John\",\"KEY1\":\"Johnny\"}]}"); assertInvalidCast("CAST(unchecked_to_json('{\"a\":1,\"b\":2,\"a\":3}') AS ROW(a BIGINT, b BIGINT))", "Cannot cast to row(a bigint,b bigint). Duplicate field: a\n{\"a\":1,\"b\":2,\"a\":3}"); assertInvalidCast("CAST(unchecked_to_json('[{\"a\":1,\"b\":2,\"a\":3}]') AS ARRAY)", "Cannot cast to array(row(a bigint,b bigint)). Duplicate field: a\n[{\"a\":1,\"b\":2,\"a\":3}]"); } @@ -503,7 +594,11 @@ public void testRowCast() assertFunction("cast(row(1,null,3) as row(aa bigint, bb boolean, cc boolean)).aa", BIGINT, 1L); assertFunction("cast(row(null,null,null) as row(aa bigint, bb boolean, cc boolean)).aa", BIGINT, null); - assertInvalidFunction("CAST(ROW(1, 2) AS ROW(a BIGINT, A DOUBLE)).a"); + // invalid type definition + assertInvalidTypeDefinition("CAST(ROW(1, 2) AS ROW(a BIGINT, A DOUBLE)).a", "Duplicate field: a"); + assertInvalidTypeDefinition("CAST(ROW(1, 2) AS ROW(KEY1 VARCHAR, \"key1\" VARCHAR))", "Duplicate field: key1"); + assertInvalidTypeDefinition("TYPEOF(CAST(row(1, 2) AS ROW(KEY1 VARCHAR, \"key1\" VARCHAR)))", "Duplicate field: key1"); + assertInvalidTypeDefinition("CAST(ROW(1, 2) AS ROW(KEY1 VARCHAR, \"key1\" VARCHAR)).key1", "Duplicate field: key1"); // there are totally 7 field names String longFieldNameCast = "CAST(row(1.2E0, ARRAY[row(233, 6.9E0)], row(1000, 6.3E0)) AS ROW(%s VARCHAR, %s ARRAY(ROW(%s VARCHAR, %s VARCHAR)), %s ROW(%s VARCHAR, %s VARCHAR))).%s[1].%s"; @@ -525,6 +620,14 @@ public void testRowCast() RowType.field("d", VARCHAR), RowType.field("e", new ArrayType(BIGINT)))), asList(2L, 1.5, true, "abc", ImmutableList.of(1L, 2L))); + + assertFunction( + "MAP(ARRAY['myFirstRow', 'mySecondRow'], ARRAY[cast(row('row1FieldValue1', 'row1FieldValue2') as row(\"firstField\" varchar, \"secondField\" varchar)), cast(row('row2FieldValue1', 'row2FieldValue2') as row(\"firstField\" varchar, \"secondField\" varchar))])", + mapType(VarcharType.createVarcharType(11), RowType.from(ImmutableList.of( + RowType.field("firstField", VARCHAR, true), + RowType.field("secondField", VARCHAR, true)))), + ImmutableMap.of("myFirstRow", asList("row1FieldValue1", "row1FieldValue2"), + "mySecondRow", asList("row2FieldValue1", "row2FieldValue2"))); } @Test diff --git a/presto-parser/src/main/java/com/facebook/presto/sql/tree/Cast.java b/presto-parser/src/main/java/com/facebook/presto/sql/tree/Cast.java index cc9bd303be275..312474f4840ea 100644 --- a/presto-parser/src/main/java/com/facebook/presto/sql/tree/Cast.java +++ b/presto-parser/src/main/java/com/facebook/presto/sql/tree/Cast.java @@ -19,7 +19,6 @@ import java.util.Objects; import java.util.Optional; -import static java.util.Locale.ENGLISH; import static java.util.Objects.requireNonNull; public final class Cast @@ -67,7 +66,7 @@ public Cast(Optional location, Expression expression, String type, requireNonNull(type, "type is null"); this.expression = expression; - this.type = type.toLowerCase(ENGLISH); + this.type = transformCase(type); this.safe = safe; this.typeOnly = typeOnly; } @@ -92,6 +91,34 @@ public boolean isTypeOnly() return typeOnly; } + /** + * We uniformly change the whole type string to lower case, + * except field names which are enclosed in double quotation marks + * */ + private static String transformCase(String input) + { + if (input == null) { + return null; + } + + StringBuilder sb = new StringBuilder(); + boolean insideQuotes = false; + for (int i = 0, l = input.length(); i < l; i++) { + char ch = input.charAt(i); + if (ch == '"') { + insideQuotes = !insideQuotes; + } + + if (insideQuotes) { + sb.append(ch); + } + else { + sb.append(Character.toLowerCase(ch)); + } + } + return sb.toString(); + } + @Override public R accept(AstVisitor visitor, C context) { diff --git a/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java b/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java index 2aef128032b8b..17169181ca3dc 100644 --- a/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java +++ b/presto-spi/src/main/java/com/facebook/presto/spi/StandardErrorCode.java @@ -70,6 +70,7 @@ public enum StandardErrorCode WARNING_AS_ERROR(0x0000_002C, USER_ERROR), INVALID_ARGUMENTS(0x0000_002D, USER_ERROR), EXCEEDED_PLAN_NODE_LIMIT(0x0000_002E, USER_ERROR), + INVALID_TYPE_DEFINITION(0x0000_002F, USER_ERROR), GENERIC_INTERNAL_ERROR(0x0001_0000, INTERNAL_ERROR), TOO_MANY_REQUESTS_FAILED(0x0001_0001, INTERNAL_ERROR, true),