diff --git a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs index c80c0b4bf54b..ca01a0657988 100644 --- a/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs +++ b/datafusion/core/tests/custom_sources_cases/provider_filter_pushdown.rs @@ -183,7 +183,7 @@ impl TableProvider for CustomProvider { Expr::Literal(ScalarValue::Int16(Some(i)), _) => *i as i64, Expr::Literal(ScalarValue::Int32(Some(i)), _) => *i as i64, Expr::Literal(ScalarValue::Int64(Some(i)), _) => *i, - Expr::Cast(Cast { expr, data_type: _ }) => match expr.deref() { + Expr::Cast(Cast { expr, field: _ }) => match expr.deref() { Expr::Literal(lit_value, _) => match lit_value { ScalarValue::Int8(Some(v)) => *v as i64, ScalarValue::Int16(Some(v)) => *v as i64, diff --git a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs index fb1371da6ceb..3ead4951c34c 100644 --- a/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs +++ b/datafusion/core/tests/user_defined/user_defined_scalar_functions.rs @@ -712,10 +712,7 @@ impl ScalarUDFImpl for CastToI64UDF { arg } else { // need to use an actual cast to get the correct type - Expr::Cast(datafusion_expr::Cast { - expr: Box::new(arg), - data_type: DataType::Int64, - }) + Expr::Cast(datafusion_expr::Cast::new(Box::new(arg), DataType::Int64)) }; // return the newly written argument to DataFusion Ok(ExprSimplifyResult::Simplified(new_expr)) diff --git a/datafusion/expr/src/expr.rs b/datafusion/expr/src/expr.rs index e1115b714053..1e9418ec659f 100644 --- a/datafusion/expr/src/expr.rs +++ b/datafusion/expr/src/expr.rs @@ -32,6 +32,8 @@ use crate::{ExprSchemable, Operator, Signature, WindowFrame, WindowUDF}; use arrow::datatypes::{DataType, Field, FieldRef}; use datafusion_common::cse::{HashNode, NormalizeEq, Normalizeable}; +use datafusion_common::datatype::DataTypeExt; +use datafusion_common::metadata::format_type_and_metadata; use datafusion_common::tree_node::{ Transformed, TransformedResult, TreeNode, TreeNodeContainer, TreeNodeRecursion, }; @@ -786,13 +788,20 @@ pub struct Cast { /// The expression being cast pub expr: Box, /// The `DataType` the expression will yield - pub data_type: DataType, + pub field: FieldRef, } impl Cast { /// Create a new Cast expression pub fn new(expr: Box, data_type: DataType) -> Self { - Self { expr, data_type } + Self { + expr, + field: data_type.into_nullable_field_ref(), + } + } + + pub fn new_from_field(expr: Box, field: FieldRef) -> Self { + Self { expr, field } } } @@ -802,13 +811,20 @@ pub struct TryCast { /// The expression being cast pub expr: Box, /// The `DataType` the expression will yield - pub data_type: DataType, + pub field: FieldRef, } impl TryCast { /// Create a new TryCast expression pub fn new(expr: Box, data_type: DataType) -> Self { - Self { expr, data_type } + Self { + expr, + field: data_type.into_nullable_field_ref(), + } + } + + pub fn new_from_field(expr: Box, field: FieldRef) -> Self { + Self { expr, field } } } @@ -2243,23 +2259,23 @@ impl NormalizeEq for Expr { ( Expr::Cast(Cast { expr: self_expr, - data_type: self_data_type, + field: self_field, }), Expr::Cast(Cast { expr: other_expr, - data_type: other_data_type, + field: other_field, }), ) | ( Expr::TryCast(TryCast { expr: self_expr, - data_type: self_data_type, + field: self_field, }), Expr::TryCast(TryCast { expr: other_expr, - data_type: other_data_type, + field: other_field, }), - ) => self_data_type == other_data_type && self_expr.normalize_eq(other_expr), + ) => self_field == other_field && self_expr.normalize_eq(other_expr), ( Expr::ScalarFunction(ScalarFunction { func: self_func, @@ -2575,15 +2591,9 @@ impl HashNode for Expr { when_then_expr: _when_then_expr, else_expr: _else_expr, }) => {} - Expr::Cast(Cast { - expr: _expr, - data_type, - }) - | Expr::TryCast(TryCast { - expr: _expr, - data_type, - }) => { - data_type.hash(state); + Expr::Cast(Cast { expr: _expr, field }) + | Expr::TryCast(TryCast { expr: _expr, field }) => { + field.hash(state); } Expr::ScalarFunction(ScalarFunction { func, args: _args }) => { func.hash(state); @@ -3274,11 +3284,15 @@ impl Display for Expr { } write!(f, "END") } - Expr::Cast(Cast { expr, data_type }) => { - write!(f, "CAST({expr} AS {data_type})") + Expr::Cast(Cast { expr, field }) => { + let formatted = + format_type_and_metadata(field.data_type(), Some(field.metadata())); + write!(f, "CAST({expr} AS {formatted})") } - Expr::TryCast(TryCast { expr, data_type }) => { - write!(f, "TRY_CAST({expr} AS {data_type})") + Expr::TryCast(TryCast { expr, field }) => { + let formatted = + format_type_and_metadata(field.data_type(), Some(field.metadata())); + write!(f, "TRY_CAST({expr} AS {formatted})") } Expr::Not(expr) => write!(f, "NOT {expr}"), Expr::Negative(expr) => write!(f, "(- {expr})"), @@ -3664,7 +3678,7 @@ mod test { fn format_cast() -> Result<()> { let expr = Expr::Cast(Cast { expr: Box::new(Expr::Literal(ScalarValue::Float32(Some(1.23)), None)), - data_type: DataType::Utf8, + field: DataType::Utf8.into_nullable_field_ref(), }); let expected_canonical = "CAST(Float32(1.23) AS Utf8)"; assert_eq!(expected_canonical, format!("{expr}")); diff --git a/datafusion/expr/src/expr_rewriter/order_by.rs b/datafusion/expr/src/expr_rewriter/order_by.rs index 6db95555502d..3b54f1b46c95 100644 --- a/datafusion/expr/src/expr_rewriter/order_by.rs +++ b/datafusion/expr/src/expr_rewriter/order_by.rs @@ -116,13 +116,13 @@ fn rewrite_in_terms_of_projection( if let Some(found) = found { return Ok(Transformed::yes(match normalized_expr { - Expr::Cast(Cast { expr: _, data_type }) => Expr::Cast(Cast { + Expr::Cast(Cast { expr: _, field }) => Expr::Cast(Cast { expr: Box::new(found), - data_type, + field, }), - Expr::TryCast(TryCast { expr: _, data_type }) => Expr::TryCast(TryCast { + Expr::TryCast(TryCast { expr: _, field }) => Expr::TryCast(TryCast { expr: Box::new(found), - data_type, + field, }), _ => found, })); diff --git a/datafusion/expr/src/expr_schema.rs b/datafusion/expr/src/expr_schema.rs index 8c557a5630f0..0f41c6967548 100644 --- a/datafusion/expr/src/expr_schema.rs +++ b/datafusion/expr/src/expr_schema.rs @@ -127,8 +127,9 @@ impl ExprSchemable for Expr { .as_ref() .map_or(Ok(DataType::Null), |e| e.get_type(schema)) } - Expr::Cast(Cast { data_type, .. }) - | Expr::TryCast(TryCast { data_type, .. }) => Ok(data_type.clone()), + Expr::Cast(Cast { field, .. }) | Expr::TryCast(TryCast { field, .. }) => { + Ok(field.data_type().clone()) + } Expr::Unnest(Unnest { expr }) => { let arg_data_type = expr.get_type(schema)?; // Unnest's output type is the inner type of the list @@ -579,9 +580,17 @@ impl ExprSchemable for Expr { func.return_field_from_args(args) } // _ => Ok((self.get_type(schema)?, self.nullable(schema)?)), - Expr::Cast(Cast { expr, data_type }) => expr + Expr::Cast(Cast { expr, field }) => expr .to_field(schema) - .map(|(_, f)| f.as_ref().clone().with_data_type(data_type.clone())) + .map(|(_, f)| { + // This currently propagates the nullability of the input + // expression as the resulting physical expression does + // not currently consider the nullability specified here + f.as_ref() + .clone() + .with_data_type(field.data_type().clone()) + .with_metadata(f.metadata().clone()) + }) .map(Arc::new), Expr::Placeholder(Placeholder { id: _, diff --git a/datafusion/expr/src/tree_node.rs b/datafusion/expr/src/tree_node.rs index 81846b4f8060..2fbd57a11ecb 100644 --- a/datafusion/expr/src/tree_node.rs +++ b/datafusion/expr/src/tree_node.rs @@ -220,12 +220,12 @@ impl TreeNode for Expr { .update_data(|(new_expr, new_when_then_expr, new_else_expr)| { Expr::Case(Case::new(new_expr, new_when_then_expr, new_else_expr)) }), - Expr::Cast(Cast { expr, data_type }) => expr + Expr::Cast(Cast { expr, field }) => expr .map_elements(f)? - .update_data(|be| Expr::Cast(Cast::new(be, data_type))), - Expr::TryCast(TryCast { expr, data_type }) => expr + .update_data(|be| Expr::Cast(Cast::new_from_field(be, field))), + Expr::TryCast(TryCast { expr, field }) => expr .map_elements(f)? - .update_data(|be| Expr::TryCast(TryCast::new(be, data_type))), + .update_data(|be| Expr::TryCast(TryCast::new_from_field(be, field))), Expr::ScalarFunction(ScalarFunction { func, args }) => { args.map_elements(f)?.map_data(|new_args| { Ok(Expr::ScalarFunction(ScalarFunction::new_udf( diff --git a/datafusion/functions/src/core/arrow_cast.rs b/datafusion/functions/src/core/arrow_cast.rs index 94a41ba4bb25..016b9c4bbccd 100644 --- a/datafusion/functions/src/core/arrow_cast.rs +++ b/datafusion/functions/src/core/arrow_cast.rs @@ -19,6 +19,7 @@ use arrow::datatypes::{DataType, Field, FieldRef}; use arrow::error::ArrowError; +use datafusion_common::datatype::DataTypeExt; use datafusion_common::{ arrow_datafusion_err, exec_err, internal_err, Result, ScalarValue, }; @@ -161,7 +162,7 @@ impl ScalarUDFImpl for ArrowCastFunc { // Use an actual cast to get the correct type Expr::Cast(datafusion_expr::Cast { expr: Box::new(arg), - data_type: target_type, + field: target_type.into_nullable_field_ref(), }) }; // return the newly written argument to DataFusion diff --git a/datafusion/optimizer/src/eliminate_outer_join.rs b/datafusion/optimizer/src/eliminate_outer_join.rs index 45877642f276..160c09cde2f5 100644 --- a/datafusion/optimizer/src/eliminate_outer_join.rs +++ b/datafusion/optimizer/src/eliminate_outer_join.rs @@ -289,8 +289,8 @@ fn extract_non_nullable_columns( false, ) } - Expr::Cast(Cast { expr, data_type: _ }) - | Expr::TryCast(TryCast { expr, data_type: _ }) => extract_non_nullable_columns( + Expr::Cast(Cast { expr, field: _ }) + | Expr::TryCast(TryCast { expr, field: _ }) => extract_non_nullable_columns( expr, non_nullable_cols, left_schema, diff --git a/datafusion/physical-expr/src/planner.rs b/datafusion/physical-expr/src/planner.rs index 7790380dffd5..bc80c6df1aa2 100644 --- a/datafusion/physical-expr/src/planner.rs +++ b/datafusion/physical-expr/src/planner.rs @@ -25,7 +25,7 @@ use crate::{ use arrow::datatypes::Schema; use datafusion_common::config::ConfigOptions; -use datafusion_common::metadata::FieldMetadata; +use datafusion_common::metadata::{format_type_and_metadata, FieldMetadata}; use datafusion_common::{ exec_err, not_impl_err, plan_err, DFSchema, Result, ScalarValue, ToDFSchema, }; @@ -34,7 +34,7 @@ use datafusion_expr::expr::{Alias, Cast, InList, Placeholder, ScalarFunction}; use datafusion_expr::var_provider::is_system_variables; use datafusion_expr::var_provider::VarType; use datafusion_expr::{ - binary_expr, lit, Between, BinaryExpr, Expr, Like, Operator, TryCast, + binary_expr, lit, Between, BinaryExpr, Expr, ExprSchemable, Like, Operator, TryCast, }; /// [PhysicalExpr] evaluate DataFusion expressions such as `A + 1`, or `CAST(c1 @@ -287,16 +287,44 @@ pub fn create_physical_expr( }; Ok(expressions::case(expr, when_then_expr, else_expr)?) } - Expr::Cast(Cast { expr, data_type }) => expressions::cast( - create_physical_expr(expr, input_dfschema, execution_props)?, - input_schema, - data_type.clone(), - ), - Expr::TryCast(TryCast { expr, data_type }) => expressions::try_cast( - create_physical_expr(expr, input_dfschema, execution_props)?, - input_schema, - data_type.clone(), - ), + Expr::Cast(Cast { expr, field }) => { + if !field.metadata().is_empty() { + let (_, src_field) = expr.to_field(input_dfschema)?; + return plan_err!( + "Cast from {} to {} is not supported", + format_type_and_metadata( + src_field.data_type(), + Some(src_field.metadata()), + ), + format_type_and_metadata(field.data_type(), Some(field.metadata())) + ); + } + + expressions::cast( + create_physical_expr(expr, input_dfschema, execution_props)?, + input_schema, + field.data_type().clone(), + ) + } + Expr::TryCast(TryCast { expr, field }) => { + if !field.metadata().is_empty() { + let (_, src_field) = expr.to_field(input_dfschema)?; + return plan_err!( + "TryCast from {} to {} is not supported", + format_type_and_metadata( + src_field.data_type(), + Some(src_field.metadata()), + ), + format_type_and_metadata(field.data_type(), Some(field.metadata())) + ); + } + + expressions::try_cast( + create_physical_expr(expr, input_dfschema, execution_props)?, + input_schema, + field.data_type().clone(), + ) + } Expr::Not(expr) => { expressions::not(create_physical_expr(expr, input_dfschema, execution_props)?) } @@ -417,6 +445,7 @@ mod tests { use arrow::array::{ArrayRef, BooleanArray, RecordBatch, StringArray}; use arrow::datatypes::{DataType, Field}; + use datafusion_common::datatype::DataTypeExt; use datafusion_expr::{col, lit}; use super::*; @@ -445,4 +474,38 @@ mod tests { Ok(()) } + + #[test] + fn test_cast_to_extension_type() -> Result<()> { + let extension_field_type = Arc::new( + DataType::FixedSizeBinary(16) + .into_nullable_field() + .with_metadata( + [("ARROW:extension:name".to_string(), "arrow.uuid".to_string())] + .into(), + ), + ); + let expr = lit("3230e5d4-888e-408b-b09b-831f44aa0c58"); + let cast_expr = Expr::Cast(Cast::new_from_field( + Box::new(expr.clone()), + Arc::clone(&extension_field_type), + )); + let err = + create_physical_expr(&cast_expr, &DFSchema::empty(), &ExecutionProps::new()) + .unwrap_err(); + assert!(err.message().contains("arrow.uuid")); + + let try_cast_expr = Expr::TryCast(TryCast::new_from_field( + Box::new(expr.clone()), + Arc::clone(&extension_field_type), + )); + let err = create_physical_expr( + &try_cast_expr, + &DFSchema::empty(), + &ExecutionProps::new(), + ) + .unwrap_err(); + assert!(err.message().contains("arrow.uuid")); + Ok(()) + } } diff --git a/datafusion/proto/proto/datafusion.proto b/datafusion/proto/proto/datafusion.proto index f9400d14a59c..e8a39071f8c5 100644 --- a/datafusion/proto/proto/datafusion.proto +++ b/datafusion/proto/proto/datafusion.proto @@ -593,11 +593,15 @@ message WhenThen { message CastNode { LogicalExprNode expr = 1; datafusion_common.ArrowType arrow_type = 2; + map metadata = 3; + optional bool nullable = 4; } message TryCastNode { LogicalExprNode expr = 1; datafusion_common.ArrowType arrow_type = 2; + map metadata = 3; + optional bool nullable = 4; } message SortExprNode { diff --git a/datafusion/proto/src/generated/pbjson.rs b/datafusion/proto/src/generated/pbjson.rs index 4cf834d0601e..405eed32ed5c 100644 --- a/datafusion/proto/src/generated/pbjson.rs +++ b/datafusion/proto/src/generated/pbjson.rs @@ -1834,6 +1834,12 @@ impl serde::Serialize for CastNode { if self.arrow_type.is_some() { len += 1; } + if !self.metadata.is_empty() { + len += 1; + } + if self.nullable.is_some() { + len += 1; + } let mut struct_ser = serializer.serialize_struct("datafusion.CastNode", len)?; if let Some(v) = self.expr.as_ref() { struct_ser.serialize_field("expr", v)?; @@ -1841,6 +1847,12 @@ impl serde::Serialize for CastNode { if let Some(v) = self.arrow_type.as_ref() { struct_ser.serialize_field("arrowType", v)?; } + if !self.metadata.is_empty() { + struct_ser.serialize_field("metadata", &self.metadata)?; + } + if let Some(v) = self.nullable.as_ref() { + struct_ser.serialize_field("nullable", v)?; + } struct_ser.end() } } @@ -1854,12 +1866,16 @@ impl<'de> serde::Deserialize<'de> for CastNode { "expr", "arrow_type", "arrowType", + "metadata", + "nullable", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { Expr, ArrowType, + Metadata, + Nullable, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -1883,6 +1899,8 @@ impl<'de> serde::Deserialize<'de> for CastNode { match value { "expr" => Ok(GeneratedField::Expr), "arrowType" | "arrow_type" => Ok(GeneratedField::ArrowType), + "metadata" => Ok(GeneratedField::Metadata), + "nullable" => Ok(GeneratedField::Nullable), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -1904,6 +1922,8 @@ impl<'de> serde::Deserialize<'de> for CastNode { { let mut expr__ = None; let mut arrow_type__ = None; + let mut metadata__ = None; + let mut nullable__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::Expr => { @@ -1918,11 +1938,27 @@ impl<'de> serde::Deserialize<'de> for CastNode { } arrow_type__ = map_.next_value()?; } + GeneratedField::Metadata => { + if metadata__.is_some() { + return Err(serde::de::Error::duplicate_field("metadata")); + } + metadata__ = Some( + map_.next_value::>()? + ); + } + GeneratedField::Nullable => { + if nullable__.is_some() { + return Err(serde::de::Error::duplicate_field("nullable")); + } + nullable__ = map_.next_value()?; + } } } Ok(CastNode { expr: expr__, arrow_type: arrow_type__, + metadata: metadata__.unwrap_or_default(), + nullable: nullable__, }) } } @@ -22049,6 +22085,12 @@ impl serde::Serialize for TryCastNode { if self.arrow_type.is_some() { len += 1; } + if !self.metadata.is_empty() { + len += 1; + } + if self.nullable.is_some() { + len += 1; + } let mut struct_ser = serializer.serialize_struct("datafusion.TryCastNode", len)?; if let Some(v) = self.expr.as_ref() { struct_ser.serialize_field("expr", v)?; @@ -22056,6 +22098,12 @@ impl serde::Serialize for TryCastNode { if let Some(v) = self.arrow_type.as_ref() { struct_ser.serialize_field("arrowType", v)?; } + if !self.metadata.is_empty() { + struct_ser.serialize_field("metadata", &self.metadata)?; + } + if let Some(v) = self.nullable.as_ref() { + struct_ser.serialize_field("nullable", v)?; + } struct_ser.end() } } @@ -22069,12 +22117,16 @@ impl<'de> serde::Deserialize<'de> for TryCastNode { "expr", "arrow_type", "arrowType", + "metadata", + "nullable", ]; #[allow(clippy::enum_variant_names)] enum GeneratedField { Expr, ArrowType, + Metadata, + Nullable, } impl<'de> serde::Deserialize<'de> for GeneratedField { fn deserialize(deserializer: D) -> std::result::Result @@ -22098,6 +22150,8 @@ impl<'de> serde::Deserialize<'de> for TryCastNode { match value { "expr" => Ok(GeneratedField::Expr), "arrowType" | "arrow_type" => Ok(GeneratedField::ArrowType), + "metadata" => Ok(GeneratedField::Metadata), + "nullable" => Ok(GeneratedField::Nullable), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), } } @@ -22119,6 +22173,8 @@ impl<'de> serde::Deserialize<'de> for TryCastNode { { let mut expr__ = None; let mut arrow_type__ = None; + let mut metadata__ = None; + let mut nullable__ = None; while let Some(k) = map_.next_key()? { match k { GeneratedField::Expr => { @@ -22133,11 +22189,27 @@ impl<'de> serde::Deserialize<'de> for TryCastNode { } arrow_type__ = map_.next_value()?; } + GeneratedField::Metadata => { + if metadata__.is_some() { + return Err(serde::de::Error::duplicate_field("metadata")); + } + metadata__ = Some( + map_.next_value::>()? + ); + } + GeneratedField::Nullable => { + if nullable__.is_some() { + return Err(serde::de::Error::duplicate_field("nullable")); + } + nullable__ = map_.next_value()?; + } } } Ok(TryCastNode { expr: expr__, arrow_type: arrow_type__, + metadata: metadata__.unwrap_or_default(), + nullable: nullable__, }) } } diff --git a/datafusion/proto/src/generated/prost.rs b/datafusion/proto/src/generated/prost.rs index 12b417627411..800edf9c4a17 100644 --- a/datafusion/proto/src/generated/prost.rs +++ b/datafusion/proto/src/generated/prost.rs @@ -919,6 +919,13 @@ pub struct CastNode { pub expr: ::core::option::Option<::prost::alloc::boxed::Box>, #[prost(message, optional, tag = "2")] pub arrow_type: ::core::option::Option, + #[prost(map = "string, string", tag = "3")] + pub metadata: ::std::collections::HashMap< + ::prost::alloc::string::String, + ::prost::alloc::string::String, + >, + #[prost(bool, optional, tag = "4")] + pub nullable: ::core::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct TryCastNode { @@ -926,6 +933,13 @@ pub struct TryCastNode { pub expr: ::core::option::Option<::prost::alloc::boxed::Box>, #[prost(message, optional, tag = "2")] pub arrow_type: ::core::option::Option, + #[prost(map = "string, string", tag = "3")] + pub metadata: ::std::collections::HashMap< + ::prost::alloc::string::String, + ::prost::alloc::string::String, + >, + #[prost(bool, optional, tag = "4")] + pub nullable: ::core::option::Option, } #[derive(Clone, PartialEq, ::prost::Message)] pub struct SortExprNode { diff --git a/datafusion/proto/src/logical_plan/from_proto.rs b/datafusion/proto/src/logical_plan/from_proto.rs index 598a77f5420e..2a458f9721ed 100644 --- a/datafusion/proto/src/logical_plan/from_proto.rs +++ b/datafusion/proto/src/logical_plan/from_proto.rs @@ -528,7 +528,8 @@ pub fn parse_expr( codec, )?); let data_type = cast.arrow_type.as_ref().required("arrow_type")?; - Ok(Expr::Cast(Cast::new(expr, data_type))) + let field = Field::new("", data_type, cast.nullable.unwrap_or(true)); + Ok(Expr::Cast(Cast::new_from_field(expr, Arc::new(field)))) } ExprType::TryCast(cast) => { let expr = Box::new(parse_required_expr( @@ -538,7 +539,11 @@ pub fn parse_expr( codec, )?); let data_type = cast.arrow_type.as_ref().required("arrow_type")?; - Ok(Expr::TryCast(TryCast::new(expr, data_type))) + let field = Field::new("", data_type, cast.nullable.unwrap_or(true)); + Ok(Expr::TryCast(TryCast::new_from_field( + expr, + Arc::new(field), + ))) } ExprType::Negative(negative) => Ok(Expr::Negative(Box::new( parse_required_expr(negative.expr.as_deref(), registry, "expr", codec)?, diff --git a/datafusion/proto/src/logical_plan/to_proto.rs b/datafusion/proto/src/logical_plan/to_proto.rs index 2774b5b6ba7c..c1b9f30f2de8 100644 --- a/datafusion/proto/src/logical_plan/to_proto.rs +++ b/datafusion/proto/src/logical_plan/to_proto.rs @@ -522,19 +522,23 @@ pub fn serialize_expr( expr_type: Some(ExprType::Case(expr)), } } - Expr::Cast(Cast { expr, data_type }) => { + Expr::Cast(Cast { expr, field }) => { let expr = Box::new(protobuf::CastNode { expr: Some(Box::new(serialize_expr(expr.as_ref(), codec)?)), - arrow_type: Some(data_type.try_into()?), + arrow_type: Some(field.data_type().try_into()?), + metadata: field.metadata().clone(), + nullable: Some(field.is_nullable()), }); protobuf::LogicalExprNode { expr_type: Some(ExprType::Cast(expr)), } } - Expr::TryCast(TryCast { expr, data_type }) => { + Expr::TryCast(TryCast { expr, field }) => { let expr = Box::new(protobuf::TryCastNode { expr: Some(Box::new(serialize_expr(expr.as_ref(), codec)?)), - arrow_type: Some(data_type.try_into()?), + arrow_type: Some(field.data_type().try_into()?), + metadata: field.metadata().clone(), + nullable: Some(field.is_nullable()), }); protobuf::LogicalExprNode { expr_type: Some(ExprType::TryCast(expr)), diff --git a/datafusion/sql/src/unparser/expr.rs b/datafusion/sql/src/unparser/expr.rs index a7fe8efa153c..f51ff4c1ba9a 100644 --- a/datafusion/sql/src/unparser/expr.rs +++ b/datafusion/sql/src/unparser/expr.rs @@ -37,6 +37,7 @@ use arrow::array::{ }; use arrow::datatypes::{ DataType, Decimal128Type, Decimal256Type, Decimal32Type, Decimal64Type, DecimalType, + Field, FieldRef, }; use arrow::util::display::array_value_to_string; use datafusion_common::{ @@ -189,9 +190,7 @@ impl Unparser<'_> { end_token: AttachedToken::empty(), }) } - Expr::Cast(Cast { expr, data_type }) => { - Ok(self.cast_to_sql(expr, data_type)?) - } + Expr::Cast(Cast { expr, field }) => Ok(self.cast_to_sql(expr, field)?), Expr::Literal(value, _) => Ok(self.scalar_to_sql(value)?), Expr::Alias(Alias { expr, name: _, .. }) => self.expr_to_sql_inner(expr), Expr::WindowFunction(window_fun) => { @@ -464,12 +463,12 @@ impl Unparser<'_> { ) }) } - Expr::TryCast(TryCast { expr, data_type }) => { + Expr::TryCast(TryCast { expr, field }) => { let inner_expr = self.expr_to_sql_inner(expr)?; Ok(ast::Expr::Cast { kind: ast::CastKind::TryCast, expr: Box::new(inner_expr), - data_type: self.arrow_dtype_to_ast_dtype(data_type)?, + data_type: self.arrow_dtype_to_ast_dtype(field)?, format: None, }) } @@ -1133,24 +1132,27 @@ impl Unparser<'_> { // Explicit type cast on ast::Expr::Value is not needed by underlying engine for certain types // For example: CAST(Utf8("binary_value") AS Binary) and CAST(Utf8("dictionary_value") AS Dictionary) - fn cast_to_sql(&self, expr: &Expr, data_type: &DataType) -> Result { + fn cast_to_sql(&self, expr: &Expr, field: &FieldRef) -> Result { let inner_expr = self.expr_to_sql_inner(expr)?; + let data_type = field.data_type(); match inner_expr { ast::Expr::Value(_) => match data_type { - DataType::Dictionary(_, _) | DataType::Binary | DataType::BinaryView => { + DataType::Dictionary(_, _) | DataType::Binary | DataType::BinaryView + if field.metadata().is_empty() => + { Ok(inner_expr) } _ => Ok(ast::Expr::Cast { kind: ast::CastKind::Cast, expr: Box::new(inner_expr), - data_type: self.arrow_dtype_to_ast_dtype(data_type)?, + data_type: self.arrow_dtype_to_ast_dtype(field)?, format: None, }), }, _ => Ok(ast::Expr::Cast { kind: ast::CastKind::Cast, expr: Box::new(inner_expr), - data_type: self.arrow_dtype_to_ast_dtype(data_type)?, + data_type: self.arrow_dtype_to_ast_dtype(field)?, format: None, }), } @@ -1672,7 +1674,8 @@ impl Unparser<'_> { })) } - fn arrow_dtype_to_ast_dtype(&self, data_type: &DataType) -> Result { + fn arrow_dtype_to_ast_dtype(&self, field: &FieldRef) -> Result { + let data_type = field.data_type(); match data_type { DataType::Null => { not_impl_err!("Unsupported DataType: conversion: {data_type}") @@ -1745,7 +1748,9 @@ impl Unparser<'_> { DataType::Union(_, _) => { not_impl_err!("Unsupported DataType: conversion: {data_type}") } - DataType::Dictionary(_, val) => self.arrow_dtype_to_ast_dtype(val), + DataType::Dictionary(_, val) => self.arrow_dtype_to_ast_dtype( + &Field::new("", val.as_ref().clone(), true).into(), + ), DataType::Decimal32(precision, scale) | DataType::Decimal64(precision, scale) | DataType::Decimal128(precision, scale) @@ -1885,34 +1890,25 @@ mod tests { r#"CASE WHEN a IS NOT NULL THEN true ELSE false END"#, ), ( - Expr::Cast(Cast { - expr: Box::new(col("a")), - data_type: DataType::Date64, - }), + Expr::Cast(Cast::new(Box::new(col("a")), DataType::Date64)), r#"CAST(a AS DATETIME)"#, ), ( - Expr::Cast(Cast { - expr: Box::new(col("a")), - data_type: DataType::Timestamp( - TimeUnit::Nanosecond, - Some("+08:00".into()), - ), - }), + Expr::Cast(Cast::new( + Box::new(col("a")), + DataType::Timestamp(TimeUnit::Nanosecond, Some("+08:00".into())), + )), r#"CAST(a AS TIMESTAMP WITH TIME ZONE)"#, ), ( - Expr::Cast(Cast { - expr: Box::new(col("a")), - data_type: DataType::Timestamp(TimeUnit::Millisecond, None), - }), + Expr::Cast(Cast::new( + Box::new(col("a")), + DataType::Timestamp(TimeUnit::Millisecond, None), + )), r#"CAST(a AS TIMESTAMP)"#, ), ( - Expr::Cast(Cast { - expr: Box::new(col("a")), - data_type: DataType::UInt32, - }), + Expr::Cast(Cast::new(Box::new(col("a")), DataType::UInt32)), r#"CAST(a AS INTEGER UNSIGNED)"#, ), ( @@ -2227,10 +2223,7 @@ mod tests { r#"((a + b) > 100.123)"#, ), ( - Expr::Cast(Cast { - expr: Box::new(col("a")), - data_type: DataType::Decimal128(10, -2), - }), + Expr::Cast(Cast::new(Box::new(col("a")), DataType::Decimal128(10, -2))), r#"CAST(a AS DECIMAL(12,0))"#, ), ( @@ -2367,10 +2360,7 @@ mod tests { .build(); let unparser = Unparser::new(&dialect); - let expr = Expr::Cast(Cast { - expr: Box::new(col("a")), - data_type: DataType::Date64, - }); + let expr = Expr::Cast(Cast::new(Box::new(col("a")), DataType::Date64)); let ast = unparser.expr_to_sql(&expr)?; let actual = format!("{ast}"); @@ -2392,10 +2382,7 @@ mod tests { .build(); let unparser = Unparser::new(&dialect); - let expr = Expr::Cast(Cast { - expr: Box::new(col("a")), - data_type: DataType::Float64, - }); + let expr = Expr::Cast(Cast::new(Box::new(col("a")), DataType::Float64)); let ast = unparser.expr_to_sql(&expr)?; let actual = format!("{ast}"); @@ -2625,23 +2612,23 @@ mod tests { fn test_cast_value_to_binary_expr() { let tests = [ ( - Expr::Cast(Cast { - expr: Box::new(Expr::Literal( + Expr::Cast(Cast::new( + Box::new(Expr::Literal( ScalarValue::Utf8(Some("blah".to_string())), None, )), - data_type: DataType::Binary, - }), + DataType::Binary, + )), "'blah'", ), ( - Expr::Cast(Cast { - expr: Box::new(Expr::Literal( + Expr::Cast(Cast::new( + Box::new(Expr::Literal( ScalarValue::Utf8(Some("blah".to_string())), None, )), - data_type: DataType::BinaryView, - }), + DataType::BinaryView, + )), "'blah'", ), ]; @@ -2672,10 +2659,7 @@ mod tests { ] { let unparser = Unparser::new(dialect); - let expr = Expr::Cast(Cast { - expr: Box::new(col("a")), - data_type, - }); + let expr = Expr::Cast(Cast::new(Box::new(col("a")), data_type)); let ast = unparser.expr_to_sql(&expr)?; let actual = format!("{ast}"); @@ -2758,10 +2742,7 @@ mod tests { [(default_dialect, "BIGINT"), (mysql_dialect, "SIGNED")] { let unparser = Unparser::new(&dialect); - let expr = Expr::Cast(Cast { - expr: Box::new(col("a")), - data_type: DataType::Int64, - }); + let expr = Expr::Cast(Cast::new(Box::new(col("a")), DataType::Int64)); let ast = unparser.expr_to_sql(&expr)?; let actual = format!("{ast}"); @@ -2786,10 +2767,7 @@ mod tests { [(default_dialect, "INTEGER"), (mysql_dialect, "SIGNED")] { let unparser = Unparser::new(&dialect); - let expr = Expr::Cast(Cast { - expr: Box::new(col("a")), - data_type: DataType::Int32, - }); + let expr = Expr::Cast(Cast::new(Box::new(col("a")), DataType::Int32)); let ast = unparser.expr_to_sql(&expr)?; let actual = format!("{ast}"); @@ -2825,10 +2803,7 @@ mod tests { (&mysql_dialect, ×tamp_with_tz, "DATETIME"), ] { let unparser = Unparser::new(dialect); - let expr = Expr::Cast(Cast { - expr: Box::new(col("a")), - data_type: data_type.clone(), - }); + let expr = Expr::Cast(Cast::new(Box::new(col("a")), data_type.clone())); let ast = unparser.expr_to_sql(&expr)?; let actual = format!("{ast}"); @@ -2881,10 +2856,7 @@ mod tests { ] { let unparser = Unparser::new(dialect); - let expr = Expr::Cast(Cast { - expr: Box::new(col("a")), - data_type, - }); + let expr = Expr::Cast(Cast::new(Box::new(col("a")), data_type)); let ast = unparser.expr_to_sql(&expr)?; let actual = format!("{ast}"); @@ -2924,13 +2896,13 @@ mod tests { #[test] fn test_cast_value_to_dict_expr() { let tests = [( - Expr::Cast(Cast { - expr: Box::new(Expr::Literal( + Expr::Cast(Cast::new( + Box::new(Expr::Literal( ScalarValue::Utf8(Some("variation".to_string())), None, )), - data_type: DataType::Dictionary(Box::new(Int8), Box::new(DataType::Utf8)), - }), + DataType::Dictionary(Box::new(Int8), Box::new(DataType::Utf8)), + )), "'variation'", )]; for (value, expected) in tests { @@ -2962,10 +2934,7 @@ mod tests { datafusion_functions::math::round::RoundFunc::new(), )), args: vec![ - Expr::Cast(Cast { - expr: Box::new(col("a")), - data_type: DataType::Float64, - }), + Expr::Cast(Cast::new(Box::new(col("a")), DataType::Float64)), Expr::Literal(ScalarValue::Int64(Some(2)), None), ], }); @@ -3127,10 +3096,12 @@ mod tests { let unparser = Unparser::new(&dialect); - let ast_dtype = unparser.arrow_dtype_to_ast_dtype(&DataType::Dictionary( - Box::new(DataType::Int32), - Box::new(DataType::Utf8), - ))?; + let arrow_field = Arc::new(Field::new( + "", + DataType::Dictionary(Box::new(DataType::Int32), Box::new(DataType::Utf8)), + true, + )); + let ast_dtype = unparser.arrow_dtype_to_ast_dtype(&arrow_field)?; assert_eq!(ast_dtype, ast::DataType::Varchar(None)); @@ -3144,7 +3115,8 @@ mod tests { .build(); let unparser = Unparser::new(&dialect); - let ast_dtype = unparser.arrow_dtype_to_ast_dtype(&DataType::Utf8View)?; + let arrow_field = Arc::new(Field::new("", DataType::Utf8View, true)); + let ast_dtype = unparser.arrow_dtype_to_ast_dtype(&arrow_field)?; assert_eq!(ast_dtype, ast::DataType::Char(None)); @@ -3212,10 +3184,10 @@ mod tests { let dialect: Arc = Arc::new(SqliteDialect {}); let unparser = Unparser::new(dialect.as_ref()); - let expr = Expr::Cast(Cast { - expr: Box::new(col("a")), - data_type: DataType::Timestamp(TimeUnit::Nanosecond, None), - }); + let expr = Expr::Cast(Cast::new( + Box::new(col("a")), + DataType::Timestamp(TimeUnit::Nanosecond, None), + )); let ast = unparser.expr_to_sql(&expr)?; diff --git a/datafusion/substrait/src/logical_plan/consumer/expr/cast.rs b/datafusion/substrait/src/logical_plan/consumer/expr/cast.rs index 5e8d3d93065f..ff88464b8bb0 100644 --- a/datafusion/substrait/src/logical_plan/consumer/expr/cast.rs +++ b/datafusion/substrait/src/logical_plan/consumer/expr/cast.rs @@ -15,8 +15,9 @@ // specific language governing permissions and limitations // under the License. -use crate::logical_plan::consumer::types::from_substrait_type_without_names; -use crate::logical_plan::consumer::SubstraitConsumer; +use crate::logical_plan::consumer::{ + field_from_substrait_type_without_names, SubstraitConsumer, +}; use datafusion::common::{substrait_err, DFSchema}; use datafusion::logical_expr::{Cast, Expr, TryCast}; use substrait::proto::expression as substrait_expression; @@ -37,11 +38,11 @@ pub async fn from_cast( ) .await?, ); - let data_type = from_substrait_type_without_names(consumer, output_type)?; + let field = field_from_substrait_type_without_names(consumer, output_type)?; if cast.failure_behavior() == ReturnNull { - Ok(Expr::TryCast(TryCast::new(input_expr, data_type))) + Ok(Expr::TryCast(TryCast::new_from_field(input_expr, field))) } else { - Ok(Expr::Cast(Cast::new(input_expr, data_type))) + Ok(Expr::Cast(Cast::new_from_field(input_expr, field))) } } None => substrait_err!("Cast expression without output type is not allowed"), diff --git a/datafusion/substrait/src/logical_plan/consumer/types.rs b/datafusion/substrait/src/logical_plan/consumer/types.rs index 772ea7177ca2..738620c5f5aa 100644 --- a/datafusion/substrait/src/logical_plan/consumer/types.rs +++ b/datafusion/substrait/src/logical_plan/consumer/types.rs @@ -34,14 +34,22 @@ use crate::variation_const::{ VIEW_CONTAINER_TYPE_VARIATION_REF, }; use datafusion::arrow::datatypes::{ - DataType, Field, Fields, IntervalUnit, Schema, TimeUnit, + DataType, Field, FieldRef, Fields, IntervalUnit, Schema, TimeUnit, }; +use datafusion::common::datatype::DataTypeExt; use datafusion::common::{ not_impl_err, substrait_datafusion_err, substrait_err, DFSchema, }; use std::sync::Arc; use substrait::proto::{r#type, NamedStruct, Type}; +pub(crate) fn field_from_substrait_type_without_names( + consumer: &impl SubstraitConsumer, + dt: &Type, +) -> datafusion::common::Result { + Ok(from_substrait_type_without_names(consumer, dt)?.into_nullable_field_ref()) +} + pub(crate) fn from_substrait_type_without_names( consumer: &impl SubstraitConsumer, dt: &Type, @@ -49,6 +57,16 @@ pub(crate) fn from_substrait_type_without_names( from_substrait_type(consumer, dt, &[], &mut 0) } +pub fn field_from_substrait_type( + consumer: &impl SubstraitConsumer, + dt: &Type, + dfs_names: &[String], + name_idx: &mut usize, +) -> datafusion::common::Result { + // We could add nullability here now that we are returning a Field + Ok(from_substrait_type(consumer, dt, dfs_names, name_idx)?.into_nullable_field_ref()) +} + pub fn from_substrait_type( consumer: &impl SubstraitConsumer, dt: &Type, diff --git a/datafusion/substrait/src/logical_plan/producer/expr/cast.rs b/datafusion/substrait/src/logical_plan/producer/expr/cast.rs index 71c2140bac8b..889a285eb6a0 100644 --- a/datafusion/substrait/src/logical_plan/producer/expr/cast.rs +++ b/datafusion/substrait/src/logical_plan/producer/expr/cast.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use crate::logical_plan::producer::{to_substrait_type, SubstraitProducer}; +use crate::logical_plan::producer::{to_substrait_type_from_field, SubstraitProducer}; use crate::variation_const::DEFAULT_TYPE_VARIATION_REF; use datafusion::common::{DFSchemaRef, ScalarValue}; use datafusion::logical_expr::{Cast, Expr, TryCast}; @@ -29,7 +29,7 @@ pub fn from_cast( cast: &Cast, schema: &DFSchemaRef, ) -> datafusion::common::Result { - let Cast { expr, data_type } = cast; + let Cast { expr, field } = cast; // since substrait Null must be typed, so if we see a cast(null, dt), we make it a typed null if let Expr::Literal(lit, _) = expr.as_ref() { // only the untyped(a null scalar value) null literal need this special handling @@ -39,8 +39,8 @@ pub fn from_cast( let lit = Literal { nullable: true, type_variation_reference: DEFAULT_TYPE_VARIATION_REF, - literal_type: Some(LiteralType::Null(to_substrait_type( - producer, data_type, true, + literal_type: Some(LiteralType::Null(to_substrait_type_from_field( + producer, field, )?)), }; return Ok(Expression { @@ -51,7 +51,7 @@ pub fn from_cast( Ok(Expression { rex_type: Some(RexType::Cast(Box::new( substrait::proto::expression::Cast { - r#type: Some(to_substrait_type(producer, data_type, true)?), + r#type: Some(to_substrait_type_from_field(producer, field)?), input: Some(Box::new(producer.handle_expr(expr, schema)?)), failure_behavior: FailureBehavior::ThrowException.into(), }, @@ -64,11 +64,11 @@ pub fn from_try_cast( cast: &TryCast, schema: &DFSchemaRef, ) -> datafusion::common::Result { - let TryCast { expr, data_type } = cast; + let TryCast { expr, field } = cast; Ok(Expression { rex_type: Some(RexType::Cast(Box::new( substrait::proto::expression::Cast { - r#type: Some(to_substrait_type(producer, data_type, true)?), + r#type: Some(to_substrait_type_from_field(producer, field)?), input: Some(Box::new(producer.handle_expr(expr, schema)?)), failure_behavior: FailureBehavior::ReturnNull.into(), }, @@ -80,7 +80,7 @@ pub fn from_try_cast( mod tests { use super::*; use crate::logical_plan::producer::{ - to_substrait_extended_expr, DefaultSubstraitProducer, + to_substrait_extended_expr, to_substrait_type, DefaultSubstraitProducer, }; use datafusion::arrow::datatypes::{DataType, Field}; use datafusion::common::DFSchema; diff --git a/datafusion/substrait/src/logical_plan/producer/types.rs b/datafusion/substrait/src/logical_plan/producer/types.rs index 2079d7fd34bb..99f5ecec6690 100644 --- a/datafusion/substrait/src/logical_plan/producer/types.rs +++ b/datafusion/substrait/src/logical_plan/producer/types.rs @@ -27,7 +27,7 @@ use crate::variation_const::{ TIME_64_TYPE_VARIATION_REF, UNSIGNED_INTEGER_TYPE_VARIATION_REF, VIEW_CONTAINER_TYPE_VARIATION_REF, }; -use datafusion::arrow::datatypes::{DataType, IntervalUnit}; +use datafusion::arrow::datatypes::{DataType, Field, FieldRef, IntervalUnit}; use datafusion::common::{internal_err, not_impl_err, plan_err, DFSchemaRef}; use substrait::proto::{r#type, NamedStruct}; @@ -36,12 +36,19 @@ pub(crate) fn to_substrait_type( dt: &DataType, nullable: bool, ) -> datafusion::common::Result { - let nullability = if nullable { + to_substrait_type_from_field(producer, &Field::new("", dt.clone(), nullable).into()) +} + +pub(crate) fn to_substrait_type_from_field( + producer: &mut impl SubstraitProducer, + field: &FieldRef, +) -> datafusion::common::Result { + let nullability = if field.is_nullable() { r#type::Nullability::Nullable as i32 } else { r#type::Nullability::Required as i32 }; - match dt { + match field.data_type() { DataType::Null => internal_err!("Null cast is not valid"), DataType::Boolean => Ok(substrait::proto::Type { kind: Some(r#type::Kind::Bool(r#type::Boolean { @@ -255,8 +262,7 @@ pub(crate) fn to_substrait_type( })), }), DataType::List(inner) => { - let inner_type = - to_substrait_type(producer, inner.data_type(), inner.is_nullable())?; + let inner_type = to_substrait_type_from_field(producer, inner)?; Ok(substrait::proto::Type { kind: Some(r#type::Kind::List(Box::new(r#type::List { r#type: Some(Box::new(inner_type)), @@ -266,8 +272,7 @@ pub(crate) fn to_substrait_type( }) } DataType::LargeList(inner) => { - let inner_type = - to_substrait_type(producer, inner.data_type(), inner.is_nullable())?; + let inner_type = to_substrait_type_from_field(producer, inner)?; Ok(substrait::proto::Type { kind: Some(r#type::Kind::List(Box::new(r#type::List { r#type: Some(Box::new(inner_type)), @@ -278,16 +283,9 @@ pub(crate) fn to_substrait_type( } DataType::Map(inner, _) => match inner.data_type() { DataType::Struct(key_and_value) if key_and_value.len() == 2 => { - let key_type = to_substrait_type( - producer, - key_and_value[0].data_type(), - key_and_value[0].is_nullable(), - )?; - let value_type = to_substrait_type( - producer, - key_and_value[1].data_type(), - key_and_value[1].is_nullable(), - )?; + let key_type = to_substrait_type_from_field(producer, &key_and_value[0])?; + let value_type = + to_substrait_type_from_field(producer, &key_and_value[1])?; Ok(substrait::proto::Type { kind: Some(r#type::Kind::Map(Box::new(r#type::Map { key: Some(Box::new(key_type)), @@ -300,8 +298,14 @@ pub(crate) fn to_substrait_type( _ => plan_err!("Map fields must contain a Struct with exactly 2 fields"), }, DataType::Dictionary(key_type, value_type) => { - let key_type = to_substrait_type(producer, key_type, nullable)?; - let value_type = to_substrait_type(producer, value_type, nullable)?; + let key_type = to_substrait_type_from_field( + producer, + &Field::new("", key_type.as_ref().clone(), field.is_nullable()).into(), + )?; + let value_type = to_substrait_type_from_field( + producer, + &Field::new("", value_type.as_ref().clone(), field.is_nullable()).into(), + )?; Ok(substrait::proto::Type { kind: Some(r#type::Kind::Map(Box::new(r#type::Map { key: Some(Box::new(key_type)), @@ -314,9 +318,7 @@ pub(crate) fn to_substrait_type( DataType::Struct(fields) => { let field_types = fields .iter() - .map(|field| { - to_substrait_type(producer, field.data_type(), field.is_nullable()) - }) + .map(|field| to_substrait_type_from_field(producer, field)) .collect::>>()?; Ok(substrait::proto::Type { kind: Some(r#type::Kind::Struct(r#type::Struct { @@ -342,7 +344,7 @@ pub(crate) fn to_substrait_type( precision: *p as i32, })), }), - _ => not_impl_err!("Unsupported cast type: {dt}"), + _ => not_impl_err!("Unsupported cast type: {field}"), } } @@ -359,7 +361,7 @@ pub(crate) fn to_substrait_named_struct( types: schema .fields() .iter() - .map(|f| to_substrait_type(producer, f.data_type(), f.is_nullable())) + .map(|f| to_substrait_type_from_field(producer, f)) .collect::>()?, type_variation_reference: DEFAULT_TYPE_VARIATION_REF, nullability: r#type::Nullability::Required as i32,