diff --git a/datafusion/functions/src/datetime/to_date.rs b/datafusion/functions/src/datetime/to_date.rs index cd75ac6bed3ac..ed5b8b16320b7 100644 --- a/datafusion/functions/src/datetime/to_date.rs +++ b/datafusion/functions/src/datetime/to_date.rs @@ -61,7 +61,7 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo name = "format_n", description = r"Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression - an error will be returned." + an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL." ) )] #[derive(Debug, PartialEq, Eq, Hash)] @@ -519,4 +519,44 @@ mod tests { panic!("Conversion of {date_str} succeeded, but should have failed. "); } } + + /// A NULL format must be skipped even when its slot still holds parseable + /// bytes, otherwise it can silently win over a later valid format. + #[test] + fn test_to_date_null_format_slot_retaining_bytes() { + use arrow::buffer::NullBuffer; + + // The first format physically holds "%d/%m/%Y", but is marked NULL. + let (offsets, values, _) = + GenericStringArray::::from(vec!["%d/%m/%Y"]).into_parts(); + let formats = + GenericStringArray::new(offsets, values, Some(NullBuffer::new_null(1))); + assert!(formats.is_null(0)); + assert_eq!(formats.value(0), "%d/%m/%Y"); + + // Without the validity check, the first format parses this as 2023-02-01 + // and incorrectly wins over the valid second format. + let values = GenericStringArray::::from(vec!["01/02/2023"]); + let fallback_formats = GenericStringArray::::from(vec!["%m/%d/%Y"]); + let res = invoke_to_date_with_args( + vec![ + ColumnarValue::Array(Arc::new(values)), + ColumnarValue::Array(Arc::new(formats)), + ColumnarValue::Array(Arc::new(fallback_formats)), + ], + 1, + ) + .unwrap(); + + let ColumnarValue::Array(res) = res else { + panic!("expected an array result"); + }; + let res = res.as_any().downcast_ref::().unwrap(); + + assert!(!res.is_null(0)); + assert_eq!( + res.value(0), + Date32Type::parse_formatted("01/02/2023", "%m/%d/%Y").unwrap() + ); + } } diff --git a/datafusion/functions/src/datetime/to_timestamp.rs b/datafusion/functions/src/datetime/to_timestamp.rs index f4507ab250559..1b45910f7261c 100644 --- a/datafusion/functions/src/datetime/to_timestamp.rs +++ b/datafusion/functions/src/datetime/to_timestamp.rs @@ -81,7 +81,8 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo description = r#" Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is +parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. +Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. "# ) @@ -131,7 +132,8 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo description = r#" Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is +parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. +Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. "# ) @@ -181,7 +183,8 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo description = r#" Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is +parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. +Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. "# ) @@ -231,7 +234,8 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo description = r#" Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is +parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. +Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. "# ) @@ -280,7 +284,8 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo description = r#" Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully -parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is +parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. +Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. "# ) diff --git a/datafusion/functions/src/datetime/to_unixtime.rs b/datafusion/functions/src/datetime/to_unixtime.rs index 9fcfd254ca74d..5b9734c05d7be 100644 --- a/datafusion/functions/src/datetime/to_unixtime.rs +++ b/datafusion/functions/src/datetime/to_unixtime.rs @@ -56,7 +56,7 @@ Integers, unsigned integers, and floats are interpreted as seconds since the uni ), argument( name = "format_n", - description = "Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned." + description = "Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL." ) )] #[derive(Debug, PartialEq, Eq, Hash)] diff --git a/datafusion/sqllogictest/test_files/datetime/dates.slt b/datafusion/sqllogictest/test_files/datetime/dates.slt index 68d87eceed99e..abf92e15659e5 100644 --- a/datafusion/sqllogictest/test_files/datetime/dates.slt +++ b/datafusion/sqllogictest/test_files/datetime/dates.slt @@ -319,6 +319,14 @@ ORDER BY id 1 2020-09-08 2 NULL +# Skipping NULL formats does not mask a later parse error. +query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08' using format '%q': trailing input +SELECT to_date('2020-09-08', NULL::VARCHAR, '%q') + +# Invalid format types are rejected before NULL input propagation. +query error DataFusion error: Execution error: to_date function unsupported data type at index 1: Int64 +SELECT to_date(NULL::VARCHAR, 12345) + statement ok create table ts_utf8_data(ts varchar(100), format varchar(100)) as values ('2020-09-08 12/00/00+00:00', '%Y-%m-%d %H/%M/%S%#z'), diff --git a/docs/source/user-guide/sql/scalar_functions.md b/docs/source/user-guide/sql/scalar_functions.md index a285b9e5f5cff..a865a3d182404 100644 --- a/docs/source/user-guide/sql/scalar_functions.md +++ b/docs/source/user-guide/sql/scalar_functions.md @@ -2851,7 +2851,7 @@ to_date('2017-05-31', '%Y-%m-%d') - **expression**: String expression to operate on. Can be a constant, column, or function, and any combination of operators. - **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression - an error will be returned. + an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. #### Example @@ -3006,7 +3006,8 @@ to_timestamp(expression[, ..., format_n]) - **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is + parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. + Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. #### Example @@ -3050,7 +3051,8 @@ to_timestamp_micros(expression[, ..., format_n]) - **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is + parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. + Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. #### Example @@ -3094,7 +3096,8 @@ to_timestamp_millis(expression[, ..., format_n]) - **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is + parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. + Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. #### Example @@ -3137,7 +3140,8 @@ to_timestamp_nanos(expression[, ..., format_n]) - **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is + parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. + Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. #### Example @@ -3181,7 +3185,8 @@ to_timestamp_seconds(expression[, ..., format_n]) - **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully - parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is + parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. + Note: parsing of named timezones (e.g. 'America/New_York') using %Z is only supported at the end of the string preceded by a space. #### Example @@ -3218,7 +3223,7 @@ to_unixtime(expression[, ..., format_n]) #### Arguments - **expression**: Expression to operate on. Can be a constant, column, or function, and any combination of arithmetic operators. -- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. +- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL. #### Example