test: add regression coverage and docs for NULL format handling

datafusion/functions/src/datetime/to_date.rs

@@ -61,7 +61,7 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo
         name = "format_n",
         description = r"Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order
   they appear with the first successful one being returned. If none of the formats successfully parse the expression
-  an error will be returned."
+  an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL."
     )
 )]
 #[derive(Debug, PartialEq, Eq, Hash)]
@@ -519,4 +519,44 @@ mod tests {
             panic!("Conversion of {date_str} succeeded, but should have failed. ");
         }
     }
+
+    /// A NULL format must be skipped even when its slot still holds parseable
+    /// bytes, otherwise it can silently win over a later valid format.
+    #[test]
+    fn test_to_date_null_format_slot_retaining_bytes() {
+        use arrow::buffer::NullBuffer;
+
+        // The first format physically holds "%d/%m/%Y", but is marked NULL.
+        let (offsets, values, _) =
+            GenericStringArray::<i32>::from(vec!["%d/%m/%Y"]).into_parts();
+        let formats =
+            GenericStringArray::new(offsets, values, Some(NullBuffer::new_null(1)));
+        assert!(formats.is_null(0));
+        assert_eq!(formats.value(0), "%d/%m/%Y");
+
+        // Without the validity check, the first format parses this as 2023-02-01
+        // and incorrectly wins over the valid second format.
+        let values = GenericStringArray::<i32>::from(vec!["01/02/2023"]);
+        let fallback_formats = GenericStringArray::<i32>::from(vec!["%m/%d/%Y"]);
+        let res = invoke_to_date_with_args(
+            vec![
+                ColumnarValue::Array(Arc::new(values)),
+                ColumnarValue::Array(Arc::new(formats)),
+                ColumnarValue::Array(Arc::new(fallback_formats)),
+            ],
+            1,
+        )
+        .unwrap();
+
+        let ColumnarValue::Array(res) = res else {
+            panic!("expected an array result");
+        };
+        let res = res.as_any().downcast_ref::<Date32Array>().unwrap();
+
+        assert!(!res.is_null(0));
+        assert_eq!(
+            res.value(0),
+            Date32Type::parse_formatted("01/02/2023", "%m/%d/%Y").unwrap()
+        );
+    }
 }

datafusion/functions/src/datetime/to_timestamp.rs

@@ -81,7 +81,8 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo
         description = r#"
 Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression.
 Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully
-parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
+parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL.
+Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
 only supported at the end of the string preceded by a space.
 "#
     )
@@ -131,7 +132,8 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo
         description = r#"
 Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression.
 Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully
-parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
+parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL.
+Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
 only supported at the end of the string preceded by a space.
 "#
     )
@@ -181,7 +183,8 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo
         description = r#"
 Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression.
 Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully
-parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
+parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL.
+Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
 only supported at the end of the string preceded by a space.
 "#
     )
@@ -231,7 +234,8 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo
         description = r#"
 Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression.
 Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully
-parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
+parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL.
+Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
 only supported at the end of the string preceded by a space.
 "#
     )
@@ -280,7 +284,8 @@ Additional examples can be found [here](https://github.com/apache/datafusion/blo
         description = r#"
 Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression.
 Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully
-parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
+parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL.
+Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
 only supported at the end of the string preceded by a space.
 "#
     )

datafusion/functions/src/datetime/to_unixtime.rs

@@ -56,7 +56,7 @@ Integers, unsigned integers, and floats are interpreted as seconds since the uni
     ),
     argument(
         name = "format_n",
-        description = "Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned."
+        description = "Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL."
     )
 )]
 #[derive(Debug, PartialEq, Eq, Hash)]

datafusion/sqllogictest/test_files/datetime/dates.slt

@@ -319,6 +319,14 @@ ORDER BY id
 1 2020-09-08
 2 NULL
 
+# Skipping NULL formats does not mask a later parse error.
+query error DataFusion error: Execution error: Error parsing timestamp from '2020\-09\-08' using format '%q': trailing input
+SELECT to_date('2020-09-08', NULL::VARCHAR, '%q')
+
+# Invalid format types are rejected before NULL input propagation.
+query error DataFusion error: Execution error: to_date function unsupported data type at index 1: Int64
+SELECT to_date(NULL::VARCHAR, 12345)
+
 statement ok
 create table ts_utf8_data(ts varchar(100), format varchar(100)) as values
   ('2020-09-08 12/00/00+00:00', '%Y-%m-%d %H/%M/%S%#z'),

docs/source/user-guide/sql/scalar_functions.md

@@ -2851,7 +2851,7 @@ to_date('2017-05-31', '%Y-%m-%d')
 - **expression**: String expression to operate on. Can be a constant, column, or function, and any combination of operators.
 - **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order
   they appear with the first successful one being returned. If none of the formats successfully parse the expression
-  an error will be returned.
+  an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL.
 
 #### Example
 
@@ -3006,7 +3006,8 @@ to_timestamp(expression[, ..., format_n])
 - **format_n**:
   Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression.
   Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully
-  parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
+  parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL.
+  Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
   only supported at the end of the string preceded by a space.
 
 #### Example
@@ -3050,7 +3051,8 @@ to_timestamp_micros(expression[, ..., format_n])
 - **format_n**:
   Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression.
   Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully
-  parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
+  parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL.
+  Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
   only supported at the end of the string preceded by a space.
 
 #### Example
@@ -3094,7 +3096,8 @@ to_timestamp_millis(expression[, ..., format_n])
 - **format_n**:
   Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression.
   Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully
-  parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
+  parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL.
+  Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
   only supported at the end of the string preceded by a space.
 
 #### Example
@@ -3137,7 +3140,8 @@ to_timestamp_nanos(expression[, ..., format_n])
 - **format_n**:
   Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression.
   Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully
-  parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
+  parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL.
+  Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
   only supported at the end of the string preceded by a space.
 
 #### Example
@@ -3181,7 +3185,8 @@ to_timestamp_seconds(expression[, ..., format_n])
 - **format_n**:
   Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression.
   Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully
-  parse the expression an error will be returned. Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
+  parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL.
+  Note: parsing of named timezones (e.g. 'America/New_York') using %Z is
   only supported at the end of the string preceded by a space.
 
 #### Example
@@ -3218,7 +3223,7 @@ to_unixtime(expression[, ..., format_n])
 #### Arguments
 
 - **expression**: Expression to operate on. Can be a constant, column, or function, and any combination of arithmetic operators.
-- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned.
+- **format_n**: Optional [Chrono format](https://docs.rs/chrono/latest/chrono/format/strftime/index.html) strings to use to parse the expression. Formats will be tried in the order they appear with the first successful one being returned. If none of the formats successfully parse the expression an error will be returned. NULL formats are skipped. If every format is NULL the result is NULL.
 
 #### Example